K8S基于Jenkins实现SpringCloud微服务CI与CD实践（三）

# K8S 基于 Jenkins 实现 SpringCloud 微服务 CI 与 CD 实践（三）

CI 阶段：开发人员 -> 提交代码 ->gitlab 仓库 ->Jenkins/CI 抓取代码 -> 漏洞扫描 -> 编译 -> 构建镜像 -> 推送 Harbor-> 部署应用至 K8S 测试环境；
CD 阶段：Jenkins/CD-> 拉取 Harbor 仓库对应项目镜像 -> 部署应用至 K8S 测试环境；

# 一、环境准备

准备好 Java 代码、Dockerfile、deploy.yaml 资源清单文件提交到 Gitlab 服务器；

	$ git config --global user.name "oldxu"
	$ git config --global user.email
	$ git init
	$ git add .
	$ git commit -m "first commit"
	$ git remote remove origin
	$ git remote add origin http://gitlab.hmallleasing.com/root/ruoyi-cloud-cicd.git
	$ git remote -v
	origin http://gitlab.hmallleasing.com/root/ruoyi-cloud-cicd.gi (fetch)
	origin http://gitlab.hmallleasing.com/root/ruoyi-cloud-cicd.gi (push)
	$ git push -u origin "master"

# 二、CI 阶段

CI 阶段分为以下 6 步：
1、获取代码
2、漏洞扫描
3、检测漏洞扫描结果，如果正常则继续、否则就终止
4、使用 maven 进行编译，打包
5、制作 Docker 镜像、推送到 Harbor 仓库 (Dockerfile 文件)
6、交付应用到 K8S

# 2.1 编写 Pipeline 框架

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}
	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	sh 'echo "获取代码"'
	}
	}
	}
	stage('代码扫描') {
	steps {
	container('sonar') {
	sh 'echo "代码扫描"'
	}
	}
	}
	stage('检查扫描结果') {
	steps {
	container('sonar') {
	sh 'echo "Code Result Check"'
	}
	}
	}
	stage('代码编译') {
	steps {
	container('maven') {
	sh 'echo "代码编译"'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.2 获取代码

1、Jenkins 上配置 Gitlab 认证信息

系统管理 ->manage credentials-> 全局 ->Add Credential

2、SlavePod 访问 Gitlab

通过 Coredns 配置自定义域名解析，如果 sgitlab 解析至公网，无需配置此步骤；

	[root@k8s-master01 ~]# kubectl edit cm coredns -n kube-system
	...
	ready

	hosts {
	192.168.1.74 gitlab.hmallleasing.com
	fallthrough
	}
	kubernetes cluster.local in-addr.arpa ip6.arpa {
	...

测试解析是否生效；

	[root@k8s-master01 ~]# kubectl exec -it gitlab-0 -n ops -- /bin/bash
	root@gitlab-0:/# ping gitlab.hmallleasing.com
	PING gitlab.hmallleasing.com (192.168.1.74): 56 data bytes
	64 bytes from 192.168.1.74: seq=0 ttl=63 time=0.381 ms
	64 bytes from 192.168.1.74: seq=1 ttl=63 time=0.203 ms

3、生成流水线脚本获取代码

通过流水线语法 -> 片段生成器 -> 生成流水线脚本，生成 Pipeline 流水线

4、测试获取代码 Pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/ruoyi-cloud-cicd.git"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}
	stage('代码扫描') {
	steps {
	container('sonar') {
	sh 'echo "代码扫描"'
	}
	}
	}
	stage('检查扫描结果') {
	steps {
	container('sonar') {
	sh 'echo "Code Result Check"'
	}
	}
	}
	stage('代码编译') {
	steps {
	container('maven') {
	sh 'echo "代码编译"'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.3 Sonarqube 漏洞扫描

1、配置 dns 解析（如果 sonarqube 解析至公网，无需配置此步骤）
slavePod 需要访问 sonarqube 服务端，sonarqube 服务端需要回调 jenkins，获取扫描结果。

	[root@k8s-master01 ~]# kubectl edit cm -n kube-system coredns
	...
	ready

	hosts {
	192.168.1.74 gitlab.hmallleasing.com #gitalb
	192.168.1.74 sonar.hmallleasing.com #sonar
	192.168.1.74 jenkins.hmallleasing.com #jenkins
	fallthrough
	}
	kubernetes cluster.local in-addr.arpa ip6.arpa {
	...

测试解析是否生效；

	[root@k8s-master01 ~]# kubectl exec -it gitlab-0 -n ops -- /bin/bash
	root@gitlab-0:/# ping sonar.hmallleasing.com
	PING sonar.hmallleasing.com (192.168.1.74): 56 data bytes
	64 bytes from 192.168.1.74: seq=0 ttl=63 time=0.187 ms
	64 bytes from 192.168.1.74: seq=1 ttl=63 time=0.202 ms

	root@gitlab-0:/# ping jenkins.hmallleasing.com
	PING jenkins.hmallleasing.com (192.168.1.74): 56 data bytes
	64 bytes from 192.168.1.74: seq=0 ttl=63 time=0.189 ms
	64 bytes from 192.168.1.74: seq=1 ttl=63 time=0.202 ms

2、Jenkins 集成 Sonarqube

生成 sonarqube 令牌：用户 -> 我的账号 -> 安全

保存令牌信息：squ_6e57e496a22ebdf0723c3683bd489df57f967f49

配置 sonarqube 认证信息：系统管理 ->manage credentials-> 全局 ->Add Credential

Jenkins 集成 Sonarqube：系统管理 -> 系统配置 ->SonarQube servers->Add SonarQube

3、获取 sonar-scanner 扫描命令（此步骤不用执行）

项目 -> 手工 -> 显示名

#获取 sonar-scanner 扫描命令，需要在项目目录下进行扫描

	sonar-scanner \
	-Dsonar.projectKey=springboot \
	-Dsonar.sources=. \
	-Dsonar.host.url=http://sonar.hmallleasing.com \
	-Dsonar.login=squ_6e57e496a22ebdf0723c3683bd489df57f967f49

4、测试代码扫描 Pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/ruoyi-cloud-cicd.git"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	environment {
	AppName="ruoyi-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Auth'){
	environment {
	AppName="ruoyi-auth"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	environment {
	AppName="ruoyi-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	environment {
	AppName="ruoyi-monitor"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测UI'){
	environment {
	AppName="ruoyi-ui"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查扫描结果') {
	steps {
	container('sonar') {
	sh 'echo "Code Result Check"'
	}
	}
	}
	stage('代码编译') {
	steps {
	container('maven') {
	sh 'echo "代码编译"'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

5、此处遇到问题总结

由于上传数据太大，在 ingress 中 client_max_body_size 默认值为 1m , 该参数的作用是设置最大允许客户端请求体的大小，如果超过了此值，客户端会收到 413 状态码，意思是请求的实体太大。

处理方法：

	#1. 针对单个 ingress
	$ cat 03-sonarqube-ingress.yaml
	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: sonarqube-ingress
	namespace: ops
	annotations:
	nginx.ingress.kubernetes.io/proxy-body-size: "200m"
	spec:
	ingressClassName: "nginx"
	rules:
	- host: "sonar.hmallleasing.com"
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: sonarqube-svc
	port:
	name: web

	#2. 针对所有 ingress
	$ kubectl edit configmap -n ingress-nginx ingress-nginx-controller
	apiVersion: v1
	data:
	proxy-body-size: "200m"
	...

# 2.4 检查扫描结果

1、配置 sonarquber, 让其将检查结果通知 jiekins，如果正常则继续、否则就终止执行

URL：http://admin:talent@jenkins.hmallleasing.com/sonarqube-webhook
Jenkins 访问地址：jenkins.hamlllleasing.com
Jenkins 用户名：admin
Jenkins 密码：talent

2、检查扫描结果 pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/ruoyi-cloud-cicd.git"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	environment {
	AppName="ruoyi-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Auth'){
	environment {
	AppName="ruoyi-auth"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	environment {
	AppName="ruoyi-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	environment {
	AppName="ruoyi-monitor"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测UI'){
	environment {
	AppName="ruoyi-ui"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查漏洞扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube 代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}

	stage('代码编译') {
	steps {
	container('maven') {
	sh 'echo "代码编译"'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.5 代码编译

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/ruoyi-cloud-cicd.git"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	environment {
	AppName="ruoyi-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Auth'){
	environment {
	AppName="ruoyi-auth"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	environment {
	AppName="ruoyi-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	environment {
	AppName="ruoyi-monitor"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测UI'){
	environment {
	AppName="ruoyi-ui"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查漏洞扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube 代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}

	stage('代码编译'){
	parallel {
	stage('编译Gateway'){
	environment{
	AppName = "ruoyi-gateway"
	}
	steps {
	container('maven'){
	sh '''
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

	stage('编译Auth'){
	environment{
	AppName = "ruoyi-auth"
	}
	steps {
	container('maven'){
	sh '''
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

	stage('编译System'){
	environment{
	AppName = "ruoyi-system"
	}
	steps {
	container('maven'){
	sh '''
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

	stage('编译Monitor'){
	environment{
	AppName = "ruoyi-monitor"
	}
	steps {
	container('maven'){
	sh '''
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

	stage('编译UI'){
	environment{
	AppName = "ruoyi-ui"
	}
	steps {
	container('nodejs'){
	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	pwd && ls -l && \
	npm install --registry=https://registry.npmmirror.com && \
	npm run build:prod && \
	pwd && ls -l ./dist/*
	'''
	}
	}
	}
	} //parallel end
	} //代码编译stage end

	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.6 制作 Docker 镜像

制作 Docker 镜像、推送到 Harbor 仓库，根据 Dockerfile 文件生成 Docker 镜像

1、配置 dns 解析（如果 harbor 解析至公网，无需配置此步骤）

	[root@k8s-master01 ~]# kubectl edit cm -n kube-system coredns
	...
	ready

	hosts {
	192.168.1.74 gitlab.hmallleasing.com #gitalb
	192.168.1.74 sonar.hmallleasing.com #sonar
	192.168.1.74 jenkins.hmallleasing.com #jenkins
	192.168.1.68 s.hmallleasing.com #harbor
	fallthrough
	}
	kubernetes cluster.local in-addr.arpa ip6.arpa {
	...

测试解析是否生效；

	[root@k8s-master01 ~]# kubectl exec -it gitlab-0 -n ops -- /bin/bash
	root@gitlab-0:/# ping s.hmallleasing.com
	PING s.hmallleasing.com (192.168.1.68): 56 data bytes
	64 bytes from 192.168.1.68: seq=0 ttl=63 time=0.257 ms
	64 bytes from 192.168.1.68: seq=1 ttl=63 time=0.198 ms

2、配置 harbor 认证信息：系统管理 ->manage credentials-> 全局 ->Add Credential

3、制作 docker 镜像 Pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/nf-flms.git"

	//Harbor相关的全局变量
	Url = "s.hmallleasing.com"
	Pro = "nf-flms"
	HARBOR_ID = "harbor-auth"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/release']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	environment {
	AppName="nf-flms-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Order'){
	environment {
	AppName="nf-flms-order"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	environment {
	AppName="nf-flms-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	environment {
	AppName="nf-flms-admin"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Statistics'){
	environment {
	AppName="nf-flms-statistics"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Openapi'){
	environment {
	AppName="nf-flms-openapi"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查漏洞扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube 代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}

	stage('代码编译') {
	steps {
	container('maven') {
	sh '''
	pwd && ls -l && \
	cd nf-flms && \
	echo "*******************开始编译代码*******************" && \
	mvn clean package && \
	pwd && ls -l nf-flms-order/target/*.jar && \
	pwd && ls -l nf-flms-gateway/target/*.jar && \
	pwd && ls -l nf-flms-statistics/target/*.jar && \
	pwd && ls -l nf-flms-system/target/*.jar && \
	pwd && ls -l nf-flms-openapi/target/*.jar && \
	pwd && ls -l nf-flms-admin/target/*.jar &&
	echo "*******************代码编译成功*******************"
	'''
	}
	}
	}

	stage('生成镜像Tag'){
	steps {
	container('maven') {
	script {
	//本次git提交的commid (git log -n1 --pretty=format:'%h')
	env.COMMITID = sh(returnStdout: true, script: "git log -n1 --pretty=format:'%h'").trim()
	//构建的时间 (date +%Y%m%d_%H%M%S)
	env.BuildTime = sh(returnStdout: true, script: "date +%Y%m%d_%H%M%S").trim()

	//完整的镜像Tag (c106654_20221115_133911)
	env.ImageTag = COMMITID + "_" + BuildTime
	}

	sh 'echo "镜像的Tag: ${ImageTag}"'
	}
	}
	}

	stage('制作Docker镜像'){
	parallel{
	stage('构建Gateway镜像') {
	environment {
	AppName = "nf-flms-gateway"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建System镜像') {
	environment {
	AppName = "nf-flms-system"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Order镜像') {
	environment {
	AppName = "nf-flms-order"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Monitor镜像') {
	environment {
	AppName = "nf-flms-admin"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Openapi镜像') {
	environment {
	AppName = "nf-flms-openapi"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Statistics镜像') {
	environment {
	AppName = "nf-flms-statistics"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}
	} //parallel end
	} // stage end

	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

5、编译报错总结

编译过程出现报错，由于多个服务同时进行编译，CPU 负载过高，解决方案让部分服务 sleep 60；

	stage('编译Gateway'){
	environment{
	AppName = "ruoyi-gateway"
	}
	steps {
	container('maven'){
	sh '''
	sleep 60 && \
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

# 2.7 交付应用到 K8S

1、将 K8S 的 kubeconfig 文件下载至本地，配置 Jenkins，将 config 文件制作为一个 Scretfile；

[root@k8s-master01 ~]# sz /root/.kube/config

配置 k8s 认证信息：系统管理 ->manage credentials-> 全局 ->Add Credential

2、编写 pipeline 流水线

在 Stage 中引用这个 config 文件，就可以操作对应的集群，部署前需做以下操作；

	$ kubectl create ns test
	$ kubectl create secret docker-registry harbor-admin --docker-server=s.hmallleasing.com --docker-username=admin --docker-password=Talent*19871988 -n test

部署的名称空间需要提前创建
需要创建对应 namespace 的 Harbor 认证 Secrets ->harbor-admin

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.125
	path: /data/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/ops/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/ops/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/ops/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/ops/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/ops/kubectl:1.22.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "https://gitee.com/chinagei/ruoyi-cloud-cicd.git"

	//Harbor相关的全局变量
	Url = "s.hmallleasing.com"
	Pro = "ruoyi"
	HARBOR_ID = "harbor-auth"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]]])
	sh 'pwd && ls -l'
	}
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	environment {
	AppName="ruoyi-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Auth'){
	environment {
	AppName="ruoyi-auth"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	environment {
	AppName="ruoyi-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	environment {
	AppName="ruoyi-monitor"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测UI'){
	environment {
	AppName="ruoyi-ui"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查漏洞扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube 代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}

	stage('代码编译'){
	parallel {
	stage('编译Gateway'){
	environment{
	AppName = "ruoyi-gateway"
	}
	steps {
	container('maven'){
	sh '''
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

	stage('编译Auth'){
	environment{
	AppName = "ruoyi-auth"
	}
	steps {
	container('maven'){
	sh '''
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

	stage('编译System'){
	environment{
	AppName = "ruoyi-system"
	}
	steps {
	container('maven'){
	sh '''
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

	stage('编译Monitor'){
	environment{
	AppName = "ruoyi-monitor"
	}
	steps {
	container('maven'){
	sh '''
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

	stage('编译UI'){
	environment{
	AppName = "ruoyi-ui"
	}
	steps {
	container('nodejs'){
	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	pwd && ls -l && \
	npm install --registry=https://registry.npmmirror.com && \
	npm run build:prod && \
	pwd && ls -l ./dist/*
	'''
	}
	}
	}
	} //parallel end
	} //代码编译stage end

	stage('生成镜像Tag'){
	steps {
	container('maven') {
	script {
	//本次git提交的commid (git log -n1 --pretty=format:'%h')
	env.COMMITID = sh(returnStdout: true, script: "git log -n1 --pretty=format:'%h'").trim()
	//构建的时间 (date +%Y%m%d_%H%M%S)
	env.BuildTime = sh(returnStdout: true, script: "date +%Y%m%d_%H%M%S").trim()

	//完整的镜像Tag (c106654_20221115_133911)
	env.ImageTag = COMMITID + "_" + BuildTime
	}

	sh 'echo "镜像的Tag: ${ImageTag}"'
	}
	}
	}

	stage('制作Docker镜像'){
	parallel{
	stage('构建Gateway镜像') {
	environment {
	AppName = "ruoyi-gateway"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建System镜像') {
	environment {
	AppName = "ruoyi-system"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Auth镜像') {
	environment {
	AppName = "ruoyi-auth"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Monitor镜像') {
	environment {
	AppName = "ruoyi-monitor"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建UI镜像') {
	environment {
	AppName = "ruoyi-ui"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}
	} //parallel end
	} // stage end

	stage('部署微服务至K8S') {
	parallel {
	stage('交付Gateway') {
	environment {
	AppName = "ruoyi-gateway"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#test#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Auth') {
	environment {
	AppName = "ruoyi-auth"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#test#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}


	stage('交付System') {
	environment {
	AppName = "ruoyi-system"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#test#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Monitor') {
	environment {
	AppName = "ruoyi-monitor"
	ImageName = "${Url}/${Pro}/${AppName}"
	IngressHost = "monitor.hmallleasing.com"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#test#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	sed -i "s#{host}#${IngressHost}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付UI') {
	environment {
	AppName = "ruoyi-ui"
	ImageName = "${Url}/${Pro}/${AppName}"
	IngressHost = "ruoyi.hmallleasing.com"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#test#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	sed -i "s#{host}#${IngressHost}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}
	} //parallel end
	} //stage

	} //stages end
	} //pipeline end

# 2.8 选择制定微服组件部署

选择指定微服务进行部署（Deploy_UI == "true" 或者 Deploy_All == "true"）

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.125
	path: /data/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/ops/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/ops/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/ops/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/ops/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/ops/kubectl:1.22.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "https://gitee.com/chinagei/ruoyi-cloud-cicd.git"

	//Harbor相关的全局变量
	Url = "s.hmallleasing.com"
	Pro = "ruoyi"
	HARBOR_ID = "harbor-auth"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]]])
	sh 'pwd && ls -l'
	}
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	when { expression {Deploy_GATEWAY == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName="ruoyi-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Auth'){
	when { expression {Deploy_AUTH == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName="ruoyi-auth"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	when { expression {Deploy_SYSTEM == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName="ruoyi-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	when { expression {Deploy_MONITOR == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName="ruoyi-monitor"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测UI'){
	when { expression {Deploy_UI == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName="ruoyi-ui"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查漏洞扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube 代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}

	stage('代码编译'){
	parallel {
	stage('编译Gateway'){
	when { expression {Deploy_GATEWAY == "true" \|\| Deploy_All == "true" } }
	environment{
	AppName = "ruoyi-gateway"
	}
	steps {
	container('maven'){
	sh '''
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

	stage('编译Auth'){
	when { expression {Deploy_AUTH == "true" \|\| Deploy_All == "true" } }
	environment{
	AppName = "ruoyi-auth"
	}
	steps {
	container('maven'){
	sh '''
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

	stage('编译System'){
	when { expression {Deploy_SYSTEM == "true" \|\| Deploy_All == "true" } }
	environment{
	AppName = "ruoyi-system"
	}
	steps {
	container('maven'){
	sh '''
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

	stage('编译Monitor'){
	when { expression {Deploy_MONITOR == "true" \|\| Deploy_All == "true" }
	environment{
	AppName = "ruoyi-monitor"
	}
	steps {
	container('maven'){
	sh '''
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

	stage('编译UI'){
	when { expression {Deploy_UI == "true" \|\| Deploy_All == "true" } }
	environment{
	AppName = "ruoyi-ui"
	}
	steps {
	container('nodejs'){
	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	pwd && ls -l && \
	npm install --registry=https://registry.npmmirror.com && \
	npm run build:prod && \
	pwd && ls -l ./dist/*
	'''
	}
	}
	}
	} //parallel end
	} //代码编译stage end

	stage('生成镜像Tag'){
	steps {
	container('maven') {
	script {
	//本次git提交的commid (git log -n1 --pretty=format:'%h')
	env.COMMITID = sh(returnStdout: true, script: "git log -n1 --pretty=format:'%h'").trim()
	//构建的时间 (date +%Y%m%d_%H%M%S)
	env.BuildTime = sh(returnStdout: true, script: "date +%Y%m%d_%H%M%S").trim()

	//完整的镜像Tag (c106654_20221115_133911)
	env.ImageTag = COMMITID + "_" + BuildTime
	}

	sh 'echo "镜像的Tag: ${ImageTag}"'
	}
	}
	}

	stage('制作Docker镜像'){
	parallel{
	stage('构建Gateway镜像') {
	when { expression {Deploy_GATEWAY == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-gateway"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建System镜像') {
	when { expression {Deploy_SYSTEM == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-system"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Auth镜像') {
	when { expression {Deploy_AUTH == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-auth"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Monitor镜像') {
	when { expression {Deploy_MONITOR == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-monitor"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建UI镜像') {
	when { expression {Deploy_UI == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-ui"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}
	} //parallel end
	} // stage end

	stage('部署微服务至K8S') {
	parallel {
	stage('交付Gateway') {
	when { expression {Deploy_GATEWAY == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-gateway"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#test#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Auth') {
	when { expression {Deploy_AUTH == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-auth"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#test#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}


	stage('交付System') {
	when { expression {Deploy_SYSTEM == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-system"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#test#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Monitor') {
	when { expression {Deploy_MONITOR == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-monitor"
	ImageName = "${Url}/${Pro}/${AppName}"
	IngressHost = "monitor.hmallleasing.com"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#test#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	sed -i "s#{host}#${IngressHost}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付UI') {
	when { expression {Deploy_UI == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-ui"
	ImageName = "${Url}/${Pro}/${AppName}"
	IngressHost = "ruoyi.hmallleasing.com"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#test#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	sed -i "s#{host}#${IngressHost}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}
	} //parallel end
	} //stage

	} //stages end
	} //pipeline end

# 2.9 配置自动触发构建

Jenkins 配置：

#Gitee 配置参考连接 https://blog.csdn.net/hali90s/article/details/126991514

#Gitlab 配置参考连接 https://blog.csdn.net/LlZzSss/article/details/123595996

1、安装插件：Gitee

2、配置 -> 构建触发器 ->Gitee webhook 触发构建

Gitee 配置：

	//交付微服务的CI流水线

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 10.0.0.206
	path: /data/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: harbor.oldxu.net/ops/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: harbor.oldxu.net/ops/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: harbor.oldxu.net/ops/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: harbor.oldxu.net/ops/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: harbor.oldxu.net/ops/kubectl:1.22.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}


	//参数化构建 Deploy_All
	parameters {
	choice choices: ['false', 'true'], description: '是否部署所有的微服务', name: 'Deploy_All'
	}


	environment{
	Gitlab_Id = "gitlab-root-token"
	Gitlab_Pro = "http://gitlab.oldxu.net/root/ruo-yi-cloud.git"

	//Harbor相关的全局变量
	Url = "harbor.oldxu.net"
	Pro = "base"
	HARBOR_ID = "harbor-auth"


	//对外暴露的域名
	Ingress_Host = "spring-dev.oldxu.net"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout([$class: 'GitSCM', branches: [[name: '*/k8s']], extensions: [], userRemoteConfigs: [[credentialsId: "${Gitlab_Id}", url: "${Gitlab_Pro}"]]])
	sh 'pwd && ls -l'
	}
	}
	}

	stage('检查变动的模块名称') {
	steps {
	sh '''#!/bin/bash
	# 定义微服务模块名称
	Init_Module=(ruoyi-gateway ruoyi-auth ruoyi-system ruoyi-monitor ruoyi-ui)

	# 变动的模块
	Change_Module=$(git diff --name-only $(git log -n2 --pretty=format:"%h") \|cut -d / -f1,2)

	# 外循环 {提取变动的模块名称}
	for Change in ${Change_Module}
	do
	# echo "改变的微服务模块是: $Change"
	# 内循环
	for init in ${Init_Module[@]}
	do
	if [[ "${Change}" =~ "${init}" ]];then
	echo "${init}=ok" >> var.txt
	fi
	done
	done
	'''
	//制作自定义变量
	script {
	env.GATEWAY = sh(returnStdout: true, script: "grep 'ruoyi-gateway' var.txt \| awk -F '=' '{print \$2}'").trim()
	env.AUTH = sh(returnStdout: true, script: "grep 'ruoyi-auth' var.txt \| awk -F '=' '{print \$2}'").trim()
	env.SYSTEM = sh(returnStdout: true, script: "grep 'ruoyi-system' var.txt \| awk -F '=' '{print \$2}'").trim()
	env.MONITOR = sh(returnStdout: true, script: "grep 'ruoyi-monitor' var.txt \| awk -F '=' '{print \$2}'").trim()
	env.UI = sh(returnStdout: true, script: "grep 'ruoyi-ui' var.txt \| awk -F '=' '{print \$2}'").trim()
	}
	}
	}

	stage('输出更新的模块状态'){
	steps {
	sh 'echo "gateway: ${GATEWAY}"'
	sh 'echo "auth: ${AUTH}"'
	sh 'echo "system: ${SYSTEM}"'
	sh 'echo "monitor: ${MONITOR}"'
	sh 'echo "ui: ${UI}"'
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	when { expression {GATEWAY == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName="ruoyi-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Auth'){
	when { expression {AUTH == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName="ruoyi-auth"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	when { expression {SYSTEM == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName="ruoyi-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	when { expression {MONITOR == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName="ruoyi-monitor"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测UI'){
	when { expression {UI == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName="ruoyi-ui"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查漏洞扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube 代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}

	stage('代码编译'){
	parallel {
	stage('编译Gateway'){
	when { expression {GATEWAY == "ok" \|\| Deploy_All == "true" } }
	environment{
	AppName = "ruoyi-gateway"
	}
	steps {
	container('maven'){
	sh '''
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am
	'''
	}
	}
	}

	stage('编译Auth'){
	when { expression {AUTH == "ok" \|\| Deploy_All == "true" } }
	environment{
	AppName = "ruoyi-auth"
	}
	steps {
	container('maven'){
	sh '''
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am
	'''
	}
	}
	}

	stage('编译System'){
	when { expression {SYSTEM == "ok" \|\| Deploy_All == "true" } }
	environment{
	AppName = "ruoyi-system"
	}
	steps {
	container('maven'){
	sh '''
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am
	'''
	}
	}
	}

	stage('编译Monitor'){
	when { expression {MONITOR == "ok" \|\| Deploy_All == "true" } }
	environment{
	AppName = "ruoyi-monitor"
	}
	steps {
	container('maven'){
	sh '''
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am
	'''
	}
	}
	}

	stage('编译UI'){
	when { expression {UI == "ok" \|\| Deploy_All == "true" } }
	environment{
	AppName = "ruoyi-ui"
	}
	steps {
	container('nodejs'){
	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	npm install --registry=https://registry.npmmirror.com && \
	npm run build:prod
	'''
	}
	}
	}
	} //parallel end
	} //代码编译stage end


	stage('生成镜像Tag'){
	steps {
	container('maven') {
	script {
	//本次git提交的commid (git log -n1 --pretty=format:'%h')
	env.COMMITID = sh(returnStdout: true, script: "git log -n1 --pretty=format:'%h'").trim()
	//构建的时间 (date +%Y%m%d_%H%M%S)
	env.BuildTime = sh(returnStdout: true, script: "date +%Y%m%d_%H%M%S").trim()

	//完整的镜像Tag (c106654_20221115_133911)
	env.ImageTag = COMMITID + "_" + BuildTime
	}

	sh 'echo "镜像的Tag: ${ImageTag}"'
	}
	}
	}

	stage('制作Docker镜像'){

	parallel{
	stage('构建Gateway镜像') {
	when { expression {GATEWAY == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-gateway"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建System镜像') {
	when { expression {SYSTEM == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-system"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Auth镜像') {
	when { expression {AUTH == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-auth"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Monitor镜像') {
	when { expression {MONITOR == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-monitor"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建UI镜像') {
	when { expression {UI == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-ui"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}






	} //parallel end
	} // stage end


	stage('交付微服务至K8S') {
	parallel {

	stage('交付Gateway') {
	when { expression {GATEWAY == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-gateway"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && cp ${KUBECONFIG} ~/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	sed -i "s#{namespace}#dev#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Auth') {
	when { expression {AUTH == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-auth"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && cp ${KUBECONFIG} ~/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	sed -i "s#{namespace}#dev#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}


	stage('交付System') {
	when { expression {SYSTEM == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-system"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && cp ${KUBECONFIG} ~/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	sed -i "s#{namespace}#dev#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Monitor') {
	when { expression {MONITOR == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-monitor"
	ImageName = "${Url}/${Pro}/${AppName}"
	IngressHost = "monitor-dev.oldxu.net"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && cp ${KUBECONFIG} ~/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	sed -i "s#{namespace}#dev#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	sed -i "s#{host}#${IngressHost}#g" deploy.yaml
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付UI') {
	when { expression {UI == "ok" \|\| Deploy_All == "true" } }
	environment {
	AppName = "ruoyi-ui"
	ImageName = "${Url}/${Pro}/${AppName}"
	IngressHost = "ui-dev.oldxu.net"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && cp ${KUBECONFIG} ~/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	sed -i "s#{namespace}#dev#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	sed -i "s#{host}#${IngressHost}#g" deploy.yaml
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	} //parallel end
	}
	} //stages end
	} //pipeline end

# 三、CD 阶段

CD 阶段：Jenkins/CD-> 拉取 Harbor 仓库对应项目镜像 -> 部署应用至 K8S 生产环境；

# 3.1 获取完整的镜像名称

1、获取 Harbor 镜像 Tag

	[root@k8s-master01 ~]# curl -s -uadmin:passwd -H'Content-Type: application/json' -X GET https://s.hmallleasing.com/v2/base/springboot/tags/list \| sed -r 's#(\{.\[)(.)(\]\})#\2#g' \| xargs -d "," -n1 \| xargs -n1 \| sort -t "_" -k2 -k3 -nr \| head -5
	3e24684_20230326_153018
	5041fbd_20230326_144343
	5da8d15_20230326_143533
	5da8d15_20230326_005207
	5da8d15_20230326_004940

2、安装插件

系统管理 -> 插件管理 ->Active Choices

3、级联变量的方式来提取对应的 tag
变量：Harbor_Url
项目 -> 配置 -> 参数化构建 ->Active Choices Parameter
语法：return ["s.hmallleasing.com","harbor.hmallleasing.com"]

变量：Harbor_Pro
项目 -> 配置 -> 参数化构建 ->Active Choices Parameter
语法：return ["base","nf-flms","ops","xx"]

变量：Image_Name
项目 -> 配置 -> 参数化构建 ->Active Choices Parameter
语法：return ["springboot","nf-flms-statistics","nf-flms-order","xx"]

级联变量：Image_Tag

项目 -> 配置 -> 参数化构建 ->Active Choices Reactive Parameter
语法：

def get_tag = [ "bash", "-c", "curl -s -uadmin:passwd -H 'Content-Type: application/json' -X GET https://${Harbor_Url}/v2/${Harbor_Pro}/${Image_Name}/tags/list | sed -r 's#(\\{.*\\[)(.*)(\\]\\})#\\2#g' | xargs -d ',' -n1 | xargs -n1 | sort -t '_' -k2 -k3 -nr | head -5"]

return get_tag.execute().text.tokenize("\n")

#级联变量：Harbor_Url+Harbor_Pro+Image_Name

4、编写 pipeline 输出完整的镜像名称

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: kubectl
	image: s.hmallleasing.com/ops/kubectl:1.22.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment {
	Full_Image = "${Harbor_Url}/${Harbor_Pro}/${Image_Name}:${Image_Tag}"
	}
	stages {
	stage('输出完整的镜像名称') {
	steps {
	sh 'echo 镜像名称-tag: ${Full_Image}'
	}
	}
	}
	}

# 3.2 生产环境中部署对应至 K8S

1、编写 yaml

	[root@k8s-master01 java]# cat deploy-prod.yaml
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: springboot
	namespace: prod #修改为 {} 特殊字符，后期好替换
	spec:
	replicas: 2
	selector:
	matchLabels:
	app: spring
	template:
	metadata:
	labels:
	app: spring
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: springboot
	image: s.hmallleasing.com/base/springboot:5da8d15_20230326_004659 # 修改为 {} 特殊字符，后期好替换
	ports:
	- name: http
	containerPort: 8080
	env: # 传递初始堆内存和最大堆内存占用
	- name: XMS_OPTS
	valueFrom:
	resourceFieldRef:
	resource: requests.memory
	- name: XMX_OPTS
	valueFrom:
	resourceFieldRef:
	resource: limits.memory
	resources:
	requests:
	memory: 150Mi
	limits:
	memory: 300Mi
	readinessProbe: # 就绪探针；如果端口不存活，则从负载均衡中移除
	tcpSocket:
	port: http # http 是一个名字；它会获取这个名字对应的端口；
	initialDelaySeconds: 10
	failureThreshold: 3
	livenessProbe: # 存活探针；获取 url，状态码不对那么则触发重启操作
	httpGet:
	path: /
	port: http
	initialDelaySeconds: 10
	failureThreshold: 3

	---

	apiVersion: v1
	kind: Service
	metadata:
	name: spring-svc
	namespace: prod
	spec:
	selector:
	app: spring
	ports:
	- port: 8080
	targetPort: 8080

	---

	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: springboot-ingress
	namespace: prod
	spec:
	ingressClassName: "nginx"
	rules:
	- host: "spring-prod.hmallleasing.com"
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: spring-svc
	port:
	number: 8080

2、部署应用

	[root@k8s-master01 java]# kubectl create ns prod
	[root@k8s-master01 java]# kubectl create secret docker-registry harbor-admin --docker-server=s.hmallleasing.com --docker-username=admin --docker-password=Talent*19871988 -n prod

	[root@k8s-master01 java]# kubectl apply -f deploy-prod.yaml

# 3.3 CD 流水线部署指定 Tag 版本到 K8S 生产环境

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: kubectl
	image: s.hmallleasing.com/ops/kubectl:1.22.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment {
	Full_Image = "${Harbor_Url}/${Harbor_Pro}/${Image_Name}:${Image_Tag}"
	}
	stages {
	stage('输出完整的镜像名称') {
	steps {
	sh 'echo 镜像名称-tag: ${Full_Image}'
	}
	}
	stage('部署应用至K8S') {
	steps {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	container('kubectl'){
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >> ~/.kube/config'
	sh 'kubectl set image deployment/${Image_Name} ${Image_Name}=${Full_Image} -n prod'
	}
	}
	}
	}
	}
	}

# 3.4 询问是否需要回退

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: kubectl
	image: s.hmallleasing.com/ops/kubectl:1.22.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment {
	Full_Image = "${Harbor_Url}/${Harbor_Pro}/${Image_Name}:${Image_Tag}"
	}
	stages {
	stage('输出完整的镜像名称') {
	steps {
	sh 'echo 镜像名称-tag: ${Full_Image}'
	}
	}
	stage('部署应用至K8S') {
	steps {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	container('kubectl'){
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >> ~/.kube/config'
	sh 'kubectl set image deployment/${Image_Name} ${Image_Name}=${Full_Image} -n prod'
	}
	}
	}
	}

	stage('快速回滚') {
	steps {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	container('kubectl'){
	script {
	timeout(time:1 , unit: 'HOURS'){
	def UserInput = input message: '是否回退至上一个版本', parameters: [choice(choices: ['No', 'Yes'], name: 'rollback')]
	if (UserInput == "Yes"){
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >> ~/.kube/config'
	sh 'kubectl rollout undo deployment ${Image_Name} -n prod'
	}else {
	echo "没有选择回退"
	}
	}
	}
	}
	}
	}
	}

	}
	}

#如果选择 “Yes” 继续，则回退至上一个版本，等待时间为 1 小时

#如果选择 “No” 继续，则不回退至上一个版本

#如果没有选择，则不回退至上一个版本

DevOps

# K8S 基于 Jenkins 实现 SpringCloud 微服务 CI 与 CD 实践（三）

# 一、环境准备

# 二、CI 阶段

# 2.1 编写 Pipeline 框架

# 2.2 获取代码

# 2.3 Sonarqube 漏洞扫描

# 2.4 检查扫描结果

# 2.5 代码编译

# 2.6 制作 Docker 镜像

# 2.7 交付应用到 K8S

# 2.8 选择制定微服组件部署

# 2.9 配置自动触发构建

# 三、CD 阶段

# 3.1 获取完整的镜像名称

# 3.2 生产环境中部署对应至 K8S

# 3.3 CD 流水线部署指定 Tag 版本到 K8S 生产环境

# 3.4 询问是否需要回退

K8S基于Jenkins实现Java项目CI与CD实践（二）

K8S基于Jenkins实现微服务应用CICD实践（四）