K8S基于Jenkins实现微服务应用CICD实践（四）

# K8S 基于 Jenkins 实现微服务应用 CICD 实践（四）

CI 阶段：开发人员 -> 提交代码 ->gitlab 仓库 ->Jenkins/CI 抓取代码 -> 漏洞扫描 -> 编译 -> 构建镜像 -> 推送 Harbor-> 部署应用至 K8S 测试环境；
CD 阶段：Jenkins/CD-> 拉取 Harbor 仓库对应项目镜像 -> 部署应用至 K8S 测试环境；

# 一、环境准备

准备好 Java 代码、Dockerfile、deploy.yaml 资源清单文件提交到 Gitlab 服务器；

	$ git config --global user.name "oldxu"
	$ git config --global user.email
	$ git init
	$ git add .
	$ git commit -m "first commit"
	$ git remote remove origin
	$ git remote add origin http://gitlab.hmallleasing.com/root/ruoyi-cloud-cicd.git
	$ git remote -v
	origin http://gitlab.hmallleasing.com/root/ruoyi-cloud-cicd.gi (fetch)
	origin http://gitlab.hmallleasing.com/root/ruoyi-cloud-cicd.gi (push)
	$ git push -u origin "master"

# 二、CI 阶段

CI 阶段分为以下 6 步：
1、获取代码
2、漏洞扫描
3、检测漏洞扫描结果，如果正常则继续、否则就终止
4、使用 maven 进行编译，打包
5、制作 Docker 镜像、推送到 Harbor 仓库 (Dockerfile 文件)
6、交付应用到 K8S

# 2.1 编写 Pipeline 框架

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}
	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	sh 'echo "获取代码"'
	}
	}
	}
	stage('代码扫描') {
	steps {
	container('sonar') {
	sh 'echo "代码扫描"'
	}
	}
	}
	stage('检查扫描结果') {
	steps {
	container('sonar') {
	sh 'echo "Code Result Check"'
	}
	}
	}
	stage('代码编译') {
	steps {
	container('maven') {
	sh 'echo "代码编译"'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.2 获取代码

1、Jenkins 上配置 Gitlab 认证信息

系统管理 ->manage credentials-> 全局 ->Add Credential

2、SlavePod 访问 gitlab

通过 Coredns 配置自定义域名解析，如果 sgitlab 解析至公网，无需配置此步骤；

	[root@k8s-master01 ~]# kubectl edit cm coredns -n kube-system
	...
	ready

	hosts {
	192.168.1.74 gitlab.hmallleasing.com
	fallthrough
	}
	kubernetes cluster.local in-addr.arpa ip6.arpa {
	...

测试解析是否生效；

	[root@k8s-master01 ~]# kubectl exec -it gitlab-0 -n ops -- /bin/bash
	root@gitlab-0:/# ping gitlab.hmallleasing.com
	PING gitlab.hmallleasing.com (192.168.1.74): 56 data bytes
	64 bytes from 192.168.1.74: seq=0 ttl=63 time=0.381 ms
	64 bytes from 192.168.1.74: seq=1 ttl=63 time=0.203 ms

3、生成流水线脚本获取代码

通过流水线语法 -> 片段生成器 -> 生成流水线脚本，生成 Pipeline 流水线

4、测试获取代码 Pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/nf-flms.git"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/release']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}
	stage('代码扫描') {
	steps {
	container('sonar') {
	sh 'echo "代码扫描"'
	}
	}
	}
	stage('检查扫描结果') {
	steps {
	container('sonar') {
	sh 'echo "Code Result Check"'
	}
	}
	}
	stage('代码编译') {
	steps {
	container('maven') {
	sh 'echo "代码编译"'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.3 Sonarqube 漏洞扫描

1、配置 dns 解析（如果 sonarqube 解析至公网，无需配置此步骤）
slavePod 需要访问 sonarqube 服务端，sonarqube 服务端需要回调 jenkins，获取扫描结果。

	[root@k8s-master01 ~]# kubectl edit cm -n kube-system coredns
	...
	ready

	hosts {
	192.168.1.74 gitlab.hmallleasing.com #gitalb
	192.168.1.74 sonar.hmallleasing.com #sonar
	192.168.1.74 jenkins.hmallleasing.com #jenkins
	fallthrough
	}
	kubernetes cluster.local in-addr.arpa ip6.arpa {
	...

测试解析是否生效；

	[root@k8s-master01 ~]# kubectl exec -it gitlab-0 -n ops -- /bin/bash
	root@gitlab-0:/# ping sonar.hmallleasing.com
	PING sonar.hmallleasing.com (192.168.1.74): 56 data bytes
	64 bytes from 192.168.1.74: seq=0 ttl=63 time=0.187 ms
	64 bytes from 192.168.1.74: seq=1 ttl=63 time=0.202 ms

	root@gitlab-0:/# ping jenkins.hmallleasing.com
	PING jenkins.hmallleasing.com (192.168.1.74): 56 data bytes
	64 bytes from 192.168.1.74: seq=0 ttl=63 time=0.189 ms
	64 bytes from 192.168.1.74: seq=1 ttl=63 time=0.202 ms

2、Jenkins 集成 Sonarqube

生成 sonarqube 令牌：用户 -> 我的账号 -> 安全

保存令牌信息：squ_6e57e496a22ebdf0723c3683bd489df57f967f49

配置 sonarqube 认证信息：系统管理 ->manage credentials-> 全局 ->Add Credential

Jenkins 集成 Sonarqube：系统管理 -> 系统配置 ->SonarQube servers->Add SonarQube

3、获取 sonar-scanner 扫描命令（此步骤不用执行）

项目 -> 手工 -> 显示名

#获取 sonar-scanner 扫描命令，需要在项目目录下进行扫描

	sonar-scanner \
	-Dsonar.projectKey=springboot \
	-Dsonar.sources=. \
	-Dsonar.host.url=http://sonar.hmallleasing.com \
	-Dsonar.login=squ_6e57e496a22ebdf0723c3683bd489df57f967f49

4、测试代码扫描 Pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/nf-flms.git"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/release']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	environment {
	AppName="nf-flms-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Order'){
	environment {
	AppName="nf-flms-order"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	environment {
	AppName="nf-flms-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	environment {
	AppName="nf-flms-admin"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Statistics'){
	environment {
	AppName="nf-flms-statistics"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Openapi'){
	environment {
	AppName="nf-flms-openapi"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查扫描结果') {
	steps {
	container('sonar') {
	sh 'echo "Code Result Check"'
	}
	}
	}
	stage('代码编译') {
	steps {
	container('maven') {
	sh 'echo "代码编译"'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

5、此处遇到问题总结

由于上传数据太大，在 ingress 中 client_max_body_size 默认值为 1m , 该参数的作用是设置最大允许客户端请求体的大小，如果超过了此值，客户端会收到 413 状态码，意思是请求的实体太大。

处理方法：

	#1. 针对单个 ingress
	$ cat 03-sonarqube-ingress.yaml
	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: sonarqube-ingress
	namespace: ops
	annotations:
	nginx.ingress.kubernetes.io/proxy-body-size: "200m"
	spec:
	ingressClassName: "nginx"
	rules:
	- host: "sonar.hmallleasing.com"
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: sonarqube-svc
	port:
	name: web

	#2. 针对所有 ingress
	$ kubectl edit configmap -n ingress-nginx ingress-nginx-controller
	apiVersion: v1
	data:
	proxy-body-size: "200m"
	...

# 2.4 检查扫描结果

1、配置 sonarquber, 让其将检查结果通知 jenkins，如果正常则继续、否则就终止执行

URL：http://admin:talent@jenkins.hmallleasing.com/sonarqube-webhook
Jenkins 访问地址：jenkins.hamlllleasing.com
Jenkins 用户名：admin
Jenkins 密码：talent

2、检查扫描结果 pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/nf-flms.git"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/release']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	environment {
	AppName="nf-flms-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Order'){
	environment {
	AppName="nf-flms-order"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	environment {
	AppName="nf-flms-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	environment {
	AppName="nf-flms-admin"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Statistics'){
	environment {
	AppName="nf-flms-statistics"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Openapi'){
	environment {
	AppName="nf-flms-openapi"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查漏洞扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube 代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}

	stage('代码编译') {
	steps {
	container('maven') {
	sh 'echo "代码编译"'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.5 代码编译

1、导入 mvn 依赖

	[root@jenkins repository]# pwd
	/root/.m2/repository
	[root@jenkins repository]# scp -rp ./* root@192.168.1.75:/data/nfs/maven/repository/

2、代码编译 pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/nf-flms.git"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/release']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	environment {
	AppName="nf-flms-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Order'){
	environment {
	AppName="nf-flms-order"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	environment {
	AppName="nf-flms-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	environment {
	AppName="nf-flms-admin"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Statistics'){
	environment {
	AppName="nf-flms-statistics"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Openapi'){
	environment {
	AppName="nf-flms-openapi"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查漏洞扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube 代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}

	stage('代码编译') {
	steps {
	container('maven') {
	sh '''
	pwd && ls -l && \
	cd nf-flms && \
	echo "*******************开始编译代码*******************" && \
	mvn clean package && \
	pwd && ls -l nf-flms-order/target/*.jar && \
	pwd && ls -l nf-flms-gateway/target/*.jar && \
	pwd && ls -l nf-flms-statistics/target/*.jar && \
	pwd && ls -l nf-flms-system/target/*.jar && \
	pwd && ls -l nf-flms-openapi/target/*.jar && \
	pwd && ls -l nf-flms-admin/target/*.jar &&
	echo "*******************代码编译成功*******************"
	'''
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.6 制作 Docker 镜像

制作 Docker 镜像、推送到 Harbor 仓库，根据 Dockerfile 文件生成 Docker 镜像

1、配置 dns 解析（如果 harbor 解析至公网，无需配置此步骤）

	[root@k8s-master01 ~]# kubectl edit cm -n kube-system coredns
	...
	ready

	hosts {
	192.168.1.74 gitlab.hmallleasing.com #gitalb
	192.168.1.74 sonar.hmallleasing.com #sonar
	192.168.1.74 jenkins.hmallleasing.com #jenkins
	192.168.1.68 s.hmallleasing.com #harbor
	fallthrough
	}
	kubernetes cluster.local in-addr.arpa ip6.arpa {
	...

测试解析是否生效；

	[root@k8s-master01 ~]# kubectl exec -it gitlab-0 -n ops -- /bin/bash
	root@gitlab-0:/# ping s.hmallleasing.com
	PING s.hmallleasing.com (192.168.1.68): 56 data bytes
	64 bytes from 192.168.1.68: seq=0 ttl=63 time=0.257 ms
	64 bytes from 192.168.1.68: seq=1 ttl=63 time=0.198 ms

2、配置 harbor 认证信息：系统管理 ->manage credentials-> 全局 ->Add Credential

3、制作 docker 镜像 Pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/nf-flms.git"

	//Harbor相关的全局变量
	Url = "s.hmallleasing.com"
	Pro = "nf-flms"
	HARBOR_ID = "harbor-auth"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/release']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	environment {
	AppName="nf-flms-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Order'){
	environment {
	AppName="nf-flms-order"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	environment {
	AppName="nf-flms-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	environment {
	AppName="nf-flms-admin"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Statistics'){
	environment {
	AppName="nf-flms-statistics"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Openapi'){
	environment {
	AppName="nf-flms-openapi"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查漏洞扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube 代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}

	stage('代码编译') {
	steps {
	container('maven') {
	sh '''
	pwd && ls -l && \
	cd nf-flms && \
	echo "*******************开始编译代码*******************" && \
	mvn clean package && \
	pwd && ls -l nf-flms-order/target/*.jar && \
	pwd && ls -l nf-flms-gateway/target/*.jar && \
	pwd && ls -l nf-flms-statistics/target/*.jar && \
	pwd && ls -l nf-flms-system/target/*.jar && \
	pwd && ls -l nf-flms-openapi/target/*.jar && \
	pwd && ls -l nf-flms-admin/target/*.jar &&
	echo "*******************代码编译成功*******************"
	'''
	}
	}
	}

	stage('生成镜像Tag'){
	steps {
	container('maven') {
	script {
	//本次git提交的commid (git log -n1 --pretty=format:'%h')
	env.COMMITID = sh(returnStdout: true, script: "git log -n1 --pretty=format:'%h'").trim()
	//构建的时间 (date +%Y%m%d_%H%M%S)
	env.BuildTime = sh(returnStdout: true, script: "date +%Y%m%d_%H%M%S").trim()

	//完整的镜像Tag (c106654_20221115_133911)
	env.ImageTag = COMMITID + "_" + BuildTime
	}

	sh 'echo "镜像的Tag: ${ImageTag}"'
	}
	}
	}

	stage('制作Docker镜像'){
	parallel{
	stage('构建Gateway镜像') {
	environment {
	AppName = "nf-flms-gateway"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建System镜像') {
	environment {
	AppName = "nf-flms-system"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Order镜像') {
	environment {
	AppName = "nf-flms-order"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Monitor镜像') {
	environment {
	AppName = "nf-flms-admin"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Openapi镜像') {
	environment {
	AppName = "nf-flms-openapi"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Statistics镜像') {
	environment {
	AppName = "nf-flms-statistics"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}
	} //parallel end
	} // stage end

	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

5、编译报错总结

编译过程出现报错，由于多个服务同时进行编译，CPU 负载过高，解决方案让部分服务 sleep 60；

	stage('编译Gateway'){
	environment{
	AppName = "ruoyi-gateway"
	}
	steps {
	container('maven'){
	sh '''
	sleep 60 && \
	pwd && ls -l && \
	Build_Path=$(find ./ -type d -name "${AppName}")
	mvn package -Dmaven.test.skip=true -pl ${Build_Path} -am && \
	pwd && ls -l ${Build_Path}/target/*
	'''
	}
	}
	}

# 2.7 交付应用到 K8S

1、将 K8S 的 kubeconfig 文件下载至本地，配置 Jenkins，将 config 文件制作为一个 Scretfile；

[root@k8s-master01 ~]# sz /root/.kube/config

配置 k8s 认证信息：系统管理 ->manage credentials-> 全局 ->Add Credential

2、编写 pipeline 流水线

在 Stage 中引用这个 config 文件，就可以操作对应的集群，部署前需做以下操作；

	$ kubectl create ns prod
	$ kubectl create secret docker-registry harbor-admin --docker-server=s.hmallleasing.com --docker-username=admin --docker-password=Talent*passwd -n prod

部署的名称空间需要提前创建
需要创建对应 namespace 的 Harbor 认证 Secrets ->harbor-admin

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/nf-flms.git"

	//Harbor相关的全局变量
	Url = "s.hmallleasing.com"
	Pro = "nf-flms"
	HARBOR_ID = "harbor-auth"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/release']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	environment {
	AppName="nf-flms-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Order'){
	environment {
	AppName="nf-flms-order"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	environment {
	AppName="nf-flms-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	environment {
	AppName="nf-flms-admin"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Statistics'){
	environment {
	AppName="nf-flms-statistics"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Openapi'){
	environment {
	AppName="nf-flms-openapi"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查漏洞扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube 代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}

	stage('代码编译') {
	steps {
	container('maven') {
	sh '''
	pwd && ls -l && \
	cd nf-flms && \
	echo "*******************开始编译代码*******************" && \
	mvn clean package && \
	pwd && ls -l nf-flms-order/target/*.jar && \
	pwd && ls -l nf-flms-gateway/target/*.jar && \
	pwd && ls -l nf-flms-statistics/target/*.jar && \
	pwd && ls -l nf-flms-system/target/*.jar && \
	pwd && ls -l nf-flms-openapi/target/*.jar && \
	pwd && ls -l nf-flms-admin/target/*.jar &&
	echo "*******************代码编译成功*******************"
	'''
	}
	}
	}

	stage('生成镜像Tag'){
	steps {
	container('maven') {
	script {
	//本次git提交的commid (git log -n1 --pretty=format:'%h')
	env.COMMITID = sh(returnStdout: true, script: "git log -n1 --pretty=format:'%h'").trim()
	//构建的时间 (date +%Y%m%d_%H%M%S)
	env.BuildTime = sh(returnStdout: true, script: "date +%Y%m%d_%H%M%S").trim()

	//完整的镜像Tag (c106654_20221115_133911)
	env.ImageTag = COMMITID + "_" + BuildTime
	}

	sh 'echo "镜像的Tag: ${ImageTag}"'
	}
	}
	}

	stage('制作Docker镜像'){
	parallel{
	stage('构建Gateway镜像') {
	environment {
	AppName = "nf-flms-gateway"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建System镜像') {
	environment {
	AppName = "nf-flms-system"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Order镜像') {
	environment {
	AppName = "nf-flms-order"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Monitor镜像') {
	environment {
	AppName = "nf-flms-admin"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Openapi镜像') {
	environment {
	AppName = "nf-flms-openapi"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Statistics镜像') {
	environment {
	AppName = "nf-flms-statistics"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}
	} //parallel end
	} // stage end

	stage('部署微服务至K8S') {
	parallel {
	stage('交付Gateway') {
	environment {
	AppName = "nf-flms-gateway"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#prod#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Order') {
	environment {
	AppName = "nf-flms-order"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#prod#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}


	stage('交付System') {
	environment {
	AppName = "nf-flms-system"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#prod#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Monitor') {
	environment {
	AppName = "nf-flms-admin"
	ImageName = "${Url}/${Pro}/${AppName}"
	IngressHost = "monitor.hmallleasing.com"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#prod#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	sed -i "s#{host}#${IngressHost}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Statistics') {
	environment {
	AppName = "nf-flms-statistics"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#prod#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	sed -i "s#{host}#${IngressHost}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Openapi') {
	environment {
	AppName = "nf-flms-openapi"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#prod#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	sed -i "s#{host}#${IngressHost}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}
	} //parallel end
	} //stage
	}
	}

# 2.8 选择制定微服组件部署

1、安装插件：

系统管理 -> 插件管理 ->Active Choices

2、选择指定微服务进行部署（Deploy_UI == "true" 或者 Deploy_All == "true"）

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/nf-flms.git"

	//Harbor相关的全局变量
	Url = "s.hmallleasing.com"
	Pro = "nf-flms"
	HARBOR_ID = "harbor-auth"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/release']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}

	stage('代码扫描'){
	parallel {
	stage('检测Gateway'){
	when { expression {Deploy_Gateway == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName="nf-flms-gateway"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Order'){
	when { expression {Deploy_Order == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName="nf-flms-order"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}

	stage('检测System'){
	when { expression {Deploy_System == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName="nf-flms-system"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Monitor'){
	when { expression {Deploy_Monitor == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName="nf-flms-admin"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar'){
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Statistics'){
	when { expression {Deploy_Statistics == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName="nf-flms-statistics"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检测Openapi'){
	when { expression {Deploy_Openapi == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName="nf-flms-openapi"
	}
	steps {
	withSonarQubeEnv('sonar-k8s'){
	container('sonar') {
	sh 'cd $(find ./ -type d -name "${AppName}") && pwd && ls && \
	sonar-scanner \
	-Dsonar.projectKey=${AppName} \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	} //parallel end
	} //stage end

	stage('检查漏洞扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube 代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}

	stage('代码编译') {
	steps {
	container('maven') {
	sh '''
	pwd && ls -l && \
	cd nf-flms && \
	echo "*******************开始编译代码*******************" && \
	mvn clean package && \
	pwd && ls -l nf-flms-order/target/*.jar && \
	pwd && ls -l nf-flms-gateway/target/*.jar && \
	pwd && ls -l nf-flms-statistics/target/*.jar && \
	pwd && ls -l nf-flms-system/target/*.jar && \
	pwd && ls -l nf-flms-openapi/target/*.jar && \
	pwd && ls -l nf-flms-admin/target/*.jar &&
	echo "*******************代码编译成功*******************"
	'''
	}
	}
	}

	stage('生成镜像Tag'){
	steps {
	container('maven') {
	script {
	//本次git提交的commid (git log -n1 --pretty=format:'%h')
	env.COMMITID = sh(returnStdout: true, script: "git log -n1 --pretty=format:'%h'").trim()
	//构建的时间 (date +%Y%m%d_%H%M%S)
	env.BuildTime = sh(returnStdout: true, script: "date +%Y%m%d_%H%M%S").trim()

	//完整的镜像Tag (c106654_20221115_133911)
	env.ImageTag = COMMITID + "_" + BuildTime
	}

	sh 'echo "镜像的Tag: ${ImageTag}"'
	}
	}
	}

	stage('制作Docker镜像'){
	parallel{
	stage('构建Gateway镜像') {
	when { expression {Deploy_Gateway == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "nf-flms-gateway"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建System镜像') {
	when { expression {Deploy_System == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "nf-flms-system"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Order镜像') {
	when { expression {Deploy_Order == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "nf-flms-order"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Monitor镜像') {
	when { expression {Deploy_Monitor == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "nf-flms-admin"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Openapi镜像') {
	when { expression {Deploy_Openapi == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "nf-flms-openapi"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}

	stage('构建Statistics镜像') {
	when { expression {Deploy_Statistics == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "nf-flms-statistics"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'
	//构建镜像
	sh 'cd $(find ./ -type d -name "${AppName}") && docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}
	} //parallel end
	} // stage end

	stage('部署微服务至K8S') {
	parallel {
	stage('交付Gateway') {
	when { expression {Deploy_Gateway == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "nf-flms-gateway"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#prod#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Order') {
	when { expression {Deploy_Order == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "nf-flms-order"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#prod#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}


	stage('交付System') {
	when { expression {Deploy_System == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "nf-flms-system"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#prod#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Monitor') {
	when { expression {Deploy_Monitor == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "nf-flms-admin"
	ImageName = "${Url}/${Pro}/${AppName}"
	IngressHost = "monitor.hmallleasing.com"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#prod#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	sed -i "s#{host}#${IngressHost}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Statistics') {
	when { expression {Deploy_Statistics == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "nf-flms-statistics"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#prod#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	sed -i "s#{host}#${IngressHost}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}

	stage('交付Openapi') {
	when { expression {Deploy_Openapi == "true" \|\| Deploy_All == "true" } }
	environment {
	AppName = "nf-flms-openapi"
	ImageName = "${Url}/${Pro}/${AppName}"
	}
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	sh '''
	cd $(find ./ -type d -name "${AppName}") && \
	cat deploy.yaml && \
	sed -i "s#{namespace}#prod#g" deploy.yaml
	sed -i "s#{image}#${ImageName}:${ImageTag}#g" deploy.yaml
	sed -i "s#{host}#${IngressHost}#g" deploy.yaml
	cat deploy.yaml && \
	kubectl apply -f deploy.yaml
	'''
	}
	}
	}
	}
	} //parallel end
	} //stage
	}
	}

# 2.9 配置自动触发构建

Jenkins 配置：

#Gitee 配置参考连接 https://blog.csdn.net/hali90s/article/details/126991514

#Gitlab 配置参考连接 https://blog.csdn.net/weixin_58423196/article/details/146407508

1、安装插件：GitLab Plugin

2、配置 ->Triggers

点击右下角的 "Advanced"，进入触发器的高级配置，在 "Allowed branched" 部分按需选择 webhook 作用于哪些分支，然后点击 "Generate" 生成唯一的 secret token。187ad167e619cf50a063093898b6befb

Gitlab 配置：

1、设置 -> 网络 -> 外部请求 -> 允许 Webhook 和服务对本地网络的请求

1、Gitlab 点击对应项目 -> 设置 ->Webhooks

# 三、CD 阶段

CD 阶段：Jenkins/CD-> 拉取 Harbor 仓库对应项目镜像 -> 部署应用至 K8S 生产环境；

# 3.1 获取完整的镜像名称

1、获取 Harbor 镜像 Tag

	[root@k8s-master01 06-service-all]# curl -s -uadmin:passwd -H'Content-Type: application/json' -X GET https://s.hmallleasing.com/v2/nf-flms/nf-flms-statistics/tags/list \| sed -r 's#(\{.\[)(.)(\]\})#\2#g' \| xargs -d "," -n1 \| xargs -n1 \| sort -t "_" -k2 -k3 -nr \| head -5
	ab41fb8_20250619_110313
	ab41fb8_20250619_104857
	ab41fb8_20250618_143612
	ab41fb8_20250618_140623
	ab41fb8_20250618_124756

2、安装插件

系统管理 -> 插件管理 ->Active Choices、 Image Tag

3、创建流水线

创建一个新的 Job，名字为 nf-flms-order-cd，类型 Pipeline

变量：Harbor_Url
项目 -> 配置 -> 参数化构建 ->Active Choices Parameter
语法：return ["s.hmallleasing.com","harbor.hmallleasing.com"]

级联变量：Image_Tag

项目 -> 配置 -> 参数化构建 ->Image Tag Parameter

4、编写 pipeline 输出完整的镜像名称

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment {
	Full_Image = "${Harbor_Url}/${Image_Tag}"
	}
	stages {
	stage('输出完整的镜像名称') {
	steps {
	sh 'echo 镜像名称-tag: ${Full_Image}'
	}
	}
	}
	}

# 3.2 生产环境中部署服务至 K8S

1、编写 yaml

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: kubectl
	image: s.hmallleasing.com/ops/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment {
	Full_Image = "${Harbor_Url}/${Harbor_Pro}/${Image_Name}:${Image_Tag}"
	}
	stages {
	stage('输出完整的镜像名称') {
	steps {
	sh 'echo 镜像名称-tag: ${Full_Image}'
	}
	}
	stage('部署应用至K8S') {
	steps {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	container('kubectl'){
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >> ~/.kube/config'
	sh 'kubectl set image deployment/nf-flms-order nf-flms-order=${Full_Image} -n prod'
	}
	}
	}
	}
	}
	}

2、部署应用

	[root@k8s-master01 java]# kubectl create ns prod
	[root@k8s-master01 java]# kubectl create secret docker-registry harbor-admin --docker-server=s.hmallleasing.com --docker-username=admin --docker-password=PASSWD -n prod

	[root@k8s-master01 java]# kubectl apply -f deploy-prod.yaml

# 3.3 CD 流水线部署指定 Tag 版本到 K8S 生产环境

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment {
	Full_Image = "${Harbor_Url}/${Image_Tag}"
	}
	stages {
	stage('输出完整的镜像名称') {
	steps {
	sh 'echo 镜像名称-tag: ${Full_Image}'
	}
	}

	stage('部署应用至K8S') {
	steps {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	container('kubectl'){
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >> ~/.kube/config'
	sh 'kubectl set image deployment/nf-flms-order nf-flms-order=${Full_Image} -n prod'
	}
	}
	}
	}

	}
	}

	[root@k8s-master01 06-service-all]# kubectl describe deploy nf-flms-order -n prod \|grep -i image
	Image: s.hmallleasing.com/nf-flms/nf-flms-order:b526b40_20250618_094603
	/data from data-image (rw)
	Image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/filebeat_sidecar:7.17.6
	data-image:
	ClaimName: data-image

# 3.4 询问是否需要回退

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment {
	Full_Image = "${Harbor_Url}/${Image_Tag}"
	Image_Name = "nf-flms-order"
	}
	stages {
	stage('输出完整的镜像名称') {
	steps {
	sh 'echo 镜像名称-tag: ${Full_Image}'
	}
	}

	stage('部署应用至K8S') {
	steps {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	container('kubectl'){
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >> ~/.kube/config'
	sh 'kubectl set image deployment/${Image_Name} ${Image_Name}=${Full_Image} -n prod'
	}
	}
	}
	}

	stage('快速回滚') {
	steps {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	container('kubectl'){
	script {
	timeout(time:1 , unit: 'HOURS'){
	def UserInput = input message: '是否回退至上一个版本', parameters: [choice(choices: ['No', 'Yes'], name: 'rollback')]
	if (UserInput == "Yes"){
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >> ~/.kube/config'
	sh 'kubectl rollout undo deployment ${Image_Name} -n prod'
	}else {
	echo "没有选择回退"
	}
	}
	}
	}
	}
	}
	}

	}
	}

#如果选择 “Yes” 继续，则回退至上一个版本，等待时间为 1 小时

#如果选择 “No” 继续，则不回退至上一个版本

#如果没有选择，则不回退至上一个版本

DevOps

# K8S 基于 Jenkins 实现微服务应用 CICD 实践（四）

# 一、环境准备

# 二、CI 阶段

# 2.1 编写 Pipeline 框架

# 2.2 获取代码

# 2.3 Sonarqube 漏洞扫描

# 2.4 检查扫描结果

# 2.5 代码编译

# 2.6 制作 Docker 镜像

# 2.7 交付应用到 K8S

# 2.8 选择制定微服组件部署

# 2.9 配置自动触发构建

# 三、CD 阶段

# 3.1 获取完整的镜像名称

# 3.2 生产环境中部署服务至 K8S

# 3.3 CD 流水线部署指定 Tag 版本到 K8S 生产环境

# 3.4 询问是否需要回退

K8S基于Jenkins实现SpringCloud微服务CI与CD实践（三）

100个运维相关技术官网汇总