K8S基于Jenkins实现Java项目CI与CD实践（二）

# K8S 基于 Jenkins 实现 Java 项目 CI 与 CD 实践（二）

CI 阶段：开发人员 -> 提交代码 ->Gitlab 仓库 ->Jenkins/CI 抓取代码 -> 漏洞扫描 -> 编译 -> 构建镜像 -> 推送 Harbor-> 部署应用至 K8S 测试环境；
CD 阶段：Jenkins/CD-> 拉取 Harbor 仓库对应项目镜像 -> 部署应用至 K8S 生产环境；

# 一、提交代码

准备好 Java 代码、Dockerfile、deploy.yaml 资源清单文件提交到 Gitlab 服务器；

	$ git config --global user.name "xuyong"
	$ git config --global user.email "373370405@qq.com"
	$ git init
	$ git add .
	$ git commit -m "first commit"
	$ git remote remove origin
	$ git remote add origin http://gitlab.hmallleasing.com/root/springboot-cicd.git
	$ git push -u origin "master"

# 二、CI 阶段

CI 阶段分为以下 6 步：
1、获取代码
2、漏洞扫描
3、检测漏洞扫描结果，如果正常则继续、否则就终止
4、使用 maven 进行编译，打包
5、制作 Docker 镜像、推送到 Harbor 仓库 (Dockerfile 文件)
6、交付应用到 K8S

# 2.1 编写 Pipeline 框架

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}
	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	sh 'echo "获取代码"'
	}
	}
	}
	stage('代码扫描') {
	steps {
	container('sonar') {
	sh 'echo "代码扫描"'
	}
	}
	}
	stage('检查扫描结果') {
	steps {
	container('sonar') {
	sh 'echo "Code Result Check"'
	}
	}
	}
	stage('代码编译') {
	steps {
	container('maven') {
	sh 'echo "代码编译"'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.2 获取代码

1、Jenkins 上配置 Gitlab 认证信息

系统管理 ->manage credentials-> 全局 ->Add Credential

2、SlavePod 访问 gitlab

通过 Coredns 配置自定义域名解析，如果 gitlab 解析至公网，无需配置此步骤；

	[root@k8s-master01 ~]# kubectl edit cm coredns -n kube-system
	...
	ready

	hosts {
	192.168.1.74 gitlab.hmallleasing.com
	fallthrough
	}
	kubernetes cluster.local in-addr.arpa ip6.arpa {
	...

测试解析是否生效；

	[root@k8s-master01 ~]# kubectl exec -it gitlab-0 -n ops -- /bin/bash
	root@gitlab-0:/# ping gitlab.hmallleasing.com
	PING gitlab.hmallleasing.com (192.168.1.74): 56 data bytes
	64 bytes from 192.168.1.74: seq=0 ttl=63 time=0.381 ms
	64 bytes from 192.168.1.74: seq=1 ttl=63 time=0.203 ms

3、生成流水线脚本获取代码

通过流水线语法 -> 片段生成器 -> 生成流水线脚本，生成 Pipeline 流水线

3、测试获取代码 Pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/springboot-cicd.git"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}
	stage('代码扫描') {
	steps {
	container('sonar') {
	sh 'echo "代码扫描"'
	}
	}
	}
	stage('检查扫描结果') {
	steps {
	container('sonar') {
	sh 'echo "Code Result Check"'
	}
	}
	}
	stage('代码编译') {
	steps {
	container('maven') {
	sh 'echo "代码编译"'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.3 Sonarqube 漏洞扫描

1、配置 dns 解析（如果 sonarqube 解析至公网，无需配置此步骤）
slavePod 需要访问 sonarqube 服务端，sonarqube 服务端需要回调 jenkins，获取扫描结果。

	[root@k8s-master01 ~]# kubectl edit cm -n kube-system coredns
	...
	ready

	hosts {
	192.168.1.74 gitlab.hmallleasing.com #gitalb
	192.168.1.74 sonar.hmallleasing.com #sonar
	192.168.1.74 jenkins.hmallleasing.com #jenkins
	fallthrough
	}
	kubernetes cluster.local in-addr.arpa ip6.arpa {
	...

测试解析是否生效；

	[root@k8s-master01 ~]# kubectl exec -it gitlab-0 -n ops -- /bin/bash
	root@gitlab-0:/# ping sonar.hmallleasing.com
	PING sonar.hmallleasing.com (192.168.1.74): 56 data bytes
	64 bytes from 192.168.1.74: seq=0 ttl=63 time=0.187 ms
	64 bytes from 192.168.1.74: seq=1 ttl=63 time=0.202 ms

	root@gitlab-0:/# ping jenkins.hmallleasing.com
	PING jenkins.hmallleasing.com (192.168.1.74): 56 data bytes
	64 bytes from 192.168.1.74: seq=0 ttl=63 time=0.189 ms
	64 bytes from 192.168.1.74: seq=1 ttl=63 time=0.202 ms

2、Jenkins 集成 Sonarqube

生成 sonarqube 令牌：用户 -> 我的账号 -> 安全

保存令牌信息：squ_6e57e496a22ebdf0723c3683bd489df57f967f49

配置 sonarqube 认证信息：系统管理 ->manage credentials-> 全局 ->Add Credential

Jenkins 集成 Sonarqube：系统管理 -> 系统配置 ->SonarQube servers->Add SonarQube

3、获取 sonar-scanner 扫描命令（此步骤不用执行）

项目 -> 手工 -> 显示名

#获取 sonar-scanner 扫描命令，需要在项目目录下进行扫描

	sonar-scanner \
	-Dsonar.projectKey=springboot \
	-Dsonar.sources=. \
	-Dsonar.host.url=http://sonar.hmallleasing.com \
	-Dsonar.login=squ_6e57e496a22ebdf0723c3683bd489df57f967f49

4、测试代码扫描 Pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/springboot-cicd.git"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}
	stage('代码扫描'){
	steps {
	withSonarQubeEnv('sonar-k8s'){ //jenkins集成sonarqube名称sonar-k8s
	container('sonar'){
	sh 'pwd && ls -l'
	sh 'sonar-scanner \
	-Dsonar.projectKey=springboot-cicd \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检查扫描结果') {
	steps {
	container('sonar') {
	sh 'echo "Code Result Check"'
	}
	}
	}
	stage('代码编译') {
	steps {
	container('maven') {
	sh 'echo "代码编译"'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.4 检查扫描结果

1、配置 sonarquber, 让其将检查结果通知 jiekins，如果正常则继续、否则就终止执行

URL：http://admin:talent@jenkins.hmallleasing.com/sonarqube-webhook
Jenkins 访问地址：jenkins.hamlllleasing.com
Jenkins 用户名：admin
Jenkins 密码：talent

2、检查扫描结果 pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/springboot-cicd.git"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}
	stage('代码扫描'){
	steps {
	withSonarQubeEnv('sonar-k8s'){ //jenkins集成sonarqube名称sonar-k8s
	container('sonar'){
	sh 'pwd && ls -l'
	sh 'sonar-scanner \
	-Dsonar.projectKey=springboot-cicd \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检查扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}
	stage('代码编译') {
	steps {
	container('maven') {
	sh 'echo "代码编译"'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.5 代码编译

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/springboot-cicd.git"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}
	stage('代码扫描'){
	steps {
	withSonarQubeEnv('sonar-k8s'){ //jenkins集成sonarqube名称sonar-k8s
	container('sonar'){
	sh 'pwd && ls -l'
	sh 'sonar-scanner \
	-Dsonar.projectKey=springboot-cicd \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检查扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}
	stage('编译代码'){
	steps {
	container('maven'){
	sh 'pwd && ls -l'
	sh 'mvn package -Dmaven.test.skip=true'
	sh 'pwd && ls -l target/*'
	}
	}
	}
	stage('制作Docker镜像') {
	steps {
	container('docker') {
	sh 'echo "制作Docker镜像"'
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.6 制作 Docker 镜像

制作 Docker 镜像、推送到 Harbor 仓库，根据 Dockerfile 文件生成 Docker 镜像

1、配置 dns 解析（如果 harbor 解析至公网，无需配置此步骤）

	[root@k8s-master01 ~]# kubectl edit cm -n kube-system coredns
	...
	ready

	hosts {
	192.168.1.74 gitlab.hmallleasing.com #gitalb
	192.168.1.74 sonar.hmallleasing.com #sonar
	192.168.1.74 jenkins.hmallleasing.com #jenkins
	192.168.1.68 s.hmallleasing.com #harbor
	fallthrough
	}
	kubernetes cluster.local in-addr.arpa ip6.arpa {
	...

测试解析是否生效；

	[root@k8s-master01 ~]# kubectl exec -it gitlab-0 -n ops -- /bin/bash
	root@gitlab-0:/# ping s.hmallleasing.com
	PING s.hmallleasing.com (192.168.1.68): 56 data bytes
	64 bytes from 192.168.1.68: seq=0 ttl=63 time=0.257 ms
	64 bytes from 192.168.1.68: seq=1 ttl=63 time=0.198 ms

2、配置 harbor 认证信息：系统管理 ->manage credentials-> 全局 ->Add Credential

3、代码扫描 Pipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/springboot-cicd.git"

	//Harbor相关的全局变量
	Url = "s.hmallleasing.com"
	Pro = "base"
	ImageName = "${Url}/${Pro}/springboot"
	HARBOR_ID = "harbor-auth"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}
	stage('代码扫描'){
	steps {
	withSonarQubeEnv('sonar-k8s'){ //jenkins集成sonarqube名称sonar-k8s
	container('sonar'){
	sh 'pwd && ls -l'
	sh 'sonar-scanner \
	-Dsonar.projectKey=springboot-cicd \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检查扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}
	stage('编译代码'){
	steps {
	container('maven'){
	sh 'pwd && ls -l'
	sh 'mvn package -Dmaven.test.skip=true'
	sh 'pwd && ls -l target/*'
	}
	}
	}
	stage('生成镜像Tag'){
	steps {
	container('maven') {
	script {
	//本次git提交的commid (git log -n1 --pretty=format:'%h')
	env.COMMITID = sh(returnStdout: true, script: "git log -n1 --pretty=format:'%h'").trim()
	//构建的时间 (date +%Y%m%d_%H%M%S)
	env.BuildTime = sh(returnStdout: true, script: "date +%Y%m%d_%H%M%S").trim()

	//完整的镜像Tag (c106654_20221115_133911)
	env.ImageTag = COMMITID + "_" + BuildTime
	}

	sh 'echo "镜像的Tag: ${ImageTag}"'
	}
	}
	}
	stage('制作Docker镜像'){
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'

	//构建镜像
	sh 'docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	sh 'echo "交付应用至K8S"'
	}
	}
	}
	}
	}

# 2.7 交付应用到 K8S

1、将 K8S 的 kubeconfig 文件下载至本地，配置 Jenkins，将 config 文件制作为一个 Scretfile；

[root@k8s-master01 ~]# sz /root/.kube/config

配置 k8s 认证信息：系统管理 ->manage credentials-> 全局 ->Add Credential

2、编写 pipeline 流水线

在 Stage 中引用这个 config 文件，就可以操作对应的集群，部署前需做以下操作；

部署的名称空间需要提前创建
需要创建对应 namespace 的 Harbor 认证 Secrets ->harbor-admin

[root@k8s-master01 ~]# kubectl create ns dev

[root@k8s-master01 ~]# kubectl create secret docker-registry harbor-admin --docker-server=s.hmallleasing.com --docker-username=admin --docker-password=passwd -n dev

3、编写 yaml 文件

	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: springboot
	namespace: {NameSpace} #修改为 {} 特殊字符，后期好替换
	spec:
	replicas: 2
	selector:
	matchLabels:
	app: spring
	template:
	metadata:
	labels:
	app: spring
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: springboot
	image: {Image} # 修改为 {} 特殊字符，后期好替换
	ports:
	- name: http
	containerPort: 8080
	env: # 传递初始堆内存和最大堆内存占用
	- name: XMS_OPTS
	valueFrom:
	resourceFieldRef:
	resource: requests.memory
	- name: XMX_OPTS
	valueFrom:
	resourceFieldRef:
	resource: limits.memory
	resources:
	requests:
	memory: 150Mi
	limits:
	memory: 300Mi
	readinessProbe: # 就绪探针；如果端口不存活，则从负载均衡中移除
	tcpSocket:
	port: http # http 是一个名字；它会获取这个名字对应的端口；
	initialDelaySeconds: 10
	failureThreshold: 3
	livenessProbe: # 存活探针；获取 url，状态码不对那么则触发重启操作
	httpGet:
	path: /
	port: http
	initialDelaySeconds: 10
	failureThreshold: 3
	---
	apiVersion: v1
	kind: Service
	metadata:
	name: spring-svc
	namespace: {namespace}
	spec:
	selector:
	app: spring
	ports:
	- port: 8080
	targetPort: 8080

	---
	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: springboot-ingress
	namespace: {namespace}
	spec:
	ingressClassName: "nginx"
	rules:
	- host: "{host}"
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: spring-svc
	port:
	number: 8080

4、编写交付 K8spipeline

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	volumes:
	- name: data
	nfs:
	server: 192.168.1.75
	path: /data/nfs/maven
	- name: dockersocket
	hostPath:
	path: /run/docker.sock
	containers:
	- name: maven
	image: s.hmallleasing.com/base/maven:3.8.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: data
	mountPath: /root/.m2
	- name: nodejs
	image: s.hmallleasing.com/base/nodejs:14.20
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: sonar
	image: s.hmallleasing.com/base/sonar-scanner:2.3.0
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	- name: docker
	image: s.hmallleasing.com/base/docker:20.10
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	volumeMounts:
	- name: dockersocket
	mountPath: /run/docker.sock
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment{
	//定义git变量
	Git_Id = "gitlab-root-token"
	Git_Url = "http://gitlab.hmallleasing.com/root/springboot-cicd.git"

	//Harbor相关的全局变量
	Url = "s.hmallleasing.com"
	Pro = "base"
	ImageName = "${Url}/${Pro}/springboot"
	HARBOR_ID = "harbor-auth"

	//对外暴露的域名
	Ingress_Host = "spring-dev.hmallleasing.com"
	}

	stages {
	stage('获取代码') {
	steps {
	container('maven') {
	checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: "${Git_Id}", url: "${Git_Url}"]])
	sh 'pwd && ls'
	}
	}
	}
	stage('代码扫描'){
	steps {
	withSonarQubeEnv('sonar-k8s'){ //jenkins集成sonarqube名称sonar-k8s
	container('sonar'){
	sh 'pwd && ls -l'
	sh 'sonar-scanner \
	-Dsonar.projectKey=springboot-cicd \
	-Dsonar.java.binaries=src \
	-Dsonar.sources=.'
	}
	}
	}
	}
	stage('检查扫描结果'){
	steps {
	container('sonar'){
	script {
	timeout(5) {
	def qg = waitForQualityGate()
	if (qg.status != 'OK') {
	error "Sonarqube代码检查失败, error的原因 ${qg.status}"
	}
	}
	}
	}
	}
	}
	stage('编译代码'){
	steps {
	container('maven'){
	sh 'pwd && ls -l'
	sh 'mvn package -Dmaven.test.skip=true'
	sh 'pwd && ls -l target/*'
	}
	}
	}
	stage('生成镜像Tag'){
	steps {
	container('maven') {
	script {
	//本次git提交的commid (git log -n1 --pretty=format:'%h')
	env.COMMITID = sh(returnStdout: true, script: "git log -n1 --pretty=format:'%h'").trim()
	//构建的时间 (date +%Y%m%d_%H%M%S)
	env.BuildTime = sh(returnStdout: true, script: "date +%Y%m%d_%H%M%S").trim()

	//完整的镜像Tag (c106654_20221115_133911)
	env.ImageTag = COMMITID + "_" + BuildTime
	}

	sh 'echo "镜像的Tag: ${ImageTag}"'
	}
	}
	}
	stage('制作Docker镜像'){
	steps {
	container('docker'){
	withCredentials([usernamePassword(credentialsId: "${HARBOR_ID}", passwordVariable: 'HARBOR_PASSWORD', usernameVariable: 'HARBOR_USER')]) {
	//登陆harbor
	sh 'echo "${HARBOR_PASSWORD}" \| docker login ${Url} -u "${HARBOR_USER}" --password-stdin'

	//构建镜像
	sh 'docker build -t ${ImageName}:${ImageTag} .'
	//推送镜像
	sh 'docker push ${ImageName}:${ImageTag}'
	//删除镜像
	sh 'docker rmi ${ImageName}:${ImageTag}'
	}
	}
	}
	}
	stage('交付应用至K8S') {
	steps {
	container('kubectl') {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	sh 'mkdir -p /root/.kube && echo ${KUBECONFIG} >/root/.kube/config'

	//替换depoy.yaml中{namespace}{Image}{host}
	sh 'cat deploy.yaml'
	sh 'sed -i "s#{namespace}#dev#g" deploy.yaml'
	sh 'sed -i "s#{Image}#${ImageName}:${ImageTag}#g" deploy.yaml'
	sh 'sed -i "s#{host}#${Ingress_Host}#g" deploy.yaml'
	sh 'cat deploy.yaml'

	sh 'kubectl apply -f deploy.yaml'
	}
	}
	}
	}
	}
	}

# 2.8 配置自动触发构建

Jenkins 配置：

#Gitee 配置参考连接 https://blog.csdn.net/hali90s/article/details/126991514

#Gitlab 配置参考连接 https://blog.csdn.net/weixin_58423196/article/details/146407508

1、安装插件：GitLab Plugin

2、配置 ->Triggers

点击右下角的 "Advanced"，进入触发器的高级配置，在 "Allowed branched" 部分按需选择 webhook 作用于哪些分支，然后点击 "Generate" 生成唯一的 secret token。

Gitlab 配置：

1、设置 -> 网络 -> 外部请求 -> 允许 Webhook 和服务对本地网络的请求

1、Gitlab 点击对应项目 -> 设置 ->Webhooks

# 三、CD 阶段

CD 阶段：Jenkins/CD-> 拉取 Harbor 仓库对应项目镜像 -> 部署应用至 K8S 生产环境；

# 3.1 获取完整的镜像名称

1、获取 Harbor 镜像 Tag

	[root@k8s-master01 ~]# curl -s -uadmin:passwd -H'Content-Type: application/json' -X GET https://s.hmallleasing.com/v2/base/springboot/tags/list \| sed -r 's#(\{.\[)(.)(\]\})#\2#g' \| xargs -d "," -n1 \| xargs -n1 \| sort -t "_" -k2 -k3 -nr \| head -5
	3e24684_20230326_153018
	5041fbd_20230326_144343
	5da8d15_20230326_143533
	5da8d15_20230326_005207
	5da8d15_20230326_004940

2、安装插件

系统管理 -> 插件管理 ->Active Choices

3、级联变量的方式来提取对应的 tag
变量：Harbor_Url
项目 -> 配置 -> 参数化构建 ->Active Choices Parameter
语法：return ["s.hmallleasing.com","harbor.hmallleasing.com"]

变量：Harbor_Pro
项目 -> 配置 -> 参数化构建 ->Active Choices Parameter
语法：return ["base","nf-flms","ops","xx"]

变量：Image_Name
项目 -> 配置 -> 参数化构建 ->Active Choices Parameter
语法：return ["springboot","nf-flms-statistics","nf-flms-order","xx"]

级联变量：Image_Tag

项目 -> 配置 -> 参数化构建 ->Active Choices Reactive Parameter
语法：

def get_tag = [ "bash", "-c", "curl -s -uadmin:passwd -H'Content-Type: application/json' -X GET https://s.hmallleasing.com/v2/base/springboot/tags/list | sed -r 's#(\\{.*\\[)(.*)(\\]\\})#\\2#g' | xargs -d ',' -n1 | xargs -n1 | sort -t '_' -k2 -k3 -nr | head -5"]

return get_tag.execute().text.tokenize("\n")

级联变量：Harbor_Url,Harbor_Pro,Image_Name

4、编写 pipeline 输出完整的镜像名称

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: kubectl
	image: s.hmallleasing.com/base/kubectl:1.32.3
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment {
	Full_Image = "${Harbor_Url}/${Harbor_Pro}/${Image_Name}:${Image_Tag}"
	}
	stages {
	stage('输出完整的镜像名称') {
	steps {
	sh 'echo 镜像名称-tag: ${Full_Image}'
	}
	}
	}
	}

# 3.2 生产环境中部署对应至 K8S

1、编写 yaml

	[root@k8s-master01 ~]# cat deploy-prod.yaml
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: springboot
	namespace: prod #修改为 {} 特殊字符，后期好替换
	spec:
	replicas: 3
	selector:
	matchLabels:
	app: spring
	template:
	metadata:
	labels:
	app: spring
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: springboot
	image: s.hmallleasing.com/base/springboot:f04e3f0_20250611_185645 # 修改为 {} 特殊字符，后期好替换
	ports:
	- name: http
	containerPort: 8080
	env: # 传递初始堆内存和最大堆内存占用
	- name: XMS_OPTS
	valueFrom:
	resourceFieldRef:
	resource: requests.memory
	- name: XMX_OPTS
	valueFrom:
	resourceFieldRef:
	resource: limits.memory
	resources:
	requests:
	memory: 150Mi
	limits:
	memory: 300Mi
	readinessProbe: # 就绪探针；如果端口不存活，则从负载均衡中移除
	tcpSocket:
	port: http # http 是一个名字；它会获取这个名字对应的端口；
	initialDelaySeconds: 10
	failureThreshold: 3
	livenessProbe: # 存活探针；获取 url，状态码不对那么则触发重启操作
	httpGet:
	path: /
	port: http
	initialDelaySeconds: 10
	failureThreshold: 3

	---

	apiVersion: v1
	kind: Service
	metadata:
	name: spring-svc
	namespace: prod
	spec:
	selector:
	app: spring
	ports:
	- port: 8080
	targetPort: 8080

	---

	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: springboot-ingress
	namespace: prod
	spec:
	ingressClassName: "nginx"
	rules:
	- host: "spring-prod.hmallleasing.com"
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: spring-svc
	port:
	number: 8080

2、部署应用

[root@k8s-master01 java]# kubectl create ns prod

[root@k8s-master01 java]# kubectl create secret docker-registry harbor-admin --docker-server=s.hmallleasing.com --docker-username=admin --docker-password=passwd -n prod

[root@k8s-master01 java]# kubectl apply -f deploy-prod.yaml

# 3.3 CD 流水线部署指定 Tag 版本到 K8S 生产环境

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: kubectl
	image: s.hmallleasing.com/ops/kubectl:1.22.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment {
	Full_Image = "${Harbor_Url}/${Harbor_Pro}/${Image_Name}:${Image_Tag}"
	}
	stages {
	stage('输出完整的镜像名称') {
	steps {
	sh 'echo 镜像名称-tag: ${Full_Image}'
	}
	}
	stage('部署应用至K8S') {
	steps {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	container('kubectl'){
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >> ~/.kube/config'
	sh 'kubectl set image deployment/${Image_Name} ${Image_Name}=${Full_Image} -n prod'
	}
	}
	}
	}
	}
	}

# 3.4 询问是否需要回退

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	yaml '''
	apiVersion: v1
	kind: Pod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: kubectl
	image: s.hmallleasing.com/ops/kubectl:1.22.6
	imagePullPolicy: IfNotPresent
	command: ["cat"]
	tty: true
	'''
	}
	}

	environment {
	Full_Image = "${Harbor_Url}/${Harbor_Pro}/${Image_Name}:${Image_Tag}"
	}
	stages {
	stage('输出完整的镜像名称') {
	steps {
	sh 'echo 镜像名称-tag: ${Full_Image}'
	}
	}
	stage('部署应用至K8S') {
	steps {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	container('kubectl'){
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >> ~/.kube/config'
	sh 'kubectl set image deployment/${Image_Name} ${Image_Name}=${Full_Image} -n prod'
	}
	}
	}
	}

	stage('快速回滚') {
	steps {
	withCredentials([file(credentialsId: 'kubeconfig', variable: 'KUBECONFIG')]) {
	container('kubectl'){
	script {
	timeout(time:1 , unit: 'HOURS'){
	def UserInput = input message: '是否回退至上一个版本', parameters: [choice(choices: ['No', 'Yes'], name: 'rollback')]
	if (UserInput == "Yes"){
	sh 'mkdir -p ~/.kube && echo ${KUBECONFIG} >> ~/.kube/config'
	sh 'kubectl rollout undo deployment ${Image_Name} -n prod'
	}else {
	echo "没有选择回退"
	}
	}
	}
	}
	}
	}
	}

	}
	}

#如果选择 “Yes” 继续，则回退至上一个版本，等待时间为 1 小时

#如果选择 “No” 继续，则不回退至上一个版本

#如果没有选择，则不回退至上一个版本

DevOps

# K8S 基于 Jenkins 实现 Java 项目 CI 与 CD 实践（二）

# 一、提交代码

# 二、CI 阶段

# 2.1 编写 Pipeline 框架

# 2.2 获取代码

# 2.3 Sonarqube 漏洞扫描

# 2.4 检查扫描结果

# 2.5 代码编译

# 2.6 制作 Docker 镜像

# 2.7 交付应用到 K8S

# 2.8 配置自动触发构建

# 三、CD 阶段

# 3.1 获取完整的镜像名称

# 3.2 生产环境中部署对应至 K8S

# 3.3 CD 流水线部署指定 Tag 版本到 K8S 生产环境

# 3.4 询问是否需要回退

K8S基于Jenkins实现SpringCloud微服务CI与CD实践（一）

K8S基于Jenkins实现SpringCloud微服务CI与CD实践（三）