K8S基于Jenkins实现SpringCloud微服务CI与CD实践（一）

# K8S 基于 Jenkins 实现 SpringCloud 微服务 CI 与 CD 实践（一）

CI 阶段：开发人员 -> 提交代码 ->gitlab 仓库 ->Jenkins/CI 抓取代码 -> 漏洞扫描 -> 编译 -> 构建镜像 -> 推送 Harbor-> 部署应用至 K8S 测试环境；

CD 阶段：Jenkins/CD-> 拉取 Harbor 仓库对应项目镜像 -> 部署应用至 K8S 测试环境

# 一、部署 Harbor

# 1.1 安装基础环境

	[root@harbor ~]# yum remove docker*
	[root@harbor ~]# yum install -y yum-utils device-mapper-persistent-data lvm2
	[root@harbor ~]# curl -o /etc/yum.repos.d/docker-ce.repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
	[root@harbor ~]# yum list docker-ce --showduplicates \|sort -r
	[root@harbor ~]# yum install docker-ce docker-compose -y

	### 提示：No match for argument: python-pip
	#改成为 Centos 8 已经换成 python3 使用该命令解决 yum install python3-pip
	[root@harbor ~]# yum -y install python-pip
	#升级
	[root@harbor ~]# pip3 install --upgrade pip
	[root@harbor ~]# pip3 install docker-compose
	[root@Harbor ~]# docker-compose -version

# 1.2 配置 Docker 加速

	[root@harbor ~]# sudo mkdir -p /etc/docker

	#Docker 加速 1
	[root@harbor ~]# sudo tee /etc/docker/daemon.json <<-'EOF'
	{
	"registry-mirrors": [
	"https://docker.credclouds.com",
	"https://k8s.credclouds.com",
	"https://quay.credclouds.com",
	"https://gcr.credclouds.com",
	"https://k8s-gcr.credclouds.com",
	"https://ghcr.credclouds.com",
	"https://do.nark.eu.org",
	"https://docker.m.daocloud.io",
	"https://docker.nju.edu.cn",
	"https://docker.mirrors.sjtug.sjtu.edu.cn",
	"https://docker.1panel.live",
	"https://docker.rainbond.cc"
	],
	"exec-opts": ["native.cgroupdriver=systemd"]
	}
	EOF

	#Docker 加速 2
	[root@harbor ~]# sudo tee /etc/docker/daemon.json <<-'EOF'
	{
	"registry-mirrors": [
	"https://docker.1panel.live",
	"https://hub.littlediary.cn",
	"https://docker.kejilion.pro",
	"https://docker.1ms.run",
	"https://lispy.org",
	"https://docker.xiaogenban1993.com",
	"https://docker.xuanyuan.me",
	"https://docker.mybacc.com",
	"https://docker-0.unsee.tech",
	"https://dockerpull.cn"
	],
	"exec-opts": ["native.cgroupdriver=systemd"]
	}
	EOF

	[root@harbor ~]# systemctl enable docker --now

# 1.3 装 Harbor

	[root@harbor ~]# cd /soft/
	[root@harbor ~]# wget https://github.com/goharbor/harbor/releases/download/v2.6.1/harbor-offline-installer-v2.6.1.tgz
	[root@harbor soft]# tar xf harbor-offline-installer-v2.6.1.tgz
	[root@harbor soft]# cd harbor
	[root@Harbor harbor]# cp harbor.yml.tmpl harbor.yml
	[root@Harbor harbor]# cat harbor.yml
	...
	hostname: s.hmallleasing.com
	...
	http:
	# port for http, default is 80. If https enabled, this port will redirect to https port
	port: 80

	# https related config
	https:
	# https port for harbor, default is 443
	port: 443
	# The path of cert and key files for nginx
	certificate: /soft/harbor/ssl/hmallleasing.com.pem
	private_key: /soft/harbor/ssl/hmallleasing.com.key
	...
	harbor_admin_password: Harbor12345
	[root@harbor harbor]# ./install.sh

# 1.4 配置 Nginx 负载均衡调度

	[root@lb ~]# vim s.hmallleasing.com.conf
	server {
	listen 443 ssl;
	server_name harbor.hmallleasing.com;
	client_max_body_size 1G;
	ssl_prefer_server_ciphers on;
	ssl_certificate /etc/nginx/sslkey/_.hmallleasing.com_chain.crt;
	ssl_certificate_key /etc/nginx/sslkey/_.hmallleasing.com_key.key;
	location / {
	proxy_pass http://192.168.1.134;
	# include proxy_params;
	# proxy_set_header Host $http_host;
	proxy_set_header X-Real-IP $remote_addr;
	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

	proxy_connect_timeout 30;
	proxy_send_timeout 60;
	proxy_read_timeout 60;

	proxy_buffering on;
	proxy_buffer_size 32k;
	proxy_buffers 4 128k;
	proxy_temp_file_write_size 10240k;
	proxy_max_temp_file_size 10240k;
	}
	}

	server {
	listen 80;
	server_name s.hmallleasing.com;
	return 302 https://$server_name$request_uri;
	}

# 1.5 推送镜像至 Harbor

[root@harbor harbor]# docker tag beae173ccac6 harbor.hmallleasing.com/ops/busybox.v1

[root@harbor harbor]# docker login harbor.hmallleasing.com

[root@harbor harbor]# docker push harbor.hmallleasing.com/ops/busybox.v1

# 1.6 Harbor 停止与启动

	#停用 Harbor
	[root@harbor harbor]# pwd
	/soft/harbor
	[root@harbor harbor]# docker-compose down
	#启动 Harbor
	[root@harbor harbor]# docker-compose up -d
	[root@harbor harbor]# docker-compose start

# 二、部署 Gitlab

通常 Gitlab 都是在独立服务器进行部署和维护，但为了更好的掌握 Kubernetes，所以本次课程采用资源清单方式将 Gitlab 以

StatefulSet 方式交付到 Kubernetes 中；

Gitlab 以容器方式运行，需要持久化如下几个目录中的数据：

持久化本地位置	容器位置	使用
${pvc}/data	/var/opt/gitlab	用于存储应用程序数据
${pvc}/logs	/var/log/gitlab	用于存储日志
${pvc}/config	/etc/gitlab	用于存储 GitLab 配置文件

# 2.1 下载 Gitlab 镜像推送至 Harbor

	[root@harbor harbor]# docker pull gitlab/gitlab-ce:14.6.0-ce.0
	[root@harbor harbor]# docker tag gitlab/gitlab-ce:14.6.0-ce.0 s.hmallleasing.com/base/gitlab-ce:14.6.0
	[root@harbor harbor]# docker login s.hmallleasing.com
	[root@harbor harbor]# docker push s.hmallleasing.com/base/gitlab-ce:14.6.0

# 2.2 创建 Service

	[root@master01 gitlab]# cat 01-gitlab-svc.yaml
	apiVersion: v1
	kind: Service
	metadata:
	name: gitlab-svc
	namespace: ops
	spec:
	clusterIP: None
	selector:
	app: gitlab
	ports:
	- name: http
	port: 80
	targetPort: 80
	- name: https
	port: 443
	targetPort: 443

# 2.3 创建 Statefulset

	# cat 02-gitlab-sts.yaml
	apiVersion: apps/v1
	kind: StatefulSet
	metadata:
	name: gitlab
	namespace: ops
	spec:
	serviceName: "gitlab-svc" # 关联的 headlessService
	selector:
	matchLabels:
	app: gitlab
	template:
	metadata:
	labels:
	app: gitlab
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: gitlab-ce
	image: s.hmallleasing.com/base/gitlab-ce:14.6.0
	imagePullPolicy: IfNotPresent
	env:
	- name: GITLAB_ROOT_PASSWORD
	value: "admin123"
	- name: GITLAB_OMNIBUS_CONFIG
	value: \|
	external_url "http://gitlab.hmallleasing.com"
	gitlab_rails['time_zone'] = 'Asia/Shanghai'
	node_exporter['enable'] = false
	redis_exporter['enable'] = false
	postgres_exporter['enable'] = false
	gitlab_exporter['enable'] = false
	grafana['enable'] = false
	grafana['reporting_enabled'] = false
	prometheus['enable'] = false
	prometheus['monitor_kubernetes'] = false
	gitlab_rails['gitlab_email_enabled'] = true
	gitlab_rails['gitlab_email_from'] = '373370405@qq.com' #发件邮箱
	gitlab_rails['gitlab_email_display_name'] = 'Nfzl-Gitlab' #发件人显示名称
	gitlab_rails['smtp_enable'] = true
	gitlab_rails['smtp_address'] = "smtp.qq.com"
	gitlab_rails['smtp_port'] = 465
	gitlab_rails['smtp_user_name'] = "373370405@qq.com" #发件人邮箱账户
	gitlab_rails['smtp_password'] = "pohleicnfawvb" #发件人邮箱客户端授权码
	gitlab_rails['smtp_domain'] = "smtp.qq.com"
	gitlab_rails['smtp_authentication'] = "login"
	gitlab_rails['smtp_enable_starttls_auto'] = true
	gitlab_rails['smtp_tls'] = true
	ports:
	- name: http
	containerPort: 80
	- name: https
	containerPort: 443
	volumeMounts:
	- name: data
	mountPath: /etc/gitlab
	subPath: config
	- name: data
	mountPath: /var/opt/gitlab
	subPath: data
	- name: data
	mountPath: /var/log/gitlab
	subPath: logs
	volumeClaimTemplates: # pvc
	- metadata:
	name: data
	spec:
	accessModes: ["ReadWriteMany"]
	storageClassName: "nfs-storage"
	resources:
	requests:
	storage: 2Gi

# 2.4 创建 Ingress

	[root@master01 gitlab]# cat 03-gitlab-ingress.yaml
	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: gitlab-ingress
	namespace: ops
	spec:
	ingressClassName: "nginx"
	rules:
	- host: "gitlab.hmallleasing.com"
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: gitlab-svc
	port:
	name: http

# 2.5 更新资源清单

[root@master01 gitlab]# kubectl create ns ops

[root@master01 gitlab]# kubectl create secret docker-registry harbor-admin --docker-server=s.hmallleasing.com --docker-username=admin --docker-password=passwd -n ops

[root@master01 gitlab]# kubectl apply -f .

# 2.6 访问 Gitlab

# 三、部署 Sonarqube

SonarQube 是一个开源的代码质量管理系统，用于检测代码中的错误、漏洞。它可以与 Jenkins 集成，让我们能自动化进行

代码质量扫描。

Sonarqube 扫描流程：

使用 SonarScanner 客户端工具将代码源文件以 http/https 方式推送给 Sonarqube 服务端；
Sonarqube 服务端基于 ElasticSerach 对代码进行分析，而后将分析结果存储至 Database；
Sonarqube 服务端读取 Database 数据，然后将扫描结果进行前端展示；

所以，安装 Sonarqube 之前需要先安装依赖的数据库，后期进行漏洞扫描时还需要借助 SonarScanner 客户端；

# 3.1 部署 Pgsql

# 3.1.1 下载 postgresql 镜像推送至 Harbor

	[root@master01 ~]# docker pull postgres:13.8
	[root@master01 ~]# docker tag postgres:13.8 s.hmallleasing.com/base/postgres:13.8
	[root@master01 ~]# docker login s.hmallleasing.com
	[root@master01 ~]# docker push s.hmallleasing.com/base/postgres:13.8

# 3.1.2 部署 Pgsql-sts

	[root@k8s-master01 sonarqube]# cat 01-pgsql-sts.yaml
	apiVersion: v1
	kind: Service
	metadata:
	name: pgsql-svc
	namespace: ops
	spec:
	clusterIP: None
	selector:
	app: pgsql
	ports:
	- port: 5432
	---
	apiVersion: apps/v1
	kind: StatefulSet
	metadata:
	name: postgresql
	namespace: ops
	spec:
	serviceName: "pgsql-svc"
	selector:
	matchLabels:
	app: pgsql
	template:
	metadata:
	labels:
	app: pgsql
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: postgresql
	image: s.hmallleasing.com/ops/postgres:13.8
	imagePullPolicy: IfNotPresent
	env:
	- name: POSTGRES_DB # 数据库
	value: sonardb
	- name: POSTGRES_USER # 用户
	value: sonar
	- name: POSTGRES_PASSWORD # 密码
	value: "Superman*2023"
	ports:
	- containerPort: 5432
	volumeMounts:
	- name: db
	mountPath: /var/lib/postgresql/data
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	volumes:
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	volumeClaimTemplates:
	- metadata:
	name: db
	spec:
	accessModes: ["ReadWriteOnce"]
	storageClassName: "nfs-storage"
	resources:
	requests:
	storage: 5Gi

# 3.1.3 检查 postgresql

	[root@k8s-master01 sonarqube]# kubectl exec -it postgresql-0 -n ops -- /bin/bash
	root@postgresql-0:/# psql -Usonar -d sonardb
	psql (13.8 (Debian 13.8-1.pgdg110+1))
	Type "help" for help.

	sonardb=# \l
	List of databases
	Name \| Owner \| Encoding \| Collate \| Ctype \| Access privileges
	-----------+-------+----------+------------+------------+-------------------
	postgres \| sonar \| UTF8 \| en_US.utf8 \| en_US.utf8 \|
	sonardb \| sonar \| UTF8 \| en_US.utf8 \| en_US.utf8 \|
	template0 \| sonar \| UTF8 \| en_US.utf8 \| en_US.utf8 \| =c/sonar +
	\| \| \| \| \| sonar=CTc/sonar
	template1 \| sonar \| UTF8 \| en_US.utf8 \| en_US.utf8 \| =c/sonar +
	\| \| \| \| \| sonar=CTc/sonar
	(4 rows)

# 3.2 部署 Sonarqube

# 3.2.1 下载 sonarqube 镜像推送至 Harbor

	[root@master01 ~]# docker pull sonarqube:9.9.8-community
	[root@master01 ~]# docker tag sonarqube:9.9.8-community s.hmallleasing.com/base/sonarqube:9.9.8
	[root@master01 ~]# docker login s.hmallleasing.com
	[root@master01 ~]# docker push s.hmallleasing.com/base/sonarqube:9.9.8

# 3.2.2 部署 Sonarqube-sts

	# cat 02-sonarqube-sts.yaml
	apiVersion: v1
	kind: Service
	metadata:
	name: sonarqube-svc
	namespace: ops
	spec:
	clusterIP: None
	selector:
	app: sonarqube
	ports:
	- name: web
	port: 9000
	---
	apiVersion: apps/v1
	kind: StatefulSet
	metadata:
	name: sonarqube
	namespace: ops
	spec:
	serviceName: "sonarqube-svc"
	selector:
	matchLabels:
	app: sonarqube
	template:
	metadata:
	labels:
	app: sonarqube
	spec:
	imagePullSecrets:
	- name: harbor-admin
	initContainers:
	- name: set-kernel
	image: busybox
	command: ["sh", "-c", "sysctl -w vm.max_map_count=524288 ; sysctl -w fs.file-max=131072 ; ulimit -n 131072 ; ulimit -u 8192"]
	securityContext:
	privileged: true
	containers:
	- name: sonarqube
	image: s.hmallleasing.com/base/sonarqube:9.9.8
	imagePullPolicy: IfNotPresent
	env:
	- name: JAVA_OPTS
	value: -Duser.timezone=Asia/Shanghai
	- name: SONARQUBE_JDBC_USERNAME # 连接 pgsql 用户名
	value: sonar
	- name: SONARQUBE_JDBC_PASSWORD # 连接 pgsql 密码
	value: "admin123"
	- name: SONARQUBE_JDBC_URL # 连接 pgsql 地址 / 数据库
	value: jdbc:postgresql://pgsql-svc:5432/sonardb
	resources:
	limits:
	cpu: 1500m
	memory: 2048Mi
	ports:
	- name: web
	containerPort: 9000
	volumeMounts:
	- name: data
	mountPath: /opt/sonarqube/data
	subPath: data
	- name: data
	mountPath: /opt/sonarqube/logs
	subPath: logs
	- name: data
	mountPath: /opt/sonarqube/extensions
	subPath: extensions
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	volumes:
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	volumeClaimTemplates:
	- metadata:
	name: data
	spec:
	accessModes: ["ReadWriteOnce"]
	storageClassName: "nfs-storage"
	resources:
	requests:
	storage: 5G

# 3.2.3 创建 Ingress

	[root@k8s-master01 sonarqube]# cat 03-sonarqube-ingress.yaml
	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: sonarqube-ingress
	namespace: ops
	spec:
	ingressClassName: "nginx"
	rules:
	- host: "sonar.hmallleasing.com"
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: sonarqube-svc
	port:
	name: web

# 3.2.4 访问 sonarqube

通过 http://sonar.hmallleasing.com ，用户名：admin，密码：admin

安装中文插件

# 四、部署 Jenkins

既然是基于 Kubernetes 来实现 CI/CD，那么最好将 Jenkins 以 Pod 的形式运行在 Kubernetes 集群中。其次 Jenkins 没有数

据库，所有的数据都存储在本地，所以只需要将 Jenkins 的数据目录持久化下来就可以了。

# 4.1 下载 Jenkins 镜像

	[root@master01 ~]# docker pull jenkins/jenkins:2.504.2-lts
	[root@master01 ~]# docker tag jenkins/jenkins:2.504.2-lts s.hmallleasing.com/base/jenkins:2.504.2
	[root@master01 ~]# docker login s.hmallleasing.com
	[root@master01 ~]# docker push s.hmallleasing.com/base/jenkins:2.504.2

# 4.2 创建 RBAC

后期 Jenkins 需要创建 Slave Pod 来完成流水线的执行，为此我们需要一些权限。

	[root@k8s-master01 jenkins]# cat 01-jenkins-rbac.yaml
	# serviceaccount
	apiVersion: v1
	kind: ServiceAccount
	metadata:
	name: jenkins
	namespace: ops
	---
	# clusterRole
	kind: ClusterRole
	apiVersion: rbac.authorization.k8s.io/v1
	metadata:
	name: jenkins
	rules:
	- apiGroups: ["extensions", "apps"]
	resources: ["deployments", "ingresses"]
	verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
	- apiGroups: [""]
	resources: ["services"]
	verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
	- apiGroups: [""]
	resources: ["pods"]
	verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
	- apiGroups: [""]
	resources: ["pods/exec"]
	verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
	- apiGroups: [""]
	resources: ["pods/log", "events"]
	verbs: ["get", "list", "watch"]
	- apiGroups: [""]
	resources: ["secrets"]
	verbs: ["get"]
	---
	# clusterrolebinding
	apiVersion: rbac.authorization.k8s.io/v1
	kind: ClusterRoleBinding
	metadata:
	name: jenkins
	namespace: ops
	roleRef:
	apiGroup: rbac.authorization.k8s.io
	kind: ClusterRole
	name: jenkins
	subjects:
	- kind: ServiceAccount
	name: jenkins
	namespace: ops

# 4.3 创建 Service

	[root@k8s-master01 jenkins]# cat 02-jenkins-svc.yaml
	apiVersion: v1
	kind: Service
	metadata:
	name: jenkins-svc
	namespace: ops
	spec:
	clusterIP: None
	selector:
	app: jenkins
	ports:
	- name: http
	port: 8080
	targetPort: 8080
	- name: agent
	port: 50000
	targetPort: 50000

# 4.4 创建 StatefulSet

	[root@k8s-master01 jenkins]# cat 03-jenkins-sts.yaml
	apiVersion: apps/v1
	kind: StatefulSet
	metadata:
	name: jenkins
	namespace: ops
	spec:
	serviceName: "jenkins-svc"
	selector:
	matchLabels:
	app: jenkins
	template:
	metadata:
	labels:
	app: jenkins
	spec:
	serviceAccount: jenkins
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: jenkins
	image: s.hmallleasing.com/base/jenkins:2.504.2
	imagePullPolicy: IfNotPresent
	securityContext: # 添加参数启用容器 root 权限
	privileged: true
	runAsUser: 0 # root 身份运行
	env:
	- name: JAVA_OPTS
	value: -Duser.timezone=Asia/Shanghai
	ports:
	- name: http
	containerPort: 8080
	- name: agent
	containerPort: 50000
	resources:
	limits:
	cpu: 1500m
	memory: 4096Mi
	readinessProbe: # 就绪探针
	httpGet:
	path: /login
	port: 8080
	initialDelaySeconds: 60
	timeoutSeconds: 5
	failureThreshold: 12
	volumeMounts:
	- name: data
	mountPath: /var/jenkins_home
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	volumes:
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	volumeClaimTemplates:
	- metadata:
	name: data
	spec:
	accessModes: ["ReadWriteOnce"]
	storageClassName: "nfs-storage"
	resources:
	requests:
	storage: 5Gi

# 4.5 创建 Ingress

	[root@k8s-master01 jenkins]# cat 04-jenkins-ingress.yaml
	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: jenkins-ingress
	namespace: ops
	spec:
	ingressClassName: "nginx"
	rules:
	- host: jenkins.hmallleasing.com
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: jenkins-svc
	port:
	name: http

# 4.6 初始化 Jenkins

1、查看 jenkins 对应的初始化密码；

2、跳过插件安装，进入 jenkins 自行安装即可；

# 4.7 Jenkins 配置与插件

1、修改默认管理员密码

2、安装 Jenkins 插件

#1. 将 jenkins 插件的路径，更改为国内源
系统管理 -> 插件管理 -> 高级 -> 升级站点 -> 修改 URL->https://updates.jenkins.io/update-center.json 替换 https://mirrors.huaweicloud.com/jenkins/updates/update-center.json

#2. 安装 Jenkins 插件

中文插件： Localization: Chinese

Git 插件： git、gitlab

Sonar 插件： SonarQube Scanner

Pipeline 插件： pipeline、Stage View、BlueOcean

Kubernetes 插件： Kubernetes

# 五、Jenkins Pipeline

# 5.1 什么是 Pipeline

Pipeline 就是通过 “代码的方式” 将多个步骤的任务连接起来。共同来完成一件事；比如：应用发布就会牵扯到非常多的步骤，获取代码 -> 代码扫描 -> 代码编译 -> 制作镜像 -> 推送仓库 -> 部署应用，将这些步骤通过代码的方式组织在一起完成这次应用发布，这个就可以称之为流水线，亦或者是 Pipeline 流水线；

# 5.2 Pipeline 代码示例

agent: 节点

stage: 阶段

steps: 动作

	pipeline{
	agent any //目前只有一台jenkins，所以他会在本地执行

	environment { //全局变量
	Harbor_Url="s.hmallleasing.com"
	Harbor_Pro="base"
	}

	stages {
	stage('下载代码') {
	environment { //局部变量
	Image_Name="nf-flms-order:v1.1"
	Full_Image="${Harbor_Url}/${Harbor_Pro}/${Image_Name}"
	}
	steps {
	sh 'echo "Get Gitlab Code ${Full_Image}"'
	}
	}
	stage('检测代码') {
	environment { //局部变量
	Image_Name="nf-flms-static:v1.1"
	Full_Image="${Harbor_Url}/${Harbor_Pro}/${Image_Name}"
	}
	steps {
	sh 'echo "Unit Test ${Full_Image}"'
	}
	}
	stage('编译代码') {
	steps {
	sh 'echo "Build Code"'
	}
	}
	stage('制作镜像') {
	steps {
	sh 'echo "Build Docker"'
	}
	}
	stage('部署应用') {
	steps {
	sh 'echo "Deploy Code"'
	}
	}
	}
	}

# 六、 Jenkins Slave 架构

# 6.1 架构基本说明

所谓 JenkinsMaster/Slave 架构，及在 Master 上进行任务分配。然后由 Slave 来完成，不过 Slave 运行方式有两种：

静态 SLave：需要固定的节点，配置其对应环境，手动注册到 Master，然后执行任务，任务完成节点处于空闲等待状态；
动态 Slave：由 Master 动态创建 Slave 的 Pod，自动注册到 Master，然后执行任务，任务结束 Pod 自动销毁；

静态 Jenkins Slave

1、能够分担主节点上的压力，加快构建速度（所有任务都由 Master 执行，造成构建速度缓慢，且任务多会出现排队现象

2、能够将特定的任务在特定的主机上运行（比如：不同的任务需要不同的编译环境）

痛点：

1、Master 发生单点故障时，整个 Jenkins 都没办法使用；
2、每个 Slave 的环境不一样，用于完成不同项目的编译打包工作，但这些不同环境的配置管理及维护都特别困难；
3、有的 Slave 构建任务频繁，可能出现排队等待，而有的 Slave 又处于空闲状态，所以会出现资源分配严重不均衡；
4、因为每个 Slave 都需要一台虚拟机，当 Slave 空间时，等于就是空跑，资源浪费明显；

** 动态 Jenkins Slave **

所谓动态 Slave，就是根据任务进行动态供应和动态删除。Jenkins Master 和 Jenkins Slave 都是以 Pod 的形式运行在 Kubernetes 集群节点上，Master 运行在其中一个节点上，其配置数据存储在一个持久卷声明中。而 Slave 则随机运行在各个节点上，但它不会一直处于运行状态，而是根据需求动态创建并自动删除。

优势：

1、高可用性：（当 Jenkins Master 故障时，Kubernetes 会自动创建一个新的 Jenkins Master 容器，并将持久卷挂载至新创建的容器，保证数据不会丢失，从而实现 Jenkins 的高可用性。）
2、高可扩缩性：（当 Kubernetes 集群因资源不足而导致任务长时间排队等待时，可以向集群新增节点，来环节压力。）
3、资源分配合理：（Kubernetes 动态分配 Slave 至空闲节点，避免因单个节点资源利用率高而导致任务排队等待。）

# 七、Jenkins 动态 Slave 配置

# 7.1 配置 Kubernetes

系统管理 -> 节点管理 ->Cloud->Add a New Cloud->Kubernetes

Kubernetes 地址： https://kubernetes.default.svc.cluster.local
Kubernetes 命名空间：ops

由于 Jenkins 是通过 Pod 运行在 Kubernetes 集群中，所以通过 service 地址即可连接 Kubernetes 集群，正常通信。

# 7.2 配置 Jenkins

Jenkins 地址：http://jenkins-svc.ops.svc.cluster.local:8080

Jenkins 通道：jenkins-svc.ops.svc.cluster.local:50000

# 7.3 运行测试流水线

1、编写流水线

	pipeline {
	agent {
	kubernetes {
	cloud 'kubernetes'
	}
	}
	stages {
	stage('输出主机名称') {
	steps {
	sh 'hostname'
	}
	}
	stage('等待片刻') {
	steps {
	sh 'sleep 60'
	}
	}
	}
	}

2、jenkins 会自动拉起一个 Pod，运行 agent 容器注册到 Master 节点，然后进行任务的执行

	[root@k8s-master01 03-jenkins]# kubectl get pods -n ops
	NAME READY STATUS RESTARTS AGE
	gitlab-0 1/1 Running 0 20h
	jenkins-0 1/1 Running 2 (22m ago) 86m
	pipeline-daemon1-4-z9rqq-0t4nr-ql2v3 1/1 Running 0 2s #jenkins slave
	postgresql-0 1/1 Running 0 17h
	sonarqube-0 1/1 Running 0 17h

# 八、制作 Pod 模板镜像

jnlp 镜像是用来连接 JenkinsMaster 以及共享 Master 的 WORKSPACE，但该镜像并没有 maven、docker、kubectl 等常用命令，为此我们需要定制几个镜像，后期通过 Pipeline 将不同的任务交由同一个 Pod 的不同的容器来执行。

# 8.1 Maven

1、下载 settings.xml

[root@k8s-master01 maven]# wget https://linux.oldxu.net/settings_docker.xml

2、Dockerfile

	[root@k8s-master01 maven]# cat Dockerfile
	FROM maven:3.8.6-openjdk-8
	ADD ./settings_docker.xml /usr/share/maven/conf/settings.xml
	RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

3、构建镜像并推送私有仓库

	[root@k8s-master01 maven]# docker build -t s.hmallleasing.com/base/maven:3.8.6 .
	[root@k8s-master01 maven]# docker push s.hmallleasing.com/base/maven:3.8.6

# 8.2 sonar

[root@k8s-master01 ~]# docker pull emeraldsquad/sonar-scanner:2.3.0

[root@k8s-master01 ~]# docker tag emeraldsquad/sonar-scanner:2.3.0 s.hmallleasing.com/base/sonar-scanner:2.3.0

[root@k8s-master01 ~]# docker push s.hmallleasing.com/base/sonar-scanner:2.3.0

# 8.3 NodeJs

1、Dockerfile

	[root@k8s-master01 nodejs]# cat Dockerfile
	FROM centos:7
	RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
	RUN curl --silent --location https://rpm.nodesource.com/setup_14.x \|bash -
	RUN curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
	RUN yum install nodejs gcc-c++ make vim -y && \
	yum clean all

2、构建镜像并推送私有仓库

	[root@k8s-master01 nodejs]# docker build -t s.hmallleasing.com/base/nodejs:14.20 .
	[root@k8s-master01 nodejs]# docker push s.hmallleasing.com/base/nodejs:14.20

# 8.4 Docker

[root@k8s-master01 ~]# docker pull docker:20.10

[root@k8s-master01 ~]# docker tag docker:20.10 s.hmallleasing.com/base/docker:20.10

[root@k8s-master01 ~]# docker push s.hmallleasing.com/base/docker:20.10

# 8.5 kubectl

1、编写 kubernetes.repo 文件

	cat <<EOF> ./kubernetes.repo
	[kubernetes]
	name=Kubernetes
	baseurl=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.32/rpm/
	enabled=1
	gpgcheck=0
	gpgkey=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.32/rpm/repodata/repomd.xml.key
	EOF