消费租赁项目Kubernetes基于ELK日志分析与实践

# 消费租赁项目 Kubernetes 基于 ELK 日志分析与实践

# 一、ELK 创建 Namespace 和 Secrets

	# kubectl create ns logging
	# kubectl create secret docker-registry harbor-admin -n logging --docker-server=registry.cn-hangzhou.aliyuncs.com --docker-username=xyapples@163.com --docker-password=passwd

# 二、交付 Zookeeper 集群至 K8S

# 2.1 制作 ZK 集群镜像

# 2.1.1 Dockerfile

	# cat Dockerfile
	FROM openjdk:8-jre

	# 1、拷贝 Zookeeper 压缩包和配置文件
	ENV VERSION=3.8.4
	ADD ./apache-zookeeper-${VERSION}-bin.tar.gz /
	ADD ./zoo.cfg /apache-zookeeper-${VERSION}-bin/conf

	# 2、对 Zookeeper 文件夹名称重新命名
	RUN mv /apache-zookeeper-${VERSION}-bin /zookeeper

	# 3、拷贝 eentrpoint 的启动脚本文件
	ADD ./entrypoint.sh /entrypoint.sh

	# 4、暴露 Zookeeper 端口
	EXPOSE 2181 2888 3888

	# 5、执行启动脚本
	CMD ["/bin/bash","/entrypoint.sh"]

# 2.1.2 zoo.cfg

	# cat zoo.cfg
	# 服务器之间或客户端与服务器之间维持心跳的时间间隔 tickTime 以毫秒为单位。
	tickTime={ZOOK_TICKTIME}

	# 集群中的 follower 服务器 (F) 与 leader 服务器 (L) 之间的初始连接心跳数 10* tickTime
	initLimit={ZOOK_INIT_LIMIT}

	# 集群中的 follower 服务器与 leader 服务器之间请求和应答之间能容忍的最多心跳数 5 * tickTime
	syncLimit={ZOOK_SYNC_LIMIT}

	# 数据保存目录
	dataDir={ZOOK_DATA_DIR}

	# 日志保存目录
	dataLogDir={ZOOK_LOG_DIR}

	# 客户端连接端口
	clientPort={ZOOK_CLIENT_PORT}

	# 客户端最大连接数。# 根据自己实际情况设置，默认为 60 个
	maxClientCnxns={ZOOK_MAX_CLIENT_CNXNS}

	# 客户端获取 zookeeper 服务的当前状态及相关信息
	4lw.commands.whitelist=*

	# 三个接点配置，格式为： server. 服务编号 = 服务地址、LF 通信端口、选举端口

# 2.1.3 entrypoint

	# cat entrypoint.sh
	#设定变量
	ZOOK_BIN_DIR=/zookeeper/bin
	ZOOK_CONF_DIR=/zookeeper/conf/zoo.cfg

	# 2、对配置文件中的字符串进行变量替换
	sed -i s@{ZOOK_TICKTIME}@${ZOOK_TICKTIME:-2000}@g ${ZOOK_CONF_DIR}
	sed -i s@{ZOOK_INIT_LIMIT}@${ZOOK_INIT_LIMIT:-10}@g ${ZOOK_CONF_DIR}
	sed -i s@{ZOOK_SYNC_LIMIT}@${ZOOK_SYNC_LIMIT:-5}@g ${ZOOK_CONF_DIR}
	sed -i s@{ZOOK_DATA_DIR}@${ZOOK_DATA_DIR:-/data}@g ${ZOOK_CONF_DIR}
	sed -i s@{ZOOK_LOG_DIR}@${ZOOK_LOG_DIR:-/logs}@g ${ZOOK_CONF_DIR}
	sed -i s@{ZOOK_CLIENT_PORT}@${ZOOK_CLIENT_PORT:-2181}@g ${ZOOK_CONF_DIR}
	sed -i s@{ZOOK_MAX_CLIENT_CNXNS}@${ZOOK_MAX_CLIENT_CNXNS:-60}@g ${ZOOK_CONF_DIR}

	# 3、准备 ZK 的集群节点地址，后期肯定是需要通过 ENV 的方式注入进来
	for server in ${ZOOK_SERVERS}
	do
	echo ${server} >> ${ZOOK_CONF_DIR}
	done

	# 4、在 datadir 目录中创建 myid 的文件，并填入对应的编号
	ZOOK_MYID=$(( $(hostname \| sed 's#.*-##g') + 1 ))
	echo ${ZOOK_MYID:-99} > ${ZOOK_DATA_DIR:-/data}/myid

	#5、前台运行 Zookeeper
	cd ${ZOOK_BIN_DIR}
	./zkServer.sh start-foreground

# 2.1.4 构建镜像并推送仓库

# wget https://dlcdn.apache.org/zookeeper/zookeeper-3.8.4/apache-zookeeper-3.8.4-bin.tar.gz

# docker build -t registry.cn-hangzhou.aliyuncs.com/kubernetes_public/zookeeper:3.8.4 .

# docker push  registry.cn-hangzhou.aliyuncs.com/kubernetes_public/zookeeper:3.8.4

# 2.2 迁移 zookeeper 至 K8S

# 2.2.1 zookeeper-headless

	# cat 01-zookeeper-headless.yaml
	apiVersion: v1
	kind: Service
	metadata:
	name: zookeeper-svc
	namespace: logging
	spec:
	clusterIP: None
	selector:
	app: zookeeper
	ports:
	- name: client
	port: 2181
	targetPort: 2181
	- name: leader-follwer
	port: 2888
	targetPort: 2888
	- name: selection
	port: 3888
	targetPort: 3888

# 2.2.2 zookeeper-sts

	[root@k8s-master01 01-zookeeper]# vim 02-zookeeper-sts.yaml
	apiVersion: apps/v1
	kind: StatefulSet
	metadata:
	name: zookeeper
	namespace: logging
	spec:
	serviceName: "zookeeper-svc"
	replicas: 3
	selector:
	matchLabels:
	app: zookeeper
	template:
	metadata:
	labels:
	app: zookeeper
	spec:
	affinity:
	podAntiAffinity:
	requiredDuringSchedulingIgnoredDuringExecution:
	- labelSelector:
	matchExpressions:
	- key: app
	operator: In
	values: ["zookeeper"]
	topologyKey: "kubernetes.io/hostname"
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: zookeeper
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/zookeeper:3.8.4
	imagePullPolicy: Always
	ports:
	- name: client
	containerPort: 2181
	- name: leader-follwer
	containerPort: 2888
	- name: selection
	containerPort: 3888
	env:
	- name: ZOOK_SERVERS
	value: "server.1=zookeeper-0.zookeeper-svc.logging.svc.cluster.local:2888:3888 server.2=zookeeper-1.zookeeper-svc.logging.svc.cluster.local:2888:3888 server.3=zookeeper-2.zookeeper-svc.logging.svc.cluster.local:2888:3888"
	readinessProbe: # 就绪探针，不就绪则不介入流量
	exec:
	command:
	- "/bin/bash"
	- "-c"
	- '[[ "$(/zookeeper/bin/zkServer.sh status 2>/dev/null\|grep 2181)" ]] && exit 0 \|\| exit 1'
	initialDelaySeconds: 5
	livenessProbe: # 存活探针。如果不存活则根据重启策略进行重启
	exec:
	command:
	- "/bin/bash"
	- "-c"
	- '[[ "$(/zookeeper/bin/zkServer.sh status 2>/dev/null\|grep 2181)" ]] && exit 0 \|\| exit 1'
	initialDelaySeconds: 5
	volumeMounts:
	- name: data
	mountPath: /data
	subPath: data
	- name: data
	mountPath: /logs
	subPath: logs
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	volumes:
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	volumeClaimTemplates:
	- metadata:
	name: data
	spec:
	accessModes: ["ReadWriteMany"]
	storageClassName: "nfs-storage"
	resources:
	requests:
	storage: 5Gi

# 2.2.3 更新资源清单

	[root@k8s-master01 01-zookeeper]# kubectl apply -f 01-zookeeper-headless.yaml
	[root@k8s-master01 01-zookeeper]# kubectl apply -f 02-zookeeper-sts.yaml
	[root@k8s-master01 01-zookeeper]# kubectl get pods -n logging
	NAME READY STATUS RESTARTS AGE
	zookeeper-0 1/1 Running 0 17m
	zookeeper-1 1/1 Running 0 14m
	zookeeper-2 1/1 Running 0 11m

# 2.2.4 检查 zookeeper 集群状态

	# for i in 0 1 2 ; do kubectl exec zookeeper-$i -n logging -- /zookeeper/bin/zkServer.sh status; done
	ZooKeeper JMX enabled by default
	Using config: /zookeeper/bin/../conf/zoo.cfg
	Client port found: 2181. Client address: localhost. Client SSL: false.
	Mode: follower
	ZooKeeper JMX enabled by default
	Using config: /zookeeper/bin/../conf/zoo.cfg
	Client port found: 2181. Client address: localhost. Client SSL: false.
	Mode: leader
	ZooKeeper JMX enabled by default
	Using config: /zookeeper/bin/../conf/zoo.cfg
	Client port found: 2181. Client address: localhost. Client SSL: false.
	Mode: follower

# 2.2.5 连接 Zookeeper 集群

	[root@k8s-master01 01-zookeeper]# kubectl exec -it zookeeper-0 -n logging -- /bin/sh
	# /zookeeper/bin/zkCli.sh -server zookeeper-svc
	[zk: zookeeper-svc(CONNECTED) 0] create /hello oldxu
	Created /hello
	[zk: zookeeper-svc(CONNECTED) 1] get /hello
	oldxu

# 三、交付 Kafka 集群至 K8S

# 3.1 制作 Kafka 集群镜像

# 3.1.1 Dockerfile

	# cat Dockerfile
	FROM openjdk:8-jre

	# 1、调整时区
	RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
	echo 'Asia/Shanghai' > /etc/timezone

	# 2、拷贝 kafka 软件以及 kafka 的配置
	ENV VERSION=2.12-2.2.0
	ADD ./kafka_${VERSION}.tgz /
	ADD ./server.properties /kafka_${VERSION}/config/server.properties

	# 3、修改 kafka 的名称
	RUN mv /kafka_${VERSION} /kafka

	# 4、启动脚本（修改 kafka 配置）
	ADD ./entrypoint.sh /entrypoint.sh

	# 5、暴露 kafka 端口 9999 是 jmx 的端口
	EXPOSE 9092 9999

	# 6、运行启动脚本
	CMD ["/bin/bash","/entrypoint.sh"]

# 3.1.2 server.properties

	# cat server.properties
	############################# Server Basics #############################
	# broker 的 id，值为整数，且必须唯一，在一个集群中不能重复
	broker.id={BROKER_ID}

	############################# Socket Server Settings #############################
	# kafka 监听端口，默认 9092
	listeners=PLAINTEXT://{LISTENERS}:9092

	# 处理网络请求的线程数量，默认为 3 个
	num.network.threads=3

	# 执行磁盘 IO 操作的线程数量，默认为 8 个
	num.io.threads=8

	# socket 服务发送数据的缓冲区大小，默认 100KB
	socket.send.buffer.bytes=102400

	# socket 服务接受数据的缓冲区大小，默认 100KB
	socket.receive.buffer.bytes=102400

	# socket 服务所能接受的一个请求的最大大小，默认为 100M
	socket.request.max.bytes=104857600

	############################# Log Basics #############################
	# kafka 存储消息数据的目录
	log.dirs={KAFKA_DATA_DIR}

	# 每个 topic 默认的 partition
	num.partitions=1

	# 设置副本数量为 3, 当 Leader 的 Replication 故障，会进行故障自动转移。
	default.replication.factor=3

	# 在启动时恢复数据和关闭时刷新数据时每个数据目录的线程数量
	num.recovery.threads.per.data.dir=1

	############################# Log Flush Policy #############################
	# 消息刷新到磁盘中的消息条数阈值
	log.flush.interval.messages=10000

	# 消息刷新到磁盘中的最大时间间隔，1s
	log.flush.interval.ms=1000

	############################# Log Retention Policy #############################
	# 日志保留小时数，超时会自动删除，默认为 7 天
	log.retention.hours=168

	# 日志保留大小，超出大小会自动删除，默认为 1G
	#log.retention.bytes=1073741824

	# 日志分片策略，单个日志文件的大小最大为 1G，超出后则创建一个新的日志文件
	log.segment.bytes=1073741824

	# 每隔多长时间检测数据是否达到删除条件，300s
	log.retention.check.interval.ms=300000

	############################# Zookeeper #############################
	# Zookeeper 连接信息，如果是 zookeeper 集群，则以逗号隔开
	zookeeper.connect={ZOOK_SERVERS}

	# 连接 zookeeper 的超时时间，6s
	zookeeper.connection.timeout.ms=6000

# 3.1.3 entrypoint

	# cat entrypoint.sh
	# 变量
	KAFKA_DIR=/kafka
	KAFKA_CONF=/kafka/config/server.properties

	# 1、基于主机名 + 1 获取 Broker_id 这个是用来标识集群节点在整个集群中必须唯一
	BROKER_ID=$(( $(hostname \| sed 's#.*-##g') + 1 ))
	LISTENERS=$(hostname -i)

	# 2、替换配置文件内容，后期 ZK 集群的地址通过 ENV 传递
	sed -i s@{BROKER_ID}@${BROKER_ID}@g ${KAFKA_CONF}
	sed -i s@{LISTENERS}@${LISTENERS}@g ${KAFKA_CONF}
	sed -i s@{KAFKA_DATA_DIR}@${KAFKA_DATA_DIR:-/data}@g ${KAFKA_CONF}
	sed -i s@{ZOOK_SERVERS}@${ZOOK_SERVERS}@g ${KAFKA_CONF}

	# 3、启动 Kafka
	cd ${KAFKA_DIR}/bin
	sed -i '/export KAFKA_HEAP_OPTS/a export JMX_PORT="9999"' kafka-server-start.sh
	./kafka-server-start.sh ../config/server.properties

# 3.1.4 构建镜像并推送仓库

# wget https://archive.apache.org/dist/kafka/2.2.0/kafka_2.12-2.2.0.tgz

# docker build -t registry.cn-hangzhou.aliyuncs.com/kubernetes_public/kafka:2.12.2 .

# docker push registry.cn-hangzhou.aliyuncs.com/kubernetes_public/kafka:2.12.2

# 3.2 迁移 Kafka 集群至 K8S

# 3.2.1 kafka-headless

	# cat 01-kafka-headless.yaml
	apiVersion: v1
	kind: Service
	metadata:
	name: kafka-svc
	namespace: logging
	spec:
	clusterIP: None
	selector:
	app: kafka
	ports:
	- name: client
	port: 9092
	targetPort: 9092
	- name: jmx
	port: 9999
	targetPort: 9999

# 3.2.2 kafka-sts

	# cat 02-kafka-sts.yaml
	apiVersion: apps/v1
	kind: StatefulSet
	metadata:
	name: kafka
	namespace: logging
	spec:
	serviceName: "kafka-svc"
	replicas: 3
	selector:
	matchLabels:
	app: kafka
	template:
	metadata:
	labels:
	app: kafka
	spec:
	affinity:
	podAntiAffinity:
	requiredDuringSchedulingIgnoredDuringExecution:
	- labelSelector:
	matchExpressions:
	- key: app
	operator: In
	values: ["kafka"]
	topologyKey: "kubernetes.io/hostname"
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: kafka
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/kafka:2.12.2
	imagePullPolicy: Always
	ports:
	- name: client
	containerPort: 9092
	- name: jmxport
	containerPort: 9999
	env:
	- name: ZOOK_SERVERS
	value: "zookeeper-0.zookeeper-svc:2181,zookeeper-1.zookeeper-svc:2181,zookeeper-2.zookeeper-svc:2181"
	readinessProbe: # 就绪探针，不就绪则不介入流量
	tcpSocket:
	port: 9092
	initialDelaySeconds: 5
	livenessProbe: # 存活探针。如果不存活则根据重启策略进行重启
	tcpSocket:
	port: 9092
	initialDelaySeconds: 5
	volumeMounts:
	- name: data
	mountPath: /data
	volumeClaimTemplates:
	- metadata:
	name: data
	spec:
	accessModes: ["ReadWriteMany"]
	storageClassName: "nfs-storage"
	resources:
	requests:
	storage: 5Gi

# 3.2.3 更新资源清单

	[root@k8s-master01 02-kafka]# kubectl apply -f 01-kafka-headless.yaml
	[root@k8s-master01 02-kafka]# kubectl apply -f 02-kafka-sts.yaml
	[root@k8s-master01 02-kafka]# kubectl get pods -n logging
	NAME READY STATUS RESTARTS AGE
	kafka-0 1/1 Running 0 5m49s
	kafka-1 1/1 Running 0 4m43s
	kafka-2 1/1 Running 0 3m40s

	#查看 kafka 是否注册到 zookeeper
	[root@k8s-master01 02-kafka]# kubectl exec -it zookeeper-0 -n logging -- /bin/bash
	root@zookeeper-0:/# /zookeeper/bin/zkCli.sh
	[zk: localhost:2181(CONNECTED) 2] get /brokers/ids/1
	{"listener_security_protocol_map":{"PLAINTEXT":"PLAINTEXT"},"endpoints":["PLAINTEXT://172.16.85.201:9092"],"jmx_port":9999,"host":"172.16.85.201","timestamp":"1748162470218","port":9092,"version":4}
	[zk: localhost:2181(CONNECTED) 3] get /brokers/ids/2
	{"listener_security_protocol_map":{"PLAINTEXT":"PLAINTEXT"},"endpoints":["PLAINTEXT://172.16.58.205:9092"],"jmx_port":9999,"host":"172.16.58.205","timestamp":"1748162532658","port":9092,"version":4}
	[zk: localhost:2181(CONNECTED) 4] get /brokers/ids/3
	{"listener_security_protocol_map":{"PLAINTEXT":"PLAINTEXT"},"endpoints":["PLAINTEXT://172.16.195.1:9092"],"jmx_port":9999,"host":"172.16.195.1","timestamp":"1748162649250","port":9092,"version":4}

# 3.2.4 检查 Kafka 集群

	1.创建一个topic
	root@kafka-0:/# /kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-0.zookeeper-svc:2181,zookeeper-1.zookeeper-svc:2181,zookeeper-2.zookeeper-svc:2181 --partitions 1 --replication-factor 3 --topic oldxu

	2.模拟消息发布
	root@kafka-1:/# /kafka/bin/kafka-console-producer.sh --broker-list kafka-0.kafka-svc:9092,kafka-1.kafka-svc:9092,kafka-2.kafka-svc:9092 --topic oldxu
	>hello kubernetes
	>hello world

	3.模拟消息订阅
	root@kafka-2:/# /kafka/bin/kafka-console-consumer.sh --bootstrap-server kafka-0.kafka-svc:9092,kafka-1.kafka-svc:9092,kafka-2.kafka-svc:9092 --topic oldxu --from-beginning
	hello kubernetes
	hello world

# 四、交付 efak 至 K8S

# 4.1 制作 efak 镜像

# 4.1.1 Dockerfile

	[root@manager 03-efak]# cat Dockerfile
	FROM openjdk:8

	# 1、调整时区
	RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
	echo 'Asia/Shanghai' > /etc/timezone

	# 2、拷贝 kafka 软件以及 kafka 的配置
	ENV VERSION=3.0.1
	ADD ./efak-web-${VERSION}-bin.tar.gz /
	ADD ./system-config.properties /efak-web-${VERSION}/conf/system-config.properties

	# 3、修改 efak 的名称
	RUN mv /efak-web-${VERSION} /efak

	# 4、环境变量
	ENV KE_HOME=/efak
	ENV PATH=$PATH:$KE_HOME/bin

	# 5、启动脚本（修改 kafka 配置）
	ADD ./entrypoint.sh /entrypoint.sh

	# 6、暴露 kafka 端口 9999 是 jmx 的端口
	EXPOSE 8048

	# 7、运行启动脚本
	CMD ["/bin/bash","/entrypoint.sh"]

# 4.1.2 system-config

	# cat system-config.properties
	######################################
	# 填写 zookeeper 集群列表
	######################################
	efak.zk.cluster.alias=cluster1
	cluster1.zk.list={ZOOK_SERVERS}

	######################################
	# broker 最大规模数量
	######################################
	cluster1.efak.broker.size=20

	######################################
	# zk 客户端线程数
	######################################
	kafka.zk.limit.size=32

	######################################
	# EFAK webui 端口
	######################################
	efak.webui.port=8048

	######################################
	# kafka offset storage
	######################################
	cluster1.efak.offset.storage=kafka

	######################################
	# kafka jmx uri
	######################################
	cluster1.efak.jmx.uri=service:jmx:rmi:///jndi/rmi://%s/jmxrmi

	######################################
	# kafka metrics 指标，默认存储 15 天
	######################################
	efak.metrics.charts=true
	efak.metrics.retain=15

	######################################
	# kafka sql topic records max
	######################################
	efak.sql.topic.records.max=5000
	efak.sql.topic.preview.records.max=10

	######################################
	# delete kafka topic token
	######################################
	efak.topic.token=keadmin

	######################################
	# kafka sqlite 数据库地址（需要修改存储路径）
	######################################
	efak.driver=org.sqlite.JDBC
	efak.url=jdbc:sqlite:{EFAK_DATA_DIR}/db/ke.db
	efak.username=root
	efak.password=www.kafka-eagle.org

# 4.1.3 entrypoint

	# cat entrypoint.sh
	# 1、变量
	EFAK_DIR=/efak
	EFAK_CONF=/efak/conf/system-config.properties

	# 2、替换配置文件内容，后期 ZK 集群的地址通过 ENV 传递
	sed -i s@{EFAK_DATA_DIR}@${EFAK_DIR}@g ${EFAK_CONF}
	sed -i s@{ZOOK_SERVERS}@${ZOOK_SERVERS}@g ${EFAK_CONF}

	# 3、启动 efka
	${EFAK_DIR}/bin/ke.sh start
	tail -f ${EFAK_DIR}/logs/ke_console.out

# 4.1.4 构建镜像并推送仓库

# wget https://github.com/smartloli/kafka-eagle-bin/archive/v3.0.1.tar.gz

# docker build -t registry.cn-hangzhou.aliyuncs.com/kubernetes_public/efak:3.0 .

# docker push registry.cn-hangzhou.aliyuncs.com/kubernetes_public/efak:3.0

# 4.2 迁移 efak 至 K8S

# 4.2.1 efak-deploy

	# cat 01-efak-deploy.yaml
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: efak
	namespace: logging
	spec:
	replicas: 1
	selector:
	matchLabels:
	app: efak
	template:
	metadata:
	labels:
	app: efak
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: efak
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/efak:3.0
	imagePullPolicy: Always
	ports:
	- name: http
	containerPort: 8048
	env:
	- name: ZOOK_SERVERS
	value: "zookeeper-0.zookeeper-svc:2181,zookeeper-1.zookeeper-svc:2181,zookeeper-2.zookeeper-svc:2181"

# 4.2.2 efak-service

	# cat 02-efak-service.yaml
	apiVersion: v1
	kind: Service
	metadata:
	name: efak-svc
	namespace: logging
	spec:
	selector:
	app: efak
	ports:
	- port: 8048
	targetPort: 8048

# 4.2.3 efak-ingress

	# cat 03-efak-ingress.yaml
	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: efak-ingress
	namespace: logging
	spec:
	ingressClassName: "nginx"
	rules:
	- host: "efak.hmallleasing.com"
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: efak-svc
	port:
	number: 8048

# 4.2.4 更新资源清单

	[root@k8s-master01 03-efak]# kubectl apply -f 01-efak-deploy.yaml
	[root@k8s-master01 03-efak]# kubectl apply -f 02-efak-service.yaml
	[root@k8s-master01 03-efak]# kubectl apply -f 03-efak-ingress.yaml

# 4.2.5 访问 efka

1、初始用户名密码 admin 123456

2、查看 Topics

3、查看 kafka 集群状态

4、查看 Zookeeper 集群状态

# 五、交付 Elastic 集群

ES 集群是由多个节点组成的，通过 cluster.name 设置 ES 集群名称，同时用于区分其它的 ES 集群。
每个节点通过 node.name 参数来设定所在集群的节点名称。
节点使用 discovery.send_hosts 参数来设定集群节点的列表。
集群在第一次启动时，需要初始化，同时需要指定参与选举的 master 节点 IP，或节点名称。
每个节点可以通过 node.master:true 设定为 master 角色，通过 node.data:true 设定为 data 角色。

	[root@k8s-master01 ~]# grep "^[a-Z]" /etc/elasticsearch/elasticsearch.yml
	# 集群名称 cluster.name: my-oldxu
	# 节点名称 node.name: node1
	# 数据存储路径 path.data: /var/lib/elasticsearch
	# 日志存储路径 path.logs: /var/log/elasticsearch
	# 监听在本地哪个地址上 network.host: 10.0.0.100
	# 监听端口 http.port: 9200
	# 集群主机列表 discovery.seed_hosts: ["ip1", "ip2", "ip3"]
	# 仅第一次启动集群时进行选举（可以填写 node.name 的名称）cluster.initial_master_nodes: ["node01", "node02", "node03"]

# 5.1 下载 elastic 镜像

# docker pull elasticsearch:7.17.6

# docker tag elasticsearch:7.17.6 registry.cn-hangzhou.aliyuncs.com/kubernetes_public/elasticsearch:7.17.6

# docker push registry.cn-hangzhou.aliyuncs.com/kubernetes_public/elasticsearch:7.17.6

# 5.2 交付 ES-Service

创建 es-headlessService，为每个 ES Pod 设定固定的 DNS 名称，无论它是 Master 或是 Data，易或是 Coordinating

	# cat 01-es-svc.yaml
	apiVersion: v1
	kind: Service
	metadata:
	name: es-svc
	namespace: logging
	spec:
	selector:
	app: es
	clusterIP: None
	ports:
	- name: cluster
	port: 9200
	targetPort: 9200
	- name: transport
	port: 9300
	targetPort: 9300

# 5.3 交付 ES-Master 节点

ES 无法使用 root 直接启动，需要授权数据目录 UID=1000，同时还需要持久化 /usr/share/elasticsearch/data ；
ES 所有节点都需要设定 vm.max_map_count 内核参数以及 ulimit；
ES 启动是通过 ENV 环境变量传参来完成的；
- 集群名称、节点名称、角色类型；
- discovery.seed_hosts 集群地址列表；
- cluster.initial_master_nodes 初始集群参与选举的 master 节点名称；

	# cat 02-es-master.yaml
	apiVersion: apps/v1
	kind: StatefulSet
	metadata:
	name: es-master
	namespace: logging
	spec:
	serviceName: "es-svc"
	replicas: 3 # es-pod 运行的实例
	selector: # 需要管理的 ES-Pod 标签
	matchLabels:
	app: es
	role: master
	template:
	metadata:
	labels:
	app: es
	role: master
	spec: # 定义 pod 规范
	imagePullSecrets: # 镜像拉取使用的认证信息
	- name: harbor-admin
	affinity: # 设定 pod 反亲和
	podAntiAffinity:
	requiredDuringSchedulingIgnoredDuringExecution:
	- labelSelector:
	matchExpressions:
	- key: app
	operator: In
	values: ["es"]
	- key: role
	operator: In
	values: ["master"]
	topologyKey: "kubernetes.io/hostname" # 每个节点就是一个位置
	initContainers: # 初始化容器设定
	- name: fix-permissions
	image: busybox
	command: ["sh","-c","chown -R 1000:1000 /usr/share/elasticsearch/data ; sysctl -w vm.max_map_count=262144; ulimit -n 65536"]
	securityContext:
	privileged: true
	volumeMounts:
	- name: data
	mountPath: /usr/share/elasticsearch/data
	containers: # ES 主容器
	- name: es
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/elasticsearch:7.17.6
	resources:
	limits:
	cpu: 1000m
	memory: 4096Mi
	requests:
	cpu: 300m
	memory: 1024Mi
	ports:
	- name: cluster
	containerPort: 9200
	- name: transport
	containerPort: 9300
	volumeMounts:
	- name: data
	mountPath: /usr/share/elasticsearch/data
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	env:
	- name: ES_JAVA_OPTS
	value: "-Xms1g -Xmx1g"
	- name: cluster.name
	value: es-cluster
	- name: node.name
	valueFrom:
	fieldRef:
	fieldPath: metadata.name
	- name: node.master
	value: "true"
	- name: node.data
	value: "false"
	- name: discovery.seed_hosts
	value: "es-master-0.es-svc,es-master-1.es-svc,es-master-2.es-svc"
	- name: cluster.initial_master_nodes
	value: "es-master-0,es-master-1,es-master-2"
	volumes:
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	volumeClaimTemplates: # 动态 pvc
	- metadata:
	name: data
	spec:
	accessModes: ["ReadWriteOnce"]
	storageClassName: "nfs-storage"
	resources:
	requests:
	storage: 5Gi

# 5.4 交付 ES-Data 节点

ES 无法使用 root 直接启动，需要授权数据目录 UID=1000，同时还需要持久化 /usr/share/elasticsearch/data
ES 所有节点都需要设定 vm.max_map_count 内核参数以及 ulimit；
ES 启动是通过 ENV 环境变量传参来完成的
- 集群名称、节点名称、角色类型
- discovery.seed_hosts 集群地址列表

	# cat 03-es-data.yaml
	apiVersion: apps/v1
	kind: StatefulSet
	metadata:
	name: es-data
	namespace: logging
	spec:
	serviceName: "es-svc"
	replicas: 2 # es-pod 运行的实例
	selector: # 需要管理的 ES-Pod 标签
	matchLabels:
	app: es
	role: data
	template:
	metadata:
	labels:
	app: es
	role: data
	spec: # 定义 pod 规范
	imagePullSecrets: # 镜像拉取使用的认证信息
	- name: harbor-admin
	affinity: # 设定 pod 反亲和
	podAntiAffinity:
	requiredDuringSchedulingIgnoredDuringExecution:
	- labelSelector:
	matchExpressions:
	- key: app
	operator: In
	values: ["es"]
	- key: role
	operator: In
	values: ["data"]
	topologyKey: "kubernetes.io/hostname" # 每个节点就是一个位置
	initContainers: # 初始化容器设定
	- name: fix-permissions
	image: busybox
	command: ["sh","-c","chown -R 1000:1000 /usr/share/elasticsearch/data ; sysctl -w vm.max_map_count=262144; ulimit -n 65536"]
	securityContext:
	privileged: true
	volumeMounts:
	- name: data
	mountPath: /usr/share/elasticsearch/data
	containers: # ES 主容器
	- name: es
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/elasticsearch:7.17.6
	resources:
	limits:
	cpu: 1000m
	memory: 4096Mi
	requests:
	cpu: 300m
	memory: 1024Mi
	ports:
	- name: cluster
	containerPort: 9200
	- name: transport
	containerPort: 9300
	volumeMounts:
	- name: data
	mountPath: /usr/share/elasticsearch/data
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	env:
	- name: ES_JAVA_OPTS
	value: "-Xms1g -Xmx1g"
	- name: cluster.name
	value: es-cluster
	- name: node.name
	valueFrom:
	fieldRef:
	fieldPath: metadata.name
	- name: node.master
	value: "false"
	- name: node.data
	value: "true"
	- name: discovery.seed_hosts
	value: "es-master-0.es-svc,es-master-1.es-svc,es-master-2.es-svc"
	volumes:
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	volumeClaimTemplates: # 动态 pvc
	- metadata:
	name: data
	spec:
	accessModes: ["ReadWriteOnce"]
	storageClassName: "nfs-storage"
	resources:
	requests:
	storage: 5Gi

# 5.5 更新资源清单

[root@k8s-master01 04-elasticsearch]# kubectl apply -f 01-es-svc.yaml

[root@k8s-master01 04-elasticsearch]# kubectl apply -f 02-es-master.yaml

[root@k8s-master01 04-elasticsearch]# kubectl apply -f 03-es-data.yaml

# 5.6 验证 ES 集群

	#1. 解析 headlessService 获取对应 ES 集群任一节点的 IP 地址
	# dig @10.96.0.10 es-svc.logging.svc.cluster.local +short
	172.16.58.229
	172.16.122.191
	172.16.195.21
	172.16.122.129
	172.16.32.164

	#2. 通过 curl 访问 ES，检查 ES 集群是否正常（如果仅交付 Master，没有 data 节点，集群状态可能会 Red，因为没有数据节点进行数据存储；）
	# curl -XGET "http://172.16.122.129:9200/_cluster/health?pretty"
	{
	"cluster_name" : "es-cluster",
	"status" : "green",
	"timed_out" : false,
	"number_of_nodes" : 5,
	"number_of_data_nodes" : 2,
	"active_primary_shards" : 3,
	"active_shards" : 6,
	"relocating_shards" : 0,
	"initializing_shards" : 0,
	"unassigned_shards" : 0,
	"delayed_unassigned_shards" : 0,
	"number_of_pending_tasks" : 0,
	"number_of_in_flight_fetch" : 0,
	"task_max_waiting_in_queue_millis" : 0,
	"active_shards_percent_as_number" : 100.0
	}

	#3. 查看 ES 各个节点详情
	# curl -XGET "http://172.16.122.129:9200/_cat/nodes"
	172.16.122.129 16 33 20 0.38 0.56 0.38 ilmr - es-master-2
	172.16.58.229 66 33 22 0.64 0.66 0.44 ilmr * es-master-1
	172.16.122.191 52 34 15 0.38 0.56 0.38 cdfhilrstw - es-data-0
	172.16.195.21 38 35 19 0.38 0.53 0.36 cdfhilrstw - es-data-1
	172.16.32.164 31 33 12 0.28 0.50 0.59 ilmr - es-master-0

# 六、交付 Kibana 可视化

# 6.1 下载 kibana 镜像

	# docker pull kibana:7.17.6
	# docker tag kibana:7.17.6 registry.cn-hangzhou.aliyuncs.com/kubernetes_public/kibana:7.17.6
	# docker push registry.cn-hangzhou.aliyuncs.com/kubernetes_public/kibana:7.17.6

# 6.2 kibana-deploy

Kibana 需要连接 ES 集群，通过 ELASTICSEARCH_HOSTS 变量来传递 ES 集群地址
kibana 通过 I18N_LOCALE 来传递语言环境
Kibana 通过 SERVER_PUBLICBASEURL 来传递服务访问的公开地址

	# cat 01-kibana-deploy.yaml
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: kibana
	namespace: logging
	spec:
	replicas: 1
	selector:
	matchLabels:
	app: kibana
	template:
	metadata:
	labels:
	app: kibana
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: kibana
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/kibana:7.17.6
	resources:
	limits:
	cpu: 1000m
	ports:
	- containerPort: 5601
	env:
	- name: ELASTICSEARCH_HOSTS
	value: '["http://es-data-0.es-svc:9200","http://es-data-1.es-svc:9200"]'
	- name: I18N_LOCALE
	value: "zh-CN"
	- name: SERVER_PUBLICBASEURL
	value: "http://kibana.hmallleasing.com" #kibana 访问 UI
	volumeMounts:
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	volumes:
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""

# 6.3 kibana-svc

	# cat 02-kibana-svc.yaml
	apiVersion: v1
	kind: Service
	metadata:
	name: kibana-svc
	namespace: logging
	spec:
	selector:
	app: kibana
	ports:
	- name: web
	port: 5601
	targetPort: 5601

# 6.4 kibana-ingress

	# cat 03-kibana-ingress.yaml
	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: kibana-ingress
	namespace: logging
	spec:
	ingressClassName: "nginx"
	rules:
	- host: "kibana.hmallleasing.com"
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: kibana-svc
	port:
	number: 5601

# 6.5 更新资源清单

	[root@k8s-master01 05-kibana]# kubectl apply -f 01-kibana-deploy.yaml
	[root@k8s-master01 05-kibana]# kubectl apply -f 02-kibana-svc.yaml
	[root@k8s-master01 05-kibana]# kubectl apply -f 03-kibana-ingress.yaml

	[root@k8s-master01 05-kibana]# kubectl get pods -n logging
	NAME READY STATUS RESTARTS AGE
	efak-5cdc74bf59-nrhb4 1/1 Running 0 5h33m
	es-data-0 1/1 Running 0 16m
	es-data-1 1/1 Running 0 15m
	es-master-0 1/1 Running 0 17m
	es-master-1 1/1 Running 0 15m
	es-master-2 1/1 Running 0 12m
	kafka-0 1/1 Running 0 5h39m
	kafka-1 1/1 Running 0 5h39m
	kafka-2 1/1 Running 0 5h38m
	kibana-5ccc46864b-ndzx9 1/1 Running 0 118s
	zookeeper-0 1/1 Running 0 5h42m
	zookeeper-1 1/1 Running 0 5h42m
	zookeeper-2 1/1 Running 0 5h41m

# 6.6 访问 kibana

# 七、filebeat-sidecar 收集业务应用日志

# 7.1 部署架构说明

对于那些能够将日志输出到本地文件的 Pod，我们可以使用 Sidecar 模式方式运行一个日志采集 Agent，对其进行单独收集日志。

首先需要将 Pod 中的业务容器日志输出至本地文件，而后运行一个 Filebeat 边车容器，采集本地路径下的日志；
Filebeat 容器需要传递如下变量；
- ENV：了解 Pod 属于隶属于哪个环境；
- PROJECT_NAME：为了后期能在单个索引中区分出不同的项目；
- PodIP：为了让用户清楚该 Pod 属于哪个 IP；
- Node：用于获取该 Pod 所处的节点；
Logstash 根据不同的环境，拉取不同的 topic 数据，然后将数据存储至 ES 对应的索引中；
Kibana 添加不同环境的 index pattern，而后选择对应环境不同的项目进行日志探索与展示；

# 7.2 Sidecar 部署思路

制作一个业务镜像，要求镜像输出日志至本地；
制作 Filebeat 镜像，配置 Input、output 等信息；
采用边车模式运行不同环境的 Pod，确保日志信息能输出至 Kafka 集群；
准备不同环境下 Logstash 配置文件，而后读取数据写入 ES 集群；
使用 kibana 添加索引，进行日志探索与展示；

# 7.3 制作 Filebeat 镜像

7.3.1 下载 filebeat

curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.17.6-x86_64.rpm

7.3.2 编写 Dockerfile

	# cat Dockerfile
	# 1、基础镜像
	FROM centos:7

	# 2、拷贝 filebeat
	ENV VERSION=7.17.6
	ADD ./filebeat-${VERSION}-x86_64.rpm /
	RUN rpm -ivh /filebeat-${VERSION}-x86_64.rpm && \
	rm -f /filebeat-${VERSION}-x86_64.rpm

	# 3、拷贝 filebeat 配置文件（核心）
	ADD ./filebeat.yml /etc/filebeat/filebeat.yml

	# 4、拷贝启动脚本
	ADD ./entrypoint.sh /entrypoint.sh
	RUN chmod +x /entrypoint.sh

	# 5、执行启动脚本
	CMD ["/bin/bash","-c","/entrypoint.sh"]

7.3.3 编写 entrypoint

	# cat entrypoint.sh
	#启动脚本
	#1、替换 filbeat 配置文件中的内容
	Beat_Conf=/etc/filebeat/filebeat.yml

	sed -i s@{ENV}@${ENV:-test}@g ${Beat_Conf}
	sed -i s@{PodIP}@${PodIP:-"no-ip"}@g ${Beat_Conf}
	sed -i s@{Node}@${Node:-"none"}@g ${Beat_Conf}
	sed -i s@{PROJECT_NAME}@${PROJECT_NAME:-"no-define"}@g ${Beat_Conf}
	sed -i s@{MULTILINE}@${MULTILINE:-"^\\\d{2}"}@g ${Beat_Conf} # \\ 用来转义
	sed -i s@{KAFKA_HOSTS}@${KAFKA_HOSTS}@g ${Beat_Conf}

	# 2、运行 filebeat
	filebeat -e -c /etc/filebeat/filebeat.yml

7.3.4 编写 filebeat 配置

{ENV}：用于定义环境的变量；
{PROJECT_NAME}：用于定义项目名称的变量；
{MULTILINE}：用于定义多行合并的正则变量；
{KAFKA_HOSTS}：用于定义 KAFKA 集群地址的变量；
{PodIP}：用于获取该 Pod 地址的变量；
{Node}：用于获取该 Pod 所处的节点；

	[root@k8s-master01 filebeat_sidecar_dockerfile]# cat filebeat.yml
	filebeat.inputs:
	- type: log
	enabled: true
	paths:
	- /logu/*.log
	- /logu//.log
	tags: ["logu"]
	fields:
	topic: {PROJECT_NAME}
	podip: {PodIP}
	node: {Node}
	fields_under_root: true # 增加的所有字段都为顶级字段

	- type: log
	enabled: true
	paths:
	- /logm/*.log
	- /logm//.log
	tags: ["logm"]
	fields:
	topic: {PROJECT_NAME}
	podip: {PodIP}
	node: {Node}
	fields_under_root: true # 增加的所有字段都为顶级字段
	multiline.pattern: '{MULTILINE}'
	multiline.negate: true
	multiline.match: after
	multiline.max_lines: 10000 #默认最大合并行为 500，可根据实际情况调整。

	output.kafka:
	hosts: [{KAFKA_HOSTS}]
	topic: app-{ENV}-%{[topic]}
	required_acks: 1 # 保证消息可靠，0 不保证，1 等待写入主分区（默认），-1 等待写入副本分区
	compression: gzip # 压缩
	max_message_bytes: 1000000 # 每条消息最大的长度，多余的被删除

7.3.5 构建并推送镜像

	# docker build -t registry.cn-hangzhou.aliyuncs.com/kubernetes_public/filebeat_sidecar:7.17.6 .
	# docker push registry.cn-hangzhou.aliyuncs.com/kubernetes_public/filebeat_sidecar:7.17.6

# 7.4 nf-flms-gateway 日志收集

7.4.1 创建 Namespace 和 Secrets

	# sed -i "s#dev#prod#g" *.yaml
	# kubectl create ns prod
	# kubectl create secret tls prod-api.hmallleasig.com --key hmallleasing.com.key --cert hmallleasing.com.pem -n prod
	# kubectl create secret docker-registry harbor-admin --docker-server=registry.cn-hangzhou.aliyuncs.com --docker-username=xyapples@163.com --docker-password=passwd -n prod

7.4.2 创建 nf-flms-gateway

	# cat 01-nf-flms-gateway.yaml
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: nf-flms-gateway
	namespace: prod
	spec:
	replicas: 2
	selector:
	matchLabels:
	app: nf-flms-gateway
	template:
	metadata:
	labels:
	app: nf-flms-gateway
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: nf-flms-gateway
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/nf-flms-gateway:v2.2
	command:
	- "/bin/sh"
	- "-c"
	- "java -Xms256m -Xmx1024m -Dspring.profiles.active=prd -Djava.security.egd=file:/dev/./urandom -jar -Duser.timezone=GMT+08 nf-flms-gateway.jar"
	resources:
	limits:
	cpu: '1000m'
	memory: 1Gi
	requests:
	cpu: "200m"
	memory: "500Mi"
	ports:
	- containerPort: 8080
	readinessProbe: # 就绪探针，不就绪则从负载均衡移除
	tcpSocket:
	port: 8080
	initialDelaySeconds: 60
	periodSeconds: 30
	timeoutSeconds: 3
	successThreshold: 1
	failureThreshold: 2
	livenessProbe: # 存活探针，不存活会重启
	tcpSocket:
	port: 8080
	initialDelaySeconds: 60
	periodSeconds: 30
	timeoutSeconds: 3
	successThreshold: 1
	failureThreshold: 2
	volumeMounts:
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	- name: log
	mountPath: /logs # 业务容器日志目录
	- name: filebeat
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/filebeat_sidecar:7.17.6
	imagePullPolicy: Always
	volumeMounts:
	- name: log
	mountPath: /logm # 匹配多行日志
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	env:
	- name: ENV
	valueFrom:
	fieldRef:
	apiVersion: v1
	fieldPath: metadata.namespace
	- name: PodIP
	valueFrom:
	fieldRef:
	fieldPath: status.podIP
	- name: Node
	valueFrom:
	fieldRef:
	fieldPath: spec.nodeName
	- name: PROJECT_NAME
	value: "nf-flms-gateway"
	- name: KAFKA_HOSTS
	value: '"kafka-0.kafka-svc.logging:9092","kafka-1.kafka-svc.logging:9092","kafka-2.kafka-svc.logging:9092"'
	volumes:
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	- name: log
	emptyDir: {}
	---
	apiVersion: v1
	kind: Service
	metadata:
	name: gateway-svc
	namespace: prod
	spec:
	selector:
	app: nf-flms-gateway
	ports:
	- port: 8080
	targetPort: 8080

	---

	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: gateway-ingress
	namespace: prod
	annotations:
	nginx.ingress.kubernetes.io/ssl-redirect: "false" #禁用 https 强制跳转
	spec:
	ingressClassName: "nginx"
	rules:
	- host: "prod-api.hmallleasing.com"
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: gateway-svc
	port:
	number: 8080
	tls: #https
	- hosts:
	- prod-api.hmallleasing.com
	secretName: "prod-api.hmallleasig.com" #配置默认证书可不添加 secretNam

7.4.3 检查 KafkaTopic

1、检查是否有对应的 topic

2、点击对应的 Preview，查看 topic 中的最新数据

# 7.5 nf-flms-order 日志收集

7.5.1 创建 PVC

创建 PVC 存储订单合同、身份证复印件等附件

	# cat 02-data-image.yaml
	apiVersion: v1
	kind: PersistentVolumeClaim
	metadata:
	name: data-image
	namespace: prod
	spec:
	storageClassName: "nfs-storage" # 明确指定使用哪个 sc 的供应商来创建 pv
	accessModes:
	- ReadWriteMany
	resources:
	requests:
	storage: 2Gi # 根据业务实际大小进行资源申请

7.5.2 创建 nf-flms-order

	# cat 02-nf-flms-order.yaml
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: nf-flms-order
	namespace: prod
	spec:
	replicas: 2
	selector:
	matchLabels:
	app: nf-flms-order
	template:
	metadata:
	labels:
	app: nf-flms-order
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: nf-flms-order
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/nf-flms-order:v2.0
	command:
	- "/bin/sh"
	- "-c"
	- "java -Xms256m -Xmx1024m -Dspring.profiles.active=prd -Djava.security.egd=file:/dev/./urandom -jar -Duser.timezone=GMT+08 nf-flms-order.jar"
	resources:
	limits:
	cpu: '1000m'
	memory: 1Gi
	requests:
	cpu: "200m"
	memory: "500Mi"
	ports:
	- containerPort: 8080
	readinessProbe: # 就绪探针，不就绪则从负载均衡移除
	tcpSocket:
	port: 8080
	initialDelaySeconds: 60
	periodSeconds: 30
	timeoutSeconds: 3
	successThreshold: 1
	failureThreshold: 2
	livenessProbe: # 存活探针，不存活会重启
	tcpSocket:
	port: 8080
	initialDelaySeconds: 60
	periodSeconds: 30
	timeoutSeconds: 3
	successThreshold: 1
	failureThreshold: 2
	volumeMounts:
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	- name: data-image
	mountPath: /data
	- name: log
	mountPath: /logs # 业务容器日志目录
	- name: filebeat
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/filebeat_sidecar:7.17.6
	imagePullPolicy: Always
	volumeMounts:
	- name: log
	mountPath: /logm # 匹配多行日志
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	env:
	- name: ENV
	valueFrom:
	fieldRef:
	apiVersion: v1
	fieldPath: metadata.namespace
	- name: PodIP
	valueFrom:
	fieldRef:
	fieldPath: status.podIP
	- name: Node
	valueFrom:
	fieldRef:
	fieldPath: spec.nodeName
	- name: PROJECT_NAME
	value: "nf-flms-order"
	- name: KAFKA_HOSTS
	value: '"kafka-0.kafka-svc.logging:9092","kafka-1.kafka-svc.logging:9092","kafka-2.kafka-svc.logging:9092"'
	volumes:
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	- name: data-image
	persistentVolumeClaim:
	claimName: data-image
	- name: log
	emptyDir: {}

7.5.3 检查 KafkaTopic

1、检查是否有对应的 topic

2、点击对应的 Preview，查看 topic 中的最新数据

# 7.6 nf-flms-statistics 日志收集

7.6.1 创建 nf-flms-statistics

	# cat 03-nf-flms-statistics.yaml
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: nf-flms-statistics
	namespace: prod
	spec:
	replicas: 2
	selector:
	matchLabels:
	app: nf-flms-statistics
	template:
	metadata:
	labels:
	app: nf-flms-statistics
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: nf-flms-statistics
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/nf-flms-statistics:v2.0
	command:
	- "/bin/sh"
	- "-c"
	- "java -Xms256m -Xmx1024m -Dspring.profiles.active=prd -Djava.security.egd=file:/dev/./urandom -jar -Duser.timezone=GMT+08 nf-flms-statistics.jar"
	resources:
	limits:
	cpu: '1000m'
	memory: 1Gi
	requests:
	cpu: "200m"
	memory: "500Mi"
	ports:
	- containerPort: 8080
	readinessProbe: # 就绪探针，不就绪则从负载均衡移除
	tcpSocket:
	port: 8080
	initialDelaySeconds: 60
	periodSeconds: 30
	timeoutSeconds: 3
	successThreshold: 1
	failureThreshold: 2
	livenessProbe: # 存活探针，不存活会重启
	tcpSocket:
	port: 8080
	initialDelaySeconds: 60
	periodSeconds: 30
	timeoutSeconds: 3
	successThreshold: 1
	failureThreshold: 2
	volumeMounts:
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	- name: log
	mountPath: /logs # 业务容器日志目录
	- name: filebeat
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/filebeat_sidecar:7.17.6
	imagePullPolicy: Always
	volumeMounts:
	- name: log
	mountPath: /logm # 匹配多行日志
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	env:
	- name: ENV
	valueFrom:
	fieldRef:
	apiVersion: v1
	fieldPath: metadata.namespace
	- name: PodIP
	valueFrom:
	fieldRef:
	fieldPath: status.podIP
	- name: Node
	valueFrom:
	fieldRef:
	fieldPath: spec.nodeName
	- name: PROJECT_NAME
	value: "nf-flms-statistics"
	- name: KAFKA_HOSTS
	value: '"kafka-0.kafka-svc.logging:9092","kafka-1.kafka-svc.logging:9092","kafka-2.kafka-svc.logging:9092"'
	volumes:
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	- name: log
	emptyDir: {}

7.6.2 检查 KafkaTopic

1、检查是否有对应的 topic

2、点击对应的 Preview，查看 topic 中的最新数据

# 7.7 nf-flms-system 日志收集

7.7.1 创建 nf-flms-system

	# cat 04-nf-flms-system.yaml
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: nf-flms-system
	namespace: prod
	spec:
	replicas: 2
	selector:
	matchLabels:
	app: nf-flms-system
	template:
	metadata:
	labels:
	app: nf-flms-system
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: nf-flms-system
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/nf-flms-system:v2.0
	command:
	- "/bin/sh"
	- "-c"
	- "java -Xms256m -Xmx1024m -Dspring.profiles.active=prd -Djava.security.egd=file:/dev/./urandom -jar -Duser.timezone=GMT+08 nf-flms-system.jar"
	resources:
	limits:
	cpu: '1000m'
	memory: 1Gi
	requests:
	cpu: "200m"
	memory: "500Mi"
	ports:
	- containerPort: 8080
	livenessProbe:
	tcpSocket:
	port: 8080
	initialDelaySeconds: 60
	periodSeconds: 10
	timeoutSeconds: 10
	readinessProbe:
	tcpSocket:
	port: 8080
	failureThreshold: 2
	initialDelaySeconds: 60
	periodSeconds: 10
	timeoutSeconds: 10
	volumeMounts:
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	- name: log
	mountPath: /logs # 业务容器日志目录
	- name: filebeat
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/filebeat_sidecar:7.17.6
	imagePullPolicy: Always
	volumeMounts:
	- name: log
	mountPath: /logm # 匹配多行日志
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	env:
	- name: ENV
	valueFrom:
	fieldRef:
	apiVersion: v1
	fieldPath: metadata.namespace
	- name: PodIP
	valueFrom:
	fieldRef:
	fieldPath: status.podIP
	- name: Node
	valueFrom:
	fieldRef:
	fieldPath: spec.nodeName
	- name: PROJECT_NAME
	value: "nf-flms-system"
	- name: KAFKA_HOSTS
	value: '"kafka-0.kafka-svc.logging:9092","kafka-1.kafka-svc.logging:9092","kafka-2.kafka-svc.logging:9092"'
	volumes:
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	- name: log
	emptyDir: {}

7.7.2 检查 KafkaTopic

1、检查是否有对应的 topic

2、点击对应的 Preview，查看 topic 中的最新数据

# 7.8 nf-flms-openapi 日志收集

7.8.1 创建 nf-flms-openapi

	# cat 06-nf-flms-openapi.yaml
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: nf-flms-openapi
	namespace: prod
	spec:
	replicas: 2
	selector:
	matchLabels:
	app: nf-flms-openapi
	template:
	metadata:
	labels:
	app: nf-flms-openapi
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: nf-flms-openapi
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/nf-flms-openapi:v2.2
	command:
	- "/bin/sh"
	- "-c"
	- "java -Xms256m -Xmx1024m -Dspring.profiles.active=prd -Djava.security.egd=file:/dev/./urandom -jar -Duser.timezone=GMT+08 nf-flms-openapi.jar"
	resources:
	limits:
	cpu: '1000m'
	memory: 1Gi
	requests:
	cpu: "200m"
	memory: "500Mi"
	ports:
	- containerPort: 8080
	livenessProbe:
	tcpSocket:
	port: 8080
	initialDelaySeconds: 60
	periodSeconds: 10
	timeoutSeconds: 10
	readinessProbe:
	tcpSocket:
	port: 8080
	failureThreshold: 2
	initialDelaySeconds: 60
	periodSeconds: 10
	timeoutSeconds: 10
	volumeMounts:
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	- name: log
	mountPath: /logs # 业务容器日志目录
	- name: filebeat
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/filebeat_sidecar:7.17.6
	imagePullPolicy: Always
	volumeMounts:
	- name: log
	mountPath: /logm # 匹配多行日志
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	env:
	- name: ENV
	valueFrom:
	fieldRef:
	apiVersion: v1
	fieldPath: metadata.namespace
	- name: PodIP
	valueFrom:
	fieldRef:
	fieldPath: status.podIP
	- name: Node
	valueFrom:
	fieldRef:
	fieldPath: spec.nodeName
	- name: PROJECT_NAME
	value: "nf-flms-openapi"
	- name: KAFKA_HOSTS
	value: '"kafka-0.kafka-svc.logging:9092","kafka-1.kafka-svc.logging:9092","kafka-2.kafka-svc.logging:9092"'
	volumes:
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	- name: log
	emptyDir: {}

7.8.2 检查 KafkaTopic

1、检查是否有对应的 topic

2、点击对应的 Preview，查看 topic 中的最新数据

# 7.9 nf-flms-ui 日志收集

7.9.1 准备 Nginx 配置文件

	# cat prod.hmallleasing.com.conf
	server {
	listen 80;
	server_name prod.hmallleasing.com;
	root /code/prod;

	location / {
	index index.html index.htm;
	}
	}

	server {
	listen 80;
	server_name prod-api.hmallleasing.com;

	location / {
	proxy_set_header Host $http_host;
	proxy_set_header X-Real-IP $remote_addr;
	proxy_set_header REMOTE-HOST $remote_addr;
	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
	proxy_pass http://gateway-svc.prod.svc.cluster.local:8080;
	}
	}

7.9.2 创建 ConfigMap

kubectl create configmap nf-flms-ui-conf --from-file=./prod.hmallleasing.com.conf -n prod

7.9.3 创建 nf-flms-ui

	[root@k8s-master01 06-service-all]# cat 07-ui-deploy-ingress.yaml
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: nf-flms-ui
	namespace: prod
	spec:
	replicas: 1
	selector:
	matchLabels:
	app: nf-flms-ui
	template:
	metadata:
	labels:
	app: nf-flms-ui
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: nf-flms-ui
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/nf-flms-ui:v1.0
	ports:
	- containerPort: 80
	resources:
	limits:
	cpu: '1000m'
	memory: 1Gi
	requests:
	cpu: "200m"
	memory: "500Mi"
	readinessProbe: # 就绪探针，不就绪则从负载均衡移除
	tcpSocket:
	port: 80
	initialDelaySeconds: 60
	periodSeconds: 10
	timeoutSeconds: 10
	livenessProbe: # 存活探针，不存活会重启
	tcpSocket:
	port: 80
	initialDelaySeconds: 60
	periodSeconds: 10
	timeoutSeconds: 10
	volumeMounts:
	- name: ngxconfs
	mountPath: /etc/nginx/conf.d/
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	- name: log
	mountPath: /var/log/nginx/ # 业务容器日志目录
	- name: filebeat
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/filebeat_sidecar:7.17.6
	imagePullPolicy: Always
	volumeMounts:
	- name: log
	mountPath: /logu # 匹配多行日志
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	env:
	- name: ENV
	valueFrom:
	fieldRef:
	apiVersion: v1
	fieldPath: metadata.namespace
	- name: PodIP
	valueFrom:
	fieldRef:
	fieldPath: status.podIP
	- name: Node
	valueFrom:
	fieldRef:
	fieldPath: spec.nodeName
	- name: PROJECT_NAME
	value: "nf-flms-ui"
	- name: KAFKA_HOSTS
	value: '"kafka-0.kafka-svc.logging:9092","kafka-1.kafka-svc.logging:9092","kafka-2.kafka-svc.logging:9092"'
	volumes:
	- name: ngxconfs
	configMap:
	name: nf-flms-ui-conf
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	- name: log
	emptyDir: {}
	---
	apiVersion: v1
	kind: Service
	metadata:
	name: nf-flms-ui-svc
	namespace: prod
	spec:
	selector:
	app: nf-flms-ui
	ports:
	- port: 80
	targetPort: 80
	---
	apiVersion: networking.k8s.io/v1
	kind: Ingress
	metadata:
	name: nf-flms-ui-ingress
	namespace: prod
	annotations:
	nginx.ingress.kubernetes.io/ssl-redirect: "false" #禁用 https 强制跳转
	spec:
	ingressClassName: "nginx"
	rules:
	- host: "prod.hmallleasing.com"
	http:
	paths:
	- path: /
	pathType: Prefix
	backend:
	service:
	name: nf-flms-ui-svc
	port:
	number: 80
	tls: #https
	- hosts:
	- prod.hmallleasing.com
	secretName: "prod-api.hmallleasig.com" #配置默认证书可不添加 secretName

7.9.2 检查 KafkaTopic

1、检查是否有对应的 topic

2、点击对应的 Preview，查看 topic 中的最新数据

# 八、交付生产环境 Logstash

# 8.1 拉取镜像

# docker pull docker.elastic.co/logstash/logstash-oss:7.17.6

# docker tag docker.elastic.co/logstash/logstash-oss:7.17.6 registry.cn-hangzhou.aliyuncs.com/kubernetes_public/logstash-oss:7.17.6

# docker push registry.cn-hangzhou.aliyuncs.com/kubernetes_public/logstash-oss:7.17.6

# 8.2 编写 logstash 配置

	[root@k8s-master01 conf]# cat logstash-prod.conf
	input {
	kafka {
	bootstrap_servers => "kafka-0.kafka-svc:9092,kafka-1.kafka-svc:9092,kafka-2.kafka-svc:9092"
	group_id => "logstash-prod" # 消费者组名称
	consumer_threads => "3" # 理想情况下，配置与分区数一样多的线程，实现均衡
	topics_pattern => "app-prod-.*" # 通过正则表达式匹配要订阅的主题
	}
	}

	filter {
	json {
	source => "message"
	}
	}

	output {
	stdout {
	codec => rubydebug
	}
	elasticsearch {
	hosts => ["es-data-0.es-svc:9200","es-data-1.es-svc:9200"]
	index => "app-prod-%{+YYYY.MM.dd}"
	template_overwrite => true
	}
	}

# 8.3 创建生产环境 configmap

kubectl create configmap logstash-prod-conf --from-file=logstash.conf=conf/logstash-prod.conf -n logging

# 8.4 创建生产环境 Logstash

1、创建 logstash-svc

	# cat 01-logstash-svc.yaml
	apiVersion: v1
	kind: Service
	metadata:
	name: logstash-svc
	namespace: logging
	spec:
	clusterIP: None
	selector:
	app: logstash
	ports:
	- port: 9600
	targetPort: 9600

2、创建 logstash-StatefulSet

	# cat 05-logstash-prod-sts.yaml
	apiVersion: apps/v1
	kind: StatefulSet
	metadata:
	name: logstash-prod
	namespace: logging
	spec:
	serviceName: "logstash-svc" # 使用此前创建的 svc，则无需重复创建
	replicas: 1
	selector:
	matchLabels:
	app: logstash
	env: prod
	template:
	metadata:
	labels:
	app: logstash
	env: prod
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: logstash
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/logstash-oss:7.17.6
	args: ["-f","config/logstash.conf"] # 启动时指定加载的配置文件
	resources:
	limits:
	memory: 1024Mi
	env:
	- name: PIPELINE_WORKERS
	value: "2"
	- name: PIPELINE_BATCH_SIZE
	value: "10000"
	lifecycle:
	postStart: # 设定 JVM
	exec:
	command:
	- "/bin/bash"
	- "-c"
	- "sed -i -e '/^-Xms/c-Xms1024m' -e '/^-Xmx/c-Xmx1024m' /usr/share/logstash/config/jvm.options"
	volumeMounts:
	- name: data # 持久化数据目录
	mountPath: /usr/share/logstash/data
	- name: conf
	mountPath: /usr/share/logstash/config/logstash.conf
	subPath: logstash.conf
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	volumes:
	- name: conf
	configMap:
	name: logstash-prod-conf
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	volumeClaimTemplates:
	- metadata:
	name: data
	spec:
	accessModes: ["ReadWriteMany"]
	storageClassName: "nfs-storage"
	resources:
	requests:
	storage: 5Gi

# 8.5 更新资源清单

	[root@k8s-master01 08-logstash]# kubectl apply -f 01-logstash-svc.yaml
	[root@k8s-master01 08-logstash]# kubectl apply -f 05-logstash-prod-sts.yaml

# 8.6 检查 ES 生产环境索引

# 九、交付测试环境 Logstash

# 9.1 拉取镜像

# docker pull docker.elastic.co/logstash/logstash-oss:7.17.6

# docker tag docker.elastic.co/logstash/logstash-oss:7.17.6 registry.cn-hangzhou.aliyuncs.com/kubernetes_public/logstash-oss:7.17.6

# docker push registry.cn-hangzhou.aliyuncs.com/kubernetes_public/logstash-oss:7.17.6

# 9.2 编写 logstash 配置

	[root@k8s-master01 08-logstash]# cat conf/logstash-test.conf
	input {
	kafka {
	bootstrap_servers => "kafka-0.kafka-svc:9092,kafka-1.kafka-svc:9092,kafka-2.kafka-svc:9092"
	group_id => "logstash-test" # 消费者组名称
	consumer_threads => "3" # 理想情况下，配置与分区数一样多的线程，实现均衡
	topics_pattern => "app-test-.*" # 通过正则表达式匹配要订阅的主题
	}
	}

	filter {
	json {
	source => "message"
	}
	}

	output {
	stdout {
	codec => rubydebug
	}
	elasticsearch {
	hosts => ["es-data-0.es-svc:9200","es-data-1.es-svc:9200"]
	index => "app-test-%{+YYYY.MM.dd}"
	template_overwrite => true
	}
	}

# 9.3 创建测试环境 configmap

kubectl create configmap logstash-test-conf --from-file=logstash.conf=conf/logstash-test.conf -n logging

# 9.4 创建测试环境 Logstash

1、创建 logstash-svc

	# cat 01-logstash-svc.yaml
	apiVersion: v1
	kind: Service
	metadata:
	name: logstash-svc
	namespace: logging
	spec:
	clusterIP: None
	selector:
	app: logstash
	ports:
	- port: 9600
	targetPort: 9600

2、创建 logstash-StatefulSet

	[root@k8s-master01 08-logstash]# cat 03-logstash-test-sts.yaml
	apiVersion: apps/v1
	kind: StatefulSet
	metadata:
	name: logstash-test
	namespace: logging
	spec:
	serviceName: "logstash-svc" # 使用此前创建的 svc，则无需重复创建
	replicas: 1
	selector:
	matchLabels:
	app: logstash
	env: test
	template:
	metadata:
	labels:
	app: logstash
	env: test
	spec:
	imagePullSecrets:
	- name: harbor-admin
	containers:
	- name: logstash
	image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/logstash-oss:7.17.6
	args: ["-f","config/logstash.conf"] # 启动时指定加载的配置文件
	resources:
	limits:
	memory: 1024Mi
	env:
	- name: PIPELINE_WORKERS
	value: "2"
	- name: PIPELINE_BATCH_SIZE
	value: "10000"
	lifecycle:
	postStart: # 设定 JVM
	exec:
	command:
	- "/bin/bash"
	- "-c"
	- "sed -i -e '/^-Xms/c-Xms1024m' -e '/^-Xmx/c-Xmx1024m' /usr/share/logstash/config/jvm.options"
	volumeMounts:
	- name: data # 持久化数据目录
	mountPath: /usr/share/logstash/data
	- name: conf
	mountPath: /usr/share/logstash/config/logstash.conf
	subPath: logstash.conf
	- name: tz-config
	mountPath: /usr/share/zoneinfo/Asia/Shanghai
	- name: tz-config
	mountPath: /etc/localtime
	- name: timezone
	mountPath: /etc/timezone
	volumes:
	- name: conf
	configMap:
	name: logstash-test-conf
	- name: tz-config
	hostPath:
	path: /usr/share/zoneinfo/Asia/Shanghai
	type: ""
	- name: timezone
	hostPath:
	path: /etc/timezone
	type: ""
	volumeClaimTemplates:
	- metadata:
	name: data
	spec:
	accessModes: ["ReadWriteMany"]
	storageClassName: "nfs-storage"
	resources:
	requests:
	storage: 5Gi

# 9.5 更新资源清单

	[root@k8s-master01 08-logstash]# kubectl apply -f 01-logstash-svc.yaml
	[root@k8s-master01 08-logstash]# kubectl apply -f 03-logstash-test-sts.yaml

# 9.6 检查 ES 测试环境索引

# 十、Kibana 数据展示

# 10.1 添加生产环境索引

# 10.2 查看生产环境数据

kibana->Discover

①点击开发环境索引 ->②选择需要查看的字段 ->③进行 filter 筛选项目 ->④选择对应时间段

ELKStack

# 消费租赁项目 Kubernetes 基于 ELK 日志分析与实践

# 一、ELK 创建 Namespace 和 Secrets

# 二、交付 Zookeeper 集群至 K8S

# 2.1 制作 ZK 集群镜像

# 2.1.1 Dockerfile

# 2.1.2 zoo.cfg

# 2.1.3 entrypoint

# 2.1.4 构建镜像并推送仓库

# 2.2 迁移 zookeeper 至 K8S

# 2.2.1 zookeeper-headless

# 2.2.2 zookeeper-sts

# 2.2.3 更新资源清单

# 2.2.4 检查 zookeeper 集群状态

# 2.2.5 连接 Zookeeper 集群

# 三、 交付 Kafka 集群至 K8S

# 3.1 制作 Kafka 集群镜像

# 3.1.1 Dockerfile

# 3.1.2 server.properties

# 3.1.3 entrypoint

# 3.1.4 构建镜像并推送仓库

# 3.2 迁移 Kafka 集群至 K8S

# 3.2.1 kafka-headless

# 3.2.2 kafka-sts

# 3.2.3 更新资源清单

# 3.2.4 检查 Kafka 集群

# 四、交付 efak 至 K8S

# 4.1 制作 efak 镜像

# 4.1.1 Dockerfile

# 4.1.2 system-config

# 4.1.3 entrypoint

# 4.1.4 构建镜像并推送仓库

# 4.2 迁移 efak 至 K8S

# 4.2.1 efak-deploy

# 4.2.2 efak-service

# 4.2.3 efak-ingress

# 4.2.4 更新资源清单

# 4.2.5 访问 efka

# 五、交付 Elastic 集群

# 5.1 下载 elastic 镜像

# 5.2 交付 ES-Service

# 5.3 交付 ES-Master 节点

# 5.4 交付 ES-Data 节点

# 5.5 更新资源清单

# 5.6 验证 ES 集群

# 六、交付 Kibana 可视化

# 6.1 下载 kibana 镜像

# 6.2 kibana-deploy

# 6.3 kibana-svc

# 6.4 kibana-ingress

# 6.5 更新资源清单

# 6.6 访问 kibana

# 七、filebeat-sidecar 收集业务应用日志

# 7.1 部署架构说明

# 7.2 Sidecar 部署思路

# 7.3 制作 Filebeat 镜像

# 7.4 nf-flms-gateway 日志收集

# 7.5 nf-flms-order 日志收集

# 7.6 nf-flms-statistics 日志收集

# 7.7 nf-flms-system 日志收集

# 7.8 nf-flms-openapi 日志收集

# 7.9 nf-flms-ui 日志收集

# 八、交付生产环境 Logstash

# 8.1 拉取镜像

# 8.2 编写 logstash 配置

# 8.3 创建生产环境 configmap

# 8.4 创建生产环境 Logstash

# 8.5 更新资源清单

# 8.6 检查 ES 生产环境索引

# 九、交付测试环境 Logstash

# 9.1 拉取镜像

# 9.2 编写 logstash 配置

# 9.3 创建测试环境 configmap

# 9.4 创建测试环境 Logstash

# 9.5 更新资源清单

# 9.6 检查 ES 测试环境索引

# 十、Kibana 数据展示

# 10.1 添加生产环境索引

# 10.2 查看生产环境数据

消费租赁系统微服务应用交付实践

Rabbitmq集群部署

# 三、交付 Kafka 集群至 K8S