企业级负载均衡LVS场景实战（二）

# 企业级负载均衡 LVS 场景实战

角色	主机名称	外网地址	内网地址
客服端	client	eth0:192.168.40.41	/
路由器	route	eth0:192.168.40.200	eth1:172.16.1.200
LVS	lb01	/	eth1:172.16.1.3/VIP:172.16.1.100
LVS	lb02	/	eth1:172.16.1.4/VIP:172.16.1.100
proxy	proxy01	/	eth1:172.16.1.5
proxy	proxy02	/	eth1:172.16.1.6
应用服务器	web01	/	eth1:172.16.1.7
应用服务器	web02	/	eth1:172.16.1.8
MySQL	db01	/	eth1:172.16.1.51
共享存储	Nfs	/	eth1:172.16.1.32

# 一、安装 MySQL5.7

	#1、下载 MySQL 官方扩展源
	[root@db01 ~]# rpm -ivh http://repo.mysql.com/yum/mysql-5.7-community/el/7/x86_64/mysql57-community-release-el7-10.noarch.rpm

	#2、安装 mysql5.7，文件过大可能会导致下载缓慢
	[root@db01 ~]# yum install mysql-community-server -y

	#3、启动并加入开机自动启动
	[root@db01 ~]# systemctl start mysqld && systemctl enable mysqld

	#4、查看端口是否启动
	[root@db01 ~]# netstat -lntp
	Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
	tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 3788/sshd
	tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 4018/master
	tcp6 0 0 :::3306 :::* LISTEN 4628/mysqld
	tcp6 0 0 :::22 :::* LISTEN 3788/sshd
	tcp6 0 0 ::1:25 :::* LISTEN 4018/master

	#5、由于 mysql5.7 默认配置密码，需要过滤 temporary password 关键字查看对应登陆数据库密码
	[root@db01 ~]# grep 'temporary password' /var/log/mysqld.log

	#6、登录 mysql 数据库 [password 中填写上一步过滤的密码]
	[root@db01 ~]# mysql -uroot -p$(awk '/temporary password/{print $NF}' /var/log/mysqld.log)

	#6、重新修改数据库密码
	mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY 'Superman*2025';
	mysql> grant all on . to 'app'@'172.16.1.%' identified by 'Superman*2025';

# 二、 NFS 服务部署

	#1.NFS 服务安装
	[root@nfs ~]# yum -y install nfs-utils

	#2.NFS 服务配置
	[root@nfs ~]# cat /etc/exports
	/data/zrlog 172.16.1.0/24(rw,all_squash,anonuid=666,anongid=666)
	/data/zh 172.16.1.0/24(rw,all_squash,anonuid=666,anongid=666)
	/data/blog 172.16.1.0/24(rw,all_squash,anonuid=666,anongid=666)

	#3.NFS 服务初始化
	[root@nfs ~]# mkdir /data/{zh,blog,zrlog} -p
	[root@nfs ~]# groupadd -g 666 www
	[root@nfs ~]# useradd -u 666 -g 666 www
	[root@nfs ~]# chown -R www.www /data/zh/
	[root@nfs ~]# chown -R www.www /data/blog/
	[root@nfs ~]# chown -R www.www /data/zrlog/

	#4.NFS 服务启动
	[root@nfs ~]# systemctl enable nfs-server && systemctl start nfs-server

	#5. 客户端挂载 NFS
	客户端也创建一个uid为666，gid为666，统一身份，避免后续出现权限不足的情况
	[root@nfs-client ~]# groupadd -g 666 www
	[root@nfs-client ~]# useradd -g 666 -u 666 www
	[root@nfs-client ~]# yum -y install nfs-utils
	[root@nfs-client ~]# showmount -e 172.16.1.32
	/data/blog 172.16.1.0/24
	/data/zh 172.16.1.0/24
	/data/zrlog 172.16.1.0/24
	[root@nfs-client ~]# mkdir /data
	[root@nfs-client ~]# mount -t nfs 192.168.40.103:/data /data

	#6. 客户端永久挂载 NFS
	[root@nfs-client ~]# vim /etc/fstab 192.168.40.103:/data /data nfs defaults 0 0

# 三、部署 web01

# 3.1 部署 Nginx

	#1.Nginx 安装
	[root@web01 ~]# yum install nginx -y

	#2. 配置 Nginx 进程运行用户
	[root@web01 ~]# groupadd -g666 www
	[root@web01 ~]# useradd -u666 -g666 www
	[root@web01 ~]# sed -i '/^user/c user www;' /etc/nginx/nginx.conf

	#3. 启动 Nginx，并将 Nginx 加入开机自启
	[root@web01 ~]# systemctl enable nginx && systemctl start nginx

# 3.2 部署 PHP7.1

	#1、移除旧版 php
	[root@web01 ~]# yum remove php-mysql-5.4 php php-fpm php-common

	#2.2 安装扩展源
	[root@web01 ~]# yum localinstall https://mirror.webtatic.com/yum/el7/webtatic-release.rpm -y

	#3、安装 php7.1 版本
	[root@web01 ~]# yum -y install php71w php71w-cli php71w-common php71w-devel php71w-embedded php71w-gd php71w-mcrypt php71w-mbstring php71w-pdo php71w-xml php71w-fpm php71w-mysqlnd php71w-opcache php71w-pecl-memcached php71w-pecl-redis php71w-pecl-mongodb

	#4、启动 php
	[root@web01 ~]# sed -i '/^user/c user = www' /etc/php-fpm.d/www.conf
	[root@web01 ~]# sed -i '/^group/c group = www' /etc/php-fpm.d/www.conf
	[root@web01 ~]# systemctl start php-fpm && systemctl enable php-fpm

# 四、部署 web02

# 4.1 部署 Nginx

	#1.Nginx 安装
	[root@web02 ~]# yum install nginx -y

	#2. 配置 Nginx 进程运行用户
	[root@web02 ~]# groupadd -g666 www
	[root@web02 ~]# useradd -u666 -g666 www
	[root@web02 ~]# sed -i '/^user/c user www;' /etc/nginx/nginx.conf

	#3. 启动 Nginx，并将 Nginx 加入开机自启
	[root@web02 ~]# systemctl enable nginx && systemctl start nginx

# 4.2 部署 PHP7.1

	#1. 移除旧版 php
	[root@web02 ~]# yum remove php-mysql-5.4 php php-fpm php-common

	#2. 安装扩展源
	[root@web02 ~]# yum localinstall https://mirror.webtatic.com/yum/el7/webtatic-release.rpm -y

	#3. 安装 php7.1 版本
	[root@web02 ~]# yum -y install php71w php71w-cli php71w-common php71w-devel php71w-embedded php71w-gd php71w-mcrypt php71w-mbstring php71w-pdo php71w-xml php71w-fpm php71w-mysqlnd php71w-opcache php71w-pecl-memcached php71w-pecl-redis php71w-pecl-mongodb

	#4. 启动 php
	[root@web02 ~]# sed -i '/^user/c user = www' /etc/php-fpm.d/www.conf
	[root@web02 ~]# sed -i '/^group/c group = www' /etc/php-fpm.d/www.conf
	[root@web02 ~]# systemctl start php-fpm && systemctl enable php-fpm

# 五、部署博客 WeCenter

# 5.1 web01 配置

	#1. 修改 nginx 反代参数
	[root@web01 ~]# cat /etc/nginx/proxy_params
	proxy_http_version 1.1;
	proxy_set_header Connectin "";

	proxy_set_header Host $http_host;
	proxy_set_header X-Real-IP $remote_addr;
	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

	proxy_connect_timeout 60;
	proxy_send_timeout 120;
	proxy_read_timeout 120;

	proxy_buffering on;
	proxy_buffer_size 32k;
	proxy_buffers 4 128k;
	proxy_temp_file_write_size 10240k;
	proxy_max_temp_file_size 10240k;

	#2. 修改 nginx 配置文件
	[root@web01 ~]# cat /etc/nginx/conf.d/zh.hmallleasing.com.conf
	server {
	server_name zh.hmallleasing.com;
	listen 80;
	root /code/zh;

	location / {
	index index.php index.html;
	}

	location ~ \.php$ {
	fastcgi_pass 127.0.0.1:9000;
	fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
	fastcgi_param HTTPS on; #支持前端用 https, 后端用 http
	include fastcgi_params;
	}
	}

	#3. 创建网站目录
	[root@web01 conf.d]# mkdir -p /code/zh
	[root@web01 conf.d]# mkdir /code/wordpress

	#4. 重启 nginx 服务
	[root@web01 ~]# nginx -t
	[root@web01 ~]# systemctl reload ngin

	#5. 获取 WeCenter 代码
	[root@web01 ~]# wget https://cn.wordpress.org/wordpress-4.9.4-zh_CN.tar.gz
	[root@web01 zh]# ll /code/zh/
	-rw-r--r-- 1 www www 8451194 Aug 25 22:13 WeCenter_3-2-1.zip
	[root@web01 blog]# ll /code/wordpress/
	-rw-r--r-- 1 root root 9082696 Feb 8 2018 wordpress-4.9.4-zh_CN.tar.gz

	#6. 解压网站源码文件，拷贝至对应站点目录，并授权站点目录
	[root@web01 blog]# tar xf wordpress-4.9.4-zh_CN.tar.gz
	[root@web01 zh]# unzip WeCenter_3-2-1.zip
	[root@web01 ~]# chown -R www.www /code/zh
	[root@web01 ~]# chown -R www.www /code/wordpress/

	#7. 由于 wordpress 产品需要依赖数据库，所以需要手动建立数据库
	#1. 登陆数据库
	[root@db01 ~]# mysql -uroot -p

	#8. 创建 wordpress 数据库
	mysql> create database wordpress;
	mysql> create database zh;

	#9. 通过浏览器访问 wordpress, 并部署该产品
	http://zh.hmallleasing.com

	#10. 获取 Wordpress 产品的附件和图片存放的位置
	浏览器->右键->检查->Network->选择按钮->点击一下图片

	#11. 挂载 NFS
	[root@web01 ~]# mount -t nfs 172.16.1.32:/data/zh /code/zh/uploads

# 5.2 web02 配置

	#1. 修改 nginx 配置文件
	[root@web01 ~]# scp zh.hmallleasing.com.conf root@172.16.1.8:/etc/nginx/conf.d/
	[root@web01 ~]# scp /etc/nginx/proxy_params root@172.16.1.8:/etc/nginx

	#2. 创建网站目录
	[root@web02 ~]# mkdir /code/zh -p

	#3. 重启 nginx 服务
	[root@web02 ~]# nginx -t
	[root@web02 ~]# systemctl reload nginx

	#4. 获取代码
	[root@web01 ~]# scp -rp /code/zh/* root@172.16.1.8:/code/zh/
	[root@web02 ~]# chown -R www.www /code/zh

	#5. 获取 Wordpress 产品的附件和图片存放的位置
	浏览器->右键->检查->Network->选择按钮->点击一下图片

	#6. 挂载 NFS
	[root@web02 ~]# mount -t nfs 172.16.1.32:/data/zh /code/zh/uploads

# 六、配置七层负载均衡

# 6.1 配置 Lb01

	#1. 修改 nginx 配置文件
	[root@proxy01 conf.d]# cat zh.hmallleasing.com.conf
	upstream zh {
	server 172.16.1.7:80;
	server 172.16.1.8:80;
	}

	server {
	listen 443 ssl;
	server_name zh.hmallleasing.com;
	ssl_prefer_server_ciphers on;
	ssl_certificate /etc/nginx/ssl_key/hmallleasing.com.pem;
	ssl_certificate_key /etc/nginx/ssl_key/hmallleasing.com.key;

	location / {
	proxy_pass http://zh;
	include proxy_params;
	}
	}

	server {
	listen 80;
	server_name zh.hmallleasing.com;
	return 302 https://$server_name$request_uri;
	}

	#2. 修改 nginx 反代参数
	[root@proxy01 ~]# cat /etc/nginx/proxy_params
	proxy_http_version 1.1;
	proxy_set_header Connectin "";

	proxy_set_header Host $http_host;
	proxy_set_header X-Real-IP $remote_addr;
	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

	proxy_connect_timeout 60;
	proxy_send_timeout 60;
	proxy_read_timeout 120;

	proxy_buffering on;
	proxy_buffer_size 32k;
	proxy_buffers 4 128k;
	proxy_temp_file_write_size 10240k;
	proxy_max_temp_file_size 10240k;

	#3. 上传 nginx 证书
	[root@proxy01 ~]# ll /etc/nginx/ssl_key/
	-rw-r--r-- 1 root root 1675 Sep 8 17:44 hmallleasing.com.key
	-rw-r--r-- 1 root root 4784 Sep 8 17:44 hmallleasing.com.pem


	#4. 重启 nginx
	[root@proxy01 conf.d]# nginx -t
	[root@proxy01 conf.d]# systemctl restart nginx

# 6.2 VIP 和 Arp 抑制脚本

	[root@proxy01 ~]# cat lvs_rs.sh
	#!/usr/bin/bash

	VIP=172.16.1.100
	DEV=lo:0

	case $1 in
	start)
	# ARP 抑制
	echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
	echo "1" >/proc/sys/net/ipv4/conf/default/arp_ignore
	echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore

	echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
	echo "2" >/proc/sys/net/ipv4/conf/default/arp_announce
	echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce

	# VIP
	cat >/etc/sysconfig/network-scripts/ifcfg-${DEV} <<-EOF
	DEVICE=lo:0
	IPADDR=${VIP}
	NETMASK=255.0.0.0
	ONBOOT=yes
	NAME=loopback
	EOF

	ifup ${DEV} # 启动网卡
	systemctl start nginx
	;;


	stop)
	echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
	echo "0" >/proc/sys/net/ipv4/conf/default/arp_ignore
	echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore

	echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
	echo "0" >/proc/sys/net/ipv4/conf/default/arp_announce
	echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce

	ifdown ${DEV} # 停止网卡
	rm -f /etc/sysconfig/network-scripts/ifcfg-${DEV}
	systemctl stop nginx
	;;
	*)
	echo "Usage: sh $0 { start \| stop }"
	esac

# 6.3 配置 RS 节点 VIP 和 Arp 抑制

	#1. 摘掉 eht0 外网，eth1 网关指向路由器
	[root@proxy01 ~]# ifdown eth0
	[root@proxy01 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth1
	TYPE="Ethernet"
	PROXY_METHOD="none"
	BROWSER_ONLY="no"
	BOOTPROTO="none"
	DEFROUTE="yes"
	IPV4_FAILURE_FATAL="no"
	IPV6INIT="yes"
	IPV6_AUTOCONF="yes"
	IPV6_DEFROUTE="yes"
	IPV6_FAILURE_FATAL="no"
	IPV6_ADDR_GEN_MODE="stable-privacy"
	NAME="eth1"
	DEVICE="eth1"
	ONBOOT="yes"
	IPV6_PRIVACY="no"
	IPADDR="172.16.1.5"
	PREFIX="24"
	GATEWAY="172.16.1.200"
	[root@proxy01 ~]# ifdown eth1 && ifup eth1

	#2. 配置 RS 节点 VIP 和 Arp 抑制
	[root@proxy01 ~]# chmod +x lvs_rs.sh
	[root@proxy01 ~]# sh lvs_rs.sh start
	[root@proxy01 ~]# ifconfig
	eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
	ether 00:0c:29:36:d2:b6 txqueuelen 1000 (Ethernet)
	RX packets 4372 bytes 356320 (347.9 KiB)
	RX errors 0 dropped 0 overruns 0 frame 0
	TX packets 2922 bytes 300240 (293.2 KiB)
	TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

	eth1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
	inet 172.16.1.5 netmask 255.255.255.0 broadcast 172.16.1.255
	inet6 fe80::1607:3fa8:6c0d:8f7f prefixlen 64 scopeid 0x20<link>
	inet6 fe80::b64e:4e5e:1653:e542 prefixlen 64 scopeid 0x20<link>
	inet6 fe80::a7ac:65c7:1aa7:6b5a prefixlen 64 scopeid 0x20<link>
	ether 00:0c:29:36:d2:c0 txqueuelen 1000 (Ethernet)
	RX packets 1688 bytes 158485 (154.7 KiB)
	RX errors 0 dropped 0 overruns 0 frame 0
	TX packets 954 bytes 149925 (146.4 KiB)
	TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

	lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
	inet 127.0.0.1 netmask 255.0.0.0
	inet6 ::1 prefixlen 128 scopeid 0x10<host>
	loop txqueuelen 1000 (Local Loopback)
	RX packets 376 bytes 28264 (27.6 KiB)
	RX errors 0 dropped 0 overruns 0 frame 0
	TX packets 376 bytes 28264 (27.6 KiB)
	TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

	lo:0: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
	inet 172.16.1.100 netmask 255.0.0.0
	loop txqueuelen 1000 (Local Loopback)

# 6.4 配置 Lb02

	#1. 修改 nginx 配置文件
	[root@proxy02 conf.d]# cat zh.hmallleasing.com.conf
	upstream zh {
	server 172.16.1.7:80;
	server 172.16.1.8:80;
	}

	server {
	listen 443 ssl;
	server_name zh.hmallleasing.com;
	ssl_prefer_server_ciphers on;
	ssl_certificate /etc/nginx/ssl_key/hmallleasing.com.pem;
	ssl_certificate_key /etc/nginx/ssl_key/hmallleasing.com.key;

	location / {
	proxy_pass http://zh;
	include proxy_params;
	}
	}

	server {
	listen 80;
	server_name zh.hmallleasing.com;
	return 302 https://$server_name$request_uri;
	}

	#2. 修改 nginx 反代参数
	[root@proxy01 ~]# cat /etc/nginx/proxy_params
	proxy_http_version 1.1;
	proxy_set_header Connectin "";

	proxy_set_header Host $http_host;
	proxy_set_header X-Real-IP $remote_addr;
	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

	proxy_connect_timeout 60;
	proxy_send_timeout 60;
	proxy_read_timeout 120;

	proxy_buffering on;
	proxy_buffer_size 32k;
	proxy_buffers 4 128k;
	proxy_temp_file_write_size 10240k;
	proxy_max_temp_file_size 10240k;

	#3. 上传 nginx 证书
	[root@proxy01 ~]# mkdir /etc/nginx/ssl_key
	[root@proxy01 ~]# ll /etc/nginx/ssl_key/
	-rw-r--r-- 1 root root 1675 Sep 8 17:44 hmallleasing.com.key
	-rw-r--r-- 1 root root 4784 Sep 8 17:44 hmallleasing.com.pem

	#4. 重启 nginx
	[root@proxy01 ~]# nginx -t
	[root@proxy01 ~]# systemctl restart nginx

# 6.5 VIP 和 Arp 抑制脚本

	[root@proxy02 ~]# cat lvs_rs.sh
	#!/usr/bin/bash

	VIP=172.16.1.100
	DEV=lo:0

	case $1 in
	start)
	# ARP 抑制
	echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
	echo "1" >/proc/sys/net/ipv4/conf/default/arp_ignore
	echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore

	echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
	echo "2" >/proc/sys/net/ipv4/conf/default/arp_announce
	echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce

	# VIP
	cat >/etc/sysconfig/network-scripts/ifcfg-${DEV} <<-EOF
	DEVICE=lo:0
	IPADDR=${VIP}
	NETMASK=255.0.0.0
	ONBOOT=yes
	NAME=loopback
	EOF

	ifup ${DEV} # 启动网卡
	systemctl start nginx
	;;


	stop)
	echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
	echo "0" >/proc/sys/net/ipv4/conf/default/arp_ignore
	echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore

	echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
	echo "0" >/proc/sys/net/ipv4/conf/default/arp_announce
	echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce

	ifdown ${DEV} # 停止网卡
	rm -f /etc/sysconfig/network-scripts/ifcfg-${DEV}
	systemctl stop nginx
	;;
	*)
	echo "Usage: sh $0 { start \| stop }"
	esac

# 6.6 配置 RS 节点 VIP 和 Arp 抑制

	#1. 摘掉 eht0 外网，eth1 网关指向路由器
	[root@proxy02 ~]# ifdown eth0
	[root@proxy02 ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth1
	TYPE="Ethernet"
	PROXY_METHOD="none"
	BROWSER_ONLY="no"
	BOOTPROTO="none"
	DEFROUTE="yes"
	IPV4_FAILURE_FATAL="no"
	IPV6INIT="yes"
	IPV6_AUTOCONF="yes"
	IPV6_DEFROUTE="yes"
	IPV6_FAILURE_FATAL="no"
	IPV6_ADDR_GEN_MODE="stable-privacy"
	NAME="eth1"
	DEVICE="eth1"
	ONBOOT="yes"
	IPV6_PRIVACY="no"
	IPADDR="172.16.1.6"
	PREFIX="24"
	GATEWAY="172.16.1.200"
	[root@proxy02 ~]# ifdown eth1 && ifup eth1

	#2. 配置 RS 节点 VIP 和 Arp 抑制
	[root@proxy02 ~]# chmod +x lvs_rs.sh
	[root@proxy02 ~]# sh lvs_rs.sh start
	[root@proxy02 ~]# ifconfig
	eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
	ether 00:0c:29:14:f6:61 txqueuelen 1000 (Ethernet)
	RX packets 5838 bytes 660060 (644.5 KiB)
	RX errors 0 dropped 0 overruns 0 frame 0
	TX packets 3996 bytes 3589831 (3.4 MiB)
	TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

	eth1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
	inet 172.16.1.6 netmask 255.255.255.0 broadcast 172.16.1.255
	inet6 fe80::1607:3fa8:6c0d:8f7f prefixlen 64 scopeid 0x20<link>
	inet6 fe80::b64e:4e5e:1653:e542 prefixlen 64 scopeid 0x20<link>
	inet6 fe80::a7ac:65c7:1aa7:6b5a prefixlen 64 scopeid 0x20<link>
	ether 00:0c:29:14:f6:6b txqueuelen 1000 (Ethernet)
	RX packets 4465 bytes 2655882 (2.5 MiB)
	RX errors 0 dropped 0 overruns 0 frame 0
	TX packets 2392 bytes 563355 (550.1 KiB)
	TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

	lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
	inet 127.0.0.1 netmask 255.0.0.0
	inet6 ::1 prefixlen 128 scopeid 0x10<host>
	loop txqueuelen 1000 (Local Loopback)
	RX packets 352 bytes 26400 (25.7 KiB)
	RX errors 0 dropped 0 overruns 0 frame 0
	TX packets 352 bytes 26400 (25.7 KiB)
	TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

	lo:0: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
	inet 172.16.1.100 netmask 255.0.0.0
	loop txqueuelen 1000 (Local Loopback)

# 七、接入四层负载均衡

# 7.1 安装 lvs 命令行工具

[root@lb01 ~]# yum install ipvsadm -y

# 7.2 使用脚本生成 lvs 规则

	[root@lb01 ~]# cat lvs_ds.sh
	#!/usr/bin/bash
	VIP=172.16.1.100
	RS1=172.16.1.5
	RS2=172.16.1.6
	PORT1=80
	PORT2=443
	SCHEDULER=rr
	DEV=eth1:1

	case $1 in
	start)
	# 配置虚拟 IP 地址 VIP
	cat >/etc/sysconfig/network-scripts/ifcfg-${DEV} <<-EOF
	TYPE=Ethernet
	BOOTPROTO=none
	DEFROUTE=yes
	NAME=${DEV}
	DEVICE=${DEV}
	ONBOOT=yes
	IPADDR=${VIP}
	PREFIX=24
	EOF

	# 启动网卡
	ifup ${DEV}

	# 配置 LVS 规则
	ipvsadm -C
	ipvsadm -A -t ${VIP}:${PORT1} -s ${SCHEDULER}
	ipvsadm -a -t ${VIP}:${PORT1} -r ${RS1} -g
	ipvsadm -a -t ${VIP}:${PORT1} -r ${RS2} -g
	ipvsadm -A -t ${VIP}:${PORT2} -s ${SCHEDULER}
	ipvsadm -a -t ${VIP}:${PORT2} -r ${RS1} -g
	ipvsadm -a -t ${VIP}:${PORT2} -r ${RS2} -g
	;;

	stop)
	ifdown ${DEV}
	rm -f /etc/sysconfig/network-scripts/ifcfg-${DEV}
	ipvsadm -C
	;;
	*)
	echo "Usage: sh $0 { start \| stop }"
	;;
	esac

# 7.3 命令行配置 lvs 规则

	#1. 配置虚拟 IP 地址 VIP
	[root@lb01 ~]# chmod +x lvs_ds.sh
	[root@lb01 ~]# sh lvs_ds.sh start

	#2. 配置 LVS 调度规则
	[root@lb01 ~]# ipvsadm -L -n
	IP Virtual Server version 1.2.1 (size=4096)
	Prot LocalAddress:Port Scheduler Flags
	-> RemoteAddress:Port Forward Weight ActiveConn InActConn
	TCP 172.16.1.100:80 rr
	-> 172.16.1.5:80 Route 1 0 0
	-> 172.16.1.6:80 Route 1 0 0
	TCP 172.16.1.100:443 rr
	-> 172.16.1.5:443 Route 1 0 0
	-> 172.16.1.6:443 Route 1 0 0

# 7.4 LVS+Keepalived 实现高可用

1.lvs01 和 lvs02 安装软件

[root@lb01 ~]# yum install keepalived ipvsadm -y

2. 必须关闭七层负载均衡的 keepalived

3. 删除 lvs 上的虚拟 IP，以及 ipvs 规则

	[root@lb01 ~]# sh lvs_ds.sh stop
	[root@lb01 ~]# ipvsadm -L -n

4. 配置 lvs-master

	[root@lb01 ~]# cat /etc/keepalived/keepalived.conf
	global_defs {
	router_id lb01
	}

	vrrp_instance VI_1 {
	state MASTER
	priority 200

	interface eth1
	virtual_router_id 50
	advert_int 1
	authentication {
	auth_type PASS
	auth_pass 1111
	}
	virtual_ipaddress {
	172.16.1.100
	}
	}

	# 配置集群地址访问的 IP+Port
	virtual_server 172.16.1.100 80 {
	# 健康检查的时间，单位：秒
	delay_loop 6
	# 配置负载均衡的算法
	lb_algo rr
	# 设置 LVS 的模式 NAT\|TUN\|DR
	lb_kind DR
	# 设置会话持久化的时间
	#persistence_timeout 30
	# 设置协议
	protocol TCP

	# 负载均衡后端的真实服务节点 RS-1
	real_server 172.16.1.5 80 {
	# 权重配比设置为 1
	weight 1
	# 设置健康检查
	TCP_CHECK {
	# 检测后端 80 端口
	connect_port 80
	# 超时时间
	connect_timeout 3
	# 重试次数 2 次
	nb_get_retry 2
	# 间隔时间 3s
	delay_beefore_retry 3
	}
	}
	# 负载均衡后端的真实服务节点 RS-2
	real_server 172.16.1.6 80 {
	weight 1
	TCP_CHECK {
	connect_port 80
	connect_timeout 3
	nb_get_retry 2
	delay_beefore_retry 3
	}
	}
	}


	# 配置集群地址访问的 IP+Port
	virtual_server 172.16.1.100 443 {
	delay_loop 6
	lb_algo rr
	lb_kind DR
	protocol TCP

	real_server 172.16.1.5 443 {
	weight 1
	TCP_CHECK {
	connect_port 443
	connect_timeout 3
	nb_get_retry 2
	delay_beefore_retry 3
	}
	}
	real_server 172.16.1.6 443 {
	weight 1
	TCP_CHECK {
	connect_port 80
	connect_timeout 3
	nb_get_retry 2
	delay_beefore_retry 3
	}
	}
	}

5. 配置 lvs-backup

	[root@lb02 ~]# cat /etc/keepalived/keepalived.conf
	global_defs {
	router_id lb02
	}

	vrrp_instance VI_1 {
	state BACKUP
	priority 150

	interface eth1
	virtual_router_id 50
	advert_int 1
	authentication {
	auth_type PASS
	auth_pass 1111
	}
	virtual_ipaddress {
	172.16.1.100
	}
	}

	# 配置集群地址访问的 IP+Port
	virtual_server 172.16.1.100 80 {
	# 健康检查的时间，单位：秒
	delay_loop 6
	# 配置负载均衡的算法
	lb_algo wlc
	# 设置 LVS 的模式 NAT\|TUN\|DR
	lb_kind DR
	# 设置会话持久化的时间
	#persistence_timeout 30
	# 设置协议
	protocol TCP

	# 负载均衡后端的真实服务节点 RS-1
	real_server 172.16.1.5 80 {
	# 权重配比设置为 1
	weight 1
	# 设置健康检查
	TCP_CHECK {
	# 检测后端 80 端口
	connect_port 80
	# 超时时间
	connect_timeout 3
	# 重试次数 2 次
	nb_get_retry 2
	# 间隔时间 3s
	delay_beefore_retry 3
	}
	}
	# 负载均衡后端的真实服务节点 RS-2
	real_server 172.16.1.6 80 {
	# 权重配比设置为 1
	weight 1
	# 设置健康检查
	TCP_CHECK {
	# 检测后端 80 端口
	connect_port 80
	# 超时时间
	connect_timeout 3
	# 重试次数 2 次
	nb_get_retry 2
	# 间隔时间 3s
	delay_beefore_retry 3
	}
	}
	}

	# 配置集群地址访问的 IP+Port
	virtual_server 172.16.1.100 443 {
	delay_loop 6
	lb_algo rr
	lb_kind DR
	protocol TCP

	real_server 172.16.1.5 443 {
	weight 1
	TCP_CHECK {
	connect_port 443
	connect_timeout 3
	nb_get_retry 2
	delay_beefore_retry 3
	}
	}
	real_server 172.16.1.6 443 {
	weight 1
	TCP_CHECK {
	connect_port 80
	connect_timeout 3
	nb_get_retry 2
	delay_beefore_retry 3
	}
	}
	}

6. 配置 RS 节点的 VIP 和 Arp 抑制

	[root@proxy01 ~]# sh lvs_rs.sh start
	[root@proxy02 ~]# sh lvs_rs.sh start

7. 配置路由器端口映射

	[root@route ~]# iptables -t nat -F
	[root@route ~]# iptables -t nat -L -n
	[root@route ~]# iptables -t nat -A PREROUTING -d 192.168.40.200 -p tcp --dport 80 -j DNAT --to 172.16.1.100:80
	[root@route ~]# iptables -t nat -A PREROUTING -d 192.168.40.200 -p tcp --dport 443 -j DNAT --to 172.16.1.100:443

8. 启动 keepalived

	[root@lb01 ~]# systemctl enable keepalived && systemctl start keepalived
	[root@lb02 ~]# systemctl enable keepalived && systemctl start keepalived

9. 查看 LVS 调度规则及 VIP

	[root@lb01 ~]# ip addr \| grep 172.16.1.100
	inet 172.16.1.100/32 scope global eth1
	[root@lb01 ~]# ipvsadm -L -n
	IP Virtual Server version 1.2.1 (size=4096)
	Prot LocalAddress:Port Scheduler Flags
	-> RemoteAddress:Port Forward Weight ActiveConn InActConn
	TCP 172.16.1.100:80 rr
	-> 172.16.1.5:80 Route 1 0 0
	-> 172.16.1.6:80 Route 1 0 0
	TCP 172.16.1.100:443 rr
	-> 172.16.1.5:443 Route 1 0 0
	-> 172.16.1.6:443 Route 1 2 0

8. 如果 realserver 节点故障，是否会自动将其移除

	[root@proxy01 conf.d]# systemctl stop nginx
	[root@lb01 ~]# ipvsadm -L -n
	IP Virtual Server version 1.2.1 (size=4096)
	Prot LocalAddress:Port Scheduler Flags
	-> RemoteAddress:Port Forward Weight ActiveConn InActConn
	TCP 172.16.1.100:80 rr
	-> 172.16.1.6:80 Route 1 0 0
	TCP 172.16.1.100:443 rr
	-> 172.16.1.6:443 Route 1 1 0

9. 如果 ds 服务器故障，能否切换到备用节点

	[root@lb01 ~]# systemctl stop keepalived
	[root@lb02 ~]# ip addr\|grep 172.16.1.100
	inet 172.16.1.100/32 scope global eth1
	[root@lb02 ~]# ipvsadm -L -n
	IP Virtual Server version 1.2.1 (size=4096)
	Prot LocalAddress:Port Scheduler Flags
	-> RemoteAddress:Port Forward Weight ActiveConn InActConn
	TCP 172.16.1.100:80 wlc
	-> 172.16.1.5:80 Route 1 0 0
	-> 172.16.1.6:80 Route 1 0 0
	TCP 172.16.1.100:443 rr
	-> 172.16.1.5:443 Route 1 0 0
	-> 172.16.1.6:443 Route 1 0 0

10. 验证测试

	#1. 查看 LVS 连接记录
	[root@lb01 ~]# ipvsadm -L -c -n
	IPVS connection entries
	pro expire state source virtual destination
	TCP 03:02 ESTABLISHED 192.168.40.1:60878 172.16.1.100:443 172.16.1.6:443
	TCP 10:27 ESTABLISHED 192.168.40.1:52923 172.16.1.100:443 172.16.1.5:443

	#2. 查看客户端是否与代理服务器建立握手
	[root@proxy01 conf.d]# netstat -npt
	Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
	tcp 0 0 172.16.1.100:443 192.168.40.1:60182 ESTABLISHED 34394/nginx: worker

	#3. 代理服务器是否与应用服务器建立握手
	[root@web01 conf.d]# netstat -ntp
	Active Internet connections (w/o servers)
	Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
	tcp 0 0 172.16.1.7:80 172.16.1.5:46644 ESTABLISHED -

LVS