# Redis Cluster 集群部署
# 1、环境配置
# 1.1 关闭防火墙、Selinux
systemctl disable --now firewalld
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
# 1.1 配置 yum 源
#rocky linux配置
sed -e 's|^mirrorlist=|#mirrorlist=|g' \
-e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.aliyun.com/rockylinux|g' \
-i.bak \
/etc/yum.repos.d/rocky-*.repo
yum clean all && yum makecache
mkdir /soft /data /scripts /backup
# 1.3 配置文件描述符
ulimit -SHn 65535
vim /etc/security/limits.conf
# 末尾添加如下内容
* soft nofile 65536
* hard nofile 131072
* soft nproc 65535
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
# 1.4 系统内核参数调优
# 修改/etc/sysctl.conf文件
cat >> /etc/sysctl.conf <<EOF
vm.max_map_count = 262144
vm.swappiness=1
net.ipv4.tcp_fin_timeout=2
net.ipv4.tcp_tw_reuse=1
#net.ipv4.tcp_tw_recycle=1
net.ipv4.tcp_syncookies=1
net.ipv4.tcp_keepalive_time=600
net.ipv4.ip_local_port_range=4000 65000
net.ipv4.tcp_max_syn_backlog=16384
net.ipv4.route.gc_timeout=100
net.ipv4.tcp_max_tw_buckets= 5000
net.ipv4.tcp_syn_retries=1
net.ipv4.tcp_synack_retries=1
net.core.somaxconn=16384
net.core.netdev_max_backlog=16384
net.ipv4.tcp_max_orphans=16384
# 设置最大内存共享段大小bytes
kernel.shmmax=15461882265
kernel.shmall=3774873
# 修改消息队列长度
kernel.msgmax=65535
kernel.msgmnb=65535
EOF
# 1.5 修改默认限制内存
cat >>/etc/systemd/system.conf<< EOF
DefaultLimitNOFILE=65536
DefaultLimitNPROC=32000
DefaultLimitMEMLOCK=infinity
EOF
# 1.6 执行命令生效状态
sysctl -p
# 1.7 安装基础软件包
yum install wget jq psmisc vim unzip net-tools telnet tree yum-utils device-mapper-persistent-data \
lvm2 git ntpdate nfs-utils iotop httpd-tools dos2unix lrzsz -y
# 1.8 升级系统
yum update -y --exclude=kernel* && reboot
# 2、Redis cluster 部署
| 192.168.1.135 | 192.168.1.136 | 192.168.1.137 |
|---|---|---|
| node1：7001 | node1：7001 | node1：7001 |
| node2：7002 | node2：7002 | node2：7002 |
| node3：7003 | node3：7003 | node3：7003 |
# 2.1 安装包下载
wget https://download.redis.io/releases/redis-7.2.1.tar.gz
# 2.2 安装 redis
yum install gcc-c++ -y
mkdir /soft
tar -xzvf redis-7.2.1.tar.gz -C /soft
ln -s /soft/redis-7.2.1 /soft/redis
cd /soft/redis
make
make install prefix=/soft/redis
# 2.3 生成集群配置文件
mkdir -p /soft/redis/data/7001
mkdir -p /soft/redis/data/7002
mkdir -p /soft/redis/data/7003
mkdir -p /soft/redis/log
cd /soft/redis
redis_7001.conf 配置文件
cat > /soft/redis/redis_7001.conf <<EOF
protected-mode yes
port 7001
requirepass admin123
masterauth admin123
cluster-enabled yes
cluster-config-file nodes-7001.conf
cluster-node-timeout 5000
maxmemory 2GB
maxmemory-policy volatile-lru
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
pidfile /soft/redis/data/redis7001.pid
loglevel notice
logfile "/soft/redis/log/redis7001.log"
#databases 16
always-show-logo no
set-proc-title yes
proc-title-template "{title} {listen-addr} {server-mode}"
locale-collate ""
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
rdb-del-sync-files no
dir /soft/redis/data/7001
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync yes
repl-diskless-sync-delay 5
repl-diskless-sync-max-replicas 0
repl-diskless-load disabled
repl-disable-tcp-nodelay no
replica-priority 100
acllog-max-len 128
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
lazyfree-lazy-user-del no
lazyfree-lazy-user-flush no
oom-score-adj no
oom-score-adj-values 0 200 800
disable-thp yes
appendonly no
appendfilename "appendonly.aof"
appenddirname "appendonlydir"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
aof-timestamp-enabled no
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-listpack-entries 512
hash-max-listpack-value 64
list-max-listpack-size -2
list-compress-depth 0
set-max-intset-entries 512
set-max-listpack-entries 128
set-max-listpack-value 64
zset-max-listpack-entries 128
zset-max-listpack-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
jemalloc-bg-thread yes
EOF
redis_7002.conf 配置文件
cat > /soft/redis/redis_7002.conf <<EOF
protected-mode yes
port 7002
requirepass admin123
masterauth admin123
cluster-enabled yes
cluster-config-file nodes-7002.conf
cluster-node-timeout 5000
maxmemory 2GB
maxmemory-policy volatile-lru
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
pidfile /soft/redis/data/redis7002.pid
loglevel notice
logfile "/soft/redis/log/redis7002.log"
#databases 16
always-show-logo no
set-proc-title yes
proc-title-template "{title} {listen-addr} {server-mode}"
locale-collate ""
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
rdb-del-sync-files no
dir /soft/redis/data/7002
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync yes
repl-diskless-sync-delay 5
repl-diskless-sync-max-replicas 0
repl-diskless-load disabled
repl-disable-tcp-nodelay no
replica-priority 100
acllog-max-len 128
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
lazyfree-lazy-user-del no
lazyfree-lazy-user-flush no
oom-score-adj no
oom-score-adj-values 0 200 800
disable-thp yes
appendonly no
appendfilename "appendonly.aof"
appenddirname "appendonlydir"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
aof-timestamp-enabled no
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-listpack-entries 512
hash-max-listpack-value 64
list-max-listpack-size -2
list-compress-depth 0
set-max-intset-entries 512
set-max-listpack-entries 128
set-max-listpack-value 64
zset-max-listpack-entries 128
zset-max-listpack-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
jemalloc-bg-thread yes
EOF
redis_7003.conf 配置文件
cat > /soft/redis/redis_7003.conf <<EOF
protected-mode yes
port 7003
requirepass admin123
masterauth admin123
cluster-enabled yes
cluster-config-file nodes-7003.conf
cluster-node-timeout 5000
maxmemory 2GB
maxmemory-policy volatile-lru
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
pidfile /soft/redis/data/redis7003.pid
loglevel notice
logfile "/soft/redis/log/redis7003.log"
#databases 16
always-show-logo no
set-proc-title yes
proc-title-template "{title} {listen-addr} {server-mode}"
locale-collate ""
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
rdb-del-sync-files no
dir /soft/redis/data/7003
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync yes
repl-diskless-sync-delay 5
repl-diskless-sync-max-replicas 0
repl-diskless-load disabled
repl-disable-tcp-nodelay no
replica-priority 100
acllog-max-len 128
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
lazyfree-lazy-user-del no
lazyfree-lazy-user-flush no
oom-score-adj no
oom-score-adj-values 0 200 800
disable-thp yes
appendonly no
appendfilename "appendonly.aof"
appenddirname "appendonlydir"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
aof-timestamp-enabled no
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-listpack-entries 512
hash-max-listpack-value 64
list-max-listpack-size -2
list-compress-depth 0
set-max-intset-entries 512
set-max-listpack-entries 128
set-max-listpack-value 64
zset-max-listpack-entries 128
zset-max-listpack-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
jemalloc-bg-thread yes
EOF
# 2.4 Redis 开机自启
redis_7001.service
cat << "EOF" > /usr/lib/systemd/system/redis_7001.service
[Unit]
Description=Redis 7001 service
Documentation=https://redis.io/documentation
Wants=network-online.target
After=network-online.target
[Service]
Type=forking
LimitNOFILE=10032
User=root
Group=root
ExecStart=/soft/redis/src/redis-server /soft/redis/redis_7001.conf
PrivateTmp=true
[Install]
WantedBy=multi-user.target
EOF
redis_7002.service
cat << "EOF" > /usr/lib/systemd/system/redis_7002.service
[Unit]
Description=Redis 7002 service
Documentation=https://redis.io/documentation
Wants=network-online.target
After=network-online.target
[Service]
Type=forking
LimitNOFILE=10032
User=root
Group=root
ExecStart=/soft/redis/src/redis-server /soft/redis/redis_7002.conf
PrivateTmp=true
[Install]
WantedBy=multi-user.target
EOF
redis_7003.service
cat << "EOF" > /usr/lib/systemd/system/redis_7003.service
[Unit]
Description=Redis 7003 service
Documentation=https://redis.io/documentation
Wants=network-online.target
After=network-online.target
[Service]
Type=forking
LimitNOFILE=10032
User=root
Group=root
ExecStart=/soft/redis/src/redis-server /soft/redis/redis_7003.conf
PrivateTmp=true
[Install]
WantedBy=multi-user.target
EOF
设置开机自启
chown -R root.root /soft/redis
systemctl daemon-reload
systemctl enable redis_7001.service
systemctl enable redis_7002.service
systemctl enable redis_7003.service
systemctl start redis_7001.service
systemctl start redis_7002.service
systemctl start redis_7003.service
systemctl status redis_7001.service
systemctl status redis_7002.service
systemctl status redis_7003.service
# 2.5 启动 redis 集群服务
--cluster-replicas 2 表示为集群中的每个主节点创建 2 个从节点
cd /soft/redis/src
./redis-cli --cluster create \
192.168.1.135:7001 192.168.1.135:7002 192.168.1.135:7003 \
192.168.1.136:7001 192.168.1.136:7002 192.168.1.136:7003 \
192.168.1.137:7001 192.168.1.137:7002 192.168.1.137:7003 \
--cluster-replicas 2 -a admin123
输入创建集群的命令后会出现以下提示，注意 Can I set the above configuration? (type 'yes' to accept): yes，该处请输入 yes
[root@qnyp_node01 src]# ./redis-cli --cluster create \
> 192.168.1.135:7001 192.168.1.135:7002 192.168.1.135:7003 \
> 192.168.1.136:7001 192.168.1.136:7002 192.168.1.136:7003 \
> 192.168.1.137:7001 192.168.1.137:7002 192.168.1.137:7003 \
> --cluster-replicas 2 -a admin123
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
>>> Performing hash slots allocation on 9 nodes...
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica 192.168.1.136:7002 to 192.168.1.135:7001
Adding replica 192.168.1.137:7002 to 192.168.1.135:7001
Adding replica 192.168.1.135:7003 to 192.168.1.136:7001
Adding replica 192.168.1.137:7003 to 192.168.1.136:7001
Adding replica 192.168.1.136:7003 to 192.168.1.137:7001
Adding replica 192.168.1.135:7002 to 192.168.1.137:7001
M: 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 192.168.1.135:7001
slots:[0-5460] (5461 slots) master
S: 4508bee0c33784e0d5be25b64e4c7e677cd9d396 192.168.1.135:7002
replicates f9133541e2175958117753ef4e206ea43a21f07c
S: a0e13083fcc1d6e96398f3bb2ea5581b7a64e05e 192.168.1.135:7003
replicates 06ea827f8d328d9d776c9643109317b0100727a6
M: 06ea827f8d328d9d776c9643109317b0100727a6 192.168.1.136:7001
slots:[5461-10922] (5462 slots) master
S: 1d1b9817e39ee8987a3518f62a9b91c3ab666eff 192.168.1.136:7002
replicates 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7
S: a73c099dcc63f5d46a11d0f61c91270ef61290ff 192.168.1.136:7003
replicates f9133541e2175958117753ef4e206ea43a21f07c
M: f9133541e2175958117753ef4e206ea43a21f07c 192.168.1.137:7001
slots:[10923-16383] (5461 slots) master
S: 626dc659bb1059ec40039869241f7de88a49cd87 192.168.1.137:7002
replicates 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7
S: 622f73cd06c5658f8d02056925ac708750f12c1a 192.168.1.137:7003
replicates 06ea827f8d328d9d776c9643109317b0100727a6
Can I set the above configuration? (type 'yes' to accept):
输完 yes 后，会出现如下提示，[OK] All 16384 slots covered. 说明成功啦
Can I set the above configuration? (type 'yes' to accept): yes
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join
..
>>> Performing Cluster Check (using node 192.168.1.135:7001)
M: 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 192.168.1.135:7001
slots:[0-5460] (5461 slots) master
2 additional replica(s)
M: 06ea827f8d328d9d776c9643109317b0100727a6 192.168.1.136:7001
slots:[5461-10922] (5462 slots) master
2 additional replica(s)
S: 622f73cd06c5658f8d02056925ac708750f12c1a 192.168.1.137:7003
slots: (0 slots) slave
replicates 06ea827f8d328d9d776c9643109317b0100727a6
S: 1d1b9817e39ee8987a3518f62a9b91c3ab666eff 192.168.1.136:7002
slots: (0 slots) slave
replicates 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7
S: a73c099dcc63f5d46a11d0f61c91270ef61290ff 192.168.1.136:7003
slots: (0 slots) slave
replicates f9133541e2175958117753ef4e206ea43a21f07c
S: a0e13083fcc1d6e96398f3bb2ea5581b7a64e05e 192.168.1.135:7003
slots: (0 slots) slave
replicates 06ea827f8d328d9d776c9643109317b0100727a6
M: f9133541e2175958117753ef4e206ea43a21f07c 192.168.1.137:7001
slots:[10923-16383] (5461 slots) master
2 additional replica(s)
S: 626dc659bb1059ec40039869241f7de88a49cd87 192.168.1.137:7002
slots: (0 slots) slave
replicates 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7
S: 4508bee0c33784e0d5be25b64e4c7e677cd9d396 192.168.1.135:7002
slots: (0 slots) slave
replicates f9133541e2175958117753ef4e206ea43a21f07c
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.
# 2.6 访问 reids 集群并验证
cd /data/redis/src
./redis-cli -h 192.168.1.135 -p 7001 -c -a admin123
#列出当前节点的信息：cluster info
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
192.168.1.135:7001> cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:9
cluster_size:3
cluster_current_epoch:9
cluster_my_epoch:1
cluster_stats_messages_ping_sent:344
cluster_stats_messages_pong_sent:354
cluster_stats_messages_sent:698
cluster_stats_messages_ping_received:346
cluster_stats_messages_pong_received:344
cluster_stats_messages_meet_received:8
cluster_stats_messages_received:698
total_cluster_links_buffer_limit_exceeded:0
列出集群的节点的信息：cluster nodes
192.168.1.135:7001> cluster nodes
06ea827f8d328d9d776c9643109317b0100727a6 192.168.1.136:7001@17001 master - 0 1747034145581 4 connected 5461-10922
622f73cd06c5658f8d02056925ac708750f12c1a 192.168.1.137:7003@17003 slave 06ea827f8d328d9d776c9643109317b0100727a6 0 1747034145581 4 connected
1d1b9817e39ee8987a3518f62a9b91c3ab666eff 192.168.1.136:7002@17002 slave 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 0 1747034145581 1 connected
a73c099dcc63f5d46a11d0f61c91270ef61290ff 192.168.1.136:7003@17003 slave f9133541e2175958117753ef4e206ea43a21f07c 0 1747034145581 7 connected
a0e13083fcc1d6e96398f3bb2ea5581b7a64e05e 192.168.1.135:7003@17003 slave 06ea827f8d328d9d776c9643109317b0100727a6 0 1747034144077 4 connected
f9133541e2175958117753ef4e206ea43a21f07c 192.168.1.137:7001@17001 master - 0 1747034145079 7 connected 10923-16383
626dc659bb1059ec40039869241f7de88a49cd87 192.168.1.137:7002@17002 slave 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 0 1747034144000 1 connected
928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 192.168.1.135:7001@17001 myself,master - 0 1747034144000 1 connected 0-5460
4508bee0c33784e0d5be25b64e4c7e677cd9d396 192.168.1.135:7002@17002 slave f9133541e2175958117753ef4e206ea43a21f07c 0 1747034144578 7 connected
