# Redis Cluster 集群部署
# 1、环境配置
# 1.1 关闭防火墙、Selinux
systemctl disable --now firewalld | |
setenforce 0 | |
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux | |
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config |
# 1.1 配置 yum 源
#rocky linux 配置 | |
sed -e 's|^mirrorlist=|#mirrorlist=|g' \ | |
-e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.aliyun.com/rockylinux|g' \ | |
-i.bak \ | |
/etc/yum.repos.d/rocky-*.repo | |
yum clean all && yum makecache | |
mkdir /soft /data /scripts /backup |
# 1.3 配置文件描述符
ulimit -SHn 65535 | |
vim /etc/security/limits.conf | |
# 末尾添加如下内容 | |
* soft nofile 65536 | |
* hard nofile 131072 | |
* soft nproc 65535 | |
* hard nproc 655350 | |
* soft memlock unlimited | |
* hard memlock unlimited |
# 1.4 系统内核参数调优
# 修改 /etc/sysctl.conf 文件 | |
cat >> /etc/sysctl.conf <<EOF | |
vm.max_map_count = 262144 | |
vm.swappiness=1 | |
net.ipv4.tcp_fin_timeout=2 | |
net.ipv4.tcp_tw_reuse=1 | |
#net.ipv4.tcp_tw_recycle=1 | |
net.ipv4.tcp_syncookies=1 | |
net.ipv4.tcp_keepalive_time=600 | |
net.ipv4.ip_local_port_range=4000 65000 | |
net.ipv4.tcp_max_syn_backlog=16384 | |
net.ipv4.route.gc_timeout=100 | |
net.ipv4.tcp_max_tw_buckets= 5000 | |
net.ipv4.tcp_syn_retries=1 | |
net.ipv4.tcp_synack_retries=1 | |
net.core.somaxconn=16384 | |
net.core.netdev_max_backlog=16384 | |
net.ipv4.tcp_max_orphans=16384 | |
# 设置最大内存共享段大小bytes | |
kernel.shmmax=15461882265 | |
kernel.shmall=3774873 | |
# 修改消息队列长度 | |
kernel.msgmax=65535 | |
kernel.msgmnb=65535 | |
EOF |
# 1.5 修改默认限制内存
cat >>/etc/systemd/system.conf<< EOF | |
DefaultLimitNOFILE=65536 | |
DefaultLimitNPROC=32000 | |
DefaultLimitMEMLOCK=infinity | |
EOF |
# 1.6 执行命令生效状态
sysctl -p |
# 1.7 安装基础软件包
yum install wget jq psmisc vim unzip net-tools telnet tree yum-utils device-mapper-persistent-data \ | |
lvm2 git ntpdate nfs-utils iotop httpd-tools dos2unix lrzsz -y |
# 1.8 升级系统
yum update -y --exclude=kernel* && reboot |
# 2、Redis cluster 部署
| 192.168.1.135 | 192.168.1.136 | 192.168.1.137 |
|---|---|---|
| node1：7001 | node1：7001 | node1：7001 |
| node2：7002 | node2：7002 | node2：7002 |
| node3：7003 | node3：7003 | node3：7003 |
# 2.1 安装包下载
wget https://download.redis.io/releases/redis-7.2.1.tar.gz |
# 2.2 安装 redis
yum install gcc-c++ -y | |
mkdir /soft | |
tar -xzvf redis-7.2.1.tar.gz -C /soft | |
ln -s /soft/redis-7.2.1 /soft/redis | |
cd /soft/redis | |
make | |
make install prefix=/soft/redis |
# 2.3 生成集群配置文件
mkdir -p /soft/redis/data/7001 | |
mkdir -p /soft/redis/data/7002 | |
mkdir -p /soft/redis/data/7003 | |
mkdir -p /soft/redis/log | |
cd /soft/redis |
redis_7001.conf 配置文件
cat > /soft/redis/redis_7001.conf <<EOF | |
protected-mode yes | |
port 7001 | |
requirepass admin123 | |
masterauth admin123 | |
cluster-enabled yes | |
cluster-config-file nodes-7001.conf | |
cluster-node-timeout 5000 | |
maxmemory 2GB | |
maxmemory-policy volatile-lru | |
tcp-backlog 511 | |
timeout 0 | |
tcp-keepalive 300 | |
daemonize yes | |
pidfile /soft/redis/data/redis7001.pid | |
loglevel notice | |
logfile "/soft/redis/log/redis7001.log" | |
#databases 16 | |
always-show-logo no | |
set-proc-title yes | |
proc-title-template "{title} {listen-addr} {server-mode}" | |
locale-collate "" | |
stop-writes-on-bgsave-error yes | |
rdbcompression yes | |
rdbchecksum yes | |
dbfilename dump.rdb | |
rdb-del-sync-files no | |
dir /soft/redis/data/7001 | |
replica-serve-stale-data yes | |
replica-read-only yes | |
repl-diskless-sync yes | |
repl-diskless-sync-delay 5 | |
repl-diskless-sync-max-replicas 0 | |
repl-diskless-load disabled | |
repl-disable-tcp-nodelay no | |
replica-priority 100 | |
acllog-max-len 128 | |
lazyfree-lazy-eviction no | |
lazyfree-lazy-expire no | |
lazyfree-lazy-server-del no | |
replica-lazy-flush no | |
lazyfree-lazy-user-del no | |
lazyfree-lazy-user-flush no | |
oom-score-adj no | |
oom-score-adj-values 0 200 800 | |
disable-thp yes | |
appendonly no | |
appendfilename "appendonly.aof" | |
appenddirname "appendonlydir" | |
appendfsync everysec | |
no-appendfsync-on-rewrite no | |
auto-aof-rewrite-percentage 100 | |
auto-aof-rewrite-min-size 64mb | |
aof-load-truncated yes | |
aof-use-rdb-preamble yes | |
aof-timestamp-enabled no | |
slowlog-log-slower-than 10000 | |
slowlog-max-len 128 | |
latency-monitor-threshold 0 | |
notify-keyspace-events "" | |
hash-max-listpack-entries 512 | |
hash-max-listpack-value 64 | |
list-max-listpack-size -2 | |
list-compress-depth 0 | |
set-max-intset-entries 512 | |
set-max-listpack-entries 128 | |
set-max-listpack-value 64 | |
zset-max-listpack-entries 128 | |
zset-max-listpack-value 64 | |
hll-sparse-max-bytes 3000 | |
stream-node-max-bytes 4096 | |
stream-node-max-entries 100 | |
activerehashing yes | |
client-output-buffer-limit normal 0 0 0 | |
client-output-buffer-limit replica 256mb 64mb 60 | |
client-output-buffer-limit pubsub 32mb 8mb 60 | |
hz 10 | |
dynamic-hz yes | |
aof-rewrite-incremental-fsync yes | |
rdb-save-incremental-fsync yes | |
jemalloc-bg-thread yes | |
EOF |
redis_7002.conf 配置文件
cat > /soft/redis/redis_7002.conf <<EOF | |
protected-mode yes | |
port 7002 | |
requirepass admin123 | |
masterauth admin123 | |
cluster-enabled yes | |
cluster-config-file nodes-7002.conf | |
cluster-node-timeout 5000 | |
maxmemory 2GB | |
maxmemory-policy volatile-lru | |
tcp-backlog 511 | |
timeout 0 | |
tcp-keepalive 300 | |
daemonize yes | |
pidfile /soft/redis/data/redis7002.pid | |
loglevel notice | |
logfile "/soft/redis/log/redis7002.log" | |
#databases 16 | |
always-show-logo no | |
set-proc-title yes | |
proc-title-template "{title} {listen-addr} {server-mode}" | |
locale-collate "" | |
stop-writes-on-bgsave-error yes | |
rdbcompression yes | |
rdbchecksum yes | |
dbfilename dump.rdb | |
rdb-del-sync-files no | |
dir /soft/redis/data/7002 | |
replica-serve-stale-data yes | |
replica-read-only yes | |
repl-diskless-sync yes | |
repl-diskless-sync-delay 5 | |
repl-diskless-sync-max-replicas 0 | |
repl-diskless-load disabled | |
repl-disable-tcp-nodelay no | |
replica-priority 100 | |
acllog-max-len 128 | |
lazyfree-lazy-eviction no | |
lazyfree-lazy-expire no | |
lazyfree-lazy-server-del no | |
replica-lazy-flush no | |
lazyfree-lazy-user-del no | |
lazyfree-lazy-user-flush no | |
oom-score-adj no | |
oom-score-adj-values 0 200 800 | |
disable-thp yes | |
appendonly no | |
appendfilename "appendonly.aof" | |
appenddirname "appendonlydir" | |
appendfsync everysec | |
no-appendfsync-on-rewrite no | |
auto-aof-rewrite-percentage 100 | |
auto-aof-rewrite-min-size 64mb | |
aof-load-truncated yes | |
aof-use-rdb-preamble yes | |
aof-timestamp-enabled no | |
slowlog-log-slower-than 10000 | |
slowlog-max-len 128 | |
latency-monitor-threshold 0 | |
notify-keyspace-events "" | |
hash-max-listpack-entries 512 | |
hash-max-listpack-value 64 | |
list-max-listpack-size -2 | |
list-compress-depth 0 | |
set-max-intset-entries 512 | |
set-max-listpack-entries 128 | |
set-max-listpack-value 64 | |
zset-max-listpack-entries 128 | |
zset-max-listpack-value 64 | |
hll-sparse-max-bytes 3000 | |
stream-node-max-bytes 4096 | |
stream-node-max-entries 100 | |
activerehashing yes | |
client-output-buffer-limit normal 0 0 0 | |
client-output-buffer-limit replica 256mb 64mb 60 | |
client-output-buffer-limit pubsub 32mb 8mb 60 | |
hz 10 | |
dynamic-hz yes | |
aof-rewrite-incremental-fsync yes | |
rdb-save-incremental-fsync yes | |
jemalloc-bg-thread yes | |
EOF |
redis_7003.conf 配置文件
cat > /soft/redis/redis_7003.conf <<EOF | |
protected-mode yes | |
port 7003 | |
requirepass admin123 | |
masterauth admin123 | |
cluster-enabled yes | |
cluster-config-file nodes-7003.conf | |
cluster-node-timeout 5000 | |
maxmemory 2GB | |
maxmemory-policy volatile-lru | |
tcp-backlog 511 | |
timeout 0 | |
tcp-keepalive 300 | |
daemonize yes | |
pidfile /soft/redis/data/redis7003.pid | |
loglevel notice | |
logfile "/soft/redis/log/redis7003.log" | |
#databases 16 | |
always-show-logo no | |
set-proc-title yes | |
proc-title-template "{title} {listen-addr} {server-mode}" | |
locale-collate "" | |
stop-writes-on-bgsave-error yes | |
rdbcompression yes | |
rdbchecksum yes | |
dbfilename dump.rdb | |
rdb-del-sync-files no | |
dir /soft/redis/data/7003 | |
replica-serve-stale-data yes | |
replica-read-only yes | |
repl-diskless-sync yes | |
repl-diskless-sync-delay 5 | |
repl-diskless-sync-max-replicas 0 | |
repl-diskless-load disabled | |
repl-disable-tcp-nodelay no | |
replica-priority 100 | |
acllog-max-len 128 | |
lazyfree-lazy-eviction no | |
lazyfree-lazy-expire no | |
lazyfree-lazy-server-del no | |
replica-lazy-flush no | |
lazyfree-lazy-user-del no | |
lazyfree-lazy-user-flush no | |
oom-score-adj no | |
oom-score-adj-values 0 200 800 | |
disable-thp yes | |
appendonly no | |
appendfilename "appendonly.aof" | |
appenddirname "appendonlydir" | |
appendfsync everysec | |
no-appendfsync-on-rewrite no | |
auto-aof-rewrite-percentage 100 | |
auto-aof-rewrite-min-size 64mb | |
aof-load-truncated yes | |
aof-use-rdb-preamble yes | |
aof-timestamp-enabled no | |
slowlog-log-slower-than 10000 | |
slowlog-max-len 128 | |
latency-monitor-threshold 0 | |
notify-keyspace-events "" | |
hash-max-listpack-entries 512 | |
hash-max-listpack-value 64 | |
list-max-listpack-size -2 | |
list-compress-depth 0 | |
set-max-intset-entries 512 | |
set-max-listpack-entries 128 | |
set-max-listpack-value 64 | |
zset-max-listpack-entries 128 | |
zset-max-listpack-value 64 | |
hll-sparse-max-bytes 3000 | |
stream-node-max-bytes 4096 | |
stream-node-max-entries 100 | |
activerehashing yes | |
client-output-buffer-limit normal 0 0 0 | |
client-output-buffer-limit replica 256mb 64mb 60 | |
client-output-buffer-limit pubsub 32mb 8mb 60 | |
hz 10 | |
dynamic-hz yes | |
aof-rewrite-incremental-fsync yes | |
rdb-save-incremental-fsync yes | |
jemalloc-bg-thread yes | |
EOF |
# 2.4 Redis 开机自启
redis_7001.service
cat << "EOF" > /usr/lib/systemd/system/redis_7001.service | |
[Unit] | |
Description=Redis 7001 service | |
Documentation=https://redis.io/documentation | |
Wants=network-online.target | |
After=network-online.target | |
[Service] | |
Type=forking | |
LimitNOFILE=10032 | |
User=root | |
Group=root | |
ExecStart=/soft/redis/src/redis-server /soft/redis/redis_7001.conf | |
PrivateTmp=true | |
[Install] | |
WantedBy=multi-user.target | |
EOF |
redis_7002.service
cat << "EOF" > /usr/lib/systemd/system/redis_7002.service | |
[Unit] | |
Description=Redis 7002 service | |
Documentation=https://redis.io/documentation | |
Wants=network-online.target | |
After=network-online.target | |
[Service] | |
Type=forking | |
LimitNOFILE=10032 | |
User=root | |
Group=root | |
ExecStart=/soft/redis/src/redis-server /soft/redis/redis_7002.conf | |
PrivateTmp=true | |
[Install] | |
WantedBy=multi-user.target | |
EOF |
redis_7003.service
cat << "EOF" > /usr/lib/systemd/system/redis_7003.service | |
[Unit] | |
Description=Redis 7003 service | |
Documentation=https://redis.io/documentation | |
Wants=network-online.target | |
After=network-online.target | |
[Service] | |
Type=forking | |
LimitNOFILE=10032 | |
User=root | |
Group=root | |
ExecStart=/soft/redis/src/redis-server /soft/redis/redis_7003.conf | |
PrivateTmp=true | |
[Install] | |
WantedBy=multi-user.target | |
EOF |
设置开机自启
chown -R root.root /soft/redis | |
systemctl daemon-reload | |
systemctl enable redis_7001.service | |
systemctl enable redis_7002.service | |
systemctl enable redis_7003.service | |
systemctl start redis_7001.service | |
systemctl start redis_7002.service | |
systemctl start redis_7003.service | |
systemctl status redis_7001.service | |
systemctl status redis_7002.service | |
systemctl status redis_7003.service |
# 2.5 启动 redis 集群服务
--cluster-replicas 2 表示为集群中的每个主节点创建 2 个从节点
cd /soft/redis/src | |
./redis-cli --cluster create \ | |
192.168.1.135:7001 192.168.1.135:7002 192.168.1.135:7003 \ | |
192.168.1.136:7001 192.168.1.136:7002 192.168.1.136:7003 \ | |
192.168.1.137:7001 192.168.1.137:7002 192.168.1.137:7003 \ | |
--cluster-replicas 2 -a admin123 |
输入创建集群的命令后会出现以下提示，注意 Can I set the above configuration? (type 'yes' to accept): yes，该处请输入 yes
[root@qnyp_node01 src]# ./redis-cli --cluster create \ | |
> 192.168.1.135:7001 192.168.1.135:7002 192.168.1.135:7003 \ | |
> 192.168.1.136:7001 192.168.1.136:7002 192.168.1.136:7003 \ | |
> 192.168.1.137:7001 192.168.1.137:7002 192.168.1.137:7003 \ | |
> --cluster-replicas 2 -a admin123 | |
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. | |
>>> Performing hash slots allocation on 9 nodes... | |
Master[0] -> Slots 0 - 5460 | |
Master[1] -> Slots 5461 - 10922 | |
Master[2] -> Slots 10923 - 16383 | |
Adding replica 192.168.1.136:7002 to 192.168.1.135:7001 | |
Adding replica 192.168.1.137:7002 to 192.168.1.135:7001 | |
Adding replica 192.168.1.135:7003 to 192.168.1.136:7001 | |
Adding replica 192.168.1.137:7003 to 192.168.1.136:7001 | |
Adding replica 192.168.1.136:7003 to 192.168.1.137:7001 | |
Adding replica 192.168.1.135:7002 to 192.168.1.137:7001 | |
M: 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 192.168.1.135:7001 | |
slots:[0-5460] (5461 slots) master | |
S: 4508bee0c33784e0d5be25b64e4c7e677cd9d396 192.168.1.135:7002 | |
replicates f9133541e2175958117753ef4e206ea43a21f07c | |
S: a0e13083fcc1d6e96398f3bb2ea5581b7a64e05e 192.168.1.135:7003 | |
replicates 06ea827f8d328d9d776c9643109317b0100727a6 | |
M: 06ea827f8d328d9d776c9643109317b0100727a6 192.168.1.136:7001 | |
slots:[5461-10922] (5462 slots) master | |
S: 1d1b9817e39ee8987a3518f62a9b91c3ab666eff 192.168.1.136:7002 | |
replicates 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 | |
S: a73c099dcc63f5d46a11d0f61c91270ef61290ff 192.168.1.136:7003 | |
replicates f9133541e2175958117753ef4e206ea43a21f07c | |
M: f9133541e2175958117753ef4e206ea43a21f07c 192.168.1.137:7001 | |
slots:[10923-16383] (5461 slots) master | |
S: 626dc659bb1059ec40039869241f7de88a49cd87 192.168.1.137:7002 | |
replicates 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 | |
S: 622f73cd06c5658f8d02056925ac708750f12c1a 192.168.1.137:7003 | |
replicates 06ea827f8d328d9d776c9643109317b0100727a6 | |
Can I set the above configuration? (type 'yes' to accept): |
输完 yes 后，会出现如下提示，[OK] All 16384 slots covered. 说明成功啦
Can I set the above configuration? (type 'yes' to accept): yes | |
>>> Nodes configuration updated | |
>>> Assign a different config epoch to each node | |
>>> Sending CLUSTER MEET messages to join the cluster | |
Waiting for the cluster to join | |
.. | |
>>> Performing Cluster Check (using node 192.168.1.135:7001) | |
M: 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 192.168.1.135:7001 | |
slots:[0-5460] (5461 slots) master | |
2 additional replica(s) | |
M: 06ea827f8d328d9d776c9643109317b0100727a6 192.168.1.136:7001 | |
slots:[5461-10922] (5462 slots) master | |
2 additional replica(s) | |
S: 622f73cd06c5658f8d02056925ac708750f12c1a 192.168.1.137:7003 | |
slots: (0 slots) slave | |
replicates 06ea827f8d328d9d776c9643109317b0100727a6 | |
S: 1d1b9817e39ee8987a3518f62a9b91c3ab666eff 192.168.1.136:7002 | |
slots: (0 slots) slave | |
replicates 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 | |
S: a73c099dcc63f5d46a11d0f61c91270ef61290ff 192.168.1.136:7003 | |
slots: (0 slots) slave | |
replicates f9133541e2175958117753ef4e206ea43a21f07c | |
S: a0e13083fcc1d6e96398f3bb2ea5581b7a64e05e 192.168.1.135:7003 | |
slots: (0 slots) slave | |
replicates 06ea827f8d328d9d776c9643109317b0100727a6 | |
M: f9133541e2175958117753ef4e206ea43a21f07c 192.168.1.137:7001 | |
slots:[10923-16383] (5461 slots) master | |
2 additional replica(s) | |
S: 626dc659bb1059ec40039869241f7de88a49cd87 192.168.1.137:7002 | |
slots: (0 slots) slave | |
replicates 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 | |
S: 4508bee0c33784e0d5be25b64e4c7e677cd9d396 192.168.1.135:7002 | |
slots: (0 slots) slave | |
replicates f9133541e2175958117753ef4e206ea43a21f07c | |
[OK] All nodes agree about slots configuration. | |
>>> Check for open slots... | |
>>> Check slots coverage... | |
[OK] All 16384 slots covered. |
# 2.6 访问 reids 集群并验证
cd /data/redis/src | |
./redis-cli -h 192.168.1.135 -p 7001 -c -a admin123 | |
#列出当前节点的信息：cluster info | |
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. | |
192.168.1.135:7001> cluster info | |
cluster_state:ok | |
cluster_slots_assigned:16384 | |
cluster_slots_ok:16384 | |
cluster_slots_pfail:0 | |
cluster_slots_fail:0 | |
cluster_known_nodes:9 | |
cluster_size:3 | |
cluster_current_epoch:9 | |
cluster_my_epoch:1 | |
cluster_stats_messages_ping_sent:344 | |
cluster_stats_messages_pong_sent:354 | |
cluster_stats_messages_sent:698 | |
cluster_stats_messages_ping_received:346 | |
cluster_stats_messages_pong_received:344 | |
cluster_stats_messages_meet_received:8 | |
cluster_stats_messages_received:698 | |
total_cluster_links_buffer_limit_exceeded:0 | |
列出集群的节点的信息：cluster nodes | |
192.168.1.135:7001> cluster nodes | |
06ea827f8d328d9d776c9643109317b0100727a6 192.168.1.136:7001@17001 master - 0 1747034145581 4 connected 5461-10922 | |
622f73cd06c5658f8d02056925ac708750f12c1a 192.168.1.137:7003@17003 slave 06ea827f8d328d9d776c9643109317b0100727a6 0 1747034145581 4 connected | |
1d1b9817e39ee8987a3518f62a9b91c3ab666eff 192.168.1.136:7002@17002 slave 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 0 1747034145581 1 connected | |
a73c099dcc63f5d46a11d0f61c91270ef61290ff 192.168.1.136:7003@17003 slave f9133541e2175958117753ef4e206ea43a21f07c 0 1747034145581 7 connected | |
a0e13083fcc1d6e96398f3bb2ea5581b7a64e05e 192.168.1.135:7003@17003 slave 06ea827f8d328d9d776c9643109317b0100727a6 0 1747034144077 4 connected | |
f9133541e2175958117753ef4e206ea43a21f07c 192.168.1.137:7001@17001 master - 0 1747034145079 7 connected 10923-16383 | |
626dc659bb1059ec40039869241f7de88a49cd87 192.168.1.137:7002@17002 slave 928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 0 1747034144000 1 connected | |
928637d72deb6a2e7935f8a7bb5ebd9455cf64a7 192.168.1.135:7001@17001 myself,master - 0 1747034144000 1 connected 0-5460 | |
4508bee0c33784e0d5be25b64e4c7e677cd9d396 192.168.1.135:7002@17002 slave f9133541e2175958117753ef4e206ea43a21f07c 0 1747034144578 7 connected |
