生产环境配置
2C8G三台，三主三从，一台二实例
注意事项：一台2实例部署，主从节点要交叉，避免主从节点同时宕机影响业务
部署步骤
- 下载软件包
wget http://download.redis.io/releases/redis-5.0.5.tar.gz
- 安装依赖
yum -y install gcc gcc-c++
- 解压tar.gz包
tar xf redis-5.0.5.tar.gz -C /usr/local/
- 编译
cd /usr/local/redis-5.0.5/
make && make install
cp ./src/redis-server /usr/bin/ && cp ./src/redis-cli /usr/bin/
- Redis优化
echo 511 > /proc/sys/net/core/somaxconn
echo "vm.overcommit_memory = 1" >> /etc/sysctl.conf && echo "net.core.somaxconn = 1024" >> /etc/sysctl.conf
echo never > /sys/kernel/mm/transparent_hugepage/enabled
#开机启动
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.d/rc.local
- 创建目录及配置文件
mkdir /usr/local/redis-cluster/{7000,7001}/data -pv
cp redis.conf /usr/local/redis-cluster/7000/
[root@redis-cs-node01 redis-cluster]# grep "^[a-Z]" 7000/redis.conf
bind 0.0.0.0
protected-mode yes
port 7000
tcp-backlog 511
timeout 0
tcp-keepalive 30
daemonize yes
supervised no
pidfile /var/run/redis_7000.pid
loglevel notice
logfile "/usr/local/redis-cluster/7000/redis.log"
databases 16
always-show-logo yes
save ""
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /usr/local/redis-cluster/7000/data
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
replica-priority 100
maxclients 10000
maxmemory 3g
maxmemory-policy allkeys-lru
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
lua-time-limit 5000
cluster-enabled yes
cluster-config-file nodes-7000.conf
cluster-node-timeout 5000
cluster-require-full-coverage no
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
#修改配置信息
cp 7000/redis.conf 7001/
sed -i 's/7000/7001/g' 7001/redis.conf
#同理另外两台节点同理
mkdir /usr/local/redis-cluster/{7002,7003}/data -pv
sed -i 's/7000/7001/g' 7002/redis.conf
sed -i 's/7000/7001/g' 7003/redis.conf
mkdir /usr/local/redis-cluster/{7004,7005}/data -pv
sed -i 's/7000/7004/g' 7004/redis.conf
sed -i 's/7000/7005/g' 7005/redis.conf
- 创建启动脚本及开启启动
#第一个节点
[root@redis-cs-node01 redis-cluster]# cat start_redis_cluster.sh
#!/bin/bash
cd /usr/local/redis-cluster
cd 7000 && /usr/bin/redis-server /usr/local/redis-cluster/7000/redis.conf
cd ../7001 &&/usr/bin/redis-server /usr/local/redis-cluster/7001/redis.conf
#第二个节点
[root@redis-cs-node02 redis-cluster]# cat start_redis_cluster.sh
#!/bin/bash
cd /usr/local/redis-cluster
cd 7002 && /usr/bin/redis-server /usr/local/redis-cluster/7002/redis.conf
cd ../7003 &&/usr/bin/redis-server /usr/local/redis-cluster/7003/redis.conf
#第三个节点
[root@redis-cs-node03 redis-cluster]# cat start_redis_cluster.sh
#!/bin/bash
cd /usr/local/redis-cluster
cd 7004 && /usr/bin/redis-server /usr/local/redis-cluster/7004/redis.conf
cd ../7005 &&/usr/bin/redis-server /usr/local/redis-cluster/7005/redis.conf
#启动节点
[root@redis-cs-node01 redis-cluster]# sh start_redis_cluster.sh
[root@redis-cs-node02 redis-cluster]# sh start_redis_cluster.sh
[root@redis-cs-node03 redis-cluster]# sh start_redis_cluster.sh
#配置开机启动
[root@redis-cs-node01 redis-cluster]# cat /etc/rc.d/rc.local
#!/bin/bash
# THIS FILE IS ADDED FOR COMPATIBILITY PURPOSES
#
# It is highly advisable to create own systemd services or udev rules
# to run scripts during boot instead of using this file.
#
# In contrast to previous versions due to parallel execution during boot
# this script will NOT be run after all other services.
#
# Please note that you must run 'chmod +x /etc/rc.d/rc.local' to ensure
# that this script will be executed during boot.
touch /var/lock/subsys/local
echo never > /sys/kernel/mm/transparent_hugepage/enabled
/usr/bin/sh /usr/local/redis-cluster/start_redis_cluster.sh
/usr/bin/sh /usr/local/redis-cluster/start_exporter.sh
- 创建集群
redis-cli --cluster create 172.16.83.248:7000 172.16.83.248:7001 172.16.83.249:7002 172.16.83.249:7003 172.16.83.250:7004 172.16.83.250:7005 --cluster-replicas 1
- 查看集群状态
[root@redis-cs-node01 ~]# redis-cli -c -p 7000 cluster nodes
18a0169a22e7b440213728b1fb45ae5d59bf8eef 172.16.83.249:7002@17002 master - 0 1601184964498 41 connected 6251-10922
83280280d19223180eabffb1a88ff078efdc748d 172.16.83.249:7003@17003 slave 27c6bff6b4b20d5620a81879324cf61a8e2439a1 0 1601184965000 39 connected
3933efb9b77e474ab036d7741bc4517746483496 172.16.83.248:7001@17001 slave 16ab783181c05ed6c0e3db933be6a1f56efd3317 0 1601184964097 42 connected
16ab783181c05ed6c0e3db933be6a1f56efd3317 172.16.83.250:7004@17004 master - 0 1601184965501 42 connected 88-332 5461-5794 10923-11255 11711-16383
30469e7f2ce74d6e0ff224463468909aa268c6cf 172.16.83.250:7005@17005 slave 18a0169a22e7b440213728b1fb45ae5d59bf8eef 0 1601184965501 41 connected
27c6bff6b4b20d5620a81879324cf61a8e2439a1 172.16.83.248:7000@17000 myself,master - 0 1601184965000 39 connected 0-87 333-5460 5795-6250 11256-11710
[root@redis-cs-node01 redis-cluster]# redis-cli --cluster check 127.0.0.1:7001
172.16.83.249:7002 (18a0169a...) -> 10 keys | 4672 slots | 1 slaves.
172.16.83.250:7004 (16ab7831...) -> 9 keys | 5585 slots | 1 slaves.
172.16.83.248:7000 (27c6bff6...) -> 5 keys | 6127 slots | 1 slaves.
[OK] 24 keys in 3 masters.
0.00 keys per slot on average.
>>> Performing Cluster Check (using node 127.0.0.1:7001)
S: 3933efb9b77e474ab036d7741bc4517746483496 127.0.0.1:7001
slots: (0 slots) slave
replicates 16ab783181c05ed6c0e3db933be6a1f56efd3317
M: 18a0169a22e7b440213728b1fb45ae5d59bf8eef 172.16.83.249:7002
slots:[6251-10922] (4672 slots) master
1 additional replica(s)
S: 30469e7f2ce74d6e0ff224463468909aa268c6cf 172.16.83.250:7005
slots: (0 slots) slave
replicates 18a0169a22e7b440213728b1fb45ae5d59bf8eef
S: 83280280d19223180eabffb1a88ff078efdc748d 172.16.83.249:7003
slots: (0 slots) slave
replicates 27c6bff6b4b20d5620a81879324cf61a8e2439a1
M: 16ab783181c05ed6c0e3db933be6a1f56efd3317 172.16.83.250:7004
slots:[88-332],[5461-5794],[10923-11255],[11711-16383] (5585 slots) master
1 additional replica(s)
M: 27c6bff6b4b20d5620a81879324cf61a8e2439a1 172.16.83.248:7000
slots:[0-87],[333-5460],[5795-6250],[11256-11710] (6127 slots) master
1 additional replica(s)
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.
扩容节点内存步骤
说明：线上Redis节点内存接近限制3g，进行无感知进行升级内存。
- 查看主从节点分配；
- 确认主从不在同一节点；
- 逐台操作从节点，进行扩容服务器配置；
- 动态升级内存限制；
config set maxmemory 6g
- 再进行从升级主操作
在slave的Redis执行 CLUSTER FAILOVER
slave会接管主, master会转化为salve
说明：故障转移的主服务器的客户端都已停止。同时，主服务器将其复制偏移发送到从服务器，从服务器等待到达其一侧的偏移。达到复制偏移量后，故障转移将开始，并且将向旧的主服务器通知配置切换。当客户端在旧的主服务器上解除阻止时，它们将被重定向到新的主服务器；
- 此时从变为主再进行升级主变为从节点服务器，升级配置；
- 再动态升级内存限；
config set maxmemory 6g
- 此时升级配置扩容内存已完成，注意检查主从分配信息，确保主从不在同一点，保证服务的高可用。
prometheus监控Redis
- 客户端脚本
[root@redis-cs-node01 redis-cluster]# cat start_exporter.sh
#!/bin/bash
cd /usr/local/redis-cluster
ports=`netstat -anltp|grep LISTEN|grep -v 1700|egrep "(700|6379)"|awk '{print $4}'|cut -d ":" -f 2`
echo "find ports: $ports"
for p in $ports
do
ip=`ip addr |grep 172|grep inet|awk '{print $2}'|awk -F "/" '{print $1}'`
echo "start exporter 127.0.0.1:${p} with ${ip}:5${p}"
nohup ./redis_exporter -web.listen-address ":5${p}" -redis.addr 127.0.0.1:${p} &
done
- prometheus服务端配置
[root@k8s-master test]# cat redis-test-yc.yaml
---
apiVersion: v1
kind: Service
metadata:
labels:
node: redis-exporter
name: redis-test-yc
namespace: monitoring
spec:
ports:
- name: metrics
port: 8080
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
---
apiVersion: v1
kind: Endpoints
metadata:
name: redis-test-yc
namespace: monitoring
subsets:
- addresses:
- ip: 172.16.83.248
ports:
- name: metrics
port: 57000
protocol: TCP
- addresses:
- ip: 172.16.83.248
ports:
- name: metrics
port: 57001
protocol: TCP
- addresses:
- ip: 172.16.83.249
ports:
- name: metrics
port: 57002
protocol: TCP
- addresses:
- ip: 172.16.83.249
ports:
- name: metrics
port: 57003
protocol: TCP
- addresses:
- ip: 172.16.83.250
ports:
- name: metrics
port: 57004
protocol: TCP
- addresses:
- ip: 172.16.83.250
ports:
- name: metrics
port: 57005
protocol: TCP
- 查看prometheus后台监控项
