zookeeper-3.4.14+codis-3.2.0集群部署（详细版）

前言

部署zookeeper+codis集群请先按以下链接部署zookeeper，再回来继续。
有问题可留言

zookeeper-3.4.14集群安装部署
https://blog.csdn.net/yuhuochengshi/article/details/102785456

一、服务版本说明

服务版本号备注
codis codis-3.2.0
codis3.2.0-go1.8.1 codis3.2.0-go1.8.1-linux

二、部署环境

三、下载路径

codis3.2.0下载地址：
https://github.com/CodisLabs/codis/releases
Codis官网地址：
https://github.com/CodisLabs/codis

四、安装约定并创建目录

安装包存放目录：/data/software/codis
安装目录：/usr/local/codis

[root@yhcs_1 ~]# mkdir -p /data/software/codis
[root@yhcs_1 ~]# mkdir -p /usr/local/codis

五、上传tar包并解压

[root@yhcs_1 ~]# cd /data/software/codis
[root@yhcs_1 codis]# rz -E
rz waiting to receive.
[root@yhcs_1 codis]# tar -zxvf codis-3.2.0.tar.gz -C /usr/local/codis/
[root@yhcs_1 codis]# tar -zxvf codis3.2.0-go1.8.1-linux.tar.gz -C /usr/local/codis/codis-3.2.0/
[root@yhcs_1 codis]# cd /usr/local/codis/codis-3.2.0/
[root@yhcs_1 codis-3.2.0]# ls
admin codis3.2.0-go1.8.1-linux doc extern MIT-LICENSE.txt scripts wandoujia_license.txt
ansible config Dockerfile Godeps pkg vendor
cmd deploy example Makefile README.md version
[root@yhcs_1 codis-3.2.0]# mv codis3.2.0-go1.8.1-linux/ bin
[root@yhcs_1 codis-3.2.0]# ls
admin cmd doc extern MIT-LICENSE.txt scripts wandoujia_license.txt
ansible config Dockerfile Godeps pkg vendor
bin deploy example Makefile README.md version
[root@yhcs_1 codis-3.2.0]# cd bin/
[root@yhcs_1 bin]# ls
assets codis-dashboard codis-proxy redis-benchmark version
codis-admin codis-fe codis-server redis-cli
[root@yhcs_1 bin]#

六、修改配置文件

（一）修改dashboard.toml
[root@yhcs_1 config]# cat /usr/local/codis/codis-3.2.0/config/dashboard.toml |grep -v ^#| grep -v ^$
coordinator_name = “zookeeper”
coordinator_addr = “192.168.43.176:2181,192.168.43.177:2181,192.168.43.178:2181”
product_name = “codis-zoo”
product_auth = “”
admin_addr = “192.168.43.176:18080”
migration_method = “semi-async”
migration_parallel_slots = 100
migration_async_maxbulks = 200
migration_async_maxbytes = “32mb”
migration_async_numkeys = 500
migration_timeout = “30s”
sentinel_quorum = 2
sentinel_parallel_syncs = 1
sentinel_down_after = “3s”
sentinel_failover_timeout = “5m”
sentinel_notification_script = “”
sentinel_client_reconfig_script = “”

（二）修改fe启动脚本
[root@yhcs_1 ~]# vi /usr/local/codis/codis-3.2.0/admin/codis-fe-admin.sh
… …
COORDINATOR_NAME=“zookeeper”
COORDINATOR_ADDR=“192.168.43.176:2181,192.168.43.177:2181,192.168.43.178:2181”
CODIS_FE_ADDR=“192.168.43.176:9090”
… …

（三）修改proxy.toml
[root@yhcs_1 ~]# cd /usr/local/codis/codis-3.2.0/config/
[root@yhcs_1 config]# cat proxy.toml |grep -v ^#|grep -v ^$
product_name = “codis-zoo”
product_auth = “”
session_auth = “”
admin_addr = “192.168.43.176:11080”
proto_type = “tcp4”
proxy_addr = “192.168.43.176:19000”
jodis_name = “zookeeper”
jodis_addr = “192.168.43.176:2181,192.168.43.177:2181,192.168.43.178:2181”
jodis_timeout = “20s”
jodis_compatible = false
proxy_datacenter = “”
proxy_max_clients = 1000
proxy_max_offheap_size = “1024mb”
proxy_heap_placeholder = “256mb”
backend_ping_period = “5s”
backend_recv_bufsize = “128kb”
backend_recv_timeout = “30s”
backend_send_bufsize = “128kb”
backend_send_timeout = “30s”
backend_max_pipeline = 20480
backend_primary_only = false
backend_primary_parallel = 1
backend_replica_parallel = 1
backend_keepalive_period = “75s”
backend_number_databases = 16
session_recv_bufsize = “128kb”
session_recv_timeout = “30m”
session_send_bufsize = “64kb”
session_send_timeout = “30s”
session_max_pipeline = 10000
session_keepalive_period = “75s”
session_break_on_failure = false
metrics_report_server = “”
metrics_report_period = “1s”
metrics_report_influxdb_server = “”
metrics_report_influxdb_period = “1s”
metrics_report_influxdb_username = “”
metrics_report_influxdb_password = “”
metrics_report_influxdb_database = “”
metrics_report_statsd_server = “”
metrics_report_statsd_period = “1s”
metrics_report_statsd_prefix = “”

（四）修改codis-proxy-admin.sh
[root@yhcs_1 ~]# cd /usr/local/codis/codis-3.2.0/admin/
[root@yhcs_1 admin]# vi codis-proxy-admin.sh
CODIS_DASHBOARD_ADDR=“192.168.43.176:18080”
………………
–ncpu=4 #修改此处。根据服务器机器的cpu数量填写

（五）修改配置文件redis.conf
#具体的优化还需根据需求调节
[root@yhcs_1 ~]# cd /usr/local/codis/codis-3.2.0/config
[root@yhcs_1 config]# cat redis.conf |grep -v ^#|grep -v ^$
bind 192.168.43.176
protected-mode no
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /usr/local/codis/codis-3.2.0/redis_6379.pid
loglevel notice
logfile “/usr/local/codis/codis-3.2.0/log/redis_6379.log”
databases 16
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /usr/local/codis/codis-3.2.0/
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly no
appendfilename “appendonly.aof”
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events “”
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
maxmemory 8gb
#定义redis最大内存使用情况，根据环境不同进行定义，一般为服务器内存的3/4

（六）修改codis-server-admin.sh
[root@yhcs_1 ~]# cd /usr/local/codis/codis-3.2.0/admin/
[root@yhcs_1 admin]# cat codis-server-admin.sh |grep -v ^#|grep -v ^$
CODIS_ADMIN="${BASH_SOURCE-KaTeX parse error: Expected 'EOF', got '}' at position 2: 0}̲" CODIS_ADMIN="(dirname “CODISADMIN")"CODISADMINDIR="{CODIS_ADMIN}")" CODIS_ADMIN_DIR="CODISADMIN")"CODISADMINDIR="(cd “CODISADMIN";pwd)"CODISBINDIR={CODIS_ADMIN}"; pwd)" CODIS_BIN_DIR=CODISADMIN";pwd)"CODISBINDIR=CODIS_ADMIN_DIR/…/bin
CODIS_LOG_DIR=CODISADMINDIR/../logCODISCONFDIR=CODIS_ADMIN_DIR/../log CODIS_CONF_DIR=CODISADMINDIR/../logCODISCONFDIR=CODIS_ADMIN_DIR/…/config
CODIS_SERVER_BIN=CODISBINDIR/codis−serverCODISSERVERPIDFILE=/usr/local/codis/codis−3.2.0/redis6379.pidCODISSERVERLOGFILE=/usr/local/codis/codis−3.2.0/log/redis6379.logCODISSERVERDAEMONFILE=CODIS_BIN_DIR/codis-server CODIS_SERVER_PID_FILE=/usr/local/codis/codis-3.2.0/redis_6379.pid CODIS_SERVER_LOG_FILE=/usr/local/codis/codis-3.2.0/log/redis_6379.log CODIS_SERVER_DAEMON_FILE=CODISBINDIR/codis−serverCODISSERVERPIDFILE=/usr/local/codis/codis−3.2.0/redis6379.pidCODISSERVERLOGFILE=/usr/local/codis/codis−3.2.0/log/redis6379.logCODISSERVERDAEMONFILE=CODIS_LOG_DIR/codis-server.out
CODIS_SERVER_CONF_FILE=$CODIS_CONF_DIR/redis.conf
echo $CODIS_SERVER_CONF_FILE
if [ ! -d $CODIS_LOG_DIR ]; then
mkdir -p $CODIS_LOG_DIR
fi
case 1instart)echo"startingcodis−server..."if[−f"1 in start) echo "starting codis-server ... " if [ -f "1instart)echo"startingcodis−server..."if[−f"CODIS_SERVER_PID_FILE” ]; then
if kill -0 cat "$CODIS_SERVER_PID_FILE" > /dev/null 2>&1; then
echo commandalreadyrunningasprocess‘cat"command already running as process `cat "commandalreadyrunningasprocess‘cat"CODIS_SERVER_PID_FILE”`.
exit 0
fi
fi
nohup “CODISSERVERBIN""CODIS_SERVER_BIN" "CODISSERVERBIN""{CODIS_SERVER_CONF_FILE}” > “KaTeX parse error: Expected 'EOF', got '&' at position 29: …DAEMON_FILE" 2>&̲1 < /dev/null &…CODIS_SERVER_PID_FILE” ]
then
echo “no codis-server to stop (could not find file $CODIS_SERVER_PID_FILE)”
else
kill -2 (cat"(cat "(cat"CODIS_SERVER_PID_FILE")
echo STOPPED
fi
exit 0
;;
stop-forced)
echo “stopping codis-server … "
if [ ! -f “$CODIS_SERVER_PID_FILE” ]
then
echo “no codis-server to stop (could not find file $CODIS_SERVER_PID_FILE)”
else
kill -9 (cat"(cat "(cat"CODIS_SERVER_PID_FILE”)
rm “$CODIS_SERVER_PID_FILE”
echo STOPPED
fi
exit 0
;;
restart)
shift
“$0” stop
sleep 1
“$0” start
;;
*)
echo “Usage: $0 {start|stop|stop-forced|restart}” >&2
esac

（七）修改sentinel.conf
[root@yhcs_1 config]# cat sentinel.conf |grep -v ^#|grep -v ^$
protected-mode no
port 26379
dir /usr/local/codis/codis-3.2.0
（八）编写sentinel启动脚本
#原安装不带该脚本，该脚本需要自己写(没有自己研究，借用朋友写好的)
[root@yhcs_1 admin]# cat codis-sentinel-admin.sh
#!/bin/bash
SENTINEL_BIN_DIR=/usr/local/codis/codis-3.2.0/bin/codis-server
SENTINEL_CONF_DIR=/usr/local/codis/codis-3.2.0/config/sentinel.conf
SENTINEL_PORT=26379
SENTINEL_STATUS=netstat -ntlp |grep -w 26379 |awk -F'[ /]+' '{print $7}' | wc -l
SENTINEL_PID=netstat -ntlp |grep -w 26379 |awk -F'[ /]+' '{print $7}'
case 1instart)echo"startingcodis−sentinel..."if["1 in start) echo "starting codis-sentinel ..." if [ "1instart)echo"startingcodis−sentinel..."if["SENTINEL_STATUS" -eq 0 ];then
$SENTINEL_BIN_DIR SENTINELCONFDIR−−sentinelelseecho"codis−sentinelalreadyrunningasprocess"fi;;stop)echo"stopingcodis−sentinel..."if["SENTINEL_CONF_DIR --sentinel else echo "codis-sentinel already running as process" fi ;; stop) echo "stoping codis-sentinel ..." if [ "SENTINELCONFDIR−−sentinelelseecho"codis−sentinelalreadyrunningasprocess"fi;;stop)echo"stopingcodis−sentinel..."if["SENTINEL_STATUS" -eq 0 ];then
echo “codis-sentinel already stop”
else
kill -9 $SENTINEL_PID
fi
;;
*)
echo “Usage: $0 {start|start-foreground|stop|stop-forced|restart}”
;;
esac

七、设置防火墙

#本次关闭防火墙进行测试，有需要的自己开端口，设置自启就行了
[root@yhcs_1 ~]# systemctl stop firewalld.service
[root@yhcs_1 ~]# systemctl disable firewalld.service

八、按顺序启动服务

（一）启动dashboard
#只需启动一台的dashboard即可
[root@yhcs_1 ~]# /usr/local/codis/codis-3.2.0/admin/codis-dashboard-admin.sh start
/usr/local/codis/codis-3.2.0/admin/…/config/dashboard.toml
starting codis-dashboard …
（二）启动fe
#启动一台的fe即可，也可都启动（3台IP:9090都可以访问）
[root@yhcs_1 ~]# /usr/local/codis/codis-3.2.0/admin/codis-fe-admin.sh start

starting codis-fe …
（三）启动proxy
[root@yhcs_1 ~]# /usr/local/codis/codis-3.2.0/admin/codis-proxy-admin.sh start
/usr/local/codis/codis-3.2.0/admin/…/config/proxy.toml
starting codis-proxy …
（四）启动codis-server
[root@yhcs_1 ~]# /usr/local/codis/codis-3.2.0/admin/codis-server-admin.sh start
/usr/local/codis/codis-3.2.0/admin/…/config/redis.conf
starting codis-server …

（五）启动codis-sentinel
[root@yhcs_1 admin]# /usr/local/codis/codis-3.2.0/admin/codis-sentinel-admin.sh start &
#启动哨兵后需要在web页面点击sync，把配置刷进去，具体操作下文有提到

九、日志查看路径

[root@yhcs_1 ~]# cd /usr/local/codis/codis-3.2.0/log/
[root@yhcs_1 log]# ls
codis-dashboard.log.2019-10-30 codis-dashboard.out codis-fe.log.2019-10-30 codis-fe.out codis-proxy.log.2019-10-30 codis-proxy.out codis-server.out redis_6379.log

十、Web 界面添加组件

#在以上所有的组件安装完毕之后，我们还没定义相关的redis主从状态关系。以往我们是直接在配置文件里设置主从状态和设置sentinel。Codis可以在web界面直接配置相关内容。

#登录web页面端口为fe启动文件里填写的端口。我只启动了176的fe，因此在浏览器输入192.168.43.176:9090 回车即可
（一）添加proxy

（二）添加redis
1、先确定要分为几个组，分几个组就添加几个

2、在组内添加redis

3、设置主从状态
点击黄色按钮PROMOTE设置为主；点击绿色扳手按钮设置从关系。
设置之后状态为

（三）添加sentinel

添加后点击sync，哨兵的配置文件就自动生成了

#查看下哨兵配置文件，本次不截图上来了

（四）平衡slots
Slots为codis自带的机制，可以用来数据平衡，迁移等操作。首次直接点击rebalance all slots按钮即可自动平衡。

十一、注意项

#各配置项不要有多余空格、以及注释（如集群那里的配置带注释容易受影响）
#codis集群维护时，需要关闭dashboard时，不能使用kill -9的命令来关闭，使用codis-dashboard-admin.sh stop命令来关闭。

#假如使用了kill -9命令关闭，或者服务器物理断电等极端情况出现，则通过删除这个lock的状态键值就可以了，例如：/usr/local/codis/codis-3.2.0/bin/codis-admin --remove-lock --product=codis-zoo --zookeeper=192.168.43.176:2181，然后重启dashboard即可解决。

#如果sentinel有错误，修复好后需要点击蓝色按钮sync同步sentinel状态。

十二、测试

#192.168.43.176通过fe界面设置为主，其他为从，测试如下
[root@yhcs_1 bin]# ./redis-cli -h 192.168.43.176 -p 6379
192.168.43.176:6379> set name 123
OK

[root@yhcs_2 bin]# ./redis-cli -h 192.168.43.178 -p 6379
192.168.43.178:6379> get name
“123”

#哨兵的测试自己百度吧，懒得回头再弄一次了

十三、参考文档

#部分文档
使用codis-admin搭建codis集群：https://www.cnblogs.com/zhoujinyi/p/9950105.html
codis3.2.1集群搭建与测试https://blog.51cto.com/arthur376/2051993

心不死，土不埋，学无止境