codis 3.2集群单机环境安装
codis 3.2集群单机环境安装
一、软件环境
系统：centos 6.5
Jdk：jdk-8u111-linux-x64.gz
go : go1.7.3.linux-amd64.tar.gz
zookeeper: zookeeper-3.4.6.tar.gz
本机IP：172.16.40.131
软件下载地址：
codis：https://github.com/CodisLabs/codis
zookeeper：https://zookeeper.apache.org/
go：http://golangtc.com/download
二、部署zookeeper集群
1.安装相关依赖包
yum install -y gcc makegcc-c++ automake lrzsz openssl-devel zlib-* bzip2-* readline* git nmap unzipwget lsof xz net-tools mercurial
2.安装java
tar zxvfjdk-8u111-linux-x64.gz -C /usr/local/
cd /usr/local/
ln -sv jdk1.8.0_111 java
修改环境变量
vi /etc/profile
JAVA_HOME=/usr/local/java
PATH=$JAVA_HOME/bin:$PATH
CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export JAVA_HOME
export PATH
export CLASSPATH
让环境变量生效
source /etc/profile
查看JAVA版本
java -version
java version "1.8.0_111"
Java(TM) SE Runtime Environment (build1.8.0_111-b14)
Java HotSpot(TM) 64-Bit Server VM (build25.111-b14, mixed mode)
3.安装zookeeper
mkdir /{app,appdata}
cd /app
mkdir -pv ./{zk1,zk2,zk3}/{data,log}
tar zxvf zookeeper-3.4.6.tar.gz -C /app/zk1/
tar zxvf zookeeper-3.4.6.tar.gz -C /app/zk2/
tar zxvf zookeeper-3.4.6.tar.gz -C /app/zk3/
ln -sv /app/zk1/zookeeper-3.4.6 /app/zk1/zookeeper
ln -sv /app/zk2/zookeeper-3.4.6 /app/zk2/zookeeper
ln -sv /app/zk3/zookeeper-3.4.6 /app/zk3/zookeeper
cp /app/zk1/zookeeper/conf/zoo_sample.cfg/app/zk1/zookeeper/conf/zoo.cfg
修改zoo.cfg配置文件
vi /app/zk1/zookeeper/conf/zoo.cfg
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between
# sending a request and getting anacknowledgement
syncLimit=5
# the directory where the snapshot isstored.
# do not use /tmp for storage, /tmp here isjust
# example sakes.
dataDir=/app/zk1/data
# the port at which the clients willconnect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle moreclients
#maxClientCnxns=60
#
# Be sure to read the maintenance sectionof the
# administrator guide before turning onautopurge.
#
#http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain indataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable autopurge feature
#autopurge.purgeInterval=1
server.1=172.16.40.131:2881:3888
server.2=172.16.40.131:2882:3888
server.3=172.16.40.131:2883:3888
生成myid
echo "1">/app/zk1/data/myid
配置zk2、zk3:
cp /app/zk1/zookeeper/conf/zoo.cfg /app/zk2/zookeeper/conf/
cp /app/zk1/zookeeper/conf/zoo.cfg /app/zk3/zookeeper/conf/
sed -i 's/zk1/zk2/g' /app/zk2/zookeeper/conf/zoo.cfg
sed -i 's/zk1/zk3/g' /app/zk3/zookeeper/conf/zoo.cfg
sed -i 's/2181/2182/g' /app/zk2/zookeeper/conf/zoo.cfg
sed -i 's/2181/2183/g' /app/zk3/zookeeper/conf/zoo.cfg
echo "2">/app/zk2/data/myid
echo "3">/app/zk3/data/myid
启动zookeeper服务
/app/zk1/zookeeper/bin/zkServer.sh start
/app/zk2/zookeeper/bin/zkServer.sh start
/app/zk3/zookeeper/bin/zkServer.sh start
查看zookeeper状态
/app/zk1/zookeeper/bin/zkServer.sh status
JMX enabled by default
Using config:/app/zk1/zookeeper/bin/../conf/zoo.cfg
Mode: leader
/app/zk2/zookeeper/bin/zkServer.sh status
JMX enabled by default
Using config:/app/zk2/zookeeper/bin/../conf/zoo.cfg
/app/zk3/zookeeper/bin/zkServer.sh status
JMX enabled by default
Using config:/app/zk3/zookeeper/bin/../conf/zoo.cfg
Mode: follower
二、部署codis
1.go环境部署
cd /app
tar zxvf go1.7.3.linux-amd64.tar.gz -C /app
mkdir gopkg
配置GOROOT、GOPATH
vi /etc/profile
export GOROOT=/app/go
export GOPATH=/app/gopkg
export PATH=$PATH:$GOROOT/bin
生效环境变量
source /etc/profile
查看go版本
go version
go version go1.7.3 linux/amd64
2.codis编译
创建codis编译目录
mkdir -pv /app/gopkg/src/github.com/CodisLabs/
下载codis软件包
cd /app/gopkg/src/github.com/CodisLabs/
git clone https://github.com/CodisLabs/codis.git -b release3.2
编译
cd codis/
make
编译正确如下，没有错误出现说明已编译完成
===============================================================================
go build -i -o bin/codis-dashboard./cmd/dashboard
go build -i -tags "cgo_jemalloc"-o bin/codis-proxy ./cmd/proxy
go build -i -o bin/codis-admin ./cmd/admin
go build -i -o bin/codis-fe ./cmd/fe
查看bin目录生成文件
cd /app/gopkg/src/github.com/CodisLabs/codis/bin/
[root@codis-02 bin]# ll
total 84488
drwxr-xr-x. 4 root root 4096 Jun 13 18:12 assets
-rwxr-xr-x. 1 root root 15465215 Jun 1318:12 codis-admin
-rwxr-xr-x. 1 root root 17085915 Jun 1318:12 codis-dashboard
-rwxr-xr-x. 1 root root 15358965 Jun 1318:12 codis-fe
-rwxr-xr-x. 1 root root 19311099 Jun 1318:12 codis-proxy
-rwxr-xr-x. 1 root root 7982986 Jun 13 18:11 codis-server
-rwxr-xr-x. 1 root root 5580519 Jun 13 18:11 redis-benchmark
-rwxr-xr-x. 1 root root 5712403 Jun 13 18:11 redis-cli
-rw-r--r--. 1 root root 168 Jun 13 18:11 version
3.codis实例
部署redis实例一主两从
创建codis实例所需目录,
mkdir -pv /app/codis/redis/{7001,7002,7003,7004,7005,7006}
cp -r /app/gopkg/src/github.com/CodisLabs/codis/bin/app/codis/
redis配置如下
vi /app/codis/redis/7001/redis.conf
bind 0.0.0.0
protected-mode no
port 7001
tcp-backlog 511
timeout 60
tcp-keepalive 300
daemonize yes
supervised no
pidfile "/tmp/redis_7001.pid"
loglevel notice
logfile "/app/codis/redis/7001/redis_7001.log"
databases 16
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename"dump_7001.rdb"
dir "/app/codis/redis/7001"
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb60
client-output-buffer-limit pubsub 32mb 8mb60
hz 10
aof-rewrite-incremental-fsync yes
配置7002,7003,7004,7005,7006
cp /app/codis/redis/7001/redis.conf /app/codis/redis/7002/
cp /app/codis/redis/7001/redis.conf /app/codis/redis/7003/
cp /app/codis/redis/7001/redis.conf /app/codis/redis/7004/
cp /app/codis/redis/7001/redis.conf /app/codis/redis/7005/
cp /app/codis/redis/7001/redis.conf /app/codis/redis/7006/
sed -i 's/7001/7002/g' /app/codis/redis/7002/redis.conf
sed -i 's/7001/7003/g' /app/codis/redis/7003/redis.conf
sed -i 's/7001/7004/g' /app/codis/redis/7004/redis.conf
sed -i 's/7001/7005/g' /app/codis/redis/7005/redis.conf
sed -i 's/7001/7006/g' /app/codis/redis/7006/redis.conf
启动codis-server服务
/app/codis/bin/codis-server /app/codis/redis/7001/redis.conf
/app/codis/bin/codis-server /app/codis/redis/7002/redis.conf
/app/codis/bin/codis-server /app/codis/redis/7003/redis.conf
/app/codis/bin/codis-server /app/codis/redis/7004/redis.conf
/app/codis/bin/codis-server /app/codis/redis/7005/redis.conf
/app/codis/bin/codis-server /app/codis/redis/7006/redis.conf
三、部署codis-proxy
1.生成codis-proxy配置文件
cd /app/codis/bin/
/app/codis/bin/codis-proxy --default-config|tee proxy.toml >> proxy.toml
vi proxy.tom
##################################################
# #
# Codis-Proxy #
# #
##################################################
# Set Codis Product Name/Auth.
product_name ="codis-demo"
product_auth =""
# Set auth for client session
# 1. product_auth is used for auth validation among codis-dashboard,
# codis-proxy and codis-server.
# 2. session_auth is different from product_auth, it requires clients
# to issue AUTH
session_auth = ""
# Set bind address for admin(rpc), tcponly.
admin_addr ="0.0.0.0:11080"
# Set bind address for proxy, proto_typecan be "tcp", "tcp4", "tcp6", "unix" or"unixpacket".
proto_type = "tcp4"
proxy_addr ="0.0.0.0:19000"
# Set jodis address & session timeout
# 1. jodis_name is short for jodis_coordinator_name, only accept"zookeeper" & "etcd".
# 2. jodis_addr is short for jodis_coordinator_addr
# 3. proxy will be registered as node:
# if jodis_compatible = true (not suggested):
# /zk/codis/db_{PRODUCT_NAME}/proxy-{HASHID}(compatible with Codis2.0)
# or else
# /jodis/{PRODUCT_NAME}/proxy-{HASHID}
jodis_name ="zookeeper"
jodis_addr ="172.16.40.131:2881,172.16.40.131:2882,172.16.40.131:2883"
jodis_timeout = "20s"
jodis_compatible = false
# Set datacenter of proxy.
proxy_datacenter = ""
# Set max number of alive sessions.
proxy_max_clients = 1000
# Set max offheap memory size. (0 todisable)
proxy_max_offheap_size = "1024mb"
# Set heap placeholder to reduce GCfrequency.
proxy_heap_placeholder = "256mb"
# Proxy will ping backend redis (and clear'MASTERDOWN' state) in a predefined interval. (0 to disable)
backend_ping_period = "5s"
# Set backend recv buffer size &timeout.
backend_recv_bufsize = "128kb"
backend_recv_timeout = "30s"
# Set backend send buffer & timeout.
backend_send_bufsize = "128kb"
backend_send_timeout = "30s"
# Set backend pipeline buffer size.
backend_max_pipeline = 20480
# Set backend never read replica groups,default is false
backend_primary_only = false
# Set backend parallel connections perserver
backend_primary_parallel = 1
backend_replica_parallel = 1
# Set backend tcp keepalive period. (0 todisable)
backend_keepalive_period = "75s"
# Set number of databases of backend.
backend_number_databases = 16
# If there is no request from client for along time, the connection will be closed. (0 to disable)
# Set session recv buffer size &timeout.
session_recv_bufsize = "128kb"
session_recv_timeout = "30m"
# Set session send buffer size &timeout.
session_send_bufsize = "64kb"
session_send_timeout = "30s"
# Make sure this is higher than the maxnumber of requests for each pipeline request, or your client may be blocked.
# Set session pipeline buffer size.
session_max_pipeline = 10000
# Set session tcp keepalive period. (0 todisable)
session_keepalive_period = "75s"
# Set session to be sensitive to failures.Default is false, instead of closing socket, proxy will send an error responseto client.
session_break_on_failure = false
# Set metrics server (such ashttp://localhost:28000), proxy will report json formatted metrics to specifiedserver in a predefined period.
metrics_report_server = ""
metrics_report_period = "1s"
# Set influxdb server (such ashttp://localhost:8086), proxy will report metrics to influxdb.
metrics_report_influxdb_server =""
metrics_report_influxdb_period ="1s"
metrics_report_influxdb_username =""
metrics_report_influxdb_password =""
metrics_report_influxdb_database =""
# Set statsd server (such aslocalhost:8125), proxy will report metrics to statsd.
metrics_report_statsd_server = ""
metrics_report_statsd_period ="1s"
metrics_report_statsd_prefix = ""
参数说明：
product_name 集群名称，参考 dashboard 参数说明
product_auth 集群密码，默认为空
admin_addr RESTfulAPI 端口
proto_type Redis 端口类型，接受 tcp/tcp4/tcp6/unix/unixpacket
proxy_addr Redis 端口地址或者路径
jodis_addr Jodis 注册 zookeeper 地址
jodis_timeout Jodis 注册sessiontimeout 时间，单位 second
jodis_compatible Jodis 注册 zookeeper 的路径
backend_ping_period 与codis-server 探活周期，单位 second，0 表示禁止
session_max_timeout 与 client 连接最大读超时，单位 second，0 表示禁止
session_max_bufsize 与 client 连接读写缓冲区大小，单位byte
session_max_pipeline 与 client 连接最大的 pipeline大小
session_keepalive_period 与 client 的 tcpkeepalive 周期，仅 tcp 有效，0 表示禁止
2.启动codis-proxy
nohup /app/codis/bin/codis-proxy --ncpu=4--config=/app/codis/bin/proxy.toml --log=proxy.log --log-level=WARN &
四、部署codis-dashboard
1. 生成codis-proxy配置文件
/app/codis/bin/codis-dashboard--default-config |tee dashboard.toml >>dashboard.toml
修改codis-proxy配置
vidashboard.toml
##################################################
# #
# Codis-Dashboard #
# #
##################################################
# SetCoordinator, only accept "zookeeper" & "etcd" &"filesystem".
# QuickStart
#coordinator_name= "filesystem"
#coordinator_addr= "/tmp/codis"
coordinator_name = "zookeeper"
coordinator_addr ="172.16.40.131:2181,172.16.40.131:2182,172.16.40.131:2183"
# SetCodis Product Name/Auth.
product_name = "codis-demo"
product_auth = ""
# Setbind address for admin(rpc), tcp only.
admin_addr = "0.0.0.0:18080"
# Setarguments for data migration (only accept 'sync' & 'semi-async').
migration_method= "semi-async"
migration_parallel_slots= 100
migration_async_maxbulks= 200
migration_async_maxbytes= "32mb"
migration_async_numkeys= 500
migration_timeout= "30s"
# Setconfigs for redis sentinel.
sentinel_quorum= 2
sentinel_parallel_syncs= 1
sentinel_down_after= "30s"
sentinel_failover_timeout= "5m"
sentinel_notification_script= ""
sentinel_client_reconfig_script= ""
参数说明：
coordinator_name外部存储类型，接受 zookeeper/etcd
coordinator_addr外部存储地址
product_name集群名称，满足正则 \w[\w\.\-]*
product_auth集群密码，默认为空
admin_addrRESTful API 端口
启动codis-dashboard
nohup/app/codis/bin/codis-dashboard --ncpu=4 --config=/app/codis/bin/dashboard.toml --log=dashboard.log --log-level=WARN &
五、部署codis-fe
5.1生成codis-fe配置文件
/app/codis/bin/codis-admin --dashboard-list --zookeeper=172.16.40.131:2182 |tee ./codis.json>>codis.json
[root@codis-02bin]# cat codis.json
[
{
"name":"codis-demo",
"dashboard":"172.16.40.131:18080"
}
]
启动codis-fe服务
nohup/app/codis/bin/codis-fe --ncpu=4 --log=fe.log --log-level=WARN --dashboard-list=/app/codis/bin/codis.json --listen=0.0.0.0:18090 &
查看相关服务端口
[root@codis-02bin]# ss -tunlp | grep codis
tcp LISTEN 0 128 *:19000 *:* users:(("codis-proxy",10106,7))
tcp LISTEN 0 128 *:7001 *:* users:(("codis-server",10063,4))
tcp LISTEN 0 128 *:7002 *:* users:(("codis-server",10068,4))
tcp LISTEN 0 128 *:7003 *:* users:(("codis-server",10073,4))
tcp LISTEN 0 128 *:7004 *:* users:(("codis-server",10078,4))
tcp LISTEN 0 128 *:7005 *:* users:(("codis-server",10083,4))
tcp LISTEN 0 128 *:7006 *:* users:(("codis-server",10088,4))
tcp LISTEN 0 128 :::18080 :::* users:(("codis-dashboard",10150,5))
tcp LISTEN 0 128 :::11080 :::* users:(("codis-proxy",10106,8))
tcp LISTEN 0 128 :::18090 :::* users:(("codis-fe",10221,5))
六、redis sentinel部署
创建redis-sentinel目录
mkdir -pv /app/codis/sentinel/{27001,27002}
配置sentinel文件
vi /app/codis/sentinel/27001/sentinel-27001.conf
bind 0.0.0.0
protected-mode no
port 27001
dir /app/codis/sentinel/27001/
vi /app/codis/sentinel/27002/sentinel-27002.conf
bind 0.0.0.0
protected-mode no
port 27002
dir /app/codis/sentinel/27002/
启动sentinel服务
cp/app/gopkg/src/github.com/CodisLabs/codis/extern/redis-3.2.8/src/redis-sentinel/app/codis/bin/
/app/codis/bin/redis-sentinel /app/codis/sentinel/27001/sentinel-27001.conf &
/app/codis/bin/redis-sentinel /app/codis/sentinel/27002/sentinel-27002.conf &
七、使用codis-fe管理codis集群环境
1.codis-fe管理界面
http://172.16.40.131:18090
添加codis-proxy：
采用命令增加
[root@codis-02 bin]# ./codis-admin --dashboard=127.0.0.1:18080 --create-proxy -x 127.0.0.1:11080
也可以采用界面增加
添加codis-server
先NEW GROUP
增加一主两从
增加两组
以上也可以用命令增加
添加redis-sentinel
命令增加
[root@codis-02bin]# ./codis-admin --dashboard=172.16.40.131:18080 --sentinel-add --addr=172.16.40.131:27002
分配 slot
slot：数据槽，在 codis-server 间数据分片的单元；建议将数据槽均分于多个 group；
在 codis-fe 管理界面添加 redis-sentinel，并同步，同步后 redis-sentinel 配置文件会更新
root@codis-02bin]# cat /app/codis/sentinel/27001/sentinel-27001.conf
bind 0.0.0.0
protected-mode no
port 27001
dir "/app/codis/sentinel/27001"
#Generated by CONFIG REWRITE
sentinelmyid b445aae57d1283b4d89d9ed23fd96982dbc344c3
sentinelmonitor codis-demo-2 172.16.40.131 7004 2
sentinelfailover-timeout codis-demo-2 300000
sentinelconfig-epoch codis-demo-2 0
sentinelleader-epoch codis-demo-2 0
sentinelknown-slave codis-demo-2 172.16.40.131 7005
sentinelknown-slave codis-demo-2 172.16.40.131 7006
sentinelknown-sentinel codis-demo-2 172.16.40.131 27002b236624dce5df7317e5c3e5f30401899c4961094
sentinelmonitor codis-demo-1 172.16.40.131 7001 2
sentinelfailover-timeout codis-demo-1 300000
sentinelconfig-epoch codis-demo-1 0
sentinelleader-epoch codis-demo-1 0
sentinelknown-slave codis-demo-1 172.16.40.131 7002
sentinelknown-slave codis-demo-1 172.16.40.131 7003
sentinelknown-sentinel codis-demo-1 172.16.40.131 27002 b236624dce5df7317e5c3e5f30401899c4961094
sentinelcurrent-epoch 0
