kafka集群安装及管理（二）

一、broker的迁移

1.查看zookeeper和kafka启动情况

[root@slave1 ~]# pssh -h hostlist -i 'jps'[1] 22:08:11 [SUCCESS] 20.0.5.123492 QuorumPeerMain8140 Jps6414 Kafka[2] 22:08:11 [SUCCESS] 20.0.5.133490 QuorumPeerMain4972 Kafka22972 Jps[3] 22:08:11 [SUCCESS] 20.0.5.117369 QuorumPeerMain23754 Jps11534 Kafka[4] 22:08:11 [SUCCESS] 20.0.5.1421263 Jps[5] 22:08:11 [SUCCESS] 20.0.5.1520818 Jps

2.创建一个topic

[root@slave1 ~]# /opt/kafka/bin/kafka-topics.sh --create --zookeeper 20.0.5.11:2181 --replication-factor 3 --partitions 3 --topic topic1Created topic "topic1".[root@slave1 ~]# /opt/kafka/bin/kafka-topics.sh --describe --zookeeper 20.0.5.11:2181 --topic topic1Topic:topic1    PartitionCount:3        ReplicationFactor:3     Configs:        Topic: topic1      Partition: 0       Leader: 1  Replicas: 1,3,2    Isr: 1,3,2        Topic: topic1      Partition: 1       Leader: 2  Replicas: 2,1,3    Isr: 2,1,3        Topic: topic1      Partition: 2       Leader: 3  Replicas: 3,2,1    Isr: 3,2,1

3.在要迁移的节点上启动kafka进程

[root@slave1 ~]# pssh -h hostlist '/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties > /root/kafka.log 2>1&'[1] 22:17:21 [SUCCESS] 20.0.5.12[2] 22:17:21 [SUCCESS] 20.0.5.11[3] 22:17:21 [SUCCESS] 20.0.5.13[4] 22:17:21 [SUCCESS] 20.0.5.14[5] 22:17:21 [SUCCESS] 20.0.5.15[root@slave1 ~]# pssh -h hostlist -i 'jps'[1] 22:17:29 [SUCCESS] 20.0.5.123492 QuorumPeerMain8461 Jps6414 Kafka[2] 22:17:29 [SUCCESS] 20.0.5.1421298 Kafka21609 Jps[3] 22:17:29 [SUCCESS] 20.0.5.117369 QuorumPeerMain24844 Jps11534 Kafka[4] 22:17:29 [SUCCESS] 20.0.5.133490 QuorumPeerMain4972 Kafka23292 Jps[5] 22:17:29 [SUCCESS] 20.0.5.1520854 Kafka21165 Jps

4.创建需要迁移的topic的json文件

[root@slave1 ~]# cat topic_move.json{"topics": [{"topic": "topic1"}],"version":1}

5.生成迁移规则

[root@slave1 ~]# /opt/kafka/bin/kafka-reassign-partitions.sh --zookeeper 20.0.5.11:2181 --topics-to-move-json-file topic_move.json --broker-list "3,4,5" --generateCurrent partition replica assignment{"version":1,"partitions":[{"topic":"topic1","partition":1,"replicas":[2,1,3],"log_dirs":["any","any","any"]},{"topic":"topic1","partition":0,"replicas":[1,3,2],"log_dirs":["any","any","any"]},{"topic":"topic1","partition":2,"replicas":[3,2,1],"log_dirs":["any","any","any"]}]}Proposed partition reassignment configuration{"version":1,"partitions":[{"topic":"topic1","partition":1,"replicas":[5,3,4],"log_dirs":["any","any","any"]},{"topic":"topic1","partition":0,"replicas":[4,5,3],"log_dirs":["any","any","any"]},{"topic":"topic1","partition":2,"replicas":[3,4,5],"log_dirs":["any","any","any"]}]}

6.将生成的数据写入新的json文件

[root@slave1 ~]# cat re_node.json{"version":1,"partitions":[{"topic":"topic1","partition":1,"replicas":[5,3,4],"log_dirs":["any","any","any"]},{"topic":"topic1","partition":0,"replicas":[4,5,3],"log_dirs":["any","any","any"]},{"topic":"topic1","partition":2,"replicas":[3,4,5],"log_dirs":["any","any","any"]}]}

7.开始迁移

[root@slave1 ~]# /opt/kafka/bin/kafka-reassign-partitions.sh --zookeeper 20.0.5.11:2181 --reassignment-json-file re_node.json --executeCurrent partition replica assignment{"version":1,"partitions":[{"topic":"topic1","partition":1,"replicas":[2,1,3],"log_dirs":["any","any","any"]},{"topic":"topic1","partition":0,"replicas":[1,3,2],"log_dirs":["any","any","any"]},{"topic":"topic1","partition":2,"replicas":[3,2,1],"log_dirs":["any","any","any"]}]}Save this to use as the --reassignment-json-file option during rollbackSuccessfully started reassignment of partitions.[root@slave1 ~]# /opt/kafka/bin/kafka-reassign-partitions.sh --zookeeper 20.0.5.11:2181 --reassignment-json-file re_node.json --verifyStatus of partition reassignment:Reassignment of partition topic1-1 completed successfullyReassignment of partition topic1-0 completed successfullyReassignment of partition topic1-2 completed successfully

8.查看新的分区

[root@slave1 ~]# /opt/kafka/bin/kafka-topics.sh --describe --zookeeper 20.0.5.11:2181 --topic topic1Topic:topic1    PartitionCount:3        ReplicationFactor:3     Configs:        Topic: topic1      Partition: 0       Leader: 4  Replicas: 4,5,3    Isr: 5,3,4        Topic: topic1      Partition: 1       Leader: 5  Replicas: 5,3,4    Isr: 5,3,4        Topic: topic1      Partition: 2       Leader: 3  Replicas: 3,4,5    Isr: 5,3,4

#迁移完后修改好producer和consumer指向新的连接后再停止旧的服务