Kafka集群环境搭建部署
1 Kafka的安装部署需要安装jdk
[root@kafka-1 ~]# rpm -ivh jdk-8u191-linux-x64.rpm
2 部署zookeeper
创建zookeeper的data目录和logs目录
[root@kafka-1 ~]# mkdir /opt/elk/kafka/zookeeper/{data,logs} –p
创建myid文件
[root@kafka-1 ~]# echo 1 >/opt/elk/kafka/zookeeper/data/myid
3 配置zookeeper配置文件
[root@kafka-1 config]# cat zookeeper1.properties | grep -v "#" | grep -v "^$"
maxClientCnxns=0
dataDir=/opt/elk/kafka/zookeeper/data
dataLogDir=/opt/elk/kafka/zookeeper/logs
clientPort=2181
maxClientCnxns=20
tickTime=2000
initLimit=10
syncLimit=5
server.1=192.168.101.28:2888:3888
server.2=192.168.101.24:2888:3888
server.3=192.168.101.25:2888:3888
4 配置kafka配置文件
[root@kafka-1 config]# cat server1.properties | grep -v "#" | grep -v "^$"
broker.id=1
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.101.28:2181,192.168.101.25:2181,192.168.101.24:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0
5 修改kafka集群其他节点的配置文件
将zookeeper.properties及server.properties拷贝到集群中其他节点上，修改配置文件中的broker.id，以及myid
Kafka-2节点配置文件
[root@kafka-2 config]# cat /opt/elk/kafka/zookeeper/data/myid
2
[root@kafka-2 config]# cat zookeeper2.properties | grep -v "#" | grep -v "^$"
maxClientCnxns=0
dataDir=/opt/elk/kafka/zookeeper/data
dataLogDir=/opt/elk/kafka/zookeeper/logs
clientPort=2181
maxClientCnxns=20
tickTime=2000
initLimit=10
syncLimit=5
server.1=192.168.101.28:2888:3888
server.2=192.168.101.24:2888:3888
server.3=192.168.101.25:2888:3888
[root@kafka-2 config]# cat server2.properties | grep -v "#" | grep -v "^$"
broker.id=2
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.101.28:2181,192.168.101.25:2181,192.168.101.24:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0
kafka-3配置文件
[root@kafka-3 config]# cat zookeeper3.properties | grep -v "#" | grep -v "^$"
maxClientCnxns=0
dataDir=/opt/elk/kafka/zookeeper/data
dataLogDir=/opt/elk/kafka/zookeeper/logs
clientPort=2181
maxClientCnxns=20
tickTime=2000
initLimit=10
syncLimit=5
server.1=192.168.101.28:2888:3888
server.2=192.168.101.24:2888:3888
server.3=192.168.101.25:2888:3888
[root@kafka-3 config]# cat server3.properties | grep -v "#" | grep -v "^$"
broker.id=3
port=9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=2
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.101.28:2181,192.168.101.25:2181,192.168.101.24:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0
[root@kafka-3 config]# cat /opt/elk/kafka/zookeeper/data/myid
3
6 启动zookeeper集群
[root@kafka-1 config]# nohup /root/kafka_2.11-1.1.0/bin/zookeeper-server-start.sh /root/kafka_2.11-1.1.0/config/zookeeper.properties >>/dev/null 2>&1 &
[root@kafka-2 config]# nohup /root/kafka_2.11-1.1.0/bin/zookeeper-server-start.sh /root/kafka_2.11-1.1.0/config/zookeeper.properties >>/dev/null 2>&1 &
[root@kafka-2 config]# nohup /root/kafka_2.11-1.1.0/bin/zookeeper-server-start.sh /root/kafka_2.11-1.1.0/config/zookeeper.properties >>/dev/null 2>&1 &
验证zookeeper是否正确启动
安装nc安装包
yum install nc –y
可以通过以下指令来查看集群的整体情况
echo stat | nc 127.0.0.1 2181 查看哪个节点被选做为follower或者leader
echo ruok | nc 127.0.0.1 2181 测试是否启动了server，若回复imok表示已经启动
echo dump | nc 127.0.0.1 2181 列出未经处理的会话和临时节点
echo kill | nc 127.0.0.1 2181 关掉server
echo conf | nc 127.0.0.1 2181 输出相关服务配置的详细信息
echo cons | nc 127.0.0.1 2181 列出所有连接到服务器的客户端的完全的连接/会话的详细信息
echo envi | nc 127.0.0.1 2181 输出关于服务环境的详细信息（区别于conf命令）
echo reqs | nc 127.0.0.1 2181 列出未经处理的请求
echo wchs | nc 127.0.0.1 2181 列出服务器watch的详细信息
echo wchs | nc 127.0.0.1 2181 通过session列出服务器watch的详细信息，它的输出是一个与watch相关的会话的列表
echo wchp | nc 127.0.0.1 2181 通过路径列出服务器的watch的详细信息，它输出一个与session相关信息
7 启动kafka集群
[root@kafka-1 bin]# ./kafka-server-start.sh -daemon ../config/server.properties
通过执行jps查看kafka服务是否已经启动
[root@kafka-1 config]# jps
5472 QuorumPeerMain
32643 Jps
16010 Kafka
执行如下指令验证能否创建topic
[root@kafka-1 config]# /root/kafka_2.11-1.1.0/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test