Kafka 集群部署

ip地址主机名安装软件

192.168.20.40 k8s-master zookeeper、kafka

192.168.20.43 k8s-node2 zookeeper、kafka

192.168.20.39 k8s-node3 zookeeper、kafka

三个节点安装zookeeper

[root@k8s-master data]# tar -zxvf zookeeper-3.4.10.tar.gz

[root@k8s-master data]# mv zookeeper-3.4.10 /data/zookeeper

[root@k8s-master data]# scp -r zookeeper root@192.168.20.43:/data/

[root@k8s-master data]# scp -r zookeeper root@192.168.20.39:/data/

修改三个节点的zookeeper的配置文件

[root@k8s-master ~]# mkdir /data/zookeeper/data

[root@k8s-master ~]# cp /data/zookeeper/conf/zoo_sample.cfg /data/zookeeper/conf/zoo_sample.cfg.back

[root@k8s-master ~]# cp /data/zookeeper/conf/zoo_sample.cfg /data/zookeeper/conf/zoo.cfg

[root@k8s-master ~]# vim /data/zookeeper/conf/zoo.cfg

[root@k8s-master ~]# more /data/zookeeper/conf/zoo.cfg

# The number of milliseconds of each tick

tickTime=2000

# The number of ticks that the initial

# synchronization phase can take

initLimit=10

# The number of ticks that can pass between

# sending a request and getting an acknowledgement

syncLimit=5

# the directory where the snapshot is stored.

# do not use /tmp for storage, /tmp here is just

# example sakes.

dataDir=/data/zookeeper/data/zookeeper

dataLog=/data/zookeeper/data/logs

# the port at which the clients will connect

clientPort=2181

# the maximum number of client connections.

# increase this if you need to handle more clients

maxClientCnxns=60

autopurge.snapRetainCount=3

autopurge.purgeInterval=1

server.1=192.168.20.40:2888:3888

server.2=192.168.20.43:2888:3888

server.3=192.168.20.39:2888:3888

启动zookeeper服务前，分别在三个zookeeper节点上创建myid

[root@k8s-master ~]# mkdir /data/zookeeper/data/zookeeper

[root@k8s-master ~]# echo 1 > /data/zookeeper/data/zookeeper/myid

[root@k8s-node2 ~]# mkdir /data/zookeeper/data/zookeeper

[root@k8s-node2 ~]# echo 2 > /data/zookeeper/data/zookeeper/myid

[root@k8s-node3 ~]# mkdir /data/zookeeper/data/zookeeper

[root@k8s-node3 ~]# echo 3 > /data/zookeeper/data/zookeeper/myid

[root@k8s-master ~]# /data/zookeeper/bin/zkServer.sh start

ZooKeeper JMX enabled by default

Using config: /data/zookeeper/bin/../conf/zoo.cfg

Starting zookeeper ... STARTED

[root@k8s-master ~]# ps -ef |grep zookeeper

k8s-node2

k8s-node3

[root@k8s-master ~]# netstat -nltp|grep :2181

tcp6 0 0 :::2181 :::* LISTEN 8962/java

查看zookeeper节点的状态

[root@k8s-node3 ~]# /data/zookeeper/bin/zkServer.sh status

ZooKeeper JMX enabled by default

Using config: /data/zookeeper/bin/../conf/zoo.cfg

Mode: follower

[root@k8s-master ~]# /data/zookeeper/bin/zkServer.sh status

ZooKeeper JMX enabled by default

Using config: /data/zookeeper/bin/../conf/zoo.cfg

Mode: follower

[root@k8s-node2 ~]# /data/zookeeper/bin/zkServer.sh status

ZooKeeper JMX enabled by default

Using config: /data/zookeeper/bin/../conf/zoo.cfg

Mode: leader

安装Kafka

[root@k8s-master ~]# tar -zvxf kafka_2.11-1.1.0.tgz

[root@k8s-master ~]# mv kafka_2.11-1.1.0 /data/kafka

[root@k8s-master ~ src]# cp /data/kafka/config/server.properties /data/kafka/config/server.properties.bak

[root@k8s-master ~]# vim /data/kafka/config/server.properties

broker.id=0

delete.topic.enable=true

listeners=PLAINTEXT://192.168.20.40:9092

num.network.threads=3

num.io.threads=8

socket.send.buffer.bytes=102400

socket.receive.buffer.bytes=102400

socket.request.max.bytes=104857600

log.dirs=/data/kafka/data

num.partitions=1

num.recovery.threads.per.data.dir=1

offsets.topic.replication.factor=1

transaction.state.log.replication.factor=1

transaction.state.log.min.isr=1

log.flush.interval.messages=10000

log.flush.interval.ms=1000

log.retention.hours=168

log.retention.bytes=1073741824

log.segment.bytes=1073741824

log.retention.check.interval.ms=300000

zookeeper.connect=192.168.20.40:2181,192.168.20.43:2181,192.168.20.39:2181

zookeeper.connection.timeout.ms=6000

group.initial.rebalance.delay.ms=0

其它两个节点只需要修改:broker.id=*和listeners=PLAINTEXT://IP

启动Kafka

[root@k8s-master ~]# nohup /data/kafka/bin/kafka-server-start.sh /data/kafka/config/server.properties >/dev/null 2>&1 &

推荐资讯

推荐站点