kafka集群安装(包含zookeeper安装)


kafka 官网:http://kafka.apache.org/ zookeeper官网:https://zookeeper.apache.org/releases.html

zookeeper部署(3台)
wget https://mirror.bit.edu.cn/apache/zookeeper/zookeeper-3.6.2/apache-zookeeper-3.6.2-bin.tar.gz

tar zxf apache-zookeeper-3.6.2-bin.tar.gz -C /opt/
mv  apache-zookeeper-3.6.2-bin  zookeeper
mkdir /data/zk_data     #创建zk数据存放目录
echo 43 > /data/zk_data/myid   # 创建myid文件,三台机器上不可相同
cd /opt/zookeeper/conf

vim zoo.cfg 
tickTime=2000
    initLimit=10
    syncLimit=5
    dataDir=/dat/zk_adata
    clientPort=2181
    server.43=172.16.6.43:2888:3888
    server.44=172.16.6.44:2888:3888
    server.45=172.16.6.45:2888:3888

# Zookeeper默认会将控制台信息输出到启动路径下的zookeeper.out中
# 显然在生产环境中我们不能允许Zookeeper这样做,通过如下方法,可以让Zookeeper输出按尺寸切分的日志文件:

# 修改conf/log4j.properties
zookeeper.root.logger=INFO, CONSOLE改为zookeeper.root.logger=INFO, ROLLINGFILE

/opt/zookeeper/zkServer.sh start  #启动zk
/opt/zookeeper/zkServer.sh status #查看zk启动状态  正常三台节点,应有两台follower和一台leader   

安装配置kafka
wget https://mirror.bit.edu.cn/apache/kafka/2.5.1/kafka_2.12-2.5.1.tgz 
tar zxf kafka_2.12-2.5.1.tgz -C /opt/
mv  kafka_2.12-2.5.1 kafka 
mkdir /data/kafka_data   # 创建消息文件存储的路径

vim /usr/local/kafka/config/server.properties

broker.id=43       # brokerid 每台机器不同
delete.topic.enable=true
num.network.threads=3
num.io.threads=8

post=9092
hostname=172.16.6.43

socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600

log.dirs=/data/kafka_data

num.partitions=6
num.recovery.threads.per.data.dir=1

log.retention.hours=72
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000

kafka默认启动以hostname识别
# vim /etc/hosts
    172.16.6.43 b28-6p43-ops-elk-01.bfdabc.com
    172.16.6.44 b28-6p44-ops-elk-02.bfdabc.com
    172.16.6.45 b28-6p45-ops-elk-03.bfdabc.com

cd /usr/local/kafka/
nohup  /opt/kafka_2.12-2.5.1/bin/kafka-server-start.sh   /opt/kafka_2.12-2.5.1/config/server.properties > /data/kafka_data/kafka_run.log &


测试zk和kafka之间的连通性
cd /opt/kafka/bin
./kafka-topics.sh   --create --zookeeper 172.16.6.43:2181 --replication-factor 2 --partitions 1 --topic kafka_es
--replication-factor 2 #复制两份
    --partitions 1 #创建1个分区
    --topic #主题为kafka_es

# 查看已创建的topic列表:
./kafka-topics.sh --list --zookeeper localhost:2181

# 查看对应topic的描述信息: 
./kafka-topics.sh --describe --zookeeper xx.x.xxx.xxx:2181 --topic xxxx # --zookeeper为zookeeperIP, --topic为topic名称 

# 消费消息:
./kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic xxxxx.xx.xxxxx.xxx --from-beginning    # 若没有任何返回或没有响应,则该topic中没有数据内容;