跳转到主要内容

kafka集群搭建

单主机安装

version: '3'
services:
  zookeeper:
    image: harbor.iovhm.com/hub/wurstmeister/zookeeper:3.4.6
    restart: always
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    volumes:
      - ./zookeeper/data:/data
      - ./zookeeper/datalog:/datalog


  kafka-1:
    image: harbor.iovhm.com/hub/wurstmeister/kafka:2.13-2.8.1
    restart: always
    hostname: kafka-1
    container_name: kafka-1
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://172.18.32.61:9092 # 消费者需要用此服务码链接，需要做服务发现或者修改为主机ip
      KAFKA_LISTENERS: PLAINTEXT://:9092
      KAFKA_CREATE_TOPICS: "topic.kafkapartition:compact"
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_BROKER_ID: 1
    volumes:
      - ./kafka-1/logs:/kafka/logs

  kafka-2:
    image: wurstmeister/kafka
    restart: always
    hostname: kafka-2
    container_name: kafka-2
    ports:
      - "9093:9093"
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://172.18.32.61:9093 # 消费者需要用此服务码链接，需要做服务发现或者修改为主机ip
      KAFKA_LISTENERS: PLAINTEXT://:9093
      KAFKA_CREATE_TOPICS: "topic.kafkapartition:compact"
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 # 集群方式
      KAFKA_BROKER_ID: 2
    volumes:
      - ./kafka-2/logs:/kafka/logs

多主机安装

  • 节点1

version: '3'
services:
  zookeeper-1:
    image: harbor.iovhm.com/hub/wurstmeister/zookeeper:3.4.6
    restart: always
    hostname: zookeeper-1
    container_name: zookeeper-1
    network_mode: host
    ports:
      - "2181:2181"
    volumes:
      - ./zookeeper/data:/data
      - ./zookeeper/datalog:/datalog
    environment:
      ZOO_MY_ID: 1
      ZOO_SERVERS: server.1=172.29.1.81:2888:3888 server.2=172.29.1.82:2888:3888 server.3=172.29.1.83:2888:3888

  kafka-1:
    image: harbor.iovhm.com/hub/wurstmeister/kafka:2.13-2.8.1
    restart: always
    hostname: kafka-1
    container_name: kafka-1
    network_mode: host
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://172.29.1.81:9092
      # 消费者需要用此服务码链接，需要做服务发现或者修改为主机ip
      KAFKA_LISTENERS: PLAINTEXT://:9092
      KAFKA_CREATE_TOPICS: "topic.kafkapartition:compact"
      KAFKA_ZOOKEEPER_CONNECT: 172.29.1.81:2181,172.29.1.82:2181,172.29.1.81:83:2181
      KAFKA_BROKER_ID: 1
    volumes:
      - ./kafka/logs:/kafka/logs
  • 节点2

version: '3'
services:
  zookeeper-2:
    image: harbor.iovhm.com/hub/wurstmeister/zookeeper:3.4.6
    restart: always
    hostname: zookeeper-2
    container_name: zookeeper-2
    network_mode: host
    ports:
      - "2181:2181"
    volumes:
      - ./zookeeper/data:/data
      - ./zookeeper/datalog:/datalog
    environment:
      ZOO_MY_ID: 2
      ZOO_SERVERS: server.1=172.29.1.81:2888:3888 server.2=172.29.1.82:2888:3888 server.3=172.29.1.83:2888:3888

  kafka-2:
    image: harbor.iovhm.com/hub/wurstmeister/kafka:2.13-2.8.1
    restart: always
    hostname: kafka-2
    container_name: kafka-2
    network_mode: host
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://172.29.1.82:9092
      # 消费者需要用此服务码链接，需要做服务发现或者修改为主机ip
      KAFKA_LISTENERS: PLAINTEXT://:9092
      KAFKA_CREATE_TOPICS: "topic.kafkapartition:compact"
      KAFKA_ZOOKEEPER_CONNECT: 172.29.1.81:2181,172.29.1.82:2181,172.29.1.81:83:2181
      KAFKA_BROKER_ID: 2
    volumes:
      - ./kafka/logs:/kafka/logs
  • 节点3

version: '3'
services:
  zookeeper-3:
    image: harbor.iovhm.com/hub/wurstmeister/zookeeper:3.4.6
    restart: always
    hostname: zookeeper-3
    container_name: zookeeper-3
    network_mode: host
    ports:
      - "2181:2181"
    volumes:
      - ./zookeeper/data:/data
      - ./zookeeper/datalog:/datalog
    environment:
      ZOO_MY_ID: 3
      ZOO_SERVERS: server.1=172.29.1.81:2888:3888 server.2=172.29.1.82:2888:3888 server.3=172.29.1.83:2888:3888

  kafka-3:
    image: harbor.iovhm.com/hub/wurstmeister/kafka:2.13-2.8.1
    restart: always
    hostname: kafka-3
    container_name: kafka-3
    network_mode: host
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://172.29.1.83:9092
      # 消费者需要用此服务码链接，需要做服务发现或者修改为主机ip
      KAFKA_LISTENERS: PLAINTEXT://:9092
      KAFKA_CREATE_TOPICS: "topic.kafkapartition:compact"
      KAFKA_ZOOKEEPER_CONNECT: 172.29.1.81:2181,172.29.1.82:2181,172.29.1.81:83:2181
      KAFKA_BROKER_ID: 3
    volumes:
      - ./kafka/logs:/kafka/logs

测试运行状态


# 进入到kafka工具集
/opt/kafka_2.12-2.3.0

# broker能够响应请求，通常意味着它正在运行
bin/kafka-broker-api-versions.sh --bootstrap-server kafka1:9092,kafka2:9092

# 查询消费组信息,能够响应请求并返回，通常意味着他正在运行
bin/kafka-consumer-groups.sh --bootstrap-server 172.18.32.61:9093 --list
回到顶部