update all

2024-03-31 20:12:36 +02:00
parent 0a61199bd7
commit e64ae07eee
135 changed files with 9726 additions and 0 deletions
--- a/Kafka/LICENSE
+++ b/Kafka/LICENSE
@ -0,0 +1,3 @@
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this Program, or any covered work, by linking or combining it with [name of library] (or a modified version of that library), containing parts covered by the terms of [name of library's license], the licensors of this Program grant you additional permission to convey the resulting work. Corresponding Source for a non-source form of such a combination shall include the source code for the parts of [name of library] used as well as that of the covered work.
--- a/Kafka/README.md
+++ b/Kafka/README.md
--- a/Kafka/cluster-zk/README.md
+++ b/Kafka/cluster-zk/README.md
@ -0,0 +1,33 @@
+## Apache Kafka cluster
+
+This docker compose provides 3 broker nodes and 1 zookeeper node Kafka Cluster.
+This helps to test Kafka application on local instead of a real cluster.
+
+The configurations are bare minimum to start the server and could be changed based on requirement
+
+There is also a Kafka UI application available at http://localhost:8080
+
+![](../../images/multi-broker-kafka-cluster.png)
+
+### How to start
+    
+```bash
+docker-compose up
+
+use -d to run in detached mode
+```
+> If you don't see changes to the cluster after updating the yml, try doing
+> `docker compose down` and then start the cluster again.
+
+### Port Mapping
+| Port | Description             |
+|------|-------------------------|
+| 2181 | Zookeeper               |
+| 9092 | Kafka Broker 1          |
+| 9093 | Kafka Broker 2          |
+| 9094 | Kafka Broker 3          |
+| 8080 | Kafka UI                |
+| 1099 | JMX PORT on all brokers |
+
+
+
--- a/Kafka/cluster-zk/docker-compose.yml
+++ b/Kafka/cluster-zk/docker-compose.yml
@ -0,0 +1,137 @@
+version: '3'
+
+services:
+  zookeeper-kafka:
+    image: confluentinc/cp-zookeeper:latest
+    ports:
+      - '2181:2181'
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    healthcheck:
+      # fake healthcheck just to have one to simplify status check
+      test: ['CMD', 'date']
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  # docs at: https://registry.hub.docker.com/r/wurstmeister/kafka
+  kafka-broker-1:
+    image: wurstmeister/kafka:latest
+    restart: always
+    depends_on:
+      - zookeeper-kafka
+    ports:
+      - '9092:9092'
+    volumes:
+      # It's the only way to deal with Kafka non-static exposed ports to host
+      # See: https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh#L65-L76
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-kafka:2181'
+      PORT_COMMAND: 'docker port $$(hostname) 9092/tcp | cut -d: -f2'
+      KAFKA_LISTENERS: 'LISTENER_INTERNAL://kafka-broker-1:29092,LISTENER_HOST://:9092'
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-broker-1:29092,LISTENER_HOST://localhost:_{PORT_COMMAND}
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
+      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka-broker-1 -Dcom.sun.management.jmxremote.rmi.port=1099"
+      JMX_PORT: 1099
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_CREATE_TOPICS: 'test:3:3'
+    healthcheck:
+      test:
+        [
+          'CMD',
+          '/opt/kafka/bin/kafka-topics.sh',
+          '--list',
+          '--bootstrap-server',
+          'kafka-broker-1:29092',
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  kafka-broker-2:
+    image: wurstmeister/kafka:latest
+    restart: always
+    depends_on:
+      - zookeeper-kafka
+    ports:
+      - '9093:9092'
+    volumes:
+      # It's the only way to deal with Kafka non-static exposed ports to host
+      # See: https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh#L65-L76
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      KAFKA_BROKER_ID: 2
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-kafka:2181'
+      PORT_COMMAND: 'docker port $$(hostname) 9092/tcp | cut -d: -f2'
+      KAFKA_LISTENERS: 'LISTENER_INTERNAL://kafka-broker-2:29092,LISTENER_HOST://:9092'
+      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka-broker-2 -Dcom.sun.management.jmxremote.rmi.port=1099"
+      JMX_PORT: 1099
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-broker-2:29092,LISTENER_HOST://localhost:_{PORT_COMMAND}
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+    healthcheck:
+      test:
+        [
+          'CMD',
+          '/opt/kafka/bin/kafka-topics.sh',
+          '--list',
+          '--bootstrap-server',
+          'kafka-broker-2:29092',
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  kafka-broker-3:
+    image: wurstmeister/kafka:latest
+    restart: always
+    depends_on:
+      - zookeeper-kafka
+    ports:
+      - '9094:9092'
+    volumes:
+      # It's the only way to deal with Kafka non-static exposed ports to host
+      # See: https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh#L65-L76
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      KAFKA_BROKER_ID: 3
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-kafka:2181'
+      PORT_COMMAND: 'docker port $$(hostname) 9092/tcp | cut -d: -f2'
+      KAFKA_LISTENERS: 'LISTENER_INTERNAL://kafka-broker-3:29092,LISTENER_HOST://:9092'
+      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka-broker-3 -Dcom.sun.management.jmxremote.rmi.port=1099"
+      JMX_PORT: 1099
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-broker-3:29092,LISTENER_HOST://localhost:_{PORT_COMMAND}
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+    healthcheck:
+      test:
+        [
+          'CMD',
+          '/opt/kafka/bin/kafka-topics.sh',
+          '--list',
+          '--bootstrap-server',
+          'kafka-broker-3:29092',
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  kafka-ui:
+    container_name: kafka-ui
+    image: provectuslabs/kafka-ui:latest
+    ports:
+      - "8080:8080"
+    depends_on:
+      - zookeeper-kafka
+      - kafka-broker-1
+      - kafka-broker-2
+      - kafka-broker-3
+    environment:
+      KAFKA_CLUSTERS_0_NAME: local
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka-broker-1:29092, kafka-broker-2:29092, kafka-broker-3:29092
+      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper-kafka:2181
+      KAFKA_CLUSTERS_0_JMXPORT: 1099
--- a/Kafka/docker-compose.yml
+++ b/Kafka/docker-compose.yml
@ -0,0 +1,105 @@
+---
+version: '3.9'
+
+#### NETWORKS
+networks:
+  docker-traefik_front_network:
+    external: true
+  back_network:
+    driver: bridge
+    attachable: true
+
+#### SERVICES
+services:
+
+### zookeeper-kafka
+  zookeeper-kafka:
+    container_name: zookeeper
+    hostname: zookeeper
+    image: confluentinc/cp-zookeeper:latest
+    ports:
+      - '2181:2181'
+    networks:
+    - back_network
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    healthcheck:
+      # fake healthcheck just to have one to simplify status check
+      test: ['CMD', 'date']
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+### kafka
+### docs at: https://registry.hub.docker.com/r/wurstmeister/kafka
+  kafka:
+    container_name: kafka
+    hostname: kafka
+    image: wurstmeister/kafka:latest
+    restart: always
+    depends_on:
+      - zookeeper-kafka
+    ports:
+      - '9092:9092'
+    networks:
+    - back_network
+    volumes:
+      # It's the only way to deal with Kafka non-static exposed ports to host
+      # See: https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh#L65-L76
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-kafka:2181'
+      PORT_COMMAND: 'docker port $$(hostname) 9092/tcp | cut -d: -f2'
+      KAFKA_LISTENERS: 'LISTENER_INTERNAL://kafka:29092,LISTENER_HOST://:9092'
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka:29092,LISTENER_HOST://localhost:_{PORT_COMMAND}
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_CREATE_TOPICS: 'test:1:1'
+    healthcheck:
+      test:
+        [
+          'CMD',
+          '/opt/kafka/bin/kafka-topics.sh',
+          '--list',
+          '--bootstrap-server',
+          'kafka:29092',
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+### kafka-ui
+  kafka-ui:
+    container_name: kafka-ui
+    hostname: kafka-ui
+    image: provectuslabs/kafka-ui:latest
+#    ports:
+#      - "8080:8080"
+    networks:
+    - docker-traefik_front_network
+    - back_network
+    depends_on:
+      - zookeeper-kafka
+      - kafka
+    environment:
+      KAFKA_CLUSTERS_0_NAME: local
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092
+      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper-kafka:2181
+      KAFKA_CLUSTERS_0_JMXPORT: 9997
+    labels:
+      - "traefik.enable=true"
+      - "traefik.docker.network=docker-traefik_front_network"
+# HTTP
+      - "traefik.http.routers.kafka-http.rule=Host(`kafka-ui.10.0.4.29.traefik.me`)"
+      - "traefik.http.routers.kafka-http.entrypoints=http"
+# HTTPS
+      - "traefik.http.routers.kafka-https.rule=Host(`kafka-ui.10.0.4.29.traefik.me`)"
+      - "traefik.http.routers.kafka-https.entrypoints=https"
+      - "traefik.http.routers.kafka-https.service=kafka-service"
+      - "traefik.http.routers.kafka-https.tls=true"
+# Middleware
+# Service
+      - "traefik.http.services.kafka-service.loadbalancer.server.port=8080"
--- a/Kafka/kraft/docker-compose.yml
+++ b/Kafka/kraft/docker-compose.yml
@ -0,0 +1,30 @@
+version: '3'
+
+services:
+  broker:
+    image: docker.io/bitnami/kafka:latest
+    ports:
+      - "9092:9092"
+      - "9094:9094"
+    environment:
+      - KAFKA_ENABLE_KRAFT=yes
+      - KAFKA_CFG_PROCESS_ROLES=broker,controller
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092,EXTERNAL://kafka_b:9094
+      - KAFKA_BROKER_ID=1
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@127.0.0.1:9093
+      - ALLOW_PLAINTEXT_LISTENER=yes
+      - KAFKA_CFG_NODE_ID=1
+      - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
+      - BITNAMI_DEBUG=yes
+      - KAFKA_CFG_NUM_PARTITIONS=2
+    healthcheck:
+      test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
+      
+