update integration tests kafka version

Nextdoor · Aug 20, 2024 · 2dc95a2 · 2dc95a2
1 parent c355207
commit 2dc95a2
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 32 deletions.
diff --git a/itests/containers/data-poller/app/poller-kafka.py b/itests/containers/data-poller/app/poller-kafka.py
@@ -8,7 +8,7 @@
 from confluent_kafka.admin import AdminClient, NewTopic
 
 KAFKA_BOOTSTRAP_HOST = os.getenv('KAFKA_BOOTSTRAP_HOST', 'kafka1')
-KAFKA_BOOTSTRAP_PORT = os.getenv('KAFKA_BOOTSTRAP_PORT', '19092')
+KAFKA_BOOTSTRAP_PORT = os.getenv('KAFKA_BOOTSTRAP_PORT', '9092')
 TOPIC_NAME = os.getenv('BIFROST_KAFKA_TOPIC', 'itests')
 EXPECTED_COUNT = int(os.getenv('EXPECTED_COUNT', '1'))
 WAIT_TIME = int(os.getenv('KAFKA_POLLER_WAIT_TIME', '90'))

diff --git a/itests/contexts/kafka.env b/itests/contexts/kafka.env
@@ -2,7 +2,7 @@ TRANSPORT_SINK=kafka
 
 BIFROST_KAFKA_TOPIC=itests
 KAFKA_BOOTSTRAP_HOST=kafka1
-KAFKA_BOOTSTRAP_PORT=19092
+KAFKA_BOOTSTRAP_PORT=9092
 
 # We don't use it but needs to be specified for docker compose file
 LOCALSTACK_PORT=1337
diff --git a/itests/docker-compose.yml b/itests/docker-compose.yml
@@ -27,46 +27,26 @@ services:
     networks:
     - net
 
-  zoo1:
-    profiles: ["all", "kafka" ]
-    image: confluentinc/cp-zookeeper:7.3.2
-    hostname: zoo1
-    container_name: zoo1
-    ports:
-      - "2181:2181"
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_SERVER_ID: 1
-      ZOOKEEPER_SERVERS: zoo1:2888:3888
-    networks:
-      - net
-
   kafka1:
     profiles: ["all", "kafka"]
-    image: confluentinc/cp-kafka:7.3.2
+    image: apache/kafka:3.8.0
     hostname: kafka1
     container_name: kafka1
     ports:
       - "9092:9092"
-      - "29092:29092"
-      - "19092:19092"
-      - "9999:9999"
     environment:
-      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,DOCKER://host.docker.internal:29092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
-      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
-      KAFKA_BROKER_ID: 1
-      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_NODE_ID: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_LISTENERS: PLAINTEXT://localhost:9092,CONTROLLER://localhost:9093
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-      KAFKA_JMX_PORT: 9999
-      KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1}
-      KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
-      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
-    depends_on:
-      - zoo1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_NUM_PARTITIONS: 3
     networks:
       - net