remove usage of /etc/hosts (#29)

simplesteph · web-flow · commit 690ca0878cdf · 2018-08-06T14:11:57.000+02:00
* first try

* travis fixes

* bring back addon

* better interpolation

* increase compose version

* use KAFKA_ADVERTISED_LISTENERS

* all stack changes

* compose v2.1

* correct indentation

* fixes in environment variables and kafka-rest and kafka-schema registry config
diff --git a/.travis.yml b/.travis.yml
@@ -2,25 +2,14 @@
 # it is not related to your Kafka Streams learning
 sudo: required
 
+env:
+  - DOCKER_HOST_IP=127.0.0.1
+  # - DOCKER_HOST_IP=192.168.99.100
+
 services:
   - docker
 
 addons:
-  hosts:
-    - kafka1
-    - kafka2
-    - kafka3
-    - zoo1
-    - zoo2
-    - zoo3
-    - kafka-schema-registry
-    - kafka-schema-registry-ui
-    - kafka-rest-proxy
-    - kafka-topics-ui
-    - kafka-connect-ui
-    - zoonavigator-web
-    - zoonavigator-api
-    
   # install the kafka binaries
   apt:
     sources:
diff --git a/README.md b/README.md
@@ -3,7 +3,9 @@
 
 # kafka-stack-docker-compose
 
-This replicates as well as possible real deployment configurations, where you have your zookeeper servers and kafka servers actually all distinct from each other. This solves all the networking hurdles that comes with docker-compose, and is compatible cross platform. It only requires an update to your host files.
+This replicates as well as possible real deployment configurations, where you have your zookeeper servers and kafka servers actually all distinct from each other. This solves all the networking hurdles that comes with Docker and docker-compose, and is compatible cross platform.
+
+[UPDATE]: No /etc/hosts file changes are necessary anymore. Explanations at: https://rmoff.net/2018/08/02/kafka-listeners-explained/
 
 ## Stack version
 
@@ -19,52 +21,29 @@ This replicates as well as possible real deployment configurations, where you ha
 
 # Requirements
 
-## Host File changes
-
-See https://support.rackspace.com/how-to/modify-your-hosts-file/ to detailed instructions on how to modify your host files.
+## Docker
 
-If you are using Docker for Mac >= 1.12, Docker for Linux, or Docker for Windows 10, then please add the following lines to `/etc/hosts` or `C:\Windows\System32\Drivers\etc\hosts`:
+Please export your environment before starting the stack:
 ```
-127.0.0.1     kafka1
-127.0.0.1     kafka2
-127.0.0.1     kafka3
-127.0.0.1     zoo1
-127.0.0.1     zoo2
-127.0.0.1     zoo3
-127.0.0.1     kafka-schema-registry
-127.0.0.1     kafka-schema-registry-ui
-127.0.0.1     kafka-rest-proxy
-127.0.0.1     kafka-topics-ui
-127.0.0.1     kafka-connect-ui
-127.0.0.1     zoonavigator-web
-127.0.0.1     zoonavigator-api
+export DOCKER_HOST_IP=127.0.0.1
 ```
+(that's the default value and you actually don't need to do a thing)
 
+## Docker-Toolbox
 If you are using Docker for Mac <= 1.11, or Docker Toolbox for Windows
 (your docker machine IP is usually `192.168.99.100`)
-Please add the following lines to `/etc/hosts` or `C:\Windows\System32\Drivers\etc\hosts`:
+
+Please export your environment before starting the stack:
 ```
-192.168.99.100    kafka1
-192.168.99.100    kafka2
-192.168.99.100    kafka3
-192.168.99.100    zoo1
-192.168.99.100    zoo2
-192.168.99.100    zoo3
-192.168.99.100    kafka-schema-registry
-192.168.99.100    kafka-schema-registry-ui
-192.168.99.100    kafka-rest-proxy
-192.168.99.100    kafka-topics-ui
-192.168.99.100    kafka-connect-ui
-192.168.99.100    zoonavigator-web
-192.168.99.100    zoonavigator-api
+export DOCKER_HOST_IP=192.168.99.100
 ```
 
 ## Single Zookeeper / Single Kafka
 
 This configuration fits most development requirements.
 
- - Zookeeper will be available at `zoo1:2181`
- - Kafka will be available at `kafka1:9092`
+ - Zookeeper will be available at `$DOCKER_HOST_IP:2181`
+ - Kafka will be available at `$DOCKER_HOST_IP:9092`
 
 
 Run with:
@@ -77,8 +56,8 @@ docker-compose -f zk-single-kafka-single.yml down
 
 If you want to have two brokers and experiment with replication / fault-tolerance.
 
-- Zookeeper will be available at `zoo1:2181`
-- Kafka will be available at `kafka1:9092,kafka2:9093,kafka3:9094`
+- Zookeeper will be available at `$DOCKER_HOST_IP:2181`
+- Kafka will be available at `$DOCKER_HOST_IP:9092,$DOCKER_HOST_IP:9093,$DOCKER_HOST_IP:9094`
 
 
 Run with:
@@ -91,22 +70,22 @@ docker-compose -f zk-single-kafka-multiple.yml down
 
 If you want to have three zookeeper and experiment with zookeeper fault-tolerance.
 
-- Zookeeper will be available at `zoo1:2181,zoo2:2182,zoo3:2183`
-- Kafka will be available at `kafka1:9092`
+- Zookeeper will be available at `$DOCKER_HOST_IP:2181,$DOCKER_HOST_IP:2182,$DOCKER_HOST_IP:2183`
+- Kafka will be available at `$DOCKER_HOST_IP:9092`
 
 Run with:
 ```
 docker-compose -f zk-multiple-kafka-single.yml up
-docker-compose -f zk-multiple-kafka-single.yml down
+docker-compose -f ²zk-multiple-kafka-single.yml down
 ```
 
 
 ## Multiple Zookeeper / Multiple Kafka
 
 If you want to have three zookeeper and two kafka brokers to experiment with production setup.
 
-- Zookeeper will be available at `zoo1:2181,zoo2:2182,zoo3:2183`
-- Kafka will be available at `kafka1:9092,kafka2:9093,kafka3:9094`
+- Zookeeper will be available at `$DOCKER_HOST_IP:2181,$DOCKER_HOST_IP:2182,$DOCKER_HOST_IP:2183`
+- Kafka will be available at `$DOCKER_HOST_IP:9092,$DOCKER_HOST_IP:9093,$DOCKER_HOST_IP:9094`
 
 Run with:
 ```
@@ -117,15 +96,15 @@ docker-compose -f zk-multiple-kafka-multiple.yml down
 
 ## Full stack
 
- - Single Zookeeper: `zoo1:2181`
- - Single Kafka: `kafka1:9092`
- - Kafka Schema Registry: `kafka-schema-registry:8081`
- - Kafka Schema Registry UI: `kafka-schema-registry-ui:8001`
- - Kafka Rest Proxy: `kafka-rest-proxy:8082`
- - Kafka Topics UI: `kafka-topics-ui:8000`
- - Kafka Connect: `kafka-connect:8083`
- - Kafka Connect UI: `kafka-connect-ui:8003`
- - Zoonavigator Web: `zoonavigator-web:8004`
+ - Single Zookeeper: `$DOCKER_HOST_IP:2181`
+ - Single Kafka: `$DOCKER_HOST_IP:9092`
+ - Kafka Schema Registry: `$DOCKER_HOST_IP:8081`
+ - Kafka Schema Registry UI: `$DOCKER_HOST_IP:8001`
+ - Kafka Rest Proxy: `$DOCKER_HOST_IP:8082`
+ - Kafka Topics UI: `$DOCKER_HOST_IP:8000`
+ - Kafka Connect: `$DOCKER_HOST_IP:8083`
+ - Kafka Connect UI: `$DOCKER_HOST_IP:8003`
+ - Zoonavigator Web: `$DOCKER_HOST_IP:8004`
 
 
  Run with:
diff --git a/full-stack.yml b/full-stack.yml
@@ -1,4 +1,4 @@
-version: '2'
+version: '2.1'
 
 services:
   zoo1:
@@ -22,8 +22,9 @@ services:
     ports:
       - "9092:9092"
     environment:
-      # add the entry "127.0.0.1    kafka1" to your /etc/hosts file
-      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka1:9092"
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 1
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
@@ -36,11 +37,10 @@ services:
   kafka-schema-registry:
     image: confluentinc/cp-schema-registry:4.1.0
     hostname: kafka-schema-registry
-    restart: unless-stopped
     ports:
       - "8081:8081"
     environment:
-      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zoo1:2181
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
       SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry
       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
     depends_on:
@@ -64,11 +64,11 @@ services:
     ports:
       - "8082:8082"
     environment:
-      KAFKA_REST_ZOOKEEPER_CONNECT: zoo1:2181
+      # KAFKA_REST_ZOOKEEPER_CONNECT: zoo1:2181
       KAFKA_REST_LISTENERS: http://0.0.0.0:8082/
       KAFKA_REST_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081/
       KAFKA_REST_HOST_NAME: kafka-rest-proxy
-      KAFKA_REST_BOOTSTRAP_SERVERS: kafka1:9092
+      KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
     depends_on:
       - zoo1
       - kafka1
@@ -94,7 +94,7 @@ services:
     ports:
       - "8083:8083"
     environment:
-      CONNECT_BOOTSTRAP_SERVERS: "kafka1:9092"
+      CONNECT_BOOTSTRAP_SERVERS: "kafka1:19092"
       CONNECT_REST_PORT: 8083
       CONNECT_GROUP_ID: compose-connect-group
       CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
@@ -140,12 +140,10 @@ services:
      - zoonavigator-api
     depends_on:
      - zoonavigator-api
-    restart: unless-stopped
 
   zoonavigator-api:
     image: elkozmon/zoonavigator-api:0.4.0
     environment:
       SERVER_HTTP_PORT: 9000
-    restart: unless-stopped
     depends_on:
       - zoo1
diff --git a/test.sh b/test.sh
@@ -36,12 +36,12 @@ kafka_tests(){
     echo "Testing Kafka"
     topic="testtopic"
     if grep -q kafka3 $1; then replication_factor="3"; else replication_factor="1"; fi
-    for i in 1 2 3 4 5; do echo "trying to create test topic" && kafka-topics --create --topic $topic --replication-factor $replication_factor --partitions 12 --zookeeper localhost:2181 && break || sleep 5; done
-    for x in {1..100}; do echo $x; done | kafka-console-producer --broker-list localhost:9092 --topic $topic
-    rows=`kafka-console-consumer --bootstrap-server localhost:9092 --topic $topic --from-beginning --timeout-ms 2000 | wc -l`
-    # rows=`kafkacat -C -b localhost:9092 -t $topic -o beginning -e | wc -l `
+    for i in 1 2 3 4 5; do echo "trying to create test topic" && kafka-topics --create --topic $topic --replication-factor $replication_factor --partitions 12 --zookeeper $DOCKER_HOST_IP:2181 && break || sleep 5; done
+    for x in {1..100}; do echo $x; done | kafka-console-producer --broker-list $DOCKER_HOST_IP:9092 --topic $topic
+    rows=`kafka-console-consumer --bootstrap-server $DOCKER_HOST_IP:9092 --topic $topic --from-beginning --timeout-ms 2000 | wc -l`
+    # rows=`kafkacat -C -b $DOCKER_HOST_IP:9092 -t $topic -o beginning -e | wc -l `
     if [ "$rows" != "100" ]; then
-        kafka-console-consumer --bootstrap-server localhost:9092 --topic test-topic --from-beginning --timeout-ms 2000 | wc -l
+        kafka-console-consumer --bootstrap-server $DOCKER_HOST_IP:9092 --topic test-topic --from-beginning --timeout-ms 2000 | wc -l
         exit 1
     else
         echo "Kafka Test Success"
diff --git a/zk-multiple-kafka-multiple.yml b/zk-multiple-kafka-multiple.yml
@@ -1,4 +1,4 @@
-version: '2'
+version: '2.1'
 
 services:
   zoo1:
@@ -47,8 +47,9 @@ services:
     ports:
       - "9092:9092"
     environment:
-      # add the entry "127.0.0.1    kafka1" to your /etc/hosts file
-      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka1:9092"
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 1
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
@@ -65,8 +66,9 @@ services:
     ports:
       - "9093:9093"
     environment:
-      # add the entry "127.0.0.1    kafka1" to your /etc/hosts file
-      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka2:9093"
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 2
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
@@ -83,8 +85,9 @@ services:
     ports:
       - "9094:9094"
     environment:
-      # add the entry "127.0.0.1    kafka1" to your /etc/hosts file
-      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka3:9094"
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 3
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
diff --git a/zk-multiple-kafka-single.yml b/zk-multiple-kafka-single.yml
@@ -1,4 +1,4 @@
-version: '2'
+version: '2.1'
 
 services:
   zoo1:
@@ -47,8 +47,9 @@ services:
     ports:
       - "9092:9092"
     environment:
-      # add the entry "127.0.0.1    kafka1" to your /etc/hosts file
-      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka1:9092"
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 1
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
diff --git a/zk-single-kafka-multiple.yml b/zk-single-kafka-multiple.yml
@@ -1,4 +1,4 @@
-version: '2'
+version: '2.1'
 
 services:
   zoo1:
@@ -20,8 +20,9 @@ services:
     ports:
       - "9092:9092"
     environment:
-      # add the entry "127.0.0.1    kafka1" to your /etc/hosts file
-      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka1:9092"
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 1
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
@@ -36,8 +37,9 @@ services:
     ports:
       - "9093:9093"
     environment:
-      # add the entry "127.0.0.1    kafka1" to your /etc/hosts file
-      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka2:9093"
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 2
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
@@ -53,8 +55,9 @@ services:
     ports:
       - "9094:9094"
     environment:
-      # add the entry "127.0.0.1    kafka1" to your /etc/hosts file
-      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka3:9094"
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 3
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
diff --git a/zk-single-kafka-single.yml b/zk-single-kafka-single.yml
@@ -1,4 +1,4 @@
-version: '2'
+version: '2.1'
 
 services:
   zoo1:
@@ -20,8 +20,9 @@ services:
     ports:
       - "9092:9092"
     environment:
-      # add the entry "127.0.0.1    kafka1" to your /etc/hosts file
-      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka1:9092"
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 1
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
