mapped data volumes to local filesystem for full transparence. this fixes #4

Author: simplesteph

full-stack.yml

@@ -11,6 +11,9 @@ services:
         ZOO_MY_ID: 1
         ZOO_PORT: 2181
         ZOO_SERVERS: server.1=zoo1:2888:3888
+    volumes:
+      - ./full-stack/zoo1/data:/data
+      - ./full-stack/zoo1/datalog:/datalog
 
 
   kafka1:
@@ -23,7 +26,9 @@ services:
       KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka1:9092"
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 1
-      # KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - ./full-stack/kafka1/data:/var/lib/kafka/data
     depends_on:
       - zoo1
 

zk-multiple-kafka-multiple.yml

@@ -3,36 +3,42 @@ version: '2'
 services:
   zoo1:
     image: zookeeper:3.4.9
-    restart: always
     hostname: zoo1
     ports:
       - "2181:2181"
     environment:
         ZOO_MY_ID: 1
         ZOO_PORT: 2181
         ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+    volumes:
+      - ./zk-multiple-kafka-multiple/zoo1/data:/data
+      - ./zk-multiple-kafka-multiple/zoo1/datalog:/datalog
+
   zoo2:
     image: zookeeper:3.4.9
     hostname: zoo2
-    restart: always
     ports:
       - "2182:2182"
     environment:
         ZOO_MY_ID: 2
         ZOO_PORT: 2182
         ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+    volumes:
+      - ./zk-multiple-kafka-multiple/zoo2/data:/data
+      - ./zk-multiple-kafka-multiple/zoo2/datalog:/datalog
 
   zoo3:
     image: zookeeper:3.4.9
     hostname: zoo3
-    restart: always
     ports:
       - "2183:2183"
     environment:
         ZOO_MY_ID: 3
         ZOO_PORT: 2183
         ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
-
+    volumes:
+      - ./zk-multiple-kafka-multiple/zoo3/data:/data
+      - ./zk-multiple-kafka-multiple/zoo3/datalog:/datalog
 
 
   kafka1:
@@ -45,7 +51,9 @@ services:
       KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka1:9092"
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 1
-      # KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - ./zk-multiple-kafka-multiple/kafka1/data:/var/lib/kafka/data
     depends_on:
       - zoo1
       - zoo2
@@ -61,12 +69,14 @@ services:
       KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka2:9093"
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 2
-      # KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - ./zk-multiple-kafka-multiple/kafka2/data:/var/lib/kafka/data
     depends_on:
       - zoo1
       - zoo2
       - zoo3
-      
+
   kafka3:
     image: confluentinc/cp-kafka:3.1.2
     hostname: kafka3
@@ -77,7 +87,9 @@ services:
       KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka3:9094"
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 3
-      # KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - ./zk-multiple-kafka-multiple/kafka3/data:/var/lib/kafka/data
     depends_on:
       - zoo1
       - zoo2

zk-multiple-kafka-single.yml

@@ -3,35 +3,42 @@ version: '2'
 services:
   zoo1:
     image: zookeeper:3.4.9
-    restart: always
     hostname: zoo1
     ports:
       - "2181:2181"
     environment:
         ZOO_MY_ID: 1
         ZOO_PORT: 2181
         ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+    volumes:
+      - ./zk-multiple-kafka-single/zoo1/data:/data
+      - ./zk-multiple-kafka-single/zoo1/datalog:/datalog
+
   zoo2:
     image: zookeeper:3.4.9
     hostname: zoo2
-    restart: always
     ports:
       - "2182:2182"
     environment:
         ZOO_MY_ID: 2
         ZOO_PORT: 2182
         ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+    volumes:
+      - ./zk-multiple-kafka-single/zoo2/data:/data
+      - ./zk-multiple-kafka-single/zoo2/datalog:/datalog
 
   zoo3:
     image: zookeeper:3.4.9
     hostname: zoo3
-    restart: always
     ports:
       - "2183:2183"
     environment:
         ZOO_MY_ID: 3
         ZOO_PORT: 2183
         ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
+    volumes:
+      - ./zk-multiple-kafka-single/zoo3/data:/data
+      - ./zk-multiple-kafka-single/zoo3/datalog:/datalog
 
 
   kafka1:
@@ -44,7 +51,9 @@ services:
       KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka1:9092"
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 1
-      # KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - ./zk-multiple-kafka-single/kafka1/data:/var/lib/kafka/data
     depends_on:
       - zoo1
       - zoo2

zk-single-kafka-multiple.yml

@@ -3,15 +3,16 @@ version: '2'
 services:
   zoo1:
     image: zookeeper:3.4.9
-    restart: always
     hostname: zoo1
     ports:
       - "2181:2181"
     environment:
         ZOO_MY_ID: 1
         ZOO_PORT: 2181
         ZOO_SERVERS: server.1=zoo1:2888:3888
-
+    volumes:
+      - ./zk-single-kafka-multiple/zoo1/data:/data
+      - ./zk-single-kafka-multiple/zoo1/datalog:/datalog
 
   kafka1:
     image: confluentinc/cp-kafka:3.1.2
@@ -23,7 +24,9 @@ services:
       KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka1:9092"
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 1
-      # KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - ./zk-single-kafka-multiple/kafka1/data:/var/lib/kafka/data
     depends_on:
       - zoo1
 
@@ -37,10 +40,12 @@ services:
       KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka2:9093"
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 2
-      # KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - ./zk-single-kafka-multiple/kafka2/data:/var/lib/kafka/data
     depends_on:
       - zoo1
-      
+
 
   kafka3:
     image: confluentinc/cp-kafka:3.1.2
@@ -52,6 +57,8 @@ services:
       KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka3:9093"
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 3
-      # KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - ./zk-single-kafka-multiple/kafka3/data:/var/lib/kafka/data
     depends_on:
       - zoo1

zk-single-kafka-single.yml

@@ -3,14 +3,16 @@ version: '2'
 services:
   zoo1:
     image: zookeeper:3.4.9
-    restart: always
     hostname: zoo1
     ports:
       - "2181:2181"
     environment:
         ZOO_MY_ID: 1
         ZOO_PORT: 2181
         ZOO_SERVERS: server.1=zoo1:2888:3888
+    volumes:
+      - ./zk-single-kafka-single/zoo1/data:/data
+      - ./zk-single-kafka-single/zoo1/datalog:/datalog
 
   kafka1:
     image: confluentinc/cp-kafka:3.1.2
@@ -22,6 +24,8 @@ services:
       KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka1:9092"
       KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
       KAFKA_BROKER_ID: 1
-      # KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+    volumes:
+      - ./zk-single-kafka-single/kafka1/data:/var/lib/kafka/data
     depends_on:
       - zoo1