Merge pull request #9 from Lispython/feature/prometheus

Add prometheus metrics and exporter to topics observer

Author: Lispython

Cargo.lock

@@ -30,6 +30,11 @@ env_logger = "0.7.1"
 futures = "0.3.0"
 byte-unit = "4.0.6"
 pretty-bytes = "0.2.2"
+prometheus = "0.9.0"
+lazy_static = "1.4.0"
+actix-web = "2.0.0"
+actix-rt = "1.1.1"
+ctrlc = "3.1.5"
 
 
 [profile.dev]
@@ -47,7 +52,6 @@ path = "src/lib.rs"
 name = "kafka-replicator"
 path = "src/main.rs"
 
-
 [[bin]]
 name = "kafka-producer"
 path = "src/producer.rs"

Cargo.toml

@@ -1,4 +1,4 @@
-FROM rust:1.44.1
+FROM rust:1.45.0
 
 
 RUN apt-get update -y && apt-get install -y \

Dockerfile

@@ -1,5 +1,5 @@
 shell:
-	docker-compose -f docker-compose.yml run --rm  --use-aliases --service-ports app /bin/bash
+	docker-compose -f docker-compose.yml run --rm  --use-aliases app /bin/bash
 
 build:
 	docker-compose -f docker-compose.yml build
@@ -8,7 +8,7 @@ build:
 replicator:
 	# docker-compose -f docker-compose.yml run --rm  --service-ports app cargo run --bin=replicator --verbose -- --config=fixtures/example_config.toml -p 9445 -h 0.0.0.0
 
-	docker-compose -f docker-compose.yml run --rm --use-aliases --service-ports app cargo run --bin=replicator --verbose -- examples/example_config.toml
+	docker-compose -f docker-compose.yml run --rm --use-aliases app cargo run --bin=kafka-replicator --verbose -- examples/example-config.yaml
 
 up:
 	docker-compose -f docker-compose.yml up --rm

Makefile

@@ -14,7 +14,7 @@ Lets start with an overview of features that exist in kafka-replicator:
   * [x] **Flexible topic selection:** Select topics with configurable config;
   * [ ] **Auto-create topics:** Destination topics are automatically created for strict_p2p strategy;
   * [x] **Stats:** The tool shows replication status;
-  * [ ] **Monitoring:** Kafka replicator exports stats via prometheus.
+  * [x] **Monitoring:** Kafka replicator exports stats via prometheus.
   * [ ] **Cycle detection**
 
 
@@ -105,7 +105,7 @@ routes:
     downstream_topic: 'topic2'
     repartitioning_strategy: random # strict_p2p | random
     upstream_group_id: group_22
-    progress_every_secs: 10
+    show_progress_interval_secs: 10
     limits:
       messages_per_sec: 10000
       number_of_messages:
@@ -117,7 +117,7 @@ routes:
     downstream_topic: 'topic2'
     repartitioning_strategy: strict_p2p
     upstream_group_id: group_22
-    progress_every_secs: 10
+    show_progress_interval_secs: 10
 
   - upstream_client: cl_2_client_1
     downstream_client: cl_1_client_1
@@ -127,28 +127,30 @@ routes:
     repartitioning_strategy: strict_p2p # strict_p2p | random
     default_begin_offset: earliest # optional
     upstream_group_id: group_2
-    progress_every_secs: 10
+    show_progress_interval_secs: 10
 
 
 observers:
   - client: cl_1_client_1
     name: "my name"
-    topics:
+    group_id: group_name # used for remaining metrics
+    topics: # filter by topics
       - 'topic1'
       - 'topic2'
-    fetch_timeout_secs: 5
-    show_progress_every_secs: 10
+    fetch_timeout_secs: 5 # default: 5
+    fetch_interval_secs: 5 # default: 60
+    show_progress_interval_secs: 10 # default: 60
 
   - client: cl_2_client_1
     topic: 'topic3'
     topics:
       - 'topic2'
-    show_progress_every_secs: 5
+    show_progress_interval_secs: 5
 
 
   - client: cl_1_client_1
     topic: 'topic1'
-    topics: []
+    topics: [] # fetch all topics
 ```
 
 

README.md

@@ -1,13 +1,13 @@
 clusters:
   - name: cluster_1
     hosts:
-      - replicator-kafka-1:9092
-      - replicator-kafka-1:9092
+      #- replicator-kafka-1:9092
+      #- replicator-kafka-1:9092
       - kafka_replicator_replicator_kafka_1_1:9092
   - name: cluster_2
     hosts:
       - kafka_replicator_replicator_kafka_2_1:9092
-      - replicator-kafka-2:9092
+      #- replicator-kafka-2:9092
 
 clients:
   - client: cl_1_client_1
@@ -19,14 +19,17 @@ clients:
     cluster: cluster_2
 
 routes:
-  - upstream_client: cl_1_client_1
-    downstream_client: cl_1_client_1
-    upstream_topics:
+  - upstream_client: cl_1_client_1 # Required: source client name from clients section
+    downstream_client: cl_1_client_1 # Required: target client name from clients section
+    upstream_topics: # Required: source topics
       - 'topic1'
-    downstream_topic: 'topic2'
+    downstream_topic: 'topic2' # Required: target topics
     repartitioning_strategy: random # strict_p2p | random
-    upstream_group_id: group_22
-    progress_every_secs: 10
+    upstream_group_id: group_22 # Option: upstream consumer group id
+    show_progress_interval_secs: 2 # Optional: interval between console output
+    update_metrics_interval_secs: 1 # Optional: interval between prometheus metrics updates
+    upstream_poll_interval_ms: 200 # Optional: interval between upstream poll calls
+    name: route_name # Optional:
     limits:
       messages_per_sec: 10000
       number_of_messages:
@@ -38,7 +41,7 @@ routes:
     downstream_topic: 'topic2'
     repartitioning_strategy: random # strict_p2p | random
     upstream_group_id: group_22
-    progress_every_secs: 10
+    show_progress_interval_secs: 10
 
   - upstream_client: cl_2_client_1
     downstream_client: cl_1_client_1
@@ -48,25 +51,31 @@ routes:
     repartitioning_strategy: strict_p2p # strict_p2p | random
     default_begin_offset: earliest # optional
     upstream_group_id: group_2
-    progress_every_secs: 10
+    show_progress_interval_secs: 10
 
 
 observers:
   - client: cl_1_client_1
-    name: "my name"
+    name: "Observer name"
+    group_id: group_22
     topics:
       - 'topic1'
-      - 'topic2'
-    fetch_timeout_secs: 5
-    show_progress_every_secs: 10
+      # - 'topic2'
+    fetch_timeout_secs: 5 # default: 5
+    fetch_interval_secs: 60 # default:
+    show_progress_interval_secs: 10 # default: 60
 
   - client: cl_2_client_1
-    topic: 'topic3'
+    # group_id: group_2
     topics:
       - 'topic2'
-    show_progress_every_secs: 5
-
+    show_progress_interval_secsy: 20
 
   - client: cl_1_client_1
-    topic: 'topic1'
-    topics: []
+    topics: [] # process all topics
+
+prometheus:
+#  namespace: "app:observer:" # custom prometheus metrics prefix
+  labels:
+    label_key: label_value
+    env: prod

examples/example-config.yml

@@ -39,4 +39,4 @@ observers:
       - 'experimental_topic_1'
       - 'experimental_topic_2_output'
     fetch_timeout_secs: 5
-    show_progress_every_secs: 10
+    show_progress_interval_secs: 10