Add prometheus metrics and exporter to topics observer

Author: Lispython

Cargo.lock

@@ -30,6 +30,11 @@ env_logger = "0.7.1"
 futures = "0.3.0"
 byte-unit = "4.0.6"
 pretty-bytes = "0.2.2"
+prometheus = "0.9.0"
+lazy_static = "1.4.0"
+actix-web = "2.0.0"
+actix-rt = "1.1.1"
+ctrlc = "3.1.5"
 
 
 [profile.dev]
@@ -47,7 +52,6 @@ path = "src/lib.rs"
 name = "kafka-replicator"
 path = "src/main.rs"
 
-
 [[bin]]
 name = "kafka-producer"
 path = "src/producer.rs"

Cargo.toml

@@ -1,4 +1,4 @@
-FROM rust:1.44.1
+FROM rust:1.45.0
 
 
 RUN apt-get update -y && apt-get install -y \

Dockerfile

@@ -1,5 +1,5 @@
 shell:
-	docker-compose -f docker-compose.yml run --rm  --use-aliases --service-ports app /bin/bash
+	docker-compose -f docker-compose.yml run --rm  --use-aliases app /bin/bash
 
 build:
 	docker-compose -f docker-compose.yml build
@@ -8,7 +8,7 @@ build:
 replicator:
 	# docker-compose -f docker-compose.yml run --rm  --service-ports app cargo run --bin=replicator --verbose -- --config=fixtures/example_config.toml -p 9445 -h 0.0.0.0
 
-	docker-compose -f docker-compose.yml run --rm --use-aliases --service-ports app cargo run --bin=replicator --verbose -- examples/example_config.toml
+	docker-compose -f docker-compose.yml run --rm --use-aliases app cargo run --bin=kafka-replicator --verbose -- examples/example-config.yaml
 
 up:
 	docker-compose -f docker-compose.yml up --rm

Makefile

@@ -105,7 +105,7 @@ routes:
     downstream_topic: 'topic2'
     repartitioning_strategy: random # strict_p2p | random
     upstream_group_id: group_22
-    progress_every_secs: 10
+    show_progress_interval_secs: 10
     limits:
       messages_per_sec: 10000
       number_of_messages:
@@ -117,7 +117,7 @@ routes:
     downstream_topic: 'topic2'
     repartitioning_strategy: strict_p2p
     upstream_group_id: group_22
-    progress_every_secs: 10
+    show_progress_interval_secs: 10
 
   - upstream_client: cl_2_client_1
     downstream_client: cl_1_client_1
@@ -127,7 +127,7 @@ routes:
     repartitioning_strategy: strict_p2p # strict_p2p | random
     default_begin_offset: earliest # optional
     upstream_group_id: group_2
-    progress_every_secs: 10
+    show_progress_interval_secs: 10
 
 
 observers:
@@ -137,13 +137,13 @@ observers:
       - 'topic1'
       - 'topic2'
     fetch_timeout_secs: 5
-    show_progress_every_secs: 10
+    show_progress_interval_secs: 10
 
   - client: cl_2_client_1
     topic: 'topic3'
     topics:
       - 'topic2'
-    show_progress_every_secs: 5
+    show_progress_interval_secs: 5
 
 
   - client: cl_1_client_1

README.md

@@ -26,19 +26,22 @@ routes:
     downstream_topic: 'topic2'
     repartitioning_strategy: random # strict_p2p | random
     upstream_group_id: group_22
-    progress_every_secs: 10
+    show_progress_interval_secs: 10
     limits:
       messages_per_sec: 10000
       number_of_messages:
 
+    labels:
+      label_key: label_value
+
   - upstream_client: cl_1_client_1
     downstream_client: cl_2_client_1
     upstream_topics:
       - 'topic2'
     downstream_topic: 'topic2'
     repartitioning_strategy: random # strict_p2p | random
     upstream_group_id: group_22
-    progress_every_secs: 10
+    show_progress_interval_secs: 10
 
   - upstream_client: cl_2_client_1
     downstream_client: cl_1_client_1
@@ -48,25 +51,38 @@ routes:
     repartitioning_strategy: strict_p2p # strict_p2p | random
     default_begin_offset: earliest # optional
     upstream_group_id: group_2
-    progress_every_secs: 10
+    show_progress_interval_secs: 10
 
 
 observers:
   - client: cl_1_client_1
     name: "my name"
+    group_id: "group_id"
     topics:
       - 'topic1'
       - 'topic2'
     fetch_timeout_secs: 5
-    show_progress_every_secs: 10
+    fetch_interval_secs: 5
+    show_progress_interval_secs: 5
+    labels:
+      label_key: label_value
+      env: prod
 
   - client: cl_2_client_1
     topic: 'topic3'
     topics:
       - 'topic2'
-    show_progress_every_secs: 5
-
+    show_progress_interval_secsy: 20
+    labels:
+      label_key: label_value2
+      env: stage
 
   - client: cl_1_client_1
     topic: 'topic1'
     topics: []
+
+prometheus:
+  namespace: "app:observer:"
+  labels:
+    label_key: label_value
+    env: prod

examples/example-config.yml

@@ -39,4 +39,4 @@ observers:
       - 'experimental_topic_1'
       - 'experimental_topic_2_output'
     fetch_timeout_secs: 5
-    show_progress_every_secs: 10
+    show_progress_interval_secs: 10