Put every Kafka mirror in a systemd unit for better management.

5GEVE · Aug 31, 2021 · 9077f86 · 9077f86
1 parent 85d94a3
commit 9077f86
Show file tree

Hide file tree

Showing 12 changed files with 111 additions and 18 deletions.
diff --git a/roles/dcm_service/files/create_dcm_service.sh b/roles/dcm_service/files/create_dcm_service.sh
@@ -1,14 +1,21 @@
 #!/bin/bash
 
-echo [Unit] | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
-echo Description=DCM REST Client | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
+# TODO we should change this so it reflects latest changes
+
+echo "[Unit]" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
+echo "Description=DCM REST Client" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
+echo "Requires=kafka-mirror-french-site.service kafka-mirror-greek-site.service kafka-mirror-italian-site.service kafka-mirror-spanish-site.service" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
+echo "After=kafka-mirror-french-site.service kafka-mirror-greek-site.service kafka-mirror-italian-site.service kafka-mirror-spanish-site.service" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
 echo | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
-echo [Service] | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
-echo User=$1 | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
-echo ExecStart=/usr/bin/sudo /usr/bin/python3 /usr/bin/dcm/dcm-python/dcm_rest_client.py --dcm_ip_address $2 --spanish_site_plugin_ip_port $3 --italian_site_plugin_ip_port $4 --french_site_plugin_ip_port $5 --greek_site_plugin_ip_port $6 --port 8090 --log info | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
+echo "[Service]" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
+echo "User=$1" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
+echo "ExecStart=/usr/bin/sudo /usr/bin/python3 /usr/bin/dcm/dcm-python/dcm_rest_client.py --dcm_ip_address $2 --spanish_site_plugin_ip_port $3 --italian_site_plugin_ip_port $4 --french_site_plugin_ip_port $5 --greek_site_plugin_ip_port $6 --port 8090 --log info" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
+echo "TimeoutSec=10" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
+echo "Restart=on-failure" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
+echo "RestartSec=1" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
 echo | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
-echo [Install] | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
-echo WantedBy=default.target | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
+echo "[Install]" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
+echo "WantedBy=default.target" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null
 
 sudo chmod 664 /etc/systemd/system/dcm.service
 sudo systemctl daemon-reload

diff --git a/roles/preparation/files/create_kafka_mirror.sh b/roles/preparation/files/create_kafka_mirror.sh
@@ -1,9 +1,11 @@
-#!/bin/bash
-# Arguments:
-# 1-site
-# 2-topic name
+#!/usr/bin/env bash
 
 SITE=$1
-TOPIC=$2
 
-/opt/kafka/bin/kafka-run-class.sh kafka.tools.MirrorMaker --consumer.config /usr/bin/dcm/${SITE}_consumer.config --num.streams 1 --producer.config /usr/bin/dcm/producer.config --whitelist \'${TOPIC}\' &
+/opt/kafka/bin/kafka-run-class.sh \
+  kafka.tools.MirrorMaker \
+    --abort.on.send.failure "true" \
+    --consumer.config "/usr/bin/dcm/${SITE}_consumer.config" \
+    --num.streams 4 \
+    --producer.config /usr/bin/dcm/producer.config \
+    --whitelist '^.*\.application_metric\..*$,^.*\.infrastructure_metric\..*$'
diff --git a/roles/preparation/files/french_consumer.config b/roles/preparation/files/french_consumer.config
@@ -0,0 +1,6 @@
+bootstrap.servers=BROKER_IP:9092
+group.id=dp-MirrorMaker-group
+exclude.internal.topics=true
+#mirror.topics.whitelist=.*
+client.id=mirror_maker_consumer
+enable.auto.commit=false
diff --git a/roles/preparation/files/greek_consumer.config b/roles/preparation/files/greek_consumer.config
@@ -0,0 +1,6 @@
+bootstrap.servers=BROKER_IP:9092
+group.id=dp-MirrorMaker-group
+exclude.internal.topics=true
+#mirror.topics.whitelist=.*
+client.id=mirror_maker_consumer
+enable.auto.commit=false
diff --git a/roles/preparation/files/italian_consumer.config b/roles/preparation/files/italian_consumer.config
@@ -0,0 +1,6 @@
+bootstrap.servers=BROKER_IP:9092
+group.id=dp-MirrorMaker-group
+exclude.internal.topics=true
+#mirror.topics.whitelist=.*
+client.id=mirror_maker_consumer
+enable.auto.commit=false
diff --git a/roles/preparation/files/kafka-mirror-services/kafka-mirror-french-site.service b/roles/preparation/files/kafka-mirror-services/kafka-mirror-french-site.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Kafka MirrorMaker for French Site
+PartOf=dcm.service
+
+[Service]
+ExecStart=/usr/bin/dcm/create_kafka_mirror.sh french
+TimeoutSec=10
+Restart=on-failure
+RestartSec=1
+
+[Install]
+WantedBy=default.target
diff --git a/roles/preparation/files/kafka-mirror-services/kafka-mirror-greek-site.service b/roles/preparation/files/kafka-mirror-services/kafka-mirror-greek-site.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Kafka MirrorMaker for Greek Site
+PartOf=dcm.service
+
+[Service]
+ExecStart=/usr/bin/dcm/create_kafka_mirror.sh greek
+TimeoutSec=10
+Restart=on-failure
+RestartSec=1
+
+[Install]
+WantedBy=default.target
diff --git a/roles/preparation/files/kafka-mirror-services/kafka-mirror-italian-site.service b/roles/preparation/files/kafka-mirror-services/kafka-mirror-italian-site.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Kafka MirrorMaker for Italian Site
+PartOf=dcm.service
+
+[Service]
+ExecStart=/usr/bin/dcm/create_kafka_mirror.sh italian
+TimeoutSec=10
+Restart=on-failure
+RestartSec=1
+
+[Install]
+WantedBy=default.target
diff --git a/roles/preparation/files/kafka-mirror-services/kafka-mirror-spanish-site.service b/roles/preparation/files/kafka-mirror-services/kafka-mirror-spanish-site.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Kafka MirrorMaker for Spanish Site
+PartOf=dcm.service
+
+[Service]
+ExecStart=/usr/bin/dcm/create_kafka_mirror.sh spanish
+TimeoutSec=10
+Restart=on-failure
+RestartSec=1
+
+[Install]
+WantedBy=default.target
diff --git a/roles/preparation/files/producer.config b/roles/preparation/files/producer.config
@@ -1,4 +1,7 @@
 bootstrap.servers=BROKER_IP:9092
-acks=1
+acks=-1
 batch.size=100
 client.id=mirror_maker_producer
+
+max.in.flight.requests.per.connection=1
+retries=1000
diff --git a/roles/preparation/files/spanish_consumer.config b/roles/preparation/files/spanish_consumer.config
@@ -0,0 +1,6 @@
+bootstrap.servers=BROKER_IP:9092
+group.id=dp-MirrorMaker-group
+exclude.internal.topics=true
+#mirror.topics.whitelist=.*
+client.id=mirror_maker_consumer
+enable.auto.commit=false
diff --git a/roles/preparation/tasks/main.yml b/roles/preparation/tasks/main.yml
@@ -43,7 +43,7 @@
 
 - name: Copy Spanish consumer config
   copy:
-    src: consumer.config
+    src: spanish_consumer.config
     dest: /usr/bin/dcm/spanish_consumer.config
     owner: root
     group: root
@@ -59,7 +59,7 @@
 
 - name: Copy Italian consumer config
   copy:
-    src: consumer.config
+    src: italian_consumer.config
     dest: /usr/bin/dcm/italian_consumer.config
     owner: root
     group: root
@@ -75,7 +75,7 @@
 
 - name: Copy French consumer config
   copy:
-    src: consumer.config
+    src: french_consumer.config
     dest: /usr/bin/dcm/french_consumer.config
     owner: root
     group: root
@@ -91,7 +91,7 @@
 
 - name: Copy Greek consumer config
   copy:
-    src: consumer.config
+    src: greek_consumer.config
     dest: /usr/bin/dcm/greek_consumer.config
     owner: root
     group: root
@@ -121,6 +121,15 @@
     line: "bootstrap.servers={{ansible_host}}:9092"
   when: site == "iwl"
 
+- name: Copy Kafka Mirror services
+  copy:
+    src: kafka-mirror-services/
+    dest: /etc/systemd/system/
+    owner: root
+    group: root
+    mode: 0644
+  when: site == "iwl"
+
 - name: Check if dcm.service file exists
   stat:
     path: /etc/systemd/system/dcm.service