diff --git a/roles/dcm_service/files/create_dcm_service.sh b/roles/dcm_service/files/create_dcm_service.sh index b63ef45..8d1c0e7 100644 --- a/roles/dcm_service/files/create_dcm_service.sh +++ b/roles/dcm_service/files/create_dcm_service.sh @@ -1,14 +1,21 @@ #!/bin/bash -echo [Unit] | sudo tee -a /etc/systemd/system/dcm.service > /dev/null -echo Description=DCM REST Client | sudo tee -a /etc/systemd/system/dcm.service > /dev/null +# TODO we should change this so it reflects latest changes + +echo "[Unit]" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null +echo "Description=DCM REST Client" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null +echo "Requires=kafka-mirror-french-site.service kafka-mirror-greek-site.service kafka-mirror-italian-site.service kafka-mirror-spanish-site.service" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null +echo "After=kafka-mirror-french-site.service kafka-mirror-greek-site.service kafka-mirror-italian-site.service kafka-mirror-spanish-site.service" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null echo | sudo tee -a /etc/systemd/system/dcm.service > /dev/null -echo [Service] | sudo tee -a /etc/systemd/system/dcm.service > /dev/null -echo User=$1 | sudo tee -a /etc/systemd/system/dcm.service > /dev/null -echo ExecStart=/usr/bin/sudo /usr/bin/python3 /usr/bin/dcm/dcm-python/dcm_rest_client.py --dcm_ip_address $2 --spanish_site_plugin_ip_port $3 --italian_site_plugin_ip_port $4 --french_site_plugin_ip_port $5 --greek_site_plugin_ip_port $6 --port 8090 --log info | sudo tee -a /etc/systemd/system/dcm.service > /dev/null +echo "[Service]" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null +echo "User=$1" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null +echo "ExecStart=/usr/bin/sudo /usr/bin/python3 /usr/bin/dcm/dcm-python/dcm_rest_client.py --dcm_ip_address $2 --spanish_site_plugin_ip_port $3 --italian_site_plugin_ip_port $4 --french_site_plugin_ip_port $5 --greek_site_plugin_ip_port $6 --port 8090 --log info" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null +echo "TimeoutSec=10" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null +echo "Restart=on-failure" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null +echo "RestartSec=1" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null echo | sudo tee -a /etc/systemd/system/dcm.service > /dev/null -echo [Install] | sudo tee -a /etc/systemd/system/dcm.service > /dev/null -echo WantedBy=default.target | sudo tee -a /etc/systemd/system/dcm.service > /dev/null +echo "[Install]" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null +echo "WantedBy=default.target" | sudo tee -a /etc/systemd/system/dcm.service > /dev/null sudo chmod 664 /etc/systemd/system/dcm.service sudo systemctl daemon-reload diff --git a/roles/preparation/files/create_kafka_mirror.sh b/roles/preparation/files/create_kafka_mirror.sh index 541f803..887d14d 100644 --- a/roles/preparation/files/create_kafka_mirror.sh +++ b/roles/preparation/files/create_kafka_mirror.sh @@ -1,9 +1,11 @@ -#!/bin/bash -# Arguments: -# 1-site -# 2-topic name +#!/usr/bin/env bash SITE=$1 -TOPIC=$2 -/opt/kafka/bin/kafka-run-class.sh kafka.tools.MirrorMaker --consumer.config /usr/bin/dcm/${SITE}_consumer.config --num.streams 1 --producer.config /usr/bin/dcm/producer.config --whitelist \'${TOPIC}\' & +/opt/kafka/bin/kafka-run-class.sh \ + kafka.tools.MirrorMaker \ + --abort.on.send.failure "true" \ + --consumer.config "/usr/bin/dcm/${SITE}_consumer.config" \ + --num.streams 4 \ + --producer.config /usr/bin/dcm/producer.config \ + --whitelist '^.*\.application_metric\..*$,^.*\.infrastructure_metric\..*$' diff --git a/roles/preparation/files/french_consumer.config b/roles/preparation/files/french_consumer.config new file mode 100755 index 0000000..0740109 --- /dev/null +++ b/roles/preparation/files/french_consumer.config @@ -0,0 +1,6 @@ +bootstrap.servers=BROKER_IP:9092 +group.id=dp-MirrorMaker-group +exclude.internal.topics=true +#mirror.topics.whitelist=.* +client.id=mirror_maker_consumer +enable.auto.commit=false diff --git a/roles/preparation/files/greek_consumer.config b/roles/preparation/files/greek_consumer.config new file mode 100755 index 0000000..0740109 --- /dev/null +++ b/roles/preparation/files/greek_consumer.config @@ -0,0 +1,6 @@ +bootstrap.servers=BROKER_IP:9092 +group.id=dp-MirrorMaker-group +exclude.internal.topics=true +#mirror.topics.whitelist=.* +client.id=mirror_maker_consumer +enable.auto.commit=false diff --git a/roles/preparation/files/italian_consumer.config b/roles/preparation/files/italian_consumer.config new file mode 100755 index 0000000..0740109 --- /dev/null +++ b/roles/preparation/files/italian_consumer.config @@ -0,0 +1,6 @@ +bootstrap.servers=BROKER_IP:9092 +group.id=dp-MirrorMaker-group +exclude.internal.topics=true +#mirror.topics.whitelist=.* +client.id=mirror_maker_consumer +enable.auto.commit=false diff --git a/roles/preparation/files/kafka-mirror-services/kafka-mirror-french-site.service b/roles/preparation/files/kafka-mirror-services/kafka-mirror-french-site.service new file mode 100644 index 0000000..2d4e47c --- /dev/null +++ b/roles/preparation/files/kafka-mirror-services/kafka-mirror-french-site.service @@ -0,0 +1,12 @@ +[Unit] +Description=Kafka MirrorMaker for French Site +PartOf=dcm.service + +[Service] +ExecStart=/usr/bin/dcm/create_kafka_mirror.sh french +TimeoutSec=10 +Restart=on-failure +RestartSec=1 + +[Install] +WantedBy=default.target diff --git a/roles/preparation/files/kafka-mirror-services/kafka-mirror-greek-site.service b/roles/preparation/files/kafka-mirror-services/kafka-mirror-greek-site.service new file mode 100644 index 0000000..64f1dc3 --- /dev/null +++ b/roles/preparation/files/kafka-mirror-services/kafka-mirror-greek-site.service @@ -0,0 +1,12 @@ +[Unit] +Description=Kafka MirrorMaker for Greek Site +PartOf=dcm.service + +[Service] +ExecStart=/usr/bin/dcm/create_kafka_mirror.sh greek +TimeoutSec=10 +Restart=on-failure +RestartSec=1 + +[Install] +WantedBy=default.target diff --git a/roles/preparation/files/kafka-mirror-services/kafka-mirror-italian-site.service b/roles/preparation/files/kafka-mirror-services/kafka-mirror-italian-site.service new file mode 100644 index 0000000..910dac3 --- /dev/null +++ b/roles/preparation/files/kafka-mirror-services/kafka-mirror-italian-site.service @@ -0,0 +1,12 @@ +[Unit] +Description=Kafka MirrorMaker for Italian Site +PartOf=dcm.service + +[Service] +ExecStart=/usr/bin/dcm/create_kafka_mirror.sh italian +TimeoutSec=10 +Restart=on-failure +RestartSec=1 + +[Install] +WantedBy=default.target diff --git a/roles/preparation/files/kafka-mirror-services/kafka-mirror-spanish-site.service b/roles/preparation/files/kafka-mirror-services/kafka-mirror-spanish-site.service new file mode 100644 index 0000000..2a108d1 --- /dev/null +++ b/roles/preparation/files/kafka-mirror-services/kafka-mirror-spanish-site.service @@ -0,0 +1,12 @@ +[Unit] +Description=Kafka MirrorMaker for Spanish Site +PartOf=dcm.service + +[Service] +ExecStart=/usr/bin/dcm/create_kafka_mirror.sh spanish +TimeoutSec=10 +Restart=on-failure +RestartSec=1 + +[Install] +WantedBy=default.target diff --git a/roles/preparation/files/producer.config b/roles/preparation/files/producer.config index be5d9e1..6ad6acd 100644 --- a/roles/preparation/files/producer.config +++ b/roles/preparation/files/producer.config @@ -1,4 +1,7 @@ bootstrap.servers=BROKER_IP:9092 -acks=1 +acks=-1 batch.size=100 client.id=mirror_maker_producer + +max.in.flight.requests.per.connection=1 +retries=1000 diff --git a/roles/preparation/files/spanish_consumer.config b/roles/preparation/files/spanish_consumer.config new file mode 100755 index 0000000..0740109 --- /dev/null +++ b/roles/preparation/files/spanish_consumer.config @@ -0,0 +1,6 @@ +bootstrap.servers=BROKER_IP:9092 +group.id=dp-MirrorMaker-group +exclude.internal.topics=true +#mirror.topics.whitelist=.* +client.id=mirror_maker_consumer +enable.auto.commit=false diff --git a/roles/preparation/tasks/main.yml b/roles/preparation/tasks/main.yml index 88f6528..26e7991 100644 --- a/roles/preparation/tasks/main.yml +++ b/roles/preparation/tasks/main.yml @@ -43,7 +43,7 @@ - name: Copy Spanish consumer config copy: - src: consumer.config + src: spanish_consumer.config dest: /usr/bin/dcm/spanish_consumer.config owner: root group: root @@ -59,7 +59,7 @@ - name: Copy Italian consumer config copy: - src: consumer.config + src: italian_consumer.config dest: /usr/bin/dcm/italian_consumer.config owner: root group: root @@ -75,7 +75,7 @@ - name: Copy French consumer config copy: - src: consumer.config + src: french_consumer.config dest: /usr/bin/dcm/french_consumer.config owner: root group: root @@ -91,7 +91,7 @@ - name: Copy Greek consumer config copy: - src: consumer.config + src: greek_consumer.config dest: /usr/bin/dcm/greek_consumer.config owner: root group: root @@ -121,6 +121,15 @@ line: "bootstrap.servers={{ansible_host}}:9092" when: site == "iwl" +- name: Copy Kafka Mirror services + copy: + src: kafka-mirror-services/ + dest: /etc/systemd/system/ + owner: root + group: root + mode: 0644 + when: site == "iwl" + - name: Check if dcm.service file exists stat: path: /etc/systemd/system/dcm.service