Cu 86bygcqxd images placement for data store dashboards monitoring

.github/workflows/run-tests.sh

@@ -25,6 +25,9 @@ for package in "${CHANGED_FILES[@]}"; do
     fi
 done
 
+# Run the basic funtional end to end tests for the CDR recipe
+DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":recipe
+
 if [[ ${#changed_packages[@]} -eq 0 ]] || [[ "${!changed_packages[*]}" == *"utils"* ]] || [[ "${!changed_packages[*]}" == *"features/steps"* ]] || [[ "${!changed_packages[*]}" == *"infrastructure"* ]] ; then
     openhim_ran="true"
     DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":openhim
@@ -34,7 +37,8 @@ else
 
         if [[ $folder_name == *"clickhouse"* ]]; then
             DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
-        elif [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
+        elif
+        if [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
             DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":elk
         elif [[ $folder_name == *"kafka"* ]] || [[ $folder_name == *"monitoring"* ]]; then
             DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":kafka
@@ -62,6 +66,3 @@ else
         fi
     done
 fi
-
-# Run the basic funtional end to end tests for the CDR recipe
-DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":recipe

analytics-datastore-clickhouse/docker-compose.cluster.yml

@@ -1,17 +1,17 @@
-version: '3.9'
+version: "3.9"
 
 services:
   analytics-datastore-clickhouse-01:
-    image: clickhouse/clickhouse-server
+    image: ${CLICKHOUSE_IMAGE}
     ulimits:
       noFile: 262144
     volumes:
       - clickhouse-data-01:/var/lib/clickhouse/
     hostname: analytics-datastore-clickhouse-01
-    deploy: 
-      placement: 
-        constraints: 
-          - "node.labels.name==node-1"
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT}"
     configs:
       - target: /etc/clickhouse-server/config.d/docker_related_config.xml
         source: docker_related_config.xml
@@ -36,12 +36,12 @@ services:
       default:
 
   analytics-datastore-clickhouse-02:
-    image: clickhouse/clickhouse-server
+    image: ${CLICKHOUSE_IMAGE}
     hostname: analytics-datastore-clickhouse-02
-    deploy: 
-      placement: 
-        constraints: 
-          - "node.labels.name==node-2"
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT}"
     ulimits:
       noFile: 262144
     volumes:
@@ -70,16 +70,16 @@ services:
       default:
 
   analytics-datastore-clickhouse-03:
-    image: clickhouse/clickhouse-server
+    image: ${CLICKHOUSE_IMAGE}
     hostname: analytics-datastore-clickhouse-03
-    deploy: 
-      placement: 
-        constraints: 
-          - "node.labels.name==node-3"
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT}"
     ulimits:
       noFile: 262144
     volumes:
-      - clickhouse-data-03:/var/lib/clickhouse/        
+      - clickhouse-data-03:/var/lib/clickhouse/
     configs:
       - target: /etc/clickhouse-server/config.d/docker_related_config.xml
         source: docker_related_config.xml
@@ -109,7 +109,7 @@ services:
     ulimits:
       noFile: 262144
     volumes:
-      - clickhouse-data-04:/var/lib/clickhouse/        
+      - clickhouse-data-04:/var/lib/clickhouse/
     configs:
       - target: /etc/clickhouse-server/config.d/docker_related_config.xml
         source: docker_related_config.xml
@@ -136,77 +136,77 @@ volumes:
   clickhouse-data-02:
   clickhouse-data-03:
   clickhouse-data-04:
-  
+
 configs:
   docker_related_config.xml:
     file: ./cluster_configs/docker_related_config.xml
     name: docker_related_config.xml-${docker_related_config_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_enable_keeper_01.xml:
     file: ./cluster_configs/enable_keeper_01.xml
     name: enable_keeper_01.xml-${enable_keeper_01_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_enable_keeper_02.xml:
     file: ./cluster_configs/enable_keeper_02.xml
     name: enable_keeper_02.xml-${enable_keeper_02_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_enable_keeper_03.xml:
     file: ./cluster_configs/enable_keeper_03.xml
     name: enable_keeper_03.xml-${enable_keeper_03_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_macros_01.xml:
     file: ./cluster_configs/macros_01.xml
     name: macros_01.xml-${macros_01_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_macros_02.xml:
     file: ./cluster_configs/macros_02.xml
     name: macros_02.xml-${macros_02_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_macros_03.xml:
     file: ./cluster_configs/macros_03.xml
     name: macros_03.xml-${macros_03_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_macros_04.xml:
     file: ./cluster_configs/macros_04.xml
     name: macros_04.xml-${macros_04_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_remote_servers.xml:
     file: ./cluster_configs/remote_servers.xml
     name: remote_servers.xml-${remote_servers_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_use_keeper.xml:
     file: ./cluster_configs/use_keeper.xml
     name: use_keeper.xml-${use_keeper_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_metric_log.xml:
     file: ./general_configs/metric_log.xml
     name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_part_log.xml:
     file: ./general_configs/part_log.xml
     name: part_log.xml.xml-${part_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_query_log.xml:
     file: ./general_configs/query_log.xml
     name: query_log.xml.xml-${query_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_trace_log.xml:
     file: ./general_configs/trace_log.xml
     name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
 
 networks:

analytics-datastore-clickhouse/docker-compose.yml

@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   analytics-datastore-clickhouse:
-    image: clickhouse/clickhouse-server
+    image: ${CLICKHOUSE_IMAGE}
     ulimits:
       noFile: 262144
     volumes:
@@ -15,7 +15,7 @@ services:
       - target: /etc/clickhouse-server/config.d/query_log.xml
         source: clickhouse_query_log.xml
       - target: /etc/clickhouse-server/config.d/trace_log.xml
-        source: clickhouse_trace_log.xml  
+        source: clickhouse_trace_log.xml
     networks:
       public:
       default:
@@ -27,22 +27,22 @@ configs:
   clickhouse_metric_log.xml:
     file: ./general_configs/metric_log.xml
     name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_part_log.xml:
     file: ./general_configs/part_log.xml
     name: part_log.xml.xml-${part_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_query_log.xml:
     file: ./general_configs/query_log.xml
     name: query_log.xml.xml-${query_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_trace_log.xml:
     file: ./general_configs/trace_log.xml
     name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
 
 networks:

analytics-datastore-clickhouse/package-metadata.json

@@ -7,6 +7,10 @@
   "dependencies": [],
   "environmentVariables": {
     "CLICKHOUSE_HOST": "analytics-datastore-clickhouse",
-    "CLICKHOUSE_PORT": "8123"
+    "CLICKHOUSE_PORT": "8123",
+    "CLICKHOUSE_IMAGE": "clickhouse/clickhouse-server:23.8.14.6",
+    "ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT": "node-1",
+    "ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT": "node-2",
+    "ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT": "node-3"
   }
 }

analytics-datastore-elastic-search/docker-compose.certs.yml

@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   create_certs:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+    image: ${ES_IMAGE}
     deploy:
       placement:
         constraints:

analytics-datastore-elastic-search/docker-compose.cluster.yml

@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   analytics-datastore-elastic-search-01:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+    image: ${ES_IMAGE}
     ulimits:
       memlock:
         soft: -1
@@ -16,20 +16,20 @@ services:
           memory: ${ES_MEMORY_RESERVE}
       placement:
         constraints:
-          - "node.labels.name==node-1"
+          - "node.labels.name==${ES_01_PLACEMENT}"
     environment:
       node.name: es01
       cluster.name: es-cluster
       discovery.seed_hosts: analytics-datastore-elastic-search-02,analytics-datastore-elastic-search-03
       cluster.initial_master_nodes: es01
       xpack.license.self_generated.type: basic
-      bootstrap.memory_lock: 'true'
-      xpack.security.enabled: 'true'
-      xpack.security.http.ssl.enabled: 'false'
+      bootstrap.memory_lock: "true"
+      xpack.security.enabled: "true"
+      xpack.security.http.ssl.enabled: "false"
       xpack.security.http.ssl.key: certs/es01/es01.key
       xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
       xpack.security.http.ssl.certificate: certs/es01/es01.crt
-      xpack.security.transport.ssl.enabled: 'true'
+      xpack.security.transport.ssl.enabled: "true"
       xpack.security.transport.ssl.verification_mode: certificate
       xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
       xpack.security.transport.ssl.certificate: certs/es01/es01.crt
@@ -52,7 +52,7 @@ services:
       public:
 
   analytics-datastore-elastic-search-02:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+    image: ${ES_IMAGE}
     ulimits:
       memlock:
         soft: -1
@@ -66,20 +66,20 @@ services:
           memory: ${ES_MEMORY_RESERVE}
       placement:
         constraints:
-          - "node.labels.name==node-2"
+          - "node.labels.name==${ES_02_PLACEMENT}"
     environment:
       node.name: es02
       cluster.name: es-cluster
       discovery.seed_hosts: analytics-datastore-elastic-search-01,analytics-datastore-elastic-search-03
       cluster.initial_master_nodes: es01
-      bootstrap.memory_lock: 'true'
-      xpack.security.enabled: 'true'
+      bootstrap.memory_lock: "true"
+      xpack.security.enabled: "true"
       xpack.license.self_generated.type: basic
-      xpack.security.http.ssl.enabled: 'false'
+      xpack.security.http.ssl.enabled: "false"
       xpack.security.http.ssl.key: certs/es02/es02.key
       xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
       xpack.security.http.ssl.certificate: certs/es02/es02.crt
-      xpack.security.transport.ssl.enabled: 'true'
+      xpack.security.transport.ssl.enabled: "true"
       xpack.security.transport.ssl.verification_mode: certificate
       xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
       xpack.security.transport.ssl.certificate: certs/es02/es02.crt
@@ -102,7 +102,7 @@ services:
       public:
 
   analytics-datastore-elastic-search-03:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+    image: ${ES_IMAGE}
     ulimits:
       memlock:
         soft: -1
@@ -116,20 +116,20 @@ services:
           memory: ${ES_MEMORY_RESERVE}
       placement:
         constraints:
-          - "node.labels.name==node-3"
+          - "node.labels.name==${ES_03_PLACEMENT}"
     environment:
       node.name: es03
       cluster.name: es-cluster
       discovery.seed_hosts: analytics-datastore-elastic-search-01,analytics-datastore-elastic-search-02
       cluster.initial_master_nodes: es01
-      bootstrap.memory_lock: 'true'
-      xpack.security.enabled: 'true'
+      bootstrap.memory_lock: "true"
+      xpack.security.enabled: "true"
       xpack.license.self_generated.type: basic
-      xpack.security.http.ssl.enabled: 'false'
+      xpack.security.http.ssl.enabled: "false"
       xpack.security.http.ssl.key: certs/es03/es03.key
       xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
       xpack.security.http.ssl.certificate: certs/es03/es03.crt
-      xpack.security.transport.ssl.enabled: 'true'
+      xpack.security.transport.ssl.enabled: "true"
       xpack.security.transport.ssl.verification_mode: certificate
       xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
       xpack.security.transport.ssl.certificate: certs/es03/es03.crt

analytics-datastore-elastic-search/docker-compose.yml

@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   analytics-datastore-elastic-search:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+    image: ${ES_IMAGE}
     ulimits:
       memlock:
         soft: -1
@@ -16,9 +16,9 @@ services:
     environment:
       node.name: es01
       discovery.type: single-node
-      bootstrap.memory_lock: 'true'
-      xpack.security.enabled: 'true'
-      xpack.monitoring.collection.enabled: 'true'
+      bootstrap.memory_lock: "true"
+      xpack.security.enabled: "true"
+      xpack.monitoring.collection.enabled: "true"
       ES_JAVA_OPTS: ${ES_HEAP_SIZE}
       search.max_buckets: 1000000
       search.default_search_timeout: -1