From c130c78d95e2f21a01e2c432b2e48b68762ecf8b Mon Sep 17 00:00:00 2001
From: Brett Onions <onionsllb@outlook.com>
Date: Tue, 16 Apr 2024 07:29:24 +0200
Subject: [PATCH 01/18] adding image variable to metadata.json and
 docker-compose files

---
 .../docker-compose.cluster.yml                | 60 +++++++++----------
 .../docker-compose.yml                        | 14 ++---
 .../package-metadata.json                     |  3 +-
 3 files changed, 39 insertions(+), 38 deletions(-)

diff --git a/analytics-datastore-clickhouse/docker-compose.cluster.yml b/analytics-datastore-clickhouse/docker-compose.cluster.yml
index f2c6b61b..af2f3e82 100644
--- a/analytics-datastore-clickhouse/docker-compose.cluster.yml
+++ b/analytics-datastore-clickhouse/docker-compose.cluster.yml
@@ -1,16 +1,16 @@
-version: '3.9'
+version: "3.9"
 
 services:
   analytics-datastore-clickhouse-01:
-    image: clickhouse/clickhouse-server
+    image: ${CLICKHOUSE_IMAGE}
     ulimits:
       noFile: 262144
     volumes:
       - clickhouse-data-01:/var/lib/clickhouse/
     hostname: analytics-datastore-clickhouse-01
-    deploy: 
-      placement: 
-        constraints: 
+    deploy:
+      placement:
+        constraints:
           - "node.labels.name==node-1"
     configs:
       - target: /etc/clickhouse-server/config.d/docker_related_config.xml
@@ -36,11 +36,11 @@ services:
       default:
 
   analytics-datastore-clickhouse-02:
-    image: clickhouse/clickhouse-server
+    image: ${CLICKHOUSE_IMAGE}
     hostname: analytics-datastore-clickhouse-02
-    deploy: 
-      placement: 
-        constraints: 
+    deploy:
+      placement:
+        constraints:
           - "node.labels.name==node-2"
     ulimits:
       noFile: 262144
@@ -70,16 +70,16 @@ services:
       default:
 
   analytics-datastore-clickhouse-03:
-    image: clickhouse/clickhouse-server
+    image: ${CLICKHOUSE_IMAGE}
     hostname: analytics-datastore-clickhouse-03
-    deploy: 
-      placement: 
-        constraints: 
+    deploy:
+      placement:
+        constraints:
           - "node.labels.name==node-3"
     ulimits:
       noFile: 262144
     volumes:
-      - clickhouse-data-03:/var/lib/clickhouse/        
+      - clickhouse-data-03:/var/lib/clickhouse/
     configs:
       - target: /etc/clickhouse-server/config.d/docker_related_config.xml
         source: docker_related_config.xml
@@ -109,7 +109,7 @@ services:
     ulimits:
       noFile: 262144
     volumes:
-      - clickhouse-data-04:/var/lib/clickhouse/        
+      - clickhouse-data-04:/var/lib/clickhouse/
     configs:
       - target: /etc/clickhouse-server/config.d/docker_related_config.xml
         source: docker_related_config.xml
@@ -136,77 +136,77 @@ volumes:
   clickhouse-data-02:
   clickhouse-data-03:
   clickhouse-data-04:
-  
+
 configs:
   docker_related_config.xml:
     file: ./cluster_configs/docker_related_config.xml
     name: docker_related_config.xml-${docker_related_config_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_enable_keeper_01.xml:
     file: ./cluster_configs/enable_keeper_01.xml
     name: enable_keeper_01.xml-${enable_keeper_01_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_enable_keeper_02.xml:
     file: ./cluster_configs/enable_keeper_02.xml
     name: enable_keeper_02.xml-${enable_keeper_02_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_enable_keeper_03.xml:
     file: ./cluster_configs/enable_keeper_03.xml
     name: enable_keeper_03.xml-${enable_keeper_03_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_macros_01.xml:
     file: ./cluster_configs/macros_01.xml
     name: macros_01.xml-${macros_01_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_macros_02.xml:
     file: ./cluster_configs/macros_02.xml
     name: macros_02.xml-${macros_02_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_macros_03.xml:
     file: ./cluster_configs/macros_03.xml
     name: macros_03.xml-${macros_03_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_macros_04.xml:
     file: ./cluster_configs/macros_04.xml
     name: macros_04.xml-${macros_04_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_remote_servers.xml:
     file: ./cluster_configs/remote_servers.xml
     name: remote_servers.xml-${remote_servers_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_use_keeper.xml:
     file: ./cluster_configs/use_keeper.xml
     name: use_keeper.xml-${use_keeper_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_metric_log.xml:
     file: ./general_configs/metric_log.xml
     name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_part_log.xml:
     file: ./general_configs/part_log.xml
     name: part_log.xml.xml-${part_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_query_log.xml:
     file: ./general_configs/query_log.xml
     name: query_log.xml.xml-${query_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_trace_log.xml:
     file: ./general_configs/trace_log.xml
     name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
 
 networks:
diff --git a/analytics-datastore-clickhouse/docker-compose.yml b/analytics-datastore-clickhouse/docker-compose.yml
index ce6363a8..3f175532 100644
--- a/analytics-datastore-clickhouse/docker-compose.yml
+++ b/analytics-datastore-clickhouse/docker-compose.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   analytics-datastore-clickhouse:
-    image: clickhouse/clickhouse-server
+    image: ${CLICKHOUSE_IMAGE}
     ulimits:
       noFile: 262144
     volumes:
@@ -15,7 +15,7 @@ services:
       - target: /etc/clickhouse-server/config.d/query_log.xml
         source: clickhouse_query_log.xml
       - target: /etc/clickhouse-server/config.d/trace_log.xml
-        source: clickhouse_trace_log.xml  
+        source: clickhouse_trace_log.xml
     networks:
       public:
       default:
@@ -27,22 +27,22 @@ configs:
   clickhouse_metric_log.xml:
     file: ./general_configs/metric_log.xml
     name: metric_log.xml.xml-${metric_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_part_log.xml:
     file: ./general_configs/part_log.xml
     name: part_log.xml.xml-${part_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_query_log.xml:
     file: ./general_configs/query_log.xml
     name: query_log.xml.xml-${query_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
   clickhouse_trace_log.xml:
     file: ./general_configs/trace_log.xml
     name: trace_log.xml.xml-${trace_log_xml_DIGEST:?err}
-    labels: 
+    labels:
       name: clickhouse
 
 networks:
diff --git a/analytics-datastore-clickhouse/package-metadata.json b/analytics-datastore-clickhouse/package-metadata.json
index 680a3bb7..3dadee9a 100644
--- a/analytics-datastore-clickhouse/package-metadata.json
+++ b/analytics-datastore-clickhouse/package-metadata.json
@@ -7,6 +7,7 @@
   "dependencies": [],
   "environmentVariables": {
     "CLICKHOUSE_HOST": "analytics-datastore-clickhouse",
-    "CLICKHOUSE_PORT": "8123"
+    "CLICKHOUSE_PORT": "8123",
+    "CLICKHOUSE_IMAGE": "clickhouse/clickhouse-server"
   }
 }

From 0db8a15336f4bda23ef835c0797c2a386df3e7fa Mon Sep 17 00:00:00 2001
From: Brett Onions <onionsllb@outlook.com>
Date: Tue, 16 Apr 2024 08:05:26 +0200
Subject: [PATCH 02/18] adding image definition to metadata.json file

---
 .../docker-compose.certs.yml                  |  2 +-
 .../docker-compose.cluster.yml                | 32 +++++++++----------
 .../docker-compose.yml                        | 10 +++---
 .../package-metadata.json                     |  3 +-
 4 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/analytics-datastore-elastic-search/docker-compose.certs.yml b/analytics-datastore-elastic-search/docker-compose.certs.yml
index 699bc390..3798771f 100644
--- a/analytics-datastore-elastic-search/docker-compose.certs.yml
+++ b/analytics-datastore-elastic-search/docker-compose.certs.yml
@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   create_certs:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+    image: ${ES_IMAGE}
     deploy:
       placement:
         constraints:
diff --git a/analytics-datastore-elastic-search/docker-compose.cluster.yml b/analytics-datastore-elastic-search/docker-compose.cluster.yml
index 7b802498..e6163a85 100644
--- a/analytics-datastore-elastic-search/docker-compose.cluster.yml
+++ b/analytics-datastore-elastic-search/docker-compose.cluster.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   analytics-datastore-elastic-search-01:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+    image: ${ES_IMAGE}
     ulimits:
       memlock:
         soft: -1
@@ -23,13 +23,13 @@ services:
       discovery.seed_hosts: analytics-datastore-elastic-search-02,analytics-datastore-elastic-search-03
       cluster.initial_master_nodes: es01
       xpack.license.self_generated.type: basic
-      bootstrap.memory_lock: 'true'
-      xpack.security.enabled: 'true'
-      xpack.security.http.ssl.enabled: 'false'
+      bootstrap.memory_lock: "true"
+      xpack.security.enabled: "true"
+      xpack.security.http.ssl.enabled: "false"
       xpack.security.http.ssl.key: certs/es01/es01.key
       xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
       xpack.security.http.ssl.certificate: certs/es01/es01.crt
-      xpack.security.transport.ssl.enabled: 'true'
+      xpack.security.transport.ssl.enabled: "true"
       xpack.security.transport.ssl.verification_mode: certificate
       xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
       xpack.security.transport.ssl.certificate: certs/es01/es01.crt
@@ -52,7 +52,7 @@ services:
       public:
 
   analytics-datastore-elastic-search-02:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+    image: ${ES_IMAGE}
     ulimits:
       memlock:
         soft: -1
@@ -72,14 +72,14 @@ services:
       cluster.name: es-cluster
       discovery.seed_hosts: analytics-datastore-elastic-search-01,analytics-datastore-elastic-search-03
       cluster.initial_master_nodes: es01
-      bootstrap.memory_lock: 'true'
-      xpack.security.enabled: 'true'
+      bootstrap.memory_lock: "true"
+      xpack.security.enabled: "true"
       xpack.license.self_generated.type: basic
-      xpack.security.http.ssl.enabled: 'false'
+      xpack.security.http.ssl.enabled: "false"
       xpack.security.http.ssl.key: certs/es02/es02.key
       xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
       xpack.security.http.ssl.certificate: certs/es02/es02.crt
-      xpack.security.transport.ssl.enabled: 'true'
+      xpack.security.transport.ssl.enabled: "true"
       xpack.security.transport.ssl.verification_mode: certificate
       xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
       xpack.security.transport.ssl.certificate: certs/es02/es02.crt
@@ -102,7 +102,7 @@ services:
       public:
 
   analytics-datastore-elastic-search-03:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+    image: ${ES_IMAGE}
     ulimits:
       memlock:
         soft: -1
@@ -122,14 +122,14 @@ services:
       cluster.name: es-cluster
       discovery.seed_hosts: analytics-datastore-elastic-search-01,analytics-datastore-elastic-search-02
       cluster.initial_master_nodes: es01
-      bootstrap.memory_lock: 'true'
-      xpack.security.enabled: 'true'
+      bootstrap.memory_lock: "true"
+      xpack.security.enabled: "true"
       xpack.license.self_generated.type: basic
-      xpack.security.http.ssl.enabled: 'false'
+      xpack.security.http.ssl.enabled: "false"
       xpack.security.http.ssl.key: certs/es03/es03.key
       xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
       xpack.security.http.ssl.certificate: certs/es03/es03.crt
-      xpack.security.transport.ssl.enabled: 'true'
+      xpack.security.transport.ssl.enabled: "true"
       xpack.security.transport.ssl.verification_mode: certificate
       xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
       xpack.security.transport.ssl.certificate: certs/es03/es03.crt
diff --git a/analytics-datastore-elastic-search/docker-compose.yml b/analytics-datastore-elastic-search/docker-compose.yml
index f6b86b78..e49b2916 100644
--- a/analytics-datastore-elastic-search/docker-compose.yml
+++ b/analytics-datastore-elastic-search/docker-compose.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   analytics-datastore-elastic-search:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.0
+    image: ${ES_IMAGE}
     ulimits:
       memlock:
         soft: -1
@@ -16,9 +16,9 @@ services:
     environment:
       node.name: es01
       discovery.type: single-node
-      bootstrap.memory_lock: 'true'
-      xpack.security.enabled: 'true'
-      xpack.monitoring.collection.enabled: 'true'
+      bootstrap.memory_lock: "true"
+      xpack.security.enabled: "true"
+      xpack.monitoring.collection.enabled: "true"
       ES_JAVA_OPTS: ${ES_HEAP_SIZE}
       search.max_buckets: 1000000
       search.default_search_timeout: -1
diff --git a/analytics-datastore-elastic-search/package-metadata.json b/analytics-datastore-elastic-search/package-metadata.json
index 0f305edb..48ac5da5 100644
--- a/analytics-datastore-elastic-search/package-metadata.json
+++ b/analytics-datastore-elastic-search/package-metadata.json
@@ -17,6 +17,7 @@
     "ES_HEAP_SIZE": "-Xms2048m -Xmx2048m",
     "ES_SSL": "false",
     "ES_MEMORY_LIMIT": "3G",
-    "ES_MEMORY_RESERVE": "500M"
+    "ES_MEMORY_RESERVE": "500M",
+    "ES_IMAGE": "docker.elastic.co/elasticsearch/elasticsearch:7.13.0"
   }
 }

From b889463ae938d9c4f22330c9b3b0ad32c705fab5 Mon Sep 17 00:00:00 2001
From: Brett Onions <onionsllb@outlook.com>
Date: Tue, 16 Apr 2024 15:37:50 +0200
Subject: [PATCH 03/18] add image and max replicas to metadata file

---
 .../docker-compose.yml                         | 18 +++++++++---------
 .../package-metadata.json                      |  2 ++
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/dashboard-visualiser-jsreport/docker-compose.yml b/dashboard-visualiser-jsreport/docker-compose.yml
index 3d87f3c7..1d145e0f 100644
--- a/dashboard-visualiser-jsreport/docker-compose.yml
+++ b/dashboard-visualiser-jsreport/docker-compose.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   dashboard-visualiser-jsreport:
-    image: jsreport/jsreport:3.2.0
+    image: ${JS_REPORT_IMAGE}
     healthcheck:
       test: wget --no-verbose --tries=1 --spider -q http://localhost:5488/api/ping || exit 1
       interval: 10s
@@ -11,7 +11,7 @@ services:
       start_period: 10s
     deploy:
       placement:
-        max_replicas_per_node: 1
+        max_replicas_per_node: ${JS_REPORT_MAX_REPLICAS_PER_NODE}
       replicas: ${JS_REPORT_INSTANCES}
       resources:
         limits:
@@ -21,14 +21,14 @@ services:
           cpus: ${JS_REPORT_CPU_RESERVE}
           memory: ${JS_REPORT_MEMORY_RESERVE}
     labels:
-      co.elastic.metrics/module: 'docker'
-      co.elastic.metrics/metricsets: 'cpu,memory,diskio,info,healthcheck,container'
+      co.elastic.metrics/module: "docker"
+      co.elastic.metrics/metricsets: "cpu,memory,diskio,info,healthcheck,container"
     environment:
-      allowLocalFilesAccess: 'true'
-      extensions_fsStore_dataDirectory: 'jsreport/data'
-      extensions_fsStore_externalModificationsSync: 'true'
+      allowLocalFilesAccess: "true"
+      extensions_fsStore_dataDirectory: "jsreport/data"
+      extensions_fsStore_externalModificationsSync: "true"
       extensions_authentication_cookieSession_secret: ${JS_REPORT_SECRET}
-      extensions_authentication_admin_username: 'admin'
+      extensions_authentication_admin_username: "admin"
       extensions_authentication_admin_password: ${JS_REPORT}
       ES_PASSWORD: ${ES_ELASTIC}
       licenseKey: ${JS_REPORT_LICENSE_KEY}
diff --git a/dashboard-visualiser-jsreport/package-metadata.json b/dashboard-visualiser-jsreport/package-metadata.json
index 20e1ad78..a06074c6 100644
--- a/dashboard-visualiser-jsreport/package-metadata.json
+++ b/dashboard-visualiser-jsreport/package-metadata.json
@@ -6,7 +6,9 @@
   "version": "0.0.1",
   "dependencies": ["analytics-datastore-elastic-search"],
   "environmentVariables": {
+    "JS_REPORT_IMAGE": "jsreport/jsreport:3.2.0",
     "JS_REPORT_INSTANCES": "1",
+    "JS_REPORT_MAX_REPLICAS_PER_NODE": "1",
     "JS_REPORT_USERNAME": "admin",
     "JS_REPORT_SECRET": "dev_secret_only",
     "JS_REPORT": "dev_password_only",

From 77f5f2ac192c59709cd6d50b06aedc4c0d2b0c47 Mon Sep 17 00:00:00 2001
From: Brett Onions <onionsllb@outlook.com>
Date: Tue, 16 Apr 2024 15:42:10 +0200
Subject: [PATCH 04/18] Update Kibana image and add KIBANA_IMAGE environment
 variable

---
 dashboard-visualiser-kibana/docker-compose.yml    | 6 +++---
 dashboard-visualiser-kibana/package-metadata.json | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/dashboard-visualiser-kibana/docker-compose.yml b/dashboard-visualiser-kibana/docker-compose.yml
index 6064821d..30e3634f 100644
--- a/dashboard-visualiser-kibana/docker-compose.yml
+++ b/dashboard-visualiser-kibana/docker-compose.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   dashboard-visualiser-kibana:
-    image: docker.elastic.co/kibana/kibana:7.13.0
+    image: ${KIBANA_IMAGE}
     healthcheck:
       test: curl --fail http://localhost:5601 || exit 1
       interval: 10s
@@ -33,7 +33,7 @@ configs:
   kibana-kibana-cluster.yml:
     file: ./kibana.cluster.yml
     name: kibana-kibana-cluster.yml-${kibana_kibana_cluster_yml_DIGEST:?err}
-    labels: 
+    labels:
       name: kibana
 
 networks:
diff --git a/dashboard-visualiser-kibana/package-metadata.json b/dashboard-visualiser-kibana/package-metadata.json
index 7a3bb05d..95d8b144 100644
--- a/dashboard-visualiser-kibana/package-metadata.json
+++ b/dashboard-visualiser-kibana/package-metadata.json
@@ -9,6 +9,7 @@
     "KIBANA_INSTANCES": 1,
     "ES_LEADER_NODE": "analytics-datastore-elastic-search",
     "ES_KIBANA_SYSTEM": "dev_password_only",
+    "KIBANA_IMAGE": "docker.elastic.co/kibana/kibana:7.13.0",
     "KIBANA_PASSWORD": "dev_password_only",
     "KIBANA_USERNAME": "elastic",
     "KIBANA_YML_CONFIG": "kibana-kibana.yml",

From 51f16dbf55350d4c921e79dc5580a67ea6e320fe Mon Sep 17 00:00:00 2001
From: Brett Onions <onionsllb@outlook.com>
Date: Wed, 17 Apr 2024 08:40:13 +0200
Subject: [PATCH 05/18] adding placement and image version for pg in metadata
 file

---
 .../docker-compose.postgres.cluster.yml                | 10 +++++-----
 .../docker-compose.postgres.yml                        |  2 +-
 dashboard-visualiser-superset/package-metadata.json    |  2 ++
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/dashboard-visualiser-superset/docker-compose.postgres.cluster.yml b/dashboard-visualiser-superset/docker-compose.postgres.cluster.yml
index e4c6ff84..b6e31cbd 100644
--- a/dashboard-visualiser-superset/docker-compose.postgres.cluster.yml
+++ b/dashboard-visualiser-superset/docker-compose.postgres.cluster.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   postgres-metastore:
-    deploy: 
-      placement: 
-        constraints: 
-          - "node.labels.name==node-2"
+    deploy:
+      placement:
+        constraints:
+          - "node.labels.name==${POSTGRES_METASTORE}"
diff --git a/dashboard-visualiser-superset/docker-compose.postgres.yml b/dashboard-visualiser-superset/docker-compose.postgres.yml
index cc4cceae..10c0a68a 100644
--- a/dashboard-visualiser-superset/docker-compose.postgres.yml
+++ b/dashboard-visualiser-superset/docker-compose.postgres.yml
@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   postgres-metastore:
-    image: postgres:16.2
+    image: ${POSTGRES_IMAGE}
     environment:
       POSTGRES_USER: ${SUPERSET_POSTGRESQL_USERNAME}
       POSTGRES_PASSWORD: ${SUPERSET_POSTGRESQL_PASSWORD}
diff --git a/dashboard-visualiser-superset/package-metadata.json b/dashboard-visualiser-superset/package-metadata.json
index a5068d95..69867d91 100644
--- a/dashboard-visualiser-superset/package-metadata.json
+++ b/dashboard-visualiser-superset/package-metadata.json
@@ -7,6 +7,8 @@
   "dependencies": ["analytics-datastore-clickhouse"],
   "environmentVariables": {
     "SUPERSET_IMAGE": "apache/superset:3.1.1",
+    "POSTGRES_IMAGE": "postgres:16.2",
+    "POSTGRES_METASTORE": "node-2",
     "SUPERSET_ENABLED_FEATURE_FLAGS": "DASHBOARD_RBAC",
     "SUPERSET_USERNAME": "admin",
     "SUPERSET_FIRSTNAME": "SUPERSET",

From d2f0701468c5fb8dac214efd461dbf93a01298b4 Mon Sep 17 00:00:00 2001
From: Brett Onions <onionsllb@outlook.com>
Date: Thu, 18 Apr 2024 09:24:44 +0200
Subject: [PATCH 06/18] Update docker-compose files and package metadata

---
 monitoring/docker-compose.cluster.yml | 56 +++++++++------------------
 monitoring/docker-compose.yml         | 30 ++++++--------
 monitoring/package-metadata.json      | 11 ++++++
 3 files changed, 42 insertions(+), 55 deletions(-)

diff --git a/monitoring/docker-compose.cluster.yml b/monitoring/docker-compose.cluster.yml
index e627fa1c..87eb7d62 100644
--- a/monitoring/docker-compose.cluster.yml
+++ b/monitoring/docker-compose.cluster.yml
@@ -1,20 +1,20 @@
-version: '3.9'
+version: "3.9"
 
 services:
   prometheus:
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-1"
+          - "node.labels.name==${PROMETHEUS_PLACEMENT}"
       replicas: 1
 
   prometheus_backup:
-    image: prom/prometheus:v2.38.0
+    image: ${PROMETHEUS_BACKUP_IMAGE}
     user: root
     deploy:
       placement:
         constraints:
-          - "node.labels.name!=node-1"
+          - "node.labels.name!=${PROMETHEUS_BACKUP_PLACEMENT}"
       replicas: 1
     volumes:
       - prometheus_data_backup:/prometheus
@@ -23,11 +23,11 @@ services:
       - target: /etc/prometheus/prometheus.yml
         source: prometheus.yml
     command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--storage.tsdb.path=/prometheus'
-      - '--web.console.libraries=/etc/prometheus/console_libraries'
-      - '--web.console.templates=/etc/prometheus/consoles'
-      - '--web.enable-lifecycle'
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/etc/prometheus/console_libraries"
+      - "--web.console.templates=/etc/prometheus/consoles"
+      - "--web.enable-lifecycle"
     networks:
       public:
       default:
@@ -36,23 +36,17 @@ services:
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-1"
+          - "node.labels.name==${MINIO_01_PLACEMENT}"
 
   minio-02:
-    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    image: ${MINIO_IMAGE}
     entrypoint: sh
     command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
     environment:
       MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
       MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
     healthcheck:
-      test:
-        [
-          "CMD",
-          "curl",
-          "-f",
-          "http://localhost:9000/minio/health/live"
-        ]
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
       timeout: 20s
       retries: 3
@@ -63,24 +57,18 @@ services:
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-1"
+          - "node.labels.name==${MINIO_02_PLACEMENT}"
       replicas: 1
 
   minio-03:
-    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    image: ${MINIO_IMAGE}
     entrypoint: sh
     command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
     environment:
       MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
       MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
     healthcheck:
-      test:
-        [
-          "CMD",
-          "curl",
-          "-f",
-          "http://localhost:9000/minio/health/live"
-        ]
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
       timeout: 20s
       retries: 3
@@ -91,24 +79,18 @@ services:
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-2"
+          - "node.labels.name==${MINIO_03_PLACEMENT}"
       replicas: 1
 
   minio-04:
-    image: quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z
+    image: ${MINIO_IMAGE}
     entrypoint: sh
     command: -c 'mkdir -p /data1/loki /data2/loki && minio server --console-address ":9001" http://minio-0{1...4}/data{1...2}'
     environment:
       MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
       MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
     healthcheck:
-      test:
-        [
-          "CMD",
-          "curl",
-          "-f",
-          "http://localhost:9000/minio/health/live"
-        ]
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
       timeout: 20s
       retries: 3
@@ -119,7 +101,7 @@ services:
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-3"
+          - "node.labels.name==${MINIO_04_PLACEMENT}"
       replicas: 1
 
 volumes:
diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml
index afe7d743..92fe4e36 100644
--- a/monitoring/docker-compose.yml
+++ b/monitoring/docker-compose.yml
@@ -1,8 +1,8 @@
-version: '3.9'
+version: "3.9"
 
 services:
   grafana:
-    image: grafana/grafana-oss:9.2.3
+    image: ${GRAFANA_IMAGE}
     volumes:
       - grafana-data:/var/lib/grafana
     environment:
@@ -71,11 +71,11 @@ services:
       - target: /etc/prometheus/prometheus.yml
         source: prometheus.yml
     command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--storage.tsdb.path=/prometheus'
-      - '--web.console.libraries=/etc/prometheus/console_libraries'
-      - '--web.console.templates=/etc/prometheus/consoles'
-      - '--web.enable-lifecycle'
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/etc/prometheus/console_libraries"
+      - "--web.console.templates=/etc/prometheus/consoles"
+      - "--web.enable-lifecycle"
     networks:
       public:
       default:
@@ -96,14 +96,14 @@ services:
     image: quay.io/prometheus/node-exporter:v1.3.1
     hostname: "{{.Node.ID}}"
     command:
-      - '--path.rootfs=/host'
+      - "--path.rootfs=/host"
     volumes:
-      - '/:/host:ro,rslave'
+      - "/:/host:ro,rslave"
     deploy:
       mode: global
 
   loki:
-    image: grafana/loki:2.6.1
+    image: ${LOKI_IMAGE}
     volumes:
       - loki-data:/tmp/loki
     environment:
@@ -119,7 +119,7 @@ services:
         - prometheus-address=loki:3100
 
   promtail:
-    image: grafana/promtail:2.6.1
+    image: ${PROMTAIL_IMAGE}
     volumes:
       - /var/lib/docker/containers:/host/containers
       - /var/log:/var/log:ro
@@ -138,13 +138,7 @@ services:
       MINIO_ROOT_USER: ${MO_SECURITY_ADMIN_USER}
       MINIO_ROOT_PASSWORD: ${MO_SECURITY_ADMIN_PASSWORD}
     healthcheck:
-      test:
-        [
-          "CMD",
-          "curl",
-          "-f",
-          "http://localhost:9000/minio/health/live"
-        ]
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
       timeout: 20s
       retries: 3
diff --git a/monitoring/package-metadata.json b/monitoring/package-metadata.json
index 7b4a07e5..5201cfa9 100644
--- a/monitoring/package-metadata.json
+++ b/monitoring/package-metadata.json
@@ -6,6 +6,17 @@
   "version": "0.0.1",
   "dependencies": [],
   "environmentVariables": {
+    "GRAFANA_IMAGE": "grafana/grafana-oss:9.2.3",
+    "LOKI_IMAGE": "grafana/loki:2.6.1",
+    "PROMTAIL_IMAGE": "grafana/promtail:2.6.1",
+    "PROMETHEUS_BACKUP_IMAGE": "prom/prometheus:v2.38.0",
+    "MINIO_IMAGE": "quay.io/minio/minio:RELEASE.2022-10-24T18-35-07Z",
+    "PROMETHEUS_PLACEMENT": "node-1",
+    "PROMETHEUS_BACKUP_PLACEMENT": "node-1",
+    "MINIO_01_PLACEMENT": "node-1",
+    "MINIO_02_PLACEMENT": "node-1",
+    "MINIO_03_PLACEMENT": "node-2",
+    "MINIO_04_PLACEMENT": "node-3",
     "GF_SECURITY_ADMIN_USER": "admin",
     "GF_SECURITY_ADMIN_PASSWORD": "dev_password_only",
     "GF_SMTP_ENABLED": "false",

From 4c13fe09ae8d24749251de6744390dbbe62c1d66 Mon Sep 17 00:00:00 2001
From: Brett Onions <onionsllb@outlook.com>
Date: Thu, 25 Apr 2024 07:33:45 +0200
Subject: [PATCH 07/18] adding deployment placement

---
 analytics-datastore-clickhouse/docker-compose.cluster.yml | 6 +++---
 analytics-datastore-clickhouse/package-metadata.json      | 5 ++++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/analytics-datastore-clickhouse/docker-compose.cluster.yml b/analytics-datastore-clickhouse/docker-compose.cluster.yml
index af2f3e82..4e970066 100644
--- a/analytics-datastore-clickhouse/docker-compose.cluster.yml
+++ b/analytics-datastore-clickhouse/docker-compose.cluster.yml
@@ -11,7 +11,7 @@ services:
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-1"
+          - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT}"
     configs:
       - target: /etc/clickhouse-server/config.d/docker_related_config.xml
         source: docker_related_config.xml
@@ -41,7 +41,7 @@ services:
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-2"
+          - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT}"
     ulimits:
       noFile: 262144
     volumes:
@@ -75,7 +75,7 @@ services:
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-3"
+          - "node.labels.name==${ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT}"
     ulimits:
       noFile: 262144
     volumes:
diff --git a/analytics-datastore-clickhouse/package-metadata.json b/analytics-datastore-clickhouse/package-metadata.json
index 3dadee9a..4bc99cef 100644
--- a/analytics-datastore-clickhouse/package-metadata.json
+++ b/analytics-datastore-clickhouse/package-metadata.json
@@ -8,6 +8,9 @@
   "environmentVariables": {
     "CLICKHOUSE_HOST": "analytics-datastore-clickhouse",
     "CLICKHOUSE_PORT": "8123",
-    "CLICKHOUSE_IMAGE": "clickhouse/clickhouse-server"
+    "CLICKHOUSE_IMAGE": "clickhouse/clickhouse-server",
+    "ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT": "node-1",
+    "ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT": "node-2",
+    "ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT": "node-3"
   }
 }

From d5a4d55ec9eef03eb21bae5635c5f8adcd222df4 Mon Sep 17 00:00:00 2001
From: Brett Onions <onionsllb@outlook.com>
Date: Thu, 25 Apr 2024 08:12:54 +0200
Subject: [PATCH 08/18] Update Elasticsearch cluster placement

---
 .../docker-compose.cluster.yml                              | 6 +++---
 analytics-datastore-elastic-search/package-metadata.json    | 5 ++++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/analytics-datastore-elastic-search/docker-compose.cluster.yml b/analytics-datastore-elastic-search/docker-compose.cluster.yml
index e6163a85..a35aa21b 100644
--- a/analytics-datastore-elastic-search/docker-compose.cluster.yml
+++ b/analytics-datastore-elastic-search/docker-compose.cluster.yml
@@ -16,7 +16,7 @@ services:
           memory: ${ES_MEMORY_RESERVE}
       placement:
         constraints:
-          - "node.labels.name==node-1"
+          - "node.labels.name==${ES_01_PLACEMENT}"
     environment:
       node.name: es01
       cluster.name: es-cluster
@@ -66,7 +66,7 @@ services:
           memory: ${ES_MEMORY_RESERVE}
       placement:
         constraints:
-          - "node.labels.name==node-2"
+          - "node.labels.name==${ES_02_PLACEMENT}"
     environment:
       node.name: es02
       cluster.name: es-cluster
@@ -116,7 +116,7 @@ services:
           memory: ${ES_MEMORY_RESERVE}
       placement:
         constraints:
-          - "node.labels.name==node-3"
+          - "node.labels.name==${ES_03_PLACEMENT}"
     environment:
       node.name: es03
       cluster.name: es-cluster
diff --git a/analytics-datastore-elastic-search/package-metadata.json b/analytics-datastore-elastic-search/package-metadata.json
index 48ac5da5..facad20b 100644
--- a/analytics-datastore-elastic-search/package-metadata.json
+++ b/analytics-datastore-elastic-search/package-metadata.json
@@ -18,6 +18,9 @@
     "ES_SSL": "false",
     "ES_MEMORY_LIMIT": "3G",
     "ES_MEMORY_RESERVE": "500M",
-    "ES_IMAGE": "docker.elastic.co/elasticsearch/elasticsearch:7.13.0"
+    "ES_IMAGE": "docker.elastic.co/elasticsearch/elasticsearch:7.13.0",
+    "ES_01_PLACEMENT": "node-1",
+    "ES_02_PLACEMENT": "node-2",
+    "ES_03_PLACEMENT": "node-3"
   }
 }

From d45061f2fde8d0feb1ccb51dd60ec10f1f075c54 Mon Sep 17 00:00:00 2001
From: Brett Onions <onionsllb@outlook.com>
Date: Thu, 25 Apr 2024 15:14:08 +0200
Subject: [PATCH 09/18] Update Docker Compose files for PostgreSQL and Pgpool

---
 .../docker-compose-pgpool.cluster.yml              | 14 +++++++-------
 .../docker-compose-postgres.cluster.yml            | 12 ++++++------
 database-postgres/docker-compose-postgres.yml      |  2 +-
 database-postgres/package-metadata.json            |  8 ++++++++
 4 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/database-postgres/docker-compose-pgpool.cluster.yml b/database-postgres/docker-compose-pgpool.cluster.yml
index 0addf043..2a23b101 100644
--- a/database-postgres/docker-compose-pgpool.cluster.yml
+++ b/database-postgres/docker-compose-pgpool.cluster.yml
@@ -1,12 +1,12 @@
-version: '3.9'
+version: "3.9"
 
 services:
   pgpool-1:
-    image: bitnami/pgpool:4.4.3
+    image: ${PG_POOL_IMAGE}
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-1"
+          - "node.labels.name==${PGPOOL_1_PLACEMENT}"
       replicas: 1
       resources:
         limits:
@@ -39,11 +39,11 @@ services:
       PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf"
 
   pgpool-2:
-    image: bitnami/pgpool:4.4.3
+    image: ${PG_POOL_IMAGE}
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-2"
+          - "node.labels.name==${PGPOOL_2_PLACEMENT}"
       replicas: 1
       resources:
         limits:
@@ -76,11 +76,11 @@ services:
       PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf"
 
   pgpool-3:
-    image: bitnami/pgpool:4.4.3
+    image: ${PG_POOL_IMAGE}
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-3"
+          - "node.labels.name==${PGPOOL_3_PLACEMENT}"
       replicas: 1
       resources:
         limits:
diff --git a/database-postgres/docker-compose-postgres.cluster.yml b/database-postgres/docker-compose-postgres.cluster.yml
index 88fab559..b7660c98 100644
--- a/database-postgres/docker-compose-postgres.cluster.yml
+++ b/database-postgres/docker-compose-postgres.cluster.yml
@@ -1,4 +1,4 @@
-version: '3.9'
+version: "3.9"
 
 services:
   postgres-1:
@@ -7,7 +7,7 @@ services:
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-1"
+          - "node.labels.name==${POSTGRES_1_PLACEMENT}"
 
   postgres-2:
     image: bitnami/postgresql-repmgr:14
@@ -23,14 +23,14 @@ services:
       REPMGR_FAILOVER: ${POSTGRES_FAILOVER}
       REPMGR_DEGRADED_MONITORING_TIMEOUT: ${POSTGRES_DEGRADED_MONITORING_TIMEOUT}
     volumes:
-      - 'hapi-postgres-2-data:/bitnami/postgresql'
+      - "hapi-postgres-2-data:/bitnami/postgresql"
     configs:
       - target: /bitnami/postgresql/conf/conf.d/custom_postgresql.conf
         source: postgresql.conf
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-2"
+          - "node.labels.name==${POSTGRES_2_PLACEMENT}"
       replicas: 1
       resources:
         limits:
@@ -58,14 +58,14 @@ services:
       REPMGR_FAILOVER: ${POSTGRES_FAILOVER}
       REPMGR_DEGRADED_MONITORING_TIMEOUT: ${POSTGRES_DEGRADED_MONITORING_TIMEOUT}
     volumes:
-      - 'hapi-postgres-3-data:/bitnami/postgresql'
+      - "hapi-postgres-3-data:/bitnami/postgresql"
     configs:
       - target: /bitnami/postgresql/conf/conf.d/custom_postgresql.conf
         source: postgresql.conf
     deploy:
       placement:
         constraints:
-          - "node.labels.name==node-3"
+          - "node.labels.name==${POSTGRES_3_PLACEMENT}"
       replicas: 1
       resources:
         limits:
diff --git a/database-postgres/docker-compose-postgres.yml b/database-postgres/docker-compose-postgres.yml
index 2a24c1ea..b5165127 100644
--- a/database-postgres/docker-compose-postgres.yml
+++ b/database-postgres/docker-compose-postgres.yml
@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   postgres-1:
-    image: bitnami/postgresql-repmgr:14
+    image: ${POSTGRES_IMAGE}
     environment:
       POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD}
       REPMGR_NODE_NETWORK_NAME: postgres-1
diff --git a/database-postgres/package-metadata.json b/database-postgres/package-metadata.json
index bc6c441b..c7240670 100644
--- a/database-postgres/package-metadata.json
+++ b/database-postgres/package-metadata.json
@@ -9,6 +9,14 @@
     "REPMGR_PRIMARY_HOST": "postgres-1",
     "REPMGR_PARTNER_NODES": "postgres-1",
     "REPMGR_PASSWORD": "instant101",
+    "POSTGRES_IMAGE": "bitnami/postgresql-repmgr:14",
+    "POSTGRES_1_PLACEMENT": "node-1",
+    "POSTGRES_2_PLACEMENT": "node-2",
+    "POSTGRES_3_PLACEMENT": "node-3",
+    "PG_POOL_IMAGE": "bitnami/pgpool:4.4.3",
+    "PGPOOL_1_PLACEMENT": "node-1",
+    "PGPOOL_2_PLACEMENT": "node-2",
+    "PGPOOL_3_PLACEMENT": "node-3",
     "POSTGRES_REPLICA_SET": "postgres-1:5432",
     "POSTGRES_CPU_LIMIT": "0",
     "POSTGRES_CPU_RESERVE": "0.05",

From cf9cf2770837b7ad77edefdd8ea53cfc17242e7d Mon Sep 17 00:00:00 2001
From: Brett Onions <onionsllb@outlook.com>
Date: Tue, 21 May 2024 11:45:00 +0200
Subject: [PATCH 10/18] chore: update CLICKHOUSE_IMAGE to version 23.8.14.6

---
 analytics-datastore-clickhouse/package-metadata.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/analytics-datastore-clickhouse/package-metadata.json b/analytics-datastore-clickhouse/package-metadata.json
index 4bc99cef..a7b0efb7 100644
--- a/analytics-datastore-clickhouse/package-metadata.json
+++ b/analytics-datastore-clickhouse/package-metadata.json
@@ -8,7 +8,7 @@
   "environmentVariables": {
     "CLICKHOUSE_HOST": "analytics-datastore-clickhouse",
     "CLICKHOUSE_PORT": "8123",
-    "CLICKHOUSE_IMAGE": "clickhouse/clickhouse-server",
+    "CLICKHOUSE_IMAGE": "clickhouse/clickhouse-server:23.8.14.6",
     "ANALYTICS_DATASTORE_CLICKHOUSE_01_PLACEMENT": "node-1",
     "ANALYTICS_DATASTORE_CLICKHOUSE_02_PLACEMENT": "node-2",
     "ANALYTICS_DATASTORE_CLICKHOUSE_03_PLACEMENT": "node-3"

From 3b9816caa5da0350a4d17a67eb7e707d56665b90 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Fri, 24 May 2024 16:22:13 +0200
Subject: [PATCH 11/18] Change the name of the postgres image variable

This is so the value is not overidden
---
 dashboard-visualiser-superset/docker-compose.postgres.yml | 2 +-
 dashboard-visualiser-superset/package-metadata.json       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dashboard-visualiser-superset/docker-compose.postgres.yml b/dashboard-visualiser-superset/docker-compose.postgres.yml
index 10c0a68a..912914cd 100644
--- a/dashboard-visualiser-superset/docker-compose.postgres.yml
+++ b/dashboard-visualiser-superset/docker-compose.postgres.yml
@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   postgres-metastore:
-    image: ${POSTGRES_IMAGE}
+    image: ${SS_POSTGRES_IMAGE}
     environment:
       POSTGRES_USER: ${SUPERSET_POSTGRESQL_USERNAME}
       POSTGRES_PASSWORD: ${SUPERSET_POSTGRESQL_PASSWORD}
diff --git a/dashboard-visualiser-superset/package-metadata.json b/dashboard-visualiser-superset/package-metadata.json
index 69867d91..0d63ff8b 100644
--- a/dashboard-visualiser-superset/package-metadata.json
+++ b/dashboard-visualiser-superset/package-metadata.json
@@ -7,7 +7,7 @@
   "dependencies": ["analytics-datastore-clickhouse"],
   "environmentVariables": {
     "SUPERSET_IMAGE": "apache/superset:3.1.1",
-    "POSTGRES_IMAGE": "postgres:16.2",
+    "SS_POSTGRES_IMAGE": "postgres:16.2",
     "POSTGRES_METASTORE": "node-2",
     "SUPERSET_ENABLED_FEATURE_FLAGS": "DASHBOARD_RBAC",
     "SUPERSET_USERNAME": "admin",

From dad0440e25c1d7048dca53f2b11fa70997e0cdc1 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Mon, 27 May 2024 15:36:01 +0200
Subject: [PATCH 12/18] Disable the clickhouse test case (temporarily)

To debug
---
 .github/workflows/run-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.sh b/.github/workflows/run-tests.sh
index d1f9a7d7..b2797f19 100755
--- a/.github/workflows/run-tests.sh
+++ b/.github/workflows/run-tests.sh
@@ -33,7 +33,7 @@ else
         echo "$folder_name was changed"
 
         if [[ $folder_name == *"clickhouse"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
+            # DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
         elif [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
             DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":elk
         elif [[ $folder_name == *"kafka"* ]] || [[ $folder_name == *"monitoring"* ]]; then

From 12e68e7b5fab1cad173b054a018e043b5dc2d254 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Mon, 27 May 2024 15:57:09 +0200
Subject: [PATCH 13/18] Fix syntax error

---
 .github/workflows/run-tests.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run-tests.sh b/.github/workflows/run-tests.sh
index b2797f19..9af62903 100755
--- a/.github/workflows/run-tests.sh
+++ b/.github/workflows/run-tests.sh
@@ -32,9 +32,10 @@ else
     for folder_name in "${!changed_packages[@]}"; do
         echo "$folder_name was changed"
 
-        if [[ $folder_name == *"clickhouse"* ]]; then
-            # DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
-        elif [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
+        # if [[ $folder_name == *"clickhouse"* ]]; then
+        #     # DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
+        # el
+        if [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
             DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":elk
         elif [[ $folder_name == *"kafka"* ]] || [[ $folder_name == *"monitoring"* ]]; then
             DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":kafka

From 295c48456f42dba82aebdcc14f732464d7292ac1 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Mon, 27 May 2024 19:36:43 +0200
Subject: [PATCH 14/18] Run the recipe tests first

---
 .github/workflows/run-tests.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run-tests.sh b/.github/workflows/run-tests.sh
index 9af62903..c564d063 100755
--- a/.github/workflows/run-tests.sh
+++ b/.github/workflows/run-tests.sh
@@ -25,6 +25,9 @@ for package in "${CHANGED_FILES[@]}"; do
     fi
 done
 
+# Run the basic funtional end to end tests for the CDR recipe
+DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":recipe
+
 if [[ ${#changed_packages[@]} -eq 0 ]] || [[ "${!changed_packages[*]}" == *"utils"* ]] || [[ "${!changed_packages[*]}" == *"features/steps"* ]] || [[ "${!changed_packages[*]}" == *"infrastructure"* ]] ; then
     openhim_ran="true"
     DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":openhim
@@ -32,9 +35,9 @@ else
     for folder_name in "${!changed_packages[@]}"; do
         echo "$folder_name was changed"
 
-        # if [[ $folder_name == *"clickhouse"* ]]; then
-        #     # DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
-        # el
+        if [[ $folder_name == *"clickhouse"* ]]; then
+            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
+        el
         if [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
             DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":elk
         elif [[ $folder_name == *"kafka"* ]] || [[ $folder_name == *"monitoring"* ]]; then
@@ -63,6 +66,3 @@ else
         fi
     done
 fi
-
-# Run the basic funtional end to end tests for the CDR recipe
-DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":recipe

From 598cb04c34cc9c7d460b5952e683600bda1805cd Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Mon, 27 May 2024 19:51:58 +0200
Subject: [PATCH 15/18] Fix typo

---
 .github/workflows/run-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.sh b/.github/workflows/run-tests.sh
index c564d063..1b4cfab8 100755
--- a/.github/workflows/run-tests.sh
+++ b/.github/workflows/run-tests.sh
@@ -37,7 +37,7 @@ else
 
         if [[ $folder_name == *"clickhouse"* ]]; then
             DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
-        el
+        elif
         if [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
             DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":elk
         elif [[ $folder_name == *"kafka"* ]] || [[ $folder_name == *"monitoring"* ]]; then

From 2146f4df9367b42ee29cd6f67ac741bc36c3fcbe Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Mon, 27 May 2024 22:17:59 +0200
Subject: [PATCH 16/18] Fix if statement

---
 .github/workflows/run-tests.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/run-tests.sh b/.github/workflows/run-tests.sh
index 1b4cfab8..754f830d 100755
--- a/.github/workflows/run-tests.sh
+++ b/.github/workflows/run-tests.sh
@@ -37,8 +37,7 @@ else
 
         if [[ $folder_name == *"clickhouse"* ]]; then
             DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
-        elif
-        if [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
+        elif [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
             DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":elk
         elif [[ $folder_name == *"kafka"* ]] || [[ $folder_name == *"monitoring"* ]]; then
             DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":kafka

From 025521b1e68c9a640fac48590f71fa9d27f4c0f9 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Tue, 28 May 2024 09:12:34 +0200
Subject: [PATCH 17/18] Fix typo in variable name

---
 .github/workflows/run-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.sh b/.github/workflows/run-tests.sh
index 754f830d..870c44c8 100755
--- a/.github/workflows/run-tests.sh
+++ b/.github/workflows/run-tests.sh
@@ -29,7 +29,7 @@ done
 DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":recipe
 
 if [[ ${#changed_packages[@]} -eq 0 ]] || [[ "${!changed_packages[*]}" == *"utils"* ]] || [[ "${!changed_packages[*]}" == *"features/steps"* ]] || [[ "${!changed_packages[*]}" == *"infrastructure"* ]] ; then
-    openhim_ran="true"
+    openhimRan="true"
     DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":openhim
 else
     for folder_name in "${!changed_packages[@]}"; do

From d2ca67a076295cf7bdadb23ed0fa90175d426dc7 Mon Sep 17 00:00:00 2001
From: bradsawadye <bradsawadye@gmail.com>
Date: Tue, 28 May 2024 12:30:28 +0200
Subject: [PATCH 18/18] Refactor

---
 .github/workflows/run-tests.sh | 34 +++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/run-tests.sh b/.github/workflows/run-tests.sh
index 870c44c8..71d1447e 100755
--- a/.github/workflows/run-tests.sh
+++ b/.github/workflows/run-tests.sh
@@ -25,43 +25,47 @@ for package in "${CHANGED_FILES[@]}"; do
     fi
 done
 
+function run_test() {
+   DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":$1
+}
+
 # Run the basic funtional end to end tests for the CDR recipe
-DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud HOST=$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":recipe
+run_test "recipe"
 
 if [[ ${#changed_packages[@]} -eq 0 ]] || [[ "${!changed_packages[*]}" == *"utils"* ]] || [[ "${!changed_packages[*]}" == *"features/steps"* ]] || [[ "${!changed_packages[*]}" == *"infrastructure"* ]] ; then
     openhimRan="true"
-    DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":openhim
+    run_test "openhim"
 else
     for folder_name in "${!changed_packages[@]}"; do
         echo "$folder_name was changed"
 
         if [[ $folder_name == *"clickhouse"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":clickhouse
+            run_test "clickhouse"
         elif [[ $folder_name == *"elastic"* ]] || [[ $folder_name == *"kibana"* ]] || [[ $folder_name == *"logstash"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":elk
+            run_test "elk"
         elif [[ $folder_name == *"kafka"* ]] || [[ $folder_name == *"monitoring"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":kafka
+            run_test "kafka"
         elif [[ $folder_name == *"openhim"* ]] && [[ $openhimRan == "false" ]]; then
             openhimRan="true"
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":openhim
+            run_test "openhim"
         elif [[ $folder_name == *"reverse-proxy"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":nginx
+            run_test "nginx"
         elif [[ $folder_name == *"hapi"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":hapi
+            run_test "hapi"
         elif [[ $folder_name == *"santempi"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":sante
+            run_test "sante"
         elif [[ $folder_name == *"monitoring"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":monitoring
+            run_test "monitoring"
         elif [[ $folder_name == *"keycloak"* ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":keycloak
+            run_test "keycloak"
         elif [[ $folder_name == *"superset"* ]] && [[ $NODE_MODE == "single" ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":superset
+            run_test "superset"
         elif [[ $folder_name == *"jsreport"* ]] && [[ $NODE_MODE == "single" ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":jsreport
+            run_test "jsreport"
         elif [[ $folder_name == *"mpi-mediator"* ]] && [[ $NODE_MODE == "single" ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":mpi-mediator
+            run_test "mpi-mediator"
         elif [[ $folder_name == *"jempi"* ]] && [[ $NODE_MODE == "single" ]]; then
-            DOCKER_HOST=ssh://ubuntu@$GITHUB_RUN_ID.jembi.cloud yarn test:"$NODE_MODE":jempi
+            run_test "jempi"
         fi
     done
 fi