Merge branch 'main' into TB-414-parameterise-clickhouse-password

jembi · May 30, 2024 · 4858847 · 4858847
2 parents 6da3495 + 71faa49
commit 4858847
Show file tree

Hide file tree

Showing 21 changed files with 352 additions and 279 deletions.
diff --git a/.env.cluster b/.env.cluster
@@ -45,6 +45,9 @@ ES_HOSTS="\"analytics-datastore-elastic-search-01:9200","analytics-datastore-ela
 # Analytics Datastore - Clickhouse
 CLICKHOUSE_HOST=analytics-datastore-clickhouse-01
 
+# Dashboard Vusualizer - Superset
+SUPERSET_POSTGRESQL_URL=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432
+
 # Dashboard Visualiser - Kibana
 KIBANA_INSTANCES=1
 
@@ -100,6 +103,7 @@ JEMPI_CONTROLLER_INSTANCES=1
 JEMPI_EM_CALCULATOR_INSTANCES=1
 JEMPI_LINKER_INSTANCES=1
 JEMPI_API_INSTANCES=1
+JEMPI_POSTGRES_DB=pgpool-1,pgpool-2,pgpool-3
 
 # Resource limits
 OPENHIM_MEMORY_LIMIT=4G
@@ -110,5 +114,5 @@ KAFDROP_MEMORY_LIMIT=500M
 
 # PG Pool
 # The postgres users have to be specified so that they can be added to the pg_pool authentication interceptor
-PGPOOL_POSTGRES_CUSTOM_USERS=hapi,keycloak
-PGPOOL_POSTGRES_CUSTOM_PASSWORDS=instant101,instant101
+PGPOOL_POSTGRES_CUSTOM_USERS=hapi,keycloak,jempi,superset
+PGPOOL_POSTGRES_CUSTOM_PASSWORDS=instant101,instant101,instant101,instant101
diff --git a/client-registry-jempi/docker-compose.api.yml b/client-registry-jempi/docker-compose.api.yml
@@ -5,10 +5,10 @@ services:
     image: jembi/jempi-api:${JEMPI_API_IMAGE_TAG}
     environment:
       LOG4J2_LEVEL: ${LOG4J2_LEVEL}
-      POSTGRESQL_IP: ${JEMPI_REPMGR_PRIMARY_HOST}
+      POSTGRESQL_IP: ${JEMPI_POSTGRES_DB}
       POSTGRESQL_PORT: 5432
-      POSTGRESQL_USER: ${POSTGRESQL_USERNAME}
-      POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD}
+      POSTGRESQL_USER: ${JEMPI_POSTGRESQL_USERNAME}
+      POSTGRESQL_PASSWORD: ${JEMPI_POSTGRESQL_PASSWORD}
       POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB}
       POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB}
       KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS}
@@ -34,6 +34,7 @@ services:
       kafka:
       default:
       jempi:
+      postgres:
 
 
   jempi-api-kc:
@@ -49,10 +50,10 @@ services:
       JEMPI_SESSION_SECRET: ${JEMPI_SESSION_SECRET}
       JEMPI_SESSION_SECURE: ${JEMPI_SESSION_SECURE}
       JEMPI_SESSION_DOMAIN_NAME: ${JEMPI_SESSION_DOMAIN_NAME}
-      POSTGRESQL_IP: ${JEMPI_REPMGR_PRIMARY_HOST}
+      POSTGRESQL_IP: ${JEMPI_POSTGRES_DB}
       POSTGRESQL_PORT: 5432
-      POSTGRESQL_USER: ${POSTGRESQL_USERNAME}
-      POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD}
+      POSTGRESQL_USER: ${JEMPI_POSTGRESQL_USERNAME}
+      POSTGRESQL_PASSWORD: ${JEMPI_POSTGRESQL_PASSWORD}
       POSTGRESQL_USERS_DB: ${POSTGRESQL_USERS_DB}
       POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB}
       POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB}
@@ -79,6 +80,7 @@ services:
       kafka:
       default:
       jempi:
+      postgres:
 
 
 volumes:
@@ -92,6 +94,9 @@ networks:
   jempi:
     name: jempi_public
     external: true
+  postgres:
+    name: postgres_public
+    external: true
   kafka:
     name: kafka_public
     external: true

diff --git a/client-registry-jempi/docker-compose.combined-cluster.yml b/client-registry-jempi/docker-compose.combined-cluster.yml
@@ -1,87 +1,8 @@
 version: '3.9'
 
 services:
-  jempi-postgresql-01:
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.name==node-1"
-
   jempi-bootstrapper:
     deploy:
       placement:
         constraints:
           - "node.labels.name==node-1"
-
-  jempi-postgresql-02:
-    image: bitnami/postgresql-repmgr:15.2.0
-    environment:
-      POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME}
-      POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
-      REPMGR_PASSWORD: ${JEMPI_REPMGR_PASSWORD}
-      REPMGR_PRIMARY_HOST: ${JEMPI_REPMGR_PRIMARY_HOST}
-      REPMGR_PARTNER_NODES: ${JEMPI_REPMGR_PARTNER_NODES}
-      REPMGR_NODE_NAME: jempi-postgresql-02
-      REPMGR_NODE_NETWORK_NAME: jempi-postgresql-02
-      ALLOW_EMPTY_PASSWORD: "yes"
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.name==node-2"
-      mode: replicated
-      replicas: 1
-      resources:
-        limits:
-          cpus: ${JEMPI_POSTGRES_CPU_LIMIT}
-          memory: ${JEMPI_POSTGRES_MEMORY_LIMIT}
-        reservations:
-          cpus: ${JEMPI_POSTGRES_CPU_RESERVE}
-          memory: ${JEMPI_POSTGRES_MEMORY_RESERVE}
-    volumes:
-      - "jempi-psql-02-data:/bitnami/postgresql"
-    configs:
-      - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql
-        source: jempi_psql_init_db.sql
-    networks:
-      jempi:
-
-  jempi-postgresql-03:
-    image: bitnami/postgresql-repmgr:15.2.0
-    environment:
-      POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME}
-      POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
-      REPMGR_PASSWORD: ${JEMPI_REPMGR_PASSWORD}
-      REPMGR_PRIMARY_HOST: ${JEMPI_REPMGR_PRIMARY_HOST}
-      REPMGR_PARTNER_NODES: ${JEMPI_REPMGR_PARTNER_NODES}
-      REPMGR_NODE_NAME: jempi-postgresql-03
-      REPMGR_NODE_NETWORK_NAME: jempi-postgresql-03
-      ALLOW_EMPTY_PASSWORD: "yes"
-    deploy:
-      placement:
-        constraints:
-          - "node.labels.name==node-3"
-      mode: replicated
-      replicas: 1
-      resources:
-        limits:
-          cpus: ${JEMPI_POSTGRES_CPU_LIMIT}
-          memory: ${JEMPI_POSTGRES_MEMORY_LIMIT}
-        reservations:
-          cpus: ${JEMPI_POSTGRES_CPU_RESERVE}
-          memory: ${JEMPI_POSTGRES_MEMORY_RESERVE}
-    volumes:
-      - "jempi-psql-03-data:/bitnami/postgresql"
-    configs:
-      - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql
-        source: jempi_psql_init_db.sql
-    networks:
-      jempi:
-
-volumes:
-  jempi-psql-02-data:
-  jempi-psql-03-data:
-
-networks:
-  jempi:
-    name: jempi_public
-    external: true
diff --git a/client-registry-jempi/docker-compose.combined-dev.yml b/client-registry-jempi/docker-compose.combined-dev.yml
@@ -1,7 +1,6 @@
 version: '3.9'
 
 services:
-
   jempi-controller:
     ports:
       - published: 50020
@@ -15,9 +14,3 @@ services:
         target: 50000
         protocol: tcp
         mode: host
-
-  jempi-postgresql-01:
-    ports:
-      - published: 5435
-        target: 5432
-        mode: host
diff --git a/client-registry-jempi/docker-compose.combined.yml b/client-registry-jempi/docker-compose.combined.yml
@@ -45,11 +45,11 @@ services:
     image: jembi/jempi-controller:${JEMPI_CONTROLLER_IMAGE_TAG}
     environment:
       LOG4J2_LEVEL: ${LOG4J2_LEVEL}
-      POSTGRESQL_IP: ${JEMPI_REPMGR_PRIMARY_HOST}
+      POSTGRESQL_IP: ${JEMPI_POSTGRES_DB}
       POSTGRESQL_PORT: 5432
       POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
-      POSTGRESQL_USER: ${POSTGRESQL_USERNAME}
-      POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD}
+      POSTGRESQL_USER: ${JEMPI_POSTGRESQL_USERNAME}
+      POSTGRESQL_PASSWORD: ${JEMPI_POSTGRESQL_PASSWORD}
       POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB}
       POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB}
       DGRAPH_HOSTS: ${DGRAPH_HOSTS}
@@ -73,17 +73,18 @@ services:
       kafka:
       default:
       jempi:
+      postgres:
 
 
   jempi-linker:
     image: jembi/jempi-linker:${JEMPI_LINKER_IMAGE_TAG}
     environment:
       LOG4J2_LEVEL: ${LOG4J2_LEVEL}
-      POSTGRESQL_IP: ${JEMPI_REPMGR_PRIMARY_HOST}
+      POSTGRESQL_IP: ${JEMPI_POSTGRES_DB}
       POSTGRESQL_PORT: 5432
       POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
-      POSTGRESQL_USER: ${POSTGRESQL_USERNAME}
-      POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD}
+      POSTGRESQL_USER: ${JEMPI_POSTGRESQL_USERNAME}
+      POSTGRESQL_PASSWORD: ${JEMPI_POSTGRESQL_PASSWORD}
       POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB}
       POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB}
       KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS}
@@ -108,14 +109,16 @@ services:
       kafka:
       default:
       jempi:
+      postgres:
+
 
   jempi-bootstrapper:
     image: jembi/jempi-bootstrapper:${JEMPI_BOOTSTRAPPER_IMAGE_TAG}
     environment:
-      POSTGRESQL_IP: ${JEMPI_REPMGR_PRIMARY_HOST}
+      POSTGRESQL_IP: ${JEMPI_POSTGRES_DB}
       POSTGRESQL_PORT: 5432
-      POSTGRESQL_USER: ${POSTGRESQL_USERNAME}
-      POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD}
+      POSTGRESQL_USER: ${JEMPI_POSTGRESQL_USERNAME}
+      POSTGRESQL_PASSWORD: ${JEMPI_POSTGRESQL_PASSWORD}
       POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
       POSTGRESQL_USERS_DB: ${POSTGRESQL_USERS_DB}
       POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB}
@@ -129,38 +132,10 @@ services:
       kafka:
       default:
       jempi:
+      postgres:
 
-  jempi-postgresql-01:
-    image: bitnami/postgresql-repmgr:15.2.0
-    environment:
-      POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME}
-      POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE}
-      REPMGR_PASSWORD: ${JEMPI_REPMGR_PASSWORD}
-      REPMGR_PRIMARY_HOST: ${JEMPI_REPMGR_PRIMARY_HOST}
-      REPMGR_PARTNER_NODES: ${JEMPI_REPMGR_PARTNER_NODES}
-      REPMGR_NODE_NAME: jempi-postgresql-01
-      REPMGR_NODE_NETWORK_NAME: jempi-postgresql-01
-      ALLOW_EMPTY_PASSWORD: "yes"
-    deploy:
-      mode: replicated
-      replicas: 1
-      resources:
-        limits:
-          cpus: ${JEMPI_POSTGRES_CPU_LIMIT}
-          memory: ${JEMPI_POSTGRES_MEMORY_LIMIT}
-        reservations:
-          cpus: ${JEMPI_POSTGRES_CPU_RESERVE}
-          memory: ${JEMPI_POSTGRES_MEMORY_RESERVE}
-    volumes:
-      - "jempi-psql-01-data:/bitnami/postgresql"
-    configs:
-      - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql
-        source: jempi_psql_init_db.sql
-    networks:
-      jempi:
 
 volumes:
-  jempi-psql-01-data:
   jempi-shared-data:
 
 
@@ -172,11 +147,6 @@ networks:
   jempi:
     name: jempi_public
     external: true
-
-
-configs:
-  jempi_psql_init_db.sql:
-    file: ./importer/jempi_psql_init_db.sql
-    name: jempi_psql_init_db.sql-${jempi_psql_init_db_sql_DIGEST:?err}
-    labels:
-      name: jempi
+  postgres:
+    name: postgres_public
+    external: true
diff --git a/client-registry-jempi/importer/jempi_psql_init_db.sql b/client-registry-jempi/importer/jempi_psql_init_db.sql