From 07e0374290cc9b3e67084161d1ba56eb7d3f87d1 Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Fri, 24 May 2024 12:09:17 +0200 Subject: [PATCH 1/9] Configure the database Create the postgres tables and user that will be used by the jempi service --- .../importer/postgres/create-db.js | 136 ++++++++++++++++++ .../postgres/docker-compose.config.yml | 40 ++++++ .../importer/postgres/package.json | 11 ++ client-registry-jempi/swarm.sh | 3 + 4 files changed, 190 insertions(+) create mode 100644 client-registry-jempi/importer/postgres/create-db.js create mode 100644 client-registry-jempi/importer/postgres/docker-compose.config.yml create mode 100644 client-registry-jempi/importer/postgres/package.json diff --git a/client-registry-jempi/importer/postgres/create-db.js b/client-registry-jempi/importer/postgres/create-db.js new file mode 100644 index 00000000..b491e9ec --- /dev/null +++ b/client-registry-jempi/importer/postgres/create-db.js @@ -0,0 +1,136 @@ +const { Pool } = require('pg'); + +const user = process.env.POSTGRES_USER || 'postgres' +const host = process.env.POSTGRES_SERVICE || 'postgres-1' +const database = process.env.POSTGRES_DATABASE || 'jempi' +const password = process.env.POSTGRES_PASSWORD || 'instant101' +const port = process.env.POSTGRES_PORT || 5432 +const newDb = process.env.NEW_DATABASE_NAME || 'jempi' +const newUser = process.env.NEW_DATABASE_USER || 'keycloak' +const newUserPassword = process.env.NEW_DATABASE_PASSWORD || 'instant101' + +const pool = new Pool({ + user, + host, + database, + password, + port +}); + +const tableQueries = [ + `CREATE TABLE IF NOT EXISTS Notification_Type + ( + Id uuid DEFAULT gen_random_uuid() PRIMARY KEY, + Type VARCHAR(50) + );`, + `CREATE TABLE IF NOT EXISTS Action_Type + ( + Id UUID DEFAULT gen_random_uuid() PRIMARY KEY UNIQUE, + Type VARCHAR(50) + );, + `, + `CREATE TABLE IF NOT EXISTS Notification_State + ( + Id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + State VARCHAR(50) + );`, + `CREATE TABLE IF NOT EXISTS Notification + ( + Id uuid DEFAULT gen_random_uuid() PRIMARY KEY, + Type VARCHAR(50), + Created date, + Reviewd_By uuid, + Reviewed_At timestamp without time zone, + State VARCHAR(50), + Patient_Id VARCHAR(50), + Names VARCHAR(100), + Golden_Id VARCHAR(50), + Score Numeric + );`, + `CREATE TABLE IF NOT EXISTS Action + ( + Id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + Notification_Id UUID, + Action_Type_Id UUID, + Date date, + CONSTRAINT FK_Notification + FOREIGN KEY(Notification_Id) + REFERENCES Notification(Id), + CONSTRAINT FK_Action_Type + FOREIGN KEY(Action_Type_Id) + REFERENCES Action_Type(Id) + );`, + `CREATE TABLE IF NOT EXISTS Match + ( + Notification_Id UUID, + Score Numeric, + Golden_Id VARCHAR(50), + CONSTRAINT FK_Notification + FOREIGN KEY(Notification_Id) + REFERENCES Notification(Id) + );`, + `CREATE TABLE IF NOT EXISTS candidates + ( + Notification_Id UUID, + Score Numeric, + Golden_Id VARCHAR(50), + CONSTRAINT FK_Notification + FOREIGN KEY(Notification_Id) + REFERENCES Notification(Id) + );`, + `CREATE TABLE IF NOT EXISTS users + ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY UNIQUE, + given_name VARCHAR(255), + family_name VARCHAR(255), + email VARCHAR(255) UNIQUE, + username VARCHAR(255) UNIQUE + );` +] +const insertQueries = [`INSERT INTO Notification_State(State) + VALUES ('New'), ('Seen'), ('Actioned'), ('Accepted'), ('Pending'); + `, + `INSERT INTO Notification_Type(Type) + VALUES ('THRESHOLD'), ('MARGIN'), ('UPDATE');` +]; + +(async () => { + const client = await pool.connect() + + const createDb = async db => { + //Check db exists before creating + + const result = await client.query('SELECT 1 FROM pg_database WHERE datname = $1', [db]) + + if (!result.rows.length) { + await client.query(`CREATE DATABASE ${db};`) + + console.log(`Database '${db}' created successfully`) + } else { + console.log(`Database '${db}' already exists`) + } + } + + const createUSer = async () => { + const user = await client.query('SELECT 1 FROM pg_user WHERE usename = $1', [newUser]) + + if (!user.rows.length) { + await client.query(`CREATE USER ${newUser} WITH ENCRYPTED PASSWORD '${newUserPassword}';`) + console.log(`User ${newUser} created`) + } + } + + try { + await createDb(newDb) + + await createUSer() + await Promise.all(tableQueries.map(query => client.query(query))) + + await Promise.all(insertQueries.map(query => client.query(query))) + } catch (error) { + console.error('Error in db operations:', error.message) + } finally { + client.release() + pool.end() + } +})(); diff --git a/client-registry-jempi/importer/postgres/docker-compose.config.yml b/client-registry-jempi/importer/postgres/docker-compose.config.yml new file mode 100644 index 00000000..5d01adc7 --- /dev/null +++ b/client-registry-jempi/importer/postgres/docker-compose.config.yml @@ -0,0 +1,40 @@ +version: '3.9' + +services: + jempi_db_config: + image: node:erbium-alpine + command: sh -c "cd /importer && ls && npm i && ls && node create-db.js" + configs: + - target: /importer/package.json + source: package.json + - target: /importer/create-db.js + source: create-db.js + networks: + postgres: + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_SERVICE: ${POSTGRES_SERVICE} + POSTGRES_DATABASE: ${POSTGRES_DATABASE} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PORT: ${POSTGRES_PORT} + deploy: + replicas: 1 + restart_policy: + condition: none + +networks: + postgres: + name: postgres_public + external: true + +configs: + package.json: + file: ./package.json + name: package.json-${package_json_DIGEST:?err} + labels: + name: jempi + create-db.js: + file: ./create-db.js + name: create_db.js-${create_db_js_DIGEST:?err} + labels: + name: jempi diff --git a/client-registry-jempi/importer/postgres/package.json b/client-registry-jempi/importer/postgres/package.json new file mode 100644 index 00000000..ebef53f8 --- /dev/null +++ b/client-registry-jempi/importer/postgres/package.json @@ -0,0 +1,11 @@ +{ + "name": "postgres-config", + "version": "0.0.1", + "main": "index.js", + "scripts": {}, + "author": "Jembi", + "license": "Apache-2.0", + "dependencies": { + "pg": "^8.11.3" + } +} diff --git a/client-registry-jempi/swarm.sh b/client-registry-jempi/swarm.sh index d1323691..808bdd51 100644 --- a/client-registry-jempi/swarm.sh +++ b/client-registry-jempi/swarm.sh @@ -65,6 +65,9 @@ function initialize_package() { log info "Importing mapping endpoints" docker::deploy_config_importer $STACK "$COMPOSE_FILE_PATH/importer/mapping-mediator/docker-compose.config.yml" "mapping-mediator-config-importer" "jempi" + log info "Configuring postgres database" + docker::deploy_config_importer $STACK "$COMPOSE_FILE_PATH/importer/postgres/docker-compose.config.yml" "jempi_db_config" "jempi" + log info "Deploy Dgraph" docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.dgraph-zero.yml" "$dgraph_zero_dev_compose_param" "$dgraph_zero_cluster_compose_param" From 574536364e0d999fe52e8e2422819ce5efe4a4c2 Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Fri, 24 May 2024 12:11:03 +0200 Subject: [PATCH 2/9] Use the database postgres package and clean up --- client-registry-jempi/docker-compose.api.yml | 17 ++-- .../docker-compose.combined-cluster.yml | 79 ------------------ .../docker-compose.combined-dev.yml | 7 -- .../docker-compose.combined.yml | 62 ++++---------- .../importer/jempi_psql_init_db.sql | 82 ------------------- client-registry-jempi/package-metadata.json | 18 ++-- 6 files changed, 35 insertions(+), 230 deletions(-) delete mode 100644 client-registry-jempi/importer/jempi_psql_init_db.sql diff --git a/client-registry-jempi/docker-compose.api.yml b/client-registry-jempi/docker-compose.api.yml index 7d218748..c3005fb0 100644 --- a/client-registry-jempi/docker-compose.api.yml +++ b/client-registry-jempi/docker-compose.api.yml @@ -5,10 +5,10 @@ services: image: jembi/jempi-api:${JEMPI_API_IMAGE_TAG} environment: LOG4J2_LEVEL: ${LOG4J2_LEVEL} - POSTGRESQL_IP: ${JEMPI_REPMGR_PRIMARY_HOST} + POSTGRESQL_IP: ${JEMPI_POSTGRES_DB} POSTGRESQL_PORT: 5432 - POSTGRESQL_USER: ${POSTGRESQL_USERNAME} - POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} + POSTGRESQL_USER: ${JEMPI_POSTGRESQL_USERNAME} + POSTGRESQL_PASSWORD: ${JEMPI_POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS} @@ -34,6 +34,7 @@ services: kafka: default: jempi: + postgres: jempi-api-kc: @@ -49,10 +50,10 @@ services: JEMPI_SESSION_SECRET: ${JEMPI_SESSION_SECRET} JEMPI_SESSION_SECURE: ${JEMPI_SESSION_SECURE} JEMPI_SESSION_DOMAIN_NAME: ${JEMPI_SESSION_DOMAIN_NAME} - POSTGRESQL_IP: ${JEMPI_REPMGR_PRIMARY_HOST} + POSTGRESQL_IP: ${JEMPI_POSTGRES_DB} POSTGRESQL_PORT: 5432 - POSTGRESQL_USER: ${POSTGRESQL_USERNAME} - POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} + POSTGRESQL_USER: ${JEMPI_POSTGRESQL_USERNAME} + POSTGRESQL_PASSWORD: ${JEMPI_POSTGRESQL_PASSWORD} POSTGRESQL_USERS_DB: ${POSTGRESQL_USERS_DB} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} @@ -79,6 +80,7 @@ services: kafka: default: jempi: + postgres: volumes: @@ -92,6 +94,9 @@ networks: jempi: name: jempi_public external: true + postgres: + name: postgres_public + external: true kafka: name: kafka_public external: true diff --git a/client-registry-jempi/docker-compose.combined-cluster.yml b/client-registry-jempi/docker-compose.combined-cluster.yml index 2781e809..f5928fd2 100644 --- a/client-registry-jempi/docker-compose.combined-cluster.yml +++ b/client-registry-jempi/docker-compose.combined-cluster.yml @@ -1,87 +1,8 @@ version: '3.9' services: - jempi-postgresql-01: - deploy: - placement: - constraints: - - "node.labels.name==node-1" - jempi-bootstrapper: deploy: placement: constraints: - "node.labels.name==node-1" - - jempi-postgresql-02: - image: bitnami/postgresql-repmgr:15.2.0 - environment: - POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME} - POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE} - REPMGR_PASSWORD: ${JEMPI_REPMGR_PASSWORD} - REPMGR_PRIMARY_HOST: ${JEMPI_REPMGR_PRIMARY_HOST} - REPMGR_PARTNER_NODES: ${JEMPI_REPMGR_PARTNER_NODES} - REPMGR_NODE_NAME: jempi-postgresql-02 - REPMGR_NODE_NETWORK_NAME: jempi-postgresql-02 - ALLOW_EMPTY_PASSWORD: "yes" - deploy: - placement: - constraints: - - "node.labels.name==node-2" - mode: replicated - replicas: 1 - resources: - limits: - cpus: ${JEMPI_POSTGRES_CPU_LIMIT} - memory: ${JEMPI_POSTGRES_MEMORY_LIMIT} - reservations: - cpus: ${JEMPI_POSTGRES_CPU_RESERVE} - memory: ${JEMPI_POSTGRES_MEMORY_RESERVE} - volumes: - - "jempi-psql-02-data:/bitnami/postgresql" - configs: - - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql - source: jempi_psql_init_db.sql - networks: - jempi: - - jempi-postgresql-03: - image: bitnami/postgresql-repmgr:15.2.0 - environment: - POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME} - POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE} - REPMGR_PASSWORD: ${JEMPI_REPMGR_PASSWORD} - REPMGR_PRIMARY_HOST: ${JEMPI_REPMGR_PRIMARY_HOST} - REPMGR_PARTNER_NODES: ${JEMPI_REPMGR_PARTNER_NODES} - REPMGR_NODE_NAME: jempi-postgresql-03 - REPMGR_NODE_NETWORK_NAME: jempi-postgresql-03 - ALLOW_EMPTY_PASSWORD: "yes" - deploy: - placement: - constraints: - - "node.labels.name==node-3" - mode: replicated - replicas: 1 - resources: - limits: - cpus: ${JEMPI_POSTGRES_CPU_LIMIT} - memory: ${JEMPI_POSTGRES_MEMORY_LIMIT} - reservations: - cpus: ${JEMPI_POSTGRES_CPU_RESERVE} - memory: ${JEMPI_POSTGRES_MEMORY_RESERVE} - volumes: - - "jempi-psql-03-data:/bitnami/postgresql" - configs: - - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql - source: jempi_psql_init_db.sql - networks: - jempi: - -volumes: - jempi-psql-02-data: - jempi-psql-03-data: - -networks: - jempi: - name: jempi_public - external: true diff --git a/client-registry-jempi/docker-compose.combined-dev.yml b/client-registry-jempi/docker-compose.combined-dev.yml index 80d14f9c..628aab66 100644 --- a/client-registry-jempi/docker-compose.combined-dev.yml +++ b/client-registry-jempi/docker-compose.combined-dev.yml @@ -1,7 +1,6 @@ version: '3.9' services: - jempi-controller: ports: - published: 50020 @@ -15,9 +14,3 @@ services: target: 50000 protocol: tcp mode: host - - jempi-postgresql-01: - ports: - - published: 5435 - target: 5432 - mode: host diff --git a/client-registry-jempi/docker-compose.combined.yml b/client-registry-jempi/docker-compose.combined.yml index 0e6a1a2b..7b58f260 100644 --- a/client-registry-jempi/docker-compose.combined.yml +++ b/client-registry-jempi/docker-compose.combined.yml @@ -45,11 +45,11 @@ services: image: jembi/jempi-controller:${JEMPI_CONTROLLER_IMAGE_TAG} environment: LOG4J2_LEVEL: ${LOG4J2_LEVEL} - POSTGRESQL_IP: ${JEMPI_REPMGR_PRIMARY_HOST} + POSTGRESQL_IP: ${JEMPI_POSTGRES_DB} POSTGRESQL_PORT: 5432 POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE} - POSTGRESQL_USER: ${POSTGRESQL_USERNAME} - POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} + POSTGRESQL_USER: ${JEMPI_POSTGRESQL_USERNAME} + POSTGRESQL_PASSWORD: ${JEMPI_POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} DGRAPH_HOSTS: ${DGRAPH_HOSTS} @@ -73,17 +73,18 @@ services: kafka: default: jempi: + postgres: jempi-linker: image: jembi/jempi-linker:${JEMPI_LINKER_IMAGE_TAG} environment: LOG4J2_LEVEL: ${LOG4J2_LEVEL} - POSTGRESQL_IP: ${JEMPI_REPMGR_PRIMARY_HOST} + POSTGRESQL_IP: ${JEMPI_POSTGRES_DB} POSTGRESQL_PORT: 5432 POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE} - POSTGRESQL_USER: ${POSTGRESQL_USERNAME} - POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} + POSTGRESQL_USER: ${JEMPI_POSTGRESQL_USERNAME} + POSTGRESQL_PASSWORD: ${JEMPI_POSTGRESQL_PASSWORD} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} POSTGRESQL_AUDIT_DB: ${POSTGRESQL_AUDIT_DB} KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS} @@ -108,14 +109,16 @@ services: kafka: default: jempi: + postgres: + jempi-bootstrapper: image: jembi/jempi-bootstrapper:${JEMPI_BOOTSTRAPPER_IMAGE_TAG} environment: - POSTGRESQL_IP: ${JEMPI_REPMGR_PRIMARY_HOST} + POSTGRESQL_IP: ${JEMPI_POSTGRES_DB} POSTGRESQL_PORT: 5432 - POSTGRESQL_USER: ${POSTGRESQL_USERNAME} - POSTGRESQL_PASSWORD: ${POSTGRESQL_PASSWORD} + POSTGRESQL_USER: ${JEMPI_POSTGRESQL_USERNAME} + POSTGRESQL_PASSWORD: ${JEMPI_POSTGRESQL_PASSWORD} POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE} POSTGRESQL_USERS_DB: ${POSTGRESQL_USERS_DB} POSTGRESQL_NOTIFICATIONS_DB: ${POSTGRESQL_NOTIFICATIONS_DB} @@ -129,38 +132,10 @@ services: kafka: default: jempi: + postgres: - jempi-postgresql-01: - image: bitnami/postgresql-repmgr:15.2.0 - environment: - POSTGRESQL_USERNAME: ${POSTGRESQL_USERNAME} - POSTGRESQL_DATABASE: ${POSTGRESQL_DATABASE} - REPMGR_PASSWORD: ${JEMPI_REPMGR_PASSWORD} - REPMGR_PRIMARY_HOST: ${JEMPI_REPMGR_PRIMARY_HOST} - REPMGR_PARTNER_NODES: ${JEMPI_REPMGR_PARTNER_NODES} - REPMGR_NODE_NAME: jempi-postgresql-01 - REPMGR_NODE_NETWORK_NAME: jempi-postgresql-01 - ALLOW_EMPTY_PASSWORD: "yes" - deploy: - mode: replicated - replicas: 1 - resources: - limits: - cpus: ${JEMPI_POSTGRES_CPU_LIMIT} - memory: ${JEMPI_POSTGRES_MEMORY_LIMIT} - reservations: - cpus: ${JEMPI_POSTGRES_CPU_RESERVE} - memory: ${JEMPI_POSTGRES_MEMORY_RESERVE} - volumes: - - "jempi-psql-01-data:/bitnami/postgresql" - configs: - - target: /docker-entrypoint-initdb.d/jempi_psql_init_db.sql - source: jempi_psql_init_db.sql - networks: - jempi: volumes: - jempi-psql-01-data: jempi-shared-data: @@ -172,11 +147,6 @@ networks: jempi: name: jempi_public external: true - - -configs: - jempi_psql_init_db.sql: - file: ./importer/jempi_psql_init_db.sql - name: jempi_psql_init_db.sql-${jempi_psql_init_db_sql_DIGEST:?err} - labels: - name: jempi + postgres: + name: postgres_public + external: true diff --git a/client-registry-jempi/importer/jempi_psql_init_db.sql b/client-registry-jempi/importer/jempi_psql_init_db.sql deleted file mode 100644 index 34b2cd61..00000000 --- a/client-registry-jempi/importer/jempi_psql_init_db.sql +++ /dev/null @@ -1,82 +0,0 @@ -CREATE TABLE IF NOT EXISTS Notification_Type -( - Id uuid DEFAULT gen_random_uuid() PRIMARY KEY, - Type VARCHAR(50) -); - -CREATE TABLE IF NOT EXISTS Action_Type -( - Id UUID DEFAULT gen_random_uuid() PRIMARY KEY UNIQUE, - Type VARCHAR(50) -); - -CREATE TABLE IF NOT EXISTS Notification_State -( - Id UUID DEFAULT gen_random_uuid() PRIMARY KEY, - State VARCHAR(50) -); - -CREATE TABLE IF NOT EXISTS Notification -( - Id uuid DEFAULT gen_random_uuid() PRIMARY KEY, - Type VARCHAR(50), - Created date, - Reviewd_By uuid, - Reviewed_At timestamp without time zone, - State VARCHAR(50), - Patient_Id VARCHAR(50), - Names VARCHAR(100), - Golden_Id VARCHAR(50), - Score Numeric -); - -CREATE TABLE IF NOT EXISTS Action -( - Id UUID DEFAULT gen_random_uuid() PRIMARY KEY, - Notification_Id UUID, - Action_Type_Id UUID, - Date date, - CONSTRAINT FK_Notification - FOREIGN KEY(Notification_Id) - REFERENCES Notification(Id), - CONSTRAINT FK_Action_Type - FOREIGN KEY(Action_Type_Id) - REFERENCES Action_Type(Id) -); - -CREATE TABLE IF NOT EXISTS Match -( - Notification_Id UUID, - Score Numeric, - Golden_Id VARCHAR(50), - CONSTRAINT FK_Notification - FOREIGN KEY(Notification_Id) - REFERENCES Notification(Id) -); - -CREATE TABLE IF NOT EXISTS candidates -( - Notification_Id UUID, - Score Numeric, - Golden_Id VARCHAR(50), - CONSTRAINT FK_Notification - FOREIGN KEY(Notification_Id) - REFERENCES Notification(Id) -); - -CREATE TABLE IF NOT EXISTS users -( - id UUID DEFAULT gen_random_uuid() PRIMARY KEY UNIQUE, - given_name VARCHAR(255), - family_name VARCHAR(255), - email VARCHAR(255) UNIQUE, - username VARCHAR(255) UNIQUE -); - -INSERT INTO Notification_State(State) -VALUES ('New'), ('Seen'), ('Actioned'), ('Accepted'), ('Pending'); - -INSERT INTO Notification_Type(Type) -VALUES ('THRESHOLD'), ('MARGIN'), ('UPDATE'); - -\dt; diff --git a/client-registry-jempi/package-metadata.json b/client-registry-jempi/package-metadata.json index 5f987422..4d940d2c 100644 --- a/client-registry-jempi/package-metadata.json +++ b/client-registry-jempi/package-metadata.json @@ -7,7 +7,8 @@ "dependencies": [ "openhim-mapping-mediator", "message-bus-kafka", - "identity-access-manager-keycloak" + "identity-access-manager-keycloak", + "database-postgres" ], "environmentVariables": { "KAFKA_HOSTS": "kafka-01:9092", @@ -60,23 +61,20 @@ "KC_JEMPI_CLIENT_ID": "jempi-oauth", "KC_JEMPI_CLIENT_SECRET": "Tbe3llP5OJIlqUjz7K1wPp8YDAdCOEMn", "KC_JEMPI_ROOT_URL": "http://localhost:3033", + "POSTGRES_SERVICE": "postgres-1", + "JEMPI_POSTGRES_DB": "postgres-1", + "JEMPI_POSTGRESQL_USER": "jempi", + "JEMPI_POSTGRESQL_PASSWORD": "instant101", "POSTGRESQL_USERS_DB": "users_db", "POSTGRESQL_NOTIFICATIONS_DB": "notifications_db", "POSTGRESQL_AUDIT_DB": "audit_db", "POSTGRESQL_KC_TEST_DB": "kc_test_db", - "POSTGRESQL_DATABASE": "postgres", + "POSTGRESQL_DATABASE": "jempi", "POSTGRESQL_USERNAME": "postgres", - "JEMPI_REPMGR_PASSWORD": "dev_password_only", - "JEMPI_REPMGR_PRIMARY_HOST": "jempi-postgresql-01", - "JEMPI_REPMGR_PARTNER_NODES": "jempi-postgresql-01", - "JEMPI_POSTGRES_CPU_LIMIT": "0", - "JEMPI_POSTGRES_CPU_RESERVE": "0.05", - "JEMPI_POSTGRES_MEMORY_LIMIT": "3G", - "JEMPI_POSTGRES_MEMORY_RESERVE": "500M", + "POSTGRESQL_PASSWORD": "instant101", "JEMPI_SESSION_SECURE": false, "JEMPI_SESSION_DOMAIN_NAME": "localhost", "DOMAIN_NAME": "", - "POSTGRESQL_PASSWORD": "postgres", "KAFKA_APPLICATION_ID_API": "api-app-id", "DGRAPH_HOSTS": "jempi-alpha-01,jempi-alpha-02,jempi-alpha-03", "DGRAPH_PORTS": "9080,9081,9082", From d668dee37cf4ecc3d03d1a4fc6d4c8e76eaa85ad Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Fri, 24 May 2024 12:12:46 +0200 Subject: [PATCH 3/9] Add jempi env variables for postgres database --- .env.cluster | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.env.cluster b/.env.cluster index a1215e27..e9f98732 100644 --- a/.env.cluster +++ b/.env.cluster @@ -100,6 +100,7 @@ JEMPI_CONTROLLER_INSTANCES=1 JEMPI_EM_CALCULATOR_INSTANCES=1 JEMPI_LINKER_INSTANCES=1 JEMPI_API_INSTANCES=1 +JEMPI_POSTGRES_DB=pgpool-1,pgpool-2,pgpool-3 # Resource limits OPENHIM_MEMORY_LIMIT=4G @@ -110,5 +111,5 @@ KAFDROP_MEMORY_LIMIT=500M # PG Pool # The postgres users have to be specified so that they can be added to the pg_pool authentication interceptor -PGPOOL_POSTGRES_CUSTOM_USERS=hapi,keycloak -PGPOOL_POSTGRES_CUSTOM_PASSWORDS=instant101,instant101 +PGPOOL_POSTGRES_CUSTOM_USERS=hapi,keycloak,jempi +PGPOOL_POSTGRES_CUSTOM_PASSWORDS=instant101,instant101,instant101 From 542dbe990528d22ef250fcbe36a26095dbb8e045 Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Tue, 28 May 2024 15:22:25 +0200 Subject: [PATCH 4/9] Configure superset to use the postgres database package --- .env.cluster | 7 ++- .../docker-compose.postgres.cluster.yml | 8 --- .../docker-compose.postgres.yml | 24 --------- .../docker-compose.yml | 4 ++ .../importer/postgres/create-db.js | 51 +++++++++++++++++++ .../postgres/docker-compose.config.yml | 43 ++++++++++++++++ .../importer/postgres/package.json | 11 ++++ .../package-metadata.json | 12 +++-- dashboard-visualiser-superset/swarm.sh | 8 +-- 9 files changed, 123 insertions(+), 45 deletions(-) delete mode 100644 dashboard-visualiser-superset/docker-compose.postgres.cluster.yml delete mode 100644 dashboard-visualiser-superset/docker-compose.postgres.yml create mode 100644 dashboard-visualiser-superset/importer/postgres/create-db.js create mode 100644 dashboard-visualiser-superset/importer/postgres/docker-compose.config.yml create mode 100644 dashboard-visualiser-superset/importer/postgres/package.json diff --git a/.env.cluster b/.env.cluster index e9f98732..a00d240d 100644 --- a/.env.cluster +++ b/.env.cluster @@ -45,6 +45,9 @@ ES_HOSTS="\"analytics-datastore-elastic-search-01:9200","analytics-datastore-ela # Analytics Datastore - Clickhouse CLICKHOUSE_HOST=analytics-datastore-clickhouse-01 +# Dashboard Vusualizer - Superset +SUPERSET_POSTGRESQL_URL=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432 + # Dashboard Visualiser - Kibana KIBANA_INSTANCES=1 @@ -111,5 +114,5 @@ KAFDROP_MEMORY_LIMIT=500M # PG Pool # The postgres users have to be specified so that they can be added to the pg_pool authentication interceptor -PGPOOL_POSTGRES_CUSTOM_USERS=hapi,keycloak,jempi -PGPOOL_POSTGRES_CUSTOM_PASSWORDS=instant101,instant101,instant101 +PGPOOL_POSTGRES_CUSTOM_USERS=hapi,keycloak,jempi,superset +PGPOOL_POSTGRES_CUSTOM_PASSWORDS=instant101,instant101,instant101,instant101 diff --git a/dashboard-visualiser-superset/docker-compose.postgres.cluster.yml b/dashboard-visualiser-superset/docker-compose.postgres.cluster.yml deleted file mode 100644 index e4c6ff84..00000000 --- a/dashboard-visualiser-superset/docker-compose.postgres.cluster.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: '3.9' - -services: - postgres-metastore: - deploy: - placement: - constraints: - - "node.labels.name==node-2" diff --git a/dashboard-visualiser-superset/docker-compose.postgres.yml b/dashboard-visualiser-superset/docker-compose.postgres.yml deleted file mode 100644 index cc4cceae..00000000 --- a/dashboard-visualiser-superset/docker-compose.postgres.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: "3.9" - -services: - postgres-metastore: - image: postgres:16.2 - environment: - POSTGRES_USER: ${SUPERSET_POSTGRESQL_USERNAME} - POSTGRES_PASSWORD: ${SUPERSET_POSTGRESQL_PASSWORD} - POSTGRES_DB: ${SUPERSET_POSTGRESQL_DATABASE} - volumes: - - "superset-postgres-data:/var/lib/postgresql/data" - deploy: - replicas: 1 - resources: - limits: - memory: ${SUPERSET_POSTGRES_MEMORY_LIMIT} - networks: - default: - -volumes: - superset-postgres-data: - -networks: - default: diff --git a/dashboard-visualiser-superset/docker-compose.yml b/dashboard-visualiser-superset/docker-compose.yml index 78389671..e7b892d4 100644 --- a/dashboard-visualiser-superset/docker-compose.yml +++ b/dashboard-visualiser-superset/docker-compose.yml @@ -36,6 +36,7 @@ services: clickhouse: keycloak: reverse-proxy: + postgres: default: configs: @@ -73,4 +74,7 @@ networks: reverse-proxy: name: reverse-proxy_public external: true + postgres: + name: postgres_public + external: true default: diff --git a/dashboard-visualiser-superset/importer/postgres/create-db.js b/dashboard-visualiser-superset/importer/postgres/create-db.js new file mode 100644 index 00000000..ae3be5bc --- /dev/null +++ b/dashboard-visualiser-superset/importer/postgres/create-db.js @@ -0,0 +1,51 @@ +const { Pool } = require('pg'); + +const user = process.env.POSTGRES_USER || 'postgres' +const host = process.env.POSTGRES_SERVICE || 'localhost' +const database = process.env.POSTGRES_DATABASE || 'postgres' +const password = process.env.POSTGRES_PASSWORD || 'instant101' +const port = process.env.POSTGRES_PORT || 5432 +const newDbs = process.env.NEW_DATABASE_NAME || 'superset' +const newUser = process.env.NEW_DATABASE_USER || 'superset' +const newUserPassword = process.env.NEW_DATABASE_PASSWORD || 'instant101' + +const pool = new Pool({ + user, + host, + database, + password, + port +}); + +(async () => { + const client = await pool.connect() + + const createDb = async db => { + //Check db exists before creating + const result = await client.query('SELECT 1 FROM pg_database WHERE datname = $1', [db]) + + if (!result.rows.length) { + const user = await client.query('SELECT 1 FROM pg_user WHERE usename = $1', [newUser]) + + if (!user.rows.length) { + await client.query(`CREATE USER ${newUser} WITH ENCRYPTED PASSWORD '${newUserPassword}';`) + console.log(`User ${newUser} created`) + } + + await client.query(`CREATE DATABASE ${db};`) + + console.log(`Database '${db}' created successfully`) + } else { + console.log(`Database '${db}' already exists`) + } + } + + try { + await Promise.all(newDbs.split(',').map(db => createDb(db))) + } catch (error) { + console.error('Error creating database:', error.message) + } finally { + client.release() + pool.end() + } +})(); diff --git a/dashboard-visualiser-superset/importer/postgres/docker-compose.config.yml b/dashboard-visualiser-superset/importer/postgres/docker-compose.config.yml new file mode 100644 index 00000000..665da975 --- /dev/null +++ b/dashboard-visualiser-superset/importer/postgres/docker-compose.config.yml @@ -0,0 +1,43 @@ +version: '3.9' + +services: + superset_db_config: + image: node:erbium-alpine + command: sh -c "cd /importer && ls && npm i && ls && node create-db.js" + configs: + - target: /importer/package.json + source: package.json + - target: /importer/create-db.js + source: create-db.js + networks: + postgres: + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_SERVICE: ${POSTGRES_SERVICE} + POSTGRES_DATABASE: ${POSTGRES_DATABASE} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PORT: ${POSTGRES_PORT} + NEW_DATABASE_NAME: ${SUPERSET_POSTGRESQL_DATABASE} + NEW_DATABASE_USER: ${SUPERSET_POSTGRESQL_USERNAME} + NEW_DATABASE_PASSWORD: ${SUPERSET_POSTGRESQL_PASSWORD} + deploy: + replicas: 1 + restart_policy: + condition: none + +networks: + postgres: + name: postgres_public + external: true + +configs: + package.json: + file: ./package.json + name: package.json-${package_json_DIGEST:?err} + labels: + name: superset + create-db.js: + file: ./create-db.js + name: create_db.js-${create_db_js_DIGEST:?err} + labels: + name: superset diff --git a/dashboard-visualiser-superset/importer/postgres/package.json b/dashboard-visualiser-superset/importer/postgres/package.json new file mode 100644 index 00000000..ebef53f8 --- /dev/null +++ b/dashboard-visualiser-superset/importer/postgres/package.json @@ -0,0 +1,11 @@ +{ + "name": "postgres-config", + "version": "0.0.1", + "main": "index.js", + "scripts": {}, + "author": "Jembi", + "license": "Apache-2.0", + "dependencies": { + "pg": "^8.11.3" + } +} diff --git a/dashboard-visualiser-superset/package-metadata.json b/dashboard-visualiser-superset/package-metadata.json index a5068d95..f1d9e7f2 100644 --- a/dashboard-visualiser-superset/package-metadata.json +++ b/dashboard-visualiser-superset/package-metadata.json @@ -4,7 +4,7 @@ "description": "A dashboard to interpret the data from the Clickhouse data store", "type": "infrastructure", "version": "0.0.1", - "dependencies": ["analytics-datastore-clickhouse"], + "dependencies": ["database-postgres", "analytics-datastore-clickhouse"], "environmentVariables": { "SUPERSET_IMAGE": "apache/superset:3.1.1", "SUPERSET_ENABLED_FEATURE_FLAGS": "DASHBOARD_RBAC", @@ -24,10 +24,12 @@ "KC_FRONTEND_URL": "http://localhost:9088", "KC_API_URL": "http://identity-access-manager-keycloak:8080", "AUTH_USER_REGISTRATION_ROLE": "Admin", - "SUPERSET_POSTGRES_MEMORY_LIMIT": "1G", - "SUPERSET_POSTGRESQL_USERNAME": "admin", - "SUPERSET_POSTGRESQL_PASSWORD": "admin", + "POSTGRES_SERVICE": "postgres-1", + "POSTGRES_USER": "postgres", + "POSTGRES_PASSWORD": "instant101", + "SUPERSET_POSTGRESQL_USERNAME": "superset", + "SUPERSET_POSTGRESQL_PASSWORD": "instant101", "SUPERSET_POSTGRESQL_DATABASE": "superset", - "SUPERSET_POSTGRESQL_URL": "postgres-metastore:5432" + "SUPERSET_POSTGRESQL_URL": "postgres-1:5432" } } diff --git a/dashboard-visualiser-superset/swarm.sh b/dashboard-visualiser-superset/swarm.sh index af798ae8..b61da5cd 100644 --- a/dashboard-visualiser-superset/swarm.sh +++ b/dashboard-visualiser-superset/swarm.sh @@ -33,7 +33,6 @@ function import_sources() { function initialize_package() { local superset_dev_compose_filename="" - local superset_postgres_cluster_compose_filename="" if [[ "${MODE}" == "dev" ]]; then log info "Running package in DEV mode" @@ -42,15 +41,12 @@ function initialize_package() { log info "Running package in PROD mode" fi - if [[ "${CLUSTERED_MODE}" == "true" ]]; then - superset_postgres_cluster_compose_filename="docker-compose.postgres.cluster.yml" - fi - # Replace env vars envsubst <"${COMPOSE_FILE_PATH}/config/client_secret_env.json" >"${COMPOSE_FILE_PATH}/config/client_secret.json" ( - docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.postgres.yml" "$superset_postgres_cluster_compose_filename" + # Create postgres users and tables + docker::deploy_config_importer $STACK "$COMPOSE_FILE_PATH/importer/postgres/docker-compose.config.yml" "superset-db-config" "superset" docker::deploy_service $STACK "${COMPOSE_FILE_PATH}" "docker-compose.yml" "$superset_dev_compose_filename" ) || { From eacce751c18d3efbf88e174e504e9489cab5d9e7 Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Tue, 28 May 2024 15:23:23 +0200 Subject: [PATCH 5/9] Add the environment variables required --- .../importer/postgres/docker-compose.config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client-registry-jempi/importer/postgres/docker-compose.config.yml b/client-registry-jempi/importer/postgres/docker-compose.config.yml index 5d01adc7..0ab50444 100644 --- a/client-registry-jempi/importer/postgres/docker-compose.config.yml +++ b/client-registry-jempi/importer/postgres/docker-compose.config.yml @@ -17,6 +17,9 @@ services: POSTGRES_DATABASE: ${POSTGRES_DATABASE} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_PORT: ${POSTGRES_PORT} + NEW_DATABASE_NAME: ${JEMPI_POSTGRESQL_DB} + NEW_DATABASE_USER: ${JEMPI_POSTGRESQL_USERNAME} + NEW_DATABASE_PASSWORD: ${JEMPI_POSTGRESQL_PASSWORD} deploy: replicas: 1 restart_policy: From 2abe67f8627155bc1f4e7bf9a30d4ca2c5e80898 Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Tue, 28 May 2024 15:29:33 +0200 Subject: [PATCH 6/9] Clean up --- .../docker-compose.postgres.cluster.yml | 8 ------- .../docker-compose.postgres.yml | 24 ------------------- 2 files changed, 32 deletions(-) delete mode 100644 dashboard-visualiser-superset/docker-compose.postgres.cluster.yml delete mode 100644 dashboard-visualiser-superset/docker-compose.postgres.yml diff --git a/dashboard-visualiser-superset/docker-compose.postgres.cluster.yml b/dashboard-visualiser-superset/docker-compose.postgres.cluster.yml deleted file mode 100644 index b6e31cbd..00000000 --- a/dashboard-visualiser-superset/docker-compose.postgres.cluster.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: "3.9" - -services: - postgres-metastore: - deploy: - placement: - constraints: - - "node.labels.name==${POSTGRES_METASTORE}" diff --git a/dashboard-visualiser-superset/docker-compose.postgres.yml b/dashboard-visualiser-superset/docker-compose.postgres.yml deleted file mode 100644 index 912914cd..00000000 --- a/dashboard-visualiser-superset/docker-compose.postgres.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: "3.9" - -services: - postgres-metastore: - image: ${SS_POSTGRES_IMAGE} - environment: - POSTGRES_USER: ${SUPERSET_POSTGRESQL_USERNAME} - POSTGRES_PASSWORD: ${SUPERSET_POSTGRESQL_PASSWORD} - POSTGRES_DB: ${SUPERSET_POSTGRESQL_DATABASE} - volumes: - - "superset-postgres-data:/var/lib/postgresql/data" - deploy: - replicas: 1 - resources: - limits: - memory: ${SUPERSET_POSTGRES_MEMORY_LIMIT} - networks: - default: - -volumes: - superset-postgres-data: - -networks: - default: From bb2d67ec0d2c310b6efd483095566f2e9777d595 Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Tue, 28 May 2024 16:50:53 +0200 Subject: [PATCH 7/9] Use camelcase for funtion name --- client-registry-jempi/importer/postgres/create-db.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client-registry-jempi/importer/postgres/create-db.js b/client-registry-jempi/importer/postgres/create-db.js index b491e9ec..356f0b41 100644 --- a/client-registry-jempi/importer/postgres/create-db.js +++ b/client-registry-jempi/importer/postgres/create-db.js @@ -111,7 +111,7 @@ const insertQueries = [`INSERT INTO Notification_State(State) } } - const createUSer = async () => { + const createUser = async () => { const user = await client.query('SELECT 1 FROM pg_user WHERE usename = $1', [newUser]) if (!user.rows.length) { @@ -123,7 +123,7 @@ const insertQueries = [`INSERT INTO Notification_State(State) try { await createDb(newDb) - await createUSer() + await createUser() await Promise.all(tableQueries.map(query => client.query(query))) await Promise.all(insertQueries.map(query => client.query(query))) From 5a315aa9dd21134b88697f3620498e7f534ea4f1 Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Wed, 29 May 2024 11:57:18 +0200 Subject: [PATCH 8/9] Fix tests --- test/cucumber/features/cluster-mode/jempi.cluster.feature | 2 -- test/cucumber/features/single-mode/jempi.feature | 2 -- 2 files changed, 4 deletions(-) diff --git a/test/cucumber/features/cluster-mode/jempi.cluster.feature b/test/cucumber/features/cluster-mode/jempi.cluster.feature index 83210fd8..f9257ee1 100644 --- a/test/cucumber/features/cluster-mode/jempi.cluster.feature +++ b/test/cucumber/features/cluster-mode/jempi.cluster.feature @@ -43,7 +43,6 @@ Feature: Client Registry JeMPI? And The service "mongo-1" should be started with 1 replica And The service "jempi-api" should be connected to the networks | kafka_public | jempi_default | - And The service "jempi-postgresql-01" should be started with 1 replica And The service "jempi-web" should be started with 3 replica And The service "jempi-web" should be connected to the networks | reverse-proxy_public | keycloak_public | jempi_default | @@ -73,7 +72,6 @@ Feature: Client Registry JeMPI? And The service "jempi-zero-01" should be removed And The service "jempi-api" should be removed And The service "jempi-web" should be removed - And The service "jempi-postgresql-01" should be removed And The service "mongo-1" should be removed And The service "mongo-2" should be removed And The service "mongo-3" should be removed diff --git a/test/cucumber/features/single-mode/jempi.feature b/test/cucumber/features/single-mode/jempi.feature index b9797c9e..5e32bc8e 100644 --- a/test/cucumber/features/single-mode/jempi.feature +++ b/test/cucumber/features/single-mode/jempi.feature @@ -36,7 +36,6 @@ Feature: Client Registry JeMPI? And The service "jempi-api" should be started with 1 replica And The service "jempi-api" should be connected to the networks | kafka_public | jempi_default | - And The service "jempi-postgresql-01" should be started with 1 replica And The service "jempi-web" should be started with 1 replica And The service "jempi-web" should be connected to the networks | reverse-proxy_public | keycloak_public | jempi_default | @@ -62,7 +61,6 @@ Feature: Client Registry JeMPI? And The service "jempi-zero-01" should be removed And The service "jempi-api" should be removed And The service "jempi-web" should be removed - And The service "jempi-postgresql-01" should be removed And The service "mongo-1" should be removed And The service "openhim-core" should be removed And The service "openhim-console" should be removed From f2c75efa18538bbed24a6445e8fe4518a7c4a995 Mon Sep 17 00:00:00 2001 From: bradsawadye Date: Thu, 30 May 2024 08:54:54 +0200 Subject: [PATCH 9/9] Correct the database user Co-authored-by: Drizzentic --- client-registry-jempi/importer/postgres/create-db.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-registry-jempi/importer/postgres/create-db.js b/client-registry-jempi/importer/postgres/create-db.js index 356f0b41..f1e7a035 100644 --- a/client-registry-jempi/importer/postgres/create-db.js +++ b/client-registry-jempi/importer/postgres/create-db.js @@ -6,7 +6,7 @@ const database = process.env.POSTGRES_DATABASE || 'jempi' const password = process.env.POSTGRES_PASSWORD || 'instant101' const port = process.env.POSTGRES_PORT || 5432 const newDb = process.env.NEW_DATABASE_NAME || 'jempi' -const newUser = process.env.NEW_DATABASE_USER || 'keycloak' +const newUser = process.env.NEW_DATABASE_USER || 'jempi' const newUserPassword = process.env.NEW_DATABASE_PASSWORD || 'instant101' const pool = new Pool({