From 69107d8731a7ec058087ce8d8da6f53dcee9f4f6 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 8 Aug 2022 22:35:25 +0200 Subject: [PATCH 1/5] Handle cronitor key docker secret and fix issue with duplicate jobs --- README.md | 12 +++++++++++- docker-compose-example.yml | 7 +++++++ scripts/entrypoint.sh | 15 ++++++++++++--- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3107d6c..7060847 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,13 @@ To run backups and WAL archiving to GCS (Google Cloud Storage) set the following - BACKUPS=true # switch to implement backups; defaults to false - STORAGE_BUCKET=gs://postgresql/backups # to specify the GCS bucket - GCP_CREDENTIALS=/run/secrets/gcp_credentials # to specify the docker secret with the service account key that has access to the GCS bucket + +and to setup database full backups schedules and job monitoring: + - FULL_BACKUP_SCHEDULE=* * * * * # to specify the cron schedule expression at which backups will run (if not set only the first initial base backup will be ran) \ # L-> check https://crontab.guru/ for schedule expression details. (e.g.: 00 00 * * * -> to run a daily backup at midnight)" - - CRONITOR_KEY=1a2b3cd4e56789f1234gh5ijkl67m890 # to specify cronitor API key for cron job monitoring. check https://cronitor.io/cron-job-monitoring for details + - CRONITOR_KEY_FILE=/run/secrets/cronitor_key # to specify the docker secret with the cronitor API key for cron job monitoring. check https://cronitor.io/cron-job-monitoring for details + - CRONITOR_ENV="PROD" # to specify the environment to be added as suffix to the cronitor job name (e.g.: "PROD", "DEV", "BETA"); defaults to "PROD" if not set Note: HA MASTER instances with BACKUPS disabled will only store WAL logs locally on the `pg_wal` folder under the PGDATA directory path. Running a postgres HA cluster without implementing backups is not recommended and is intended only for testing purposes. @@ -85,6 +89,8 @@ services: - STORAGE_BUCKET=gs://postgresql/backups - GCP_CREDENTIALS=/run/secrets/gcp_credentials - FULL_BACKUP_SCHEDULE:00 00 * * * + - CRONITOR_KEY_FILE=/run/secrets/cronitor_key + - CRONITOR_ENV=TEST ports: - "5432:5432" secrets: @@ -100,6 +106,10 @@ services: uid: "70" gid: "70" mode: 0550 + - source: cronitor_key + uid: "70" + gid: "70" + mode: 0550 networks: database: aliases: diff --git a/docker-compose-example.yml b/docker-compose-example.yml index 89b0350..555c241 100644 --- a/docker-compose-example.yml +++ b/docker-compose-example.yml @@ -6,6 +6,8 @@ secrets: external: true gcp_credentials: external: true + cronitor_key: + external: true services: pg_master: @@ -26,6 +28,7 @@ services: - STORAGE_BUCKET=gs://postgresql/backups - GCP_CREDENTIALS=/run/secrets/gcp_credentials - FULL_BACKUP_SCHEDULE=00 00 * * * + - CRONITOR_KEY=/run/secrets/cronitor_key ports: - "5432:5432" secrets: @@ -41,6 +44,10 @@ services: uid: "70" gid: "70" mode: 0550 + - source: cronitor_key + uid: "70" + gid: "70" + mode: 0550 networks: database: aliases: diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 2c5e09b..46d4a82 100644 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -17,7 +17,8 @@ export GCP_CREDENTIALS=$GCP_CREDENTIALS export RESTORE_BACKUP=${RESTORE_BACKUP:-false} export BACKUP_NAME=$BACKUP_NAME export FULL_BACKUP_SCHEDULE=$FULL_BACKUP_SCHEDULE -export CRONITOR_KEY=$CRONITOR_KEY +export CRONITOR_KEY_FILE=$CRONITOR_KEY_FILE +export CRONITOR_ENV=${CRONITOR_ENV:-PROD} if [[ ${PG_MASTER^^} == TRUE && ${PG_SLAVE^^} == TRUE ]]; then echo "Both \$PG_MASTER and \$PG_SLAVE cannot be true" @@ -156,10 +157,18 @@ if [[ ${BACKUPS^^} == TRUE ]] && [[ ! -z ${FULL_BACKUP_SCHEDULE} ]] && [[ $(id echo "Starting cron job scheduler" && crond echo "Database backups will be scheduled to run at ${FULL_BACKUP_SCHEDULE}. Check https://crontab.guru/ for schedule expression details" backup_cron_schedule - if [[ ! -z ${CRONITOR_KEY} ]]; then + if [[ -n ${CRONITOR_KEY} ]]; then + CRONITOR_KEY=$(cat "${CRONITOR_KEY_FILE}") + NEW_JOB_NAME="${POSTGRES_DB}-DB-FullBackup-${CRONITOR_ENV}" + for JOB_NAME in $(curl https://cronitor.io/api/monitors -u ${CRONITOR_KEY}:| jq -r '.monitors | .[].name') + do + if [ $JOB_NAME == $NEW_JOB_NAME ]; then + curl -X DELETE https://cronitor.io/api/monitors/$JOB_NAME -u ${CRONITOR_KEY}: + fi + done echo "Configuring cronitor. Check https://cronitor.io/cron-job-monitoring to see jobs monitoring" cronitor configure --api-key ${CRONITOR_KEY} > /dev/null - yes "${POSTGRES_DB} DB Full Backup" | cronitor discover + yes $NEW_JOB_NAME | cronitor discover fi fi From 1c446c5f4543787e0ec117fcac429bcbdf9389a6 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 9 Aug 2022 16:55:02 +0200 Subject: [PATCH 2/5] Add jq package installation to alpine --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 45bd090..26d433b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN set -ex \ FROM postgres:14.2-alpine3.15 -RUN apk add --update iputils htop curl busybox-suid \ +RUN apk add --update iputils htop curl busybox-suid jq \ && curl -sOL https://cronitor.io/dl/linux_amd64.tar.gz \ && tar xvf linux_amd64.tar.gz -C /usr/bin/ From 50240a715ed669721863e681d91c088d4ad972e0 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 9 Aug 2022 17:49:45 +0200 Subject: [PATCH 3/5] Update variable name CRONITOR_KEY_FILE on condition --- scripts/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 46d4a82..7828ec5 100644 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -157,7 +157,7 @@ if [[ ${BACKUPS^^} == TRUE ]] && [[ ! -z ${FULL_BACKUP_SCHEDULE} ]] && [[ $(id echo "Starting cron job scheduler" && crond echo "Database backups will be scheduled to run at ${FULL_BACKUP_SCHEDULE}. Check https://crontab.guru/ for schedule expression details" backup_cron_schedule - if [[ -n ${CRONITOR_KEY} ]]; then + if [[ -n ${CRONITOR_KEY_FILE} ]]; then CRONITOR_KEY=$(cat "${CRONITOR_KEY_FILE}") NEW_JOB_NAME="${POSTGRES_DB}-DB-FullBackup-${CRONITOR_ENV}" for JOB_NAME in $(curl https://cronitor.io/api/monitors -u ${CRONITOR_KEY}:| jq -r '.monitors | .[].name') From 343579bba60f75409de9347b28dfe80fbf11fafa Mon Sep 17 00:00:00 2001 From: Daniel Perez <58601098+danperezsan@users.noreply.github.com> Date: Tue, 9 Aug 2022 20:01:54 +0200 Subject: [PATCH 4/5] Update postgres image to use latest available postgres:14.4-alpine3.16 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 26d433b..d7f8b4c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN set -ex \ && install main/pg/wal-g / \ && /wal-g --help -FROM postgres:14.2-alpine3.15 +FROM postgres:14.4-alpine3.16 RUN apk add --update iputils htop curl busybox-suid jq \ && curl -sOL https://cronitor.io/dl/linux_amd64.tar.gz \ From 59082d438a45c2c45470ba9928a58233aa7cf5f4 Mon Sep 17 00:00:00 2001 From: Daniel Perez <58601098+danperezsan@users.noreply.github.com> Date: Tue, 9 Aug 2022 20:19:41 +0200 Subject: [PATCH 5/5] Update CRONITOR_ENV example value on README file --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7060847..baebe4f 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ and to setup database full backups schedules and job monitoring: - FULL_BACKUP_SCHEDULE=* * * * * # to specify the cron schedule expression at which backups will run (if not set only the first initial base backup will be ran) \ # L-> check https://crontab.guru/ for schedule expression details. (e.g.: 00 00 * * * -> to run a daily backup at midnight)" - CRONITOR_KEY_FILE=/run/secrets/cronitor_key # to specify the docker secret with the cronitor API key for cron job monitoring. check https://cronitor.io/cron-job-monitoring for details - - CRONITOR_ENV="PROD" # to specify the environment to be added as suffix to the cronitor job name (e.g.: "PROD", "DEV", "BETA"); defaults to "PROD" if not set + - CRONITOR_ENV=PROD # to specify the environment to be added as suffix to the cronitor job name (e.g.: PROD, DEV, BETA, TEST); defaults to PROD if not set Note: HA MASTER instances with BACKUPS disabled will only store WAL logs locally on the `pg_wal` folder under the PGDATA directory path. Running a postgres HA cluster without implementing backups is not recommended and is intended only for testing purposes. @@ -533,4 +533,4 @@ mfdk34jll34k testapp_app.1 testapp/testapp-prod:1.0.0 - [Contributing](https://github.com/mesoform/documentation/blob/main/CONTRIBUTING.md) - [Code of Conduct](https://github.com/mesoform/documentation/blob/main/CODE_OF_CONDUCT.md) - [Licence](https://github.com/mesoform/postgres-ha/blob/main/LICENSE) -- [Contact](https://mesoform.com/contact) \ No newline at end of file +- [Contact](https://mesoform.com/contact)