fix logging and add improve action

kartoza · Oct 23, 2024 · 0eb604c · 0eb604c
1 parent d470188
commit 0eb604c
Show file tree

Hide file tree

Showing 7 changed files with 79 additions and 45 deletions.
diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml
@@ -13,7 +13,7 @@ on:
     branches:
       - master
 jobs:
-  run-scenario-tests:
+  build-backup-image:
     if: |
       github.actor != 'dependabot[bot]' &&
       !(
@@ -47,16 +47,47 @@ jobs:
           push: false
           load: true
           tags: kartoza/pg-backup:manual-build
+          outputs: type=docker,dest=/tmp/pg-backup.tar
           build-args: |
             POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }}
             POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }}
             POSTGIS_MINOR_VERSION=${{ matrix.postgisMinorRelease }}
           cache-from: |
-           type=gha,scope=test
-           type=gha,scope=prod
-           type=gha,scope=base
+            type=gha,scope=test
+            type=gha,scope=prod
+            type=gha,scope=base
           cache-to: type=gha,scope=test
           target: postgis-backup-test
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: kartoza-pg-backup
+          path: /tmp/pg-backup.tar
+
+  run-scenario-tests:
+    if: |
+      github.actor != 'dependabot[bot]' &&
+      !(
+        contains(github.event.pull_request.title, '[skip-release]') ||
+        contains(github.event.comment.body, '/skiprelease')
+      )
+    runs-on: ubuntu-latest
+    needs: [ build-backup-image ]
+    strategy:
+      matrix:
+        scenario:
+          - restore
+          - s3
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: kartoza-pg-backup
+          path: /tmp
+      - name: Load image
+        run: |
+          docker load --input /tmp/pg-backup.tar
 
       - name: Run scenario test ${{ matrix.scenario }}
         working-directory: scenario_tests/${{ matrix.scenario }}
@@ -87,10 +118,14 @@ jobs:
           - 5
     steps:
       - uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: kartoza-pg-backup
+          path: /tmp
+      - name: Load image
+        run: |
+          docker load --input /tmp/pg-backup.tar
       - name: Login to DockerHub
         uses: docker/login-action@v3
         with:
@@ -106,21 +141,4 @@ jobs:
             type=ref,event=branch
             type=ref,event=pr
 
-      - name: Build image for testing
-        id: docker_build_testing_image
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: Dockerfile
-          push: true
-          tags: |
-            ${{ steps.docker_meta.outputs.tags }}-${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }}
-          build-args: |
-            POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }}
-            POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }}
-            POSTGIS_MINOR_VERSION=${{ matrix.postgisMinorRelease }}
-          cache-from: |
-           type=gha,scope=test
-           type=gha,scope=prod
-          cache-to: type=gha,scope=test
-          target: postgis-backup-test
+      
diff --git a/.github/workflows/deploy-image.yaml b/.github/workflows/deploy-image.yaml
@@ -47,9 +47,11 @@ jobs:
       - name: Check if image exists on Docker Hub
         id: check_hub_image_exists
         run: |
-          docker login --username ${{ secrets.DOCKERHUB_USERNAME }} --password ${{ secrets.DOCKERHUB_PASSWORD }}
+          echo ${{ secrets.DOCKERHUB_PASSWORD }} > /tmp/credentials.txt
+          cat /tmp/credentials.txt | docker login --username ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
           TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "'${{ secrets.DOCKERHUB_USERNAME }}'", "password": "'${{ secrets.DOCKERHUB_PASSWORD }}'"}' https://hub.docker.com/v2/users/login/ | jq -r .token)
           check_image=$(curl --silent -f --head -lL https://hub.docker.com/v2/repositories/kartoza/pg-backup/tags/${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }}/ | head -n 1 | cut -d ' ' -f2) >> $GITHUB_OUTPUT
+          rm /tmp/credentials.txt
 
       - name: Build prod image
         id: docker_build_prod

diff --git a/build_data/backups-cron b/build_data/backups-cron
@@ -1,4 +1,4 @@
 # Run the backups at 11pm each night
-${CRON_SCHEDULE} /backup-scripts/backups.sh > /var/log/cron.out 2>&1
+${CRON_SCHEDULE} /backup-scripts/backups.sh > ${CONSOLE_LOGGING_OUTPUT}
 
 # We need a blank line here for it to be a valid cron file
diff --git a/build_data/backups-cron-default b/build_data/backups-cron-default
@@ -1,4 +1,4 @@
 # Run the backups at 11pm each night
-0 23 * * * /backup-scripts/backups.sh > /var/log/cron.out 2>&1
+0 23 * * * /backup-scripts/backups.sh > ${CONSOLE_LOGGING_OUTPUT}
 
 # We need a blank line here for it to be a valid cron file
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -29,7 +29,8 @@ services:
       - POSTGRES_PASS=docker
       - POSTGRES_PORT=5432
       - RUN_AS_ROOT=true
-      #- CRON_SCHEDULE="*/5 * * * *"
+      - CRON_SCHEDULE="*/5 * * * *"
+      - CONSOLE_LOGGING=TRUE
       #- DB_DUMP_ENCRYPTION=true
     restart: on-failure
     depends_on:

diff --git a/scripts/backups.sh b/scripts/backups.sh
@@ -81,17 +81,17 @@ function backup_db() {
     else
       export FILENAME=${MYBASEDIR}/"${ARCHIVE_FILENAME}.${DB}.dmp"
     fi
-    echo "Backing up $DB" >>/var/log/cron.log
+    echo "Backing up $DB" >> ${CONSOLE_LOGGING_OUTPUT}
     if [ -z "${DB_TABLES:-}" ]; then
       if [[ "${DB_DUMP_ENCRYPTION}" =~ [Tt][Rr][Uu][Ee] ]];then
         PGPASSWORD=${POSTGRES_PASS} pg_dump ${PG_CONN_PARAMETERS} ${DUMP_ARGS} -d ${DB} | openssl enc -aes-256-cbc -pass pass:${DB_DUMP_ENCRYPTION_PASS_PHRASE} -pbkdf2 -iter 10000 -md sha256 -out ${FILENAME}
       else
         PGPASSWORD=${POSTGRES_PASS} pg_dump ${PG_CONN_PARAMETERS} ${DUMP_ARGS} -d ${DB} > ${FILENAME}
       fi
-      echo "Backing up $FILENAME done" >>/var/log/cron.log
+      echo "Backing up $FILENAME done" >> ${CONSOLE_LOGGING_OUTPUT}
       if [[ ${STORAGE_BACKEND} == "S3" ]]; then
         gzip $FILENAME
-        echo "Backing up $FILENAME to s3://${BUCKET}/" >>/var/log/cron.log
+        echo "Backing up $FILENAME to s3://${BUCKET}/" >> ${CONSOLE_LOGGING_OUTPUT}
         ${EXTRA_PARAMS}
         rm ${MYBACKUPDIR}/*.dmp.gz
       fi
@@ -118,7 +118,7 @@ if [[ ${STORAGE_BACKEND} == "S3" ]]; then
 
   # Backup globals Always get the latest
   PGPASSWORD=${POSTGRES_PASS} pg_dumpall ${PG_CONN_PARAMETERS}  --globals-only | s3cmd put - s3://${BUCKET}/globals.sql
-  echo "Sync globals.sql to ${BUCKET} bucket  " >>/var/log/cron.log
+  echo "Sync globals.sql to ${BUCKET} bucket  " >> ${CONSOLE_LOGGING_OUTPUT}
   backup_db "s3cmd sync -r ${MYBASEDIR}/* s3://${BUCKET}/"
 
 elif [[ ${STORAGE_BACKEND} =~ [Ff][Ii][Ll][Ee] ]]; then
@@ -129,14 +129,14 @@ elif [[ ${STORAGE_BACKEND} =~ [Ff][Ii][Ll][Ee] ]]; then
 
 fi
 
-echo "Backup running to $MYBACKUPDIR" >>/var/log/cron.log
+echo "Backup running to $MYBACKUPDIR" >> ${CONSOLE_LOGGING_OUTPUT}
 
 
 if [ "${REMOVE_BEFORE:-}" ]; then
   TIME_MINUTES=$((REMOVE_BEFORE * 24 * 60))
   if [[ ${STORAGE_BACKEND} == "FILE" ]]; then
-    echo "Removing following backups older than ${REMOVE_BEFORE} days" >>/var/log/cron.log
-    find ${MYBASEDIR}/* -type f -mmin +${TIME_MINUTES} -delete &>>/var/log/cron.log
+    echo "Removing following backups older than ${REMOVE_BEFORE} days" >> ${CONSOLE_LOGGING_OUTPUT}
+    find ${MYBASEDIR}/* -type f -mmin +${TIME_MINUTES} -delete &>> ${CONSOLE_LOGGING_OUTPUT}
   elif [[ ${STORAGE_BACKEND} == "S3" ]]; then
     # Credits https://shout.setfive.com/2011/12/05/deleting-files-older-than-specified-time-with-s3cmd-and-bash/
     clean_s3bucket "${BUCKET}" "${REMOVE_BEFORE} days"

diff --git a/scripts/start.sh b/scripts/start.sh
@@ -118,6 +118,10 @@ if [ -z "${DB_DUMP_ENCRYPTION}" ]; then
   DB_DUMP_ENCRYPTION=FALSE
 fi
 
+if [ -z "${CONSOLE_LOGGING}" ]; then
+  CONSOLE_LOGGING=FALSE
+fi
+
 file_env 'DB_DUMP_ENCRYPTION_PASS_PHRASE'
 if [ -z "${DB_DUMP_ENCRYPTION_PASS_PHRASE}" ]; then
   STRING_LENGTH=30
@@ -128,20 +132,16 @@ fi
 
 
 function cron_config() {
-  if [[ ! -f /backup-scripts/backups-cron ]]; then
-    # If it doesn't exists, copy from ${EXTRA_CONF_DIR} directory if exists
-    if [[ -f ${EXTRA_CONFIG_DIR}/backups-cron ]]; then
-      cp -f ${EXTRA_CONFIG_DIR}/backups-cron /backup-scripts
-    else
+  if [[ -f ${EXTRA_CONFIG_DIR}/backups-cron ]]; then
+      envsubst < ${EXTRA_CONFIG_DIR}/backups-cron > /backup-scripts/backups-cron
+  else
       # default value
       if [ -z "${CRON_SCHEDULE}" ]; then
-            cp /build_data/backups-cron-default /backup-scripts/backups-cron
+            envsubst < /build_data/backups-cron-default > /backup-scripts/backups-cron
       else
             envsubst < /build_data/backups-cron > /backup-scripts/backups-cron
        fi
-    fi
   fi
-
 }
 
 function directory_checker() {
@@ -167,8 +167,14 @@ function non_root_permission() {
 
 mkdir -p ${DEFAULT_EXTRA_CONF_DIR}
 # Copy settings for cron file
+if [[ ${CONSOLE_LOGGING} =~ [Tt][Rr][Uu][Ee] ]];then
+   export CONSOLE_LOGGING_OUTPUT='/proc/1/fd/1 2>&1'
+else
+   export CONSOLE_LOGGING_OUTPUT='/var/log/cron.out 2>&1'
+fi
 
 cron_config
+
 function configure_env_variables() {
 echo "
 export PATH=\"${PATH}\"
@@ -196,11 +202,18 @@ DB_DUMP_ENCRYPTION="${DB_DUMP_ENCRYPTION}"
 export PG_CONN_PARAMETERS=\"${PG_CONN_PARAMETERS}\"
 export DBLIST=\"${DBLIST}\"
  " > /backup-scripts/pgenv.sh
+
 echo "Start script running with these environment options"
 set | grep PG
 
 }
 configure_env_variables
+if [[ ${CONSOLE_LOGGING} =~ [Tt][Rr][Uu][Ee] ]];then
+   sed -i 's#${CONSOLE_LOGGING_OUTPUT}#/proc/1/fd/1 2>\&1#g' /backup-scripts/backups.sh
+else
+   sed -i 's#${CONSOLE_LOGGING_OUTPUT}#/var/log/cron.out 2>\&1#g' /backup-scripts/backups.sh
+fi
+
 # Fix variables not interpolated
 sed -i "s/'//g" /backup-scripts/backups-cron
 sed -i 's/\"//g' /backup-scripts/backups-cron