add RDS data refresh helm job (#476)

ministryofjustice · Nov 21, 2024 · a5411d1 · a5411d1
1 parent 115ca94
commit a5411d1
Show file tree

Hide file tree

Showing 3 changed files with 251 additions and 0 deletions.
diff --git a/.github/workflows/delius-core-cp-rds-data-refresh.yaml b/.github/workflows/delius-core-cp-rds-data-refresh.yaml
@@ -0,0 +1,114 @@
+---
+name: CP RDS Data Refresh
+
+on:
+  workflow_dispatch:
+    inputs:
+      component:
+        description: Delius Core component to refresh data for
+        required: true
+        type: choice
+        options:
+          - gdpr
+          - merge
+      source_env:
+        description: Environment to copy data from
+        required: true
+        type: choice
+        options:
+          - test
+      destination_env:
+        description: Environment to copy data to
+        required: true
+        type: choice
+        options:
+          - dev
+
+permissions:
+  contents: read
+
+jobs:
+  stop-destination-environment:
+    name: Stop ${{ github.event.inputs.destination_env }} Environment
+    runs-on: ubuntu-22.04
+    environment:
+      name: ${{ github.event.inputs.destination_env }}
+    steps:
+    - name: Configure kubectl
+      run: |
+        echo "${{ secrets.KUBE_CERT }}" > ca.crt
+        kubectl config set-cluster ${KUBE_CLUSTER} --certificate-authority=./ca.crt --server=https://${KUBE_CLUSTER}
+        kubectl config set-credentials deploy-user --token=${{ secrets.KUBE_TOKEN }}
+        kubectl config set-context ${KUBE_CLUSTER} --cluster=${KUBE_CLUSTER} --user=deploy-user --namespace=hmpps-delius-core-${{ github.event.inputs.destination_env }}
+        kubectl config use-context ${KUBE_CLUSTER}
+      env:
+        KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }}
+
+    - name: Stop ${{ github.event.inputs.destination_env }} Environment
+      run: |
+        kubectl scale deployment ${{ github.event.inputs.component }} --replicas=0
+
+  refresh-db:
+    name: Refresh DB
+    runs-on: ubuntu-22.04
+    environment:
+      name: ${{ github.event.inputs.source_env }}
+    needs: stop-destination-environment
+    steps:
+      - uses: actions/checkout@v4.1.1
+
+      - name: Install kubectl
+        uses: azure/setup-kubectl@v3.2
+        with:
+          version: 'v1.26.0' # default is latest stable
+        id: kubectl_install
+
+      - uses: azure/setup-helm@v4.2.0
+        with:
+           version: 'v3.15.3' # default is latest (stable)
+        id: install
+
+      - name: Configure kubectl
+        run: |
+          echo "${{ secrets.KUBE_CERT }}" > ca.crt
+          kubectl config set-cluster ${KUBE_CLUSTER} --certificate-authority=./ca.crt --server=https://${KUBE_CLUSTER}
+          kubectl config set-credentials deploy-user --token=${{ secrets.KUBE_TOKEN }}
+          kubectl config set-context ${KUBE_CLUSTER} --cluster=${KUBE_CLUSTER} --user=deploy-user --namespace=hmpps-delius-core-${{ github.event.inputs.destination_env }}
+          kubectl config use-context ${KUBE_CLUSTER}
+        env:
+          KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }}
+
+      - name: Uninstall DB Refresh chart
+        run: helm uninstall refresh-db-${{ github.events.inputs.component }} --ignore-not-found
+
+      - name: DB Refresh
+        working-directory: helm/refresh-db
+        run: |
+          helm install refresh-db-${{ github.events.inputs.component }} . \
+            --set sourceEnvironment=${{ github.event.inputs.source_env }} \
+            --set destinationEnvironment=${{ github.event.inputs.destination_env }} \
+            --set slackWebhookUrl=${{ secrets.SLACK_WEBHOOK_URL }} \ 
+            --set component=${{ github.events.inputs.component }}
+
+  start-destination-environment:
+    name: Start ${{ github.event.inputs.destination_env }} Environment
+    runs-on: ubuntu-22.04
+    environment:
+      name: ${{ github.event.inputs.destination_env }}
+    needs:
+      - refresh-db
+    steps:
+    - name: Configure kubectl
+      run: |
+        echo "${{ secrets.KUBE_CERT }}" > ca.crt
+        kubectl config set-cluster ${KUBE_CLUSTER} --certificate-authority=./ca.crt --server=https://${KUBE_CLUSTER}
+        kubectl config set-credentials deploy-user --token=${{ secrets.KUBE_TOKEN }}
+        kubectl config set-context ${KUBE_CLUSTER} --cluster=${KUBE_CLUSTER} --user=deploy-user --namespace=hmpps-delius-core-${{ github.event.inputs.destination_env }}
+        kubectl config use-context ${KUBE_CLUSTER}
+      env:
+        KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }}
+
+    - name: Start ${{ github.event.inputs.destination_env }} Environment
+      run: |
+
+        kubectl scale deployment ${{ github.event.inputs.component }} --replicas=1
diff --git a/helm/refresh-db/Chart.yaml b/helm/refresh-db/Chart.yaml
@@ -0,0 +1,5 @@
+apiVersion: v2
+appVersion: 0.1
+version: 0.0.1
+description: Job to refresh DB data
+name: refresh-db
diff --git a/helm/refresh-db/templates/job.yaml b/helm/refresh-db/templates/job.yaml
@@ -0,0 +1,132 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: refresh-db-script
+data:
+  entrypoint.sh: |-
+    #!/bin/bash
+    set -e
+    trap 'send_slack_notification $?' EXIT
+
+    function send_slack_notification() {
+      STATUS=$1
+      if [ "$STATUS" -eq 0 ]; then
+        JSON_PAYLOAD=$(jq -n --arg text "Refresh DB (${SRC_ENV} to ${DST_ENV}) job succeeded" '{text: $text}')
+      else
+        ERROR_MSG=$(tail -n 10 ~/error.log) || ERROR_MSG="Unknown error"
+        JSON_PAYLOAD=$(jq -n --arg text "Refresh DB (${SRC_ENV} to ${DST_ENV}) job failed with error: $ERROR_MSG" '{text: $text}')
+      fi
+      curl -X POST -H 'Content-type: application/json' --data "$JSON_PAYLOAD" $SLACK_WEBHOOK_URL
+    }
+
+    echo "${SRC_DB_HOST}:5432:${SRC_DB_NAME}:${SRC_DB_USER}:${SRC_DB_PASS}" > ~/.pgpass
+    echo "${DST_DB_HOST}:5432:${DST_DB_NAME}:${DST_DB_USER}:${DST_DB_PASS}" >> ~/.pgpass
+    cat ~/.pgpass
+    chmod 0600 ~/.pgpass
+    chown job:job ~/.pgpass
+    set -x
+
+    # Dump the source database
+    pg_dump --jobs=4 --host="$SRC_DB_HOST" --username="$SRC_DB_USER" --dbname="$SRC_DB_NAME" --no-owner --no-privileges --verbose --format=directory --file=/home/job/db-dump 2> >(tee ~/error.log >&2)
+
+    psql --host="$DST_DB_HOST" --username="$DST_DB_USER" --dbname="$DST_DB_NAME" -c "drop schema if exists public cascade;" 2> >(tee ~/error.log >&2)
+
+    psql --host="$DST_DB_HOST" --username="$DST_DB_USER" --dbname="$DST_DB_NAME" -c "create schema public;" 2> >(tee ~/error.log >&2)
+
+    # Restore the source database dump to the destination database
+    pg_restore --jobs=4 --host="$DST_DB_HOST" --username="$DST_DB_USER" --dbname="$DST_DB_NAME" --no-owner --no-privileges --verbose /home/job/db-dump 2> >(tee ~/error.log >&2)
+    rm -rv /home/job/db-dump ~/.pgpass
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: refresh-db
+spec:
+  template:
+    spec:
+      containers:
+        - name: refresh-db
+          image: ghcr.io/ministryofjustice/REPLACE ME # replace with image similar to the afresco db utils
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits:
+              cpu: 4
+              memory: 2Gi
+          command:
+            - /bin/entrypoint.sh
+          env:
+            - name: HOME
+              value: "/home/job"
+            - name: SRC_DB_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: rds-instance-output-{{ .Values.component }}
+                  key: DATABASE_NAME
+            - name: SRC_DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: rds-instance-output-{{ .Values.component }}
+                  key: DATABASE_USERNAME
+            - name: SRC_DB_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: rds-instance-output-{{ .Values.component }}
+                  key: DATABASE_PASSWORD
+            - name: SRC_DB_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: rds-instance-output-{{ .Values.component }}
+                  key: RDS_INSTANCE_ADDRESS
+            - name: DST_DB_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: rds-instance-output-{{ .Values.destinationEnvironment }}-{{ .Values.component }}
+                  key: DATABASE_NAME
+            - name: DST_DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: rds-instance-output-{{ .Values.destinationEnvironment }}-{{ .Values.component }}
+                  key: DATABASE_USERNAME
+            - name: DST_DB_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: rds-instance-output-{{ .Values.destinationEnvironment }}-{{ .Values.component }}
+                  key: DATABASE_PASSWORD
+            - name: DST_DB_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: rds-instance-output-{{ .Values.destinationEnvironment }}-{{ .Values.component }}
+                  key: RDS_INSTANCE_ADDRESS
+            - name: SLACK_WEBHOOK_URL
+              value: "{{ .Values.slackWebhookUrl }}"
+            - name: SRC_ENV
+              value: "{{ .Values.sourceEnvironment }}"
+            - name: DST_ENV
+              value: "{{ .Values.destinationEnvironment }}"
+          volumeMounts:
+            - name: refresh-db-script
+              mountPath: /bin/entrypoint.sh
+              readOnly: true
+              subPath: entrypoint.sh
+          securityContext:
+            allowPrivilegeEscalation: false
+            privileged: false
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 999
+            capabilities:
+              drop:
+                - ALL
+            seccompProfile:
+              type: RuntimeDefault
+      serviceAccount: cd-serviceaccount
+      serviceAccountName: cd-serviceaccount
+      restartPolicy: Never
+      volumes:
+        - name: refresh-db-script
+          configMap:
+            name: refresh-db-script
+            defaultMode: 0755
+  backoffLimit: 0