Skip to content

Commit 045aa86

Browse files
wmuldergovwmuldergov
committed
DBC22-2085: Backup Media and improve log handling
1 parent 41596bb commit 045aa86

35 files changed

+721
-279
lines changed

‎.github/workflows/dev.yml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -106,27 +106,27 @@ jobs:
106106
username: ${{ env.REGISTRY_USER }}
107107
password: ${{ env.REGISTRY_PASSWORD }}
108108

109-
build-logprocessing:
109+
build-openshiftjobs:
110110
runs-on: ubuntu-latest
111-
name: Build & Push Docker Log Processing Image
111+
name: Build & Push Docker OpenShift Jobs Image
112112
environment:
113113
name: dev
114114

115115
steps:
116116
- name: Checkout Code
117117
uses: actions/checkout@v4
118118

119-
- name: Build Log Processing
119+
- name: Build OpenShift Jobs
120120
id: build_image
121121
uses: redhat-actions/buildah-build@v2
122122
with:
123123
context: .
124124
layers: true
125-
image: drivebc-logprocessing
125+
image: drivebc-openshiftjobs
126126
tags: latest latest-dev ${{ github.sha }}
127127
labels: |
128128
app=drivebc
129-
containerfiles: ./compose/logprocessing/DockerFile
129+
containerfiles: ./compose/openshiftjobs/DockerFile
130130

131131
- name: Push to Github Packages
132132
uses: redhat-actions/push-to-registry@v2
@@ -138,7 +138,7 @@ jobs:
138138
password: ${{ env.REGISTRY_PASSWORD }}
139139

140140
versionUpdate:
141-
needs: [build-static, build-backend, build-redis, build-logprocessing]
141+
needs: [build-static, build-backend, build-redis, build-openshiftjobs]
142142
runs-on: ubuntu-latest
143143
name: Deploy Latest Images
144144
environment:
@@ -159,4 +159,4 @@ jobs:
159159
- name: Helm upgrade on OpenShift Environment
160160
run: |
161161
helm dependency update ./infrastructure/main
162-
helm upgrade dev-drivebc -f ./infrastructure/main/values-dev.yaml ./infrastructure/main --set django.image.tag="${{ github.sha }}" --set redis.image.tag="${{ github.sha }}" --set static.image.tag="${{ github.sha }}" --set tasks.image.tag="${{ github.sha }}" --set logprocessing.image.tag="${{ github.sha }}"
162+
helm upgrade dev-drivebc -f ./infrastructure/main/values-dev.yaml ./infrastructure/main --set django.image.tag="${{ github.sha }}" --set redis.image.tag="${{ github.sha }}" --set static.image.tag="${{ github.sha }}" --set tasks.image.tag="${{ github.sha }}" --set openshiftjobs.image.tag="${{ github.sha }}"

‎.github/workflows/prod.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,11 @@ jobs:
5959
tags: |
6060
latest-prod
6161
62-
- name: Add latest-uat tag to logprocessing image
62+
- name: Add latest-uat tag to openshiftjobs image
6363
uses: shrink/actions-docker-registry-tag@v4
6464
with:
6565
registry: ghcr.io
66-
repository: bcgov/drivebc-logprocessing
66+
repository: bcgov/drivebc-openshiftjobs
6767
target: ${{ needs.deploy-pre.outputs.tag }}
6868
tags: |
6969
latest-prod
@@ -90,4 +90,4 @@ jobs:
9090
- name: Helm upgrade on OpenShift Environment
9191
run: |
9292
helm dependency update ./infrastructure/main
93-
helm upgrade prod-drivebc -f ./infrastructure/main/values-prod.yaml ./infrastructure/main --set django.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set redis.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set static.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.deploy-pre.outputs.tag }}"
93+
helm upgrade prod-drivebc -f ./infrastructure/main/values-prod.yaml ./infrastructure/main --set django.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set redis.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set static.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set openshiftjobs.image.tag="${{ needs.deploy-pre.outputs.tag }}"

‎.github/workflows/test.yml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -152,25 +152,25 @@ jobs:
152152
username: ${{ env.REGISTRY_USER }}
153153
password: ${{ env.REGISTRY_PASSWORD }}
154154

155-
build-logprocessing:
155+
build-openshiftjobs:
156156
needs: [build-pre]
157157
runs-on: ubuntu-latest
158-
name: Build & Push Docker Log-Processing Image
158+
name: Build & Push Docker OpenShift Jobs Image
159159
environment: test
160160

161161
steps:
162162
- name: Checkout Code
163163
uses: actions/checkout@v4
164164

165-
- name: Build Log-Processing
165+
- name: Build OpenShift Jobs
166166
id: build_image
167167
uses: redhat-actions/buildah-build@v2
168168
with:
169-
image: drivebc-logprocessing
169+
image: drivebc-openshiftjobs
170170
tags: latest latest-test ${{ github.sha }} ${{ needs.build-pre.outputs.tag }}
171171
labels: |
172172
app=drivebc
173-
containerfiles: ./compose/logprocessing/DockerFile
173+
containerfiles: ./compose/openshiftjobs/DockerFile
174174

175175
- name: Push to Github Packages
176176
uses: redhat-actions/push-to-registry@v2
@@ -182,7 +182,7 @@ jobs:
182182
password: ${{ env.REGISTRY_PASSWORD }}
183183

184184
versionUpdate:
185-
needs: [build-pre, build-static, build-backend, build-redis, build-logprocessing]
185+
needs: [build-pre, build-static, build-backend, build-redis, build-openshiftjobs]
186186
runs-on: ubuntu-latest
187187
name: Deploy Latest Images
188188
environment:
@@ -205,4 +205,4 @@ jobs:
205205
- name: Helm upgrade on OpenShift Environment
206206
run: |
207207
helm dependency update ./infrastructure/main
208-
helm upgrade test-drivebc -f ./infrastructure/main/values-test.yaml ./infrastructure/main --set django.image.tag="${{ needs.build-pre.outputs.tag }}" --set redis.image.tag="${{ needs.build-pre.outputs.tag }}" --set static.image.tag="${{ needs.build-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.build-pre.outputs.tag }}" --set logprocessing.image.tag="${{ needs.build-pre.outputs.tag }}"
208+
helm upgrade test-drivebc -f ./infrastructure/main/values-test.yaml ./infrastructure/main --set django.image.tag="${{ needs.build-pre.outputs.tag }}" --set redis.image.tag="${{ needs.build-pre.outputs.tag }}" --set static.image.tag="${{ needs.build-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.build-pre.outputs.tag }}" --set openshiftjobs.image.tag="${{ needs.build-pre.outputs.tag }}"

‎.github/workflows/test_manual.yml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -131,25 +131,25 @@ jobs:
131131
username: ${{ env.REGISTRY_USER }}
132132
password: ${{ env.REGISTRY_PASSWORD }}
133133

134-
build-logprocessing:
134+
build-openshiftjobs:
135135
needs: [build-pre]
136136
runs-on: ubuntu-latest
137-
name: Build & Push Docker Log-Processing Image
137+
name: Build & Push Docker OpenShift Jobs Image
138138
environment: test
139139

140140
steps:
141141
- name: Checkout Code
142142
uses: actions/checkout@v4
143143

144-
- name: Build Log-Processing
144+
- name: Build OpenShift Jobs
145145
id: build_image
146146
uses: redhat-actions/buildah-build@v2
147147
with:
148-
image: drivebc-logprocessing
148+
image: drivebc-openshiftjobs
149149
tags: latest latest-test ${{ github.sha }} ${{ needs.build-pre.outputs.tag }}
150150
labels: |
151151
app=drivebc
152-
containerfiles: ./compose/logprocessing/DockerFile
152+
containerfiles: ./compose/openshiftjobs/DockerFile
153153

154154
- name: Push to Github Packages
155155
uses: redhat-actions/push-to-registry@v2
@@ -161,7 +161,7 @@ jobs:
161161
password: ${{ env.REGISTRY_PASSWORD }}
162162

163163
versionUpdate:
164-
needs: [build-pre, build-static, build-backend, build-redis, build-logprocessing]
164+
needs: [build-pre, build-static, build-backend, build-redis, build-openshiftjobs]
165165
runs-on: ubuntu-latest
166166
name: Deploy Latest Images
167167
environment:
@@ -184,4 +184,4 @@ jobs:
184184
- name: Helm upgrade on OpenShift Environment
185185
run: |
186186
helm dependency update ./infrastructure/main
187-
helm upgrade test-drivebc -f ./infrastructure/main/values-test.yaml ./infrastructure/main --set django.image.tag="${{ needs.build-pre.outputs.tag }}" --set redis.image.tag="${{ needs.build-pre.outputs.tag }}" --set static.image.tag="${{ needs.build-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.build-pre.outputs.tag }}" --set logprocessing.image.tag="${{ needs.build-pre.outputs.tag }}"
187+
helm upgrade test-drivebc -f ./infrastructure/main/values-test.yaml ./infrastructure/main --set django.image.tag="${{ needs.build-pre.outputs.tag }}" --set redis.image.tag="${{ needs.build-pre.outputs.tag }}" --set static.image.tag="${{ needs.build-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.build-pre.outputs.tag }}" --set openshiftjobs.image.tag="${{ needs.build-pre.outputs.tag }}"

‎.github/workflows/uat.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,11 @@ jobs:
6161
tags: |
6262
latest-uat
6363
64-
- name: Add latest-uat tag to logprocessing image
64+
- name: Add latest-uat tag to openshiftjobs image
6565
uses: shrink/actions-docker-registry-tag@v4
6666
with:
6767
registry: ghcr.io
68-
repository: bcgov/drivebc-logprocessing
68+
repository: bcgov/drivebc-openshiftjobs
6969
target: ${{ needs.deploy-pre.outputs.tag }}
7070
tags: |
7171
latest-uat
@@ -92,4 +92,4 @@ jobs:
9292
- name: Helm upgrade on OpenShift Environment
9393
run: |
9494
helm dependency update ./infrastructure/main
95-
helm upgrade uat-drivebc -f ./infrastructure/main/values-uat.yaml ./infrastructure/main --set django.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set redis.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set static.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set logprocessing.image.tag="${{ needs.deploy-pre.outputs.tag }}"
95+
helm upgrade uat-drivebc -f ./infrastructure/main/values-uat.yaml ./infrastructure/main --set django.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set redis.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set static.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set tasks.image.tag="${{ needs.deploy-pre.outputs.tag }}" --set openshiftjobs.image.tag="${{ needs.deploy-pre.outputs.tag }}"

‎compose/logprocessing/DockerFile

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎compose/logprocessing/entrypoint.sh

Lines changed: 0 additions & 86 deletions
This file was deleted.

‎compose/openshiftjobs/DockerFile

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM alpine:3
2+
RUN apk update && apk upgrade
3+
#Need goaccess 1.92 for a timezone fix. Once that version is in the regular branch, we can pull it from there.
4+
RUN apk add goaccess --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main
5+
6+
RUN apk add --no-cache \
7+
aws-cli \
8+
bash \
9+
coreutils \
10+
tzdata
11+
12+
COPY ./compose/openshiftjobs/entrypoint.sh /
13+
COPY ./compose/openshiftjobs/scripts/analyzeexportlogs.sh /scripts/
14+
COPY ./compose/openshiftjobs/scripts/backupmediapvc.sh /scripts/
15+
COPY ./compose/openshiftjobs/scripts/ziplogs.sh /scripts/
16+
17+
RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh
18+
RUN sed -i 's/\r$//g' /scripts/analyzeexportlogs.sh && chmod +x /scripts/analyzeexportlogs.sh
19+
RUN sed -i 's/\r$//g' /scripts/backupmediapvc.sh && chmod +x /scripts/backupmediapvc.sh
20+
RUN sed -i 's/\r$//g' /scripts/ziplogs.sh && chmod +x /scripts/ziplogs.sh
21+
22+
23+
ENTRYPOINT ["/entrypoint.sh"]

‎compose/openshiftjobs/entrypoint.sh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
# Check the number of arguments
4+
if [ "$#" -lt 1 ]; then
5+
echo "Usage: $0 <script_name> [args...]"
6+
exit 1
7+
fi
8+
9+
# Determine which script to run based on the first argument
10+
case "$1" in
11+
backupmediapvc)
12+
# Run backupmediapvc.sh which will backup the data from the django-media pvc to s3 storage
13+
/scripts/backupmediapvc.sh
14+
;;
15+
ziplogs)
16+
# Run ziplogs.sh which will zip all files that were created in the previous hour or older in the nginx log storage pvc
17+
/scripts/ziplogs.sh
18+
;;
19+
analyzeexportlogs)
20+
# Run analyzeexportlogs with additional arguments which will send the specified days logs through goaccess and then upload to s3.
21+
shift # Remove the first argument (script number)
22+
/scripts/analyzeexportlogs.sh "$@"
23+
;;
24+
*)
25+
echo "Invalid script"
26+
exit 1
27+
;;
28+
esac

0 commit comments

Comments
 (0)