From f120391074ac7f6649263b89ef1e09f967171001 Mon Sep 17 00:00:00 2001 From: hannah-macdonald1 Date: Thu, 1 Feb 2024 15:05:20 -0700 Subject: [PATCH] test deploy on openshift --- .github/workflows/.merge.yml | 17 ++++++- containers/db_cleanup/cleanup.js | 50 ++++++++++++------- .../openshift/templates/db-cleanup-dc.yaml | 20 ++++---- containers/db_cleanup/package-lock.json | 35 +++++++++++++ containers/db_cleanup/package.json | 1 + 5 files changed, 93 insertions(+), 30 deletions(-) diff --git a/.github/workflows/.merge.yml b/.github/workflows/.merge.yml index 706f250..74435f5 100644 --- a/.github/workflows/.merge.yml +++ b/.github/workflows/.merge.yml @@ -4,6 +4,12 @@ on: pull_request: branches: [dev] types: [closed] + # temp change to force a deployment + push: + branches: + - "*" + commit: + workflow_dispatch: concurrency: @@ -59,12 +65,16 @@ jobs: packages: write strategy: matrix: - package: [vhers-virus-scan] + package: [vhers-virus-scan, db-cleanup] include: - package: vhers-virus-scan build_context: ./ triggers: ('vhers-virus-scan/') build_file: Dockerfile + - package: db-cleanup + build_context: ./containers/db_cleanup + triggers: ('vhers-virus-scan/containers/db_cleanup') + build_file: Dockerfile timeout-minutes: 10 steps: - uses: bcgov-nr/action-builder-ghcr@v2.0.0 @@ -90,6 +100,11 @@ jobs: - name: vhers-virus-scan file: ./openshift.deploy.yml overwrite: true + name: [db-cleanup] + include: + - name: db-cleanup + file: ./containers/db_cleanup/openshift/templates/db-cleanup-dc.yaml + overwrite: true steps: - uses: bcgov-nr/action-deployer-openshift@v2.0.0 diff --git a/containers/db_cleanup/cleanup.js b/containers/db_cleanup/cleanup.js index 0077796..95bdcc5 100644 --- a/containers/db_cleanup/cleanup.js +++ b/containers/db_cleanup/cleanup.js @@ -28,28 +28,21 @@ const fileTimeString = `${now.toDate()}-${now.hour().toPrecision(2)}:${now.minut const retainUntilString = retainUntil.format('YYYY-MM-DD HH:mm:ss.SSS ZZ'); // COPY logs that are about to be deleted to csv +// \\g /deleted/vhers-audit-log.csv // TODO: Automate storing them elsewhere? -// try { -// pg.raw(`\\COPY public.vhers_audit_log TO '/deleted/vhers-audit-log.csv' WITH (FORMAT CSV, HEADER);`).then(); -// } catch (err) { -// console.log(err); -// exit(1); // cannot continue without saving backup -// } + // pg.raw(`COPY public.vhers_audit_log TO STDOUT WITH (FORMAT CSV, HEADER)`).then( + // (ret)=>{console.log(ret); process.exit(0);}, + // (err) => {console.log(err); process.exit(1);} + // ); -// pool.connect(function (err, client, done) { -// var stream = client.query(to(`COPY public.vhers_audit_log TO STDOUT`)) -// // var fileStream = fs.createReadStream('/deleted/vhers-audit-log.csv') -// // fileStream.on('error', done) -// stream.on('error', done) -// stream.on('finish', done) -// // fileStream.pipe(stream) -// }); - -// const fs = require('node:fs'); +// const fs = require('fs'); +// const csv = require('csv'); +// const path = require('path'); +// const EOL = require('os').EOL; // const { Pool } = require('pg'); // const { to } = require('pg-copy-streams'); -// var pool = new Pool({ +// const pool = new Pool({ // host: process.env.DB_HOST, // port: process.env.DB_PORT, // user: process.env.DB_USERNAME, @@ -57,7 +50,28 @@ const retainUntilString = retainUntil.format('YYYY-MM-DD HH:mm:ss.SSS ZZ'); // password: process.env.DB_PASSWORD, // }); -// const exec = require('child_process').exec; +// const outFile = path.join( __dirname, 'vhers_audit_log.csv'); +// const writeStream = fs.createWriteStream(outFile); + +// const parse = csv.parse(); + +// const transform = csv.transform((row, cb) => { +// row.push('NEW_COL'); +// result = row.join(',') + EOL; +// cb(null, result); +// }); + +// pool.connect(function (err, client, done) { +// const stream = client.query(to(`COPY public.vhers_audit_log TO STDOUT WITH (FORMAT CSV, HEADER)`)) +// // var fileStream = fs.createReadStream('/deleted/vhers-audit-log.csv') +// // // fileStream.on('error', done) +// // stream.on('error', done) +// // stream.on('finish', done) +// // // fileStream.pipe(stream) +// stream.pipe(parse).pipe(transform).pipe(writeStream); +// stream.on('end', done) +// stream.on('error', done) +// }); // Delete the logs pg('vhers_audit_log').where('created_at', '<', retainUntilString).delete().then( diff --git a/containers/db_cleanup/openshift/templates/db-cleanup-dc.yaml b/containers/db_cleanup/openshift/templates/db-cleanup-dc.yaml index 6d189c1..87c15cd 100644 --- a/containers/db_cleanup/openshift/templates/db-cleanup-dc.yaml +++ b/containers/db_cleanup/openshift/templates/db-cleanup-dc.yaml @@ -68,8 +68,10 @@ objects: metadata: name: db-cleanup-cronjob spec: - schedule: "30 2 * * SUN" - startingDeadlineSeconds: 3600 + # schedule: "30 2 * * SUN" + # Different schedule for test purposes + schedule: "*/10 * * * *" + startingDeadlineSeconds: 200 concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 @@ -90,27 +92,27 @@ objects: valueFrom: secretKeyRef: name: patroni-ha-postgres-instance - key: app-db-name + key: app-db-name-dev - name: DB_HOST valueFrom: secretKeyRef: name: patroni-ha-postgres-instance - key: app-db-hostname + key: db-hostname-dev - name: DB_PORT valueFrom: secretKeyRef: name: patroni-ha-postgres-instance - key: app-db-port + key: db-port-dev - name: DB_USER valueFrom: secretKeyRef: name: patroni-ha-postgres-instance - key: superuser-username + key: superuser-username-dev - name: DB_PASSWORD valueFrom: secretKeyRef: name: patroni-ha-postgres-instance - key: superuser-password + key: superuser-password-dev - name: RENTENTION_MONTHS valueFrom: secretKeyRef: @@ -126,8 +128,4 @@ objects: requests: memory: "${MEMORY_REQUEST}" cpu: "${CPU_REQUEST}" - volumes: - - name: output - persistentVolumeClaim: - claimName: db-deletion-dump restartPolicy: OnFailure diff --git a/containers/db_cleanup/package-lock.json b/containers/db_cleanup/package-lock.json index fa68fea..e1bb28c 100644 --- a/containers/db_cleanup/package-lock.json +++ b/containers/db_cleanup/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "csv": "^6.3.6", "dayjs": "^1.11.10", "dotenv": "^16.4.1", "knex": "^3.1.0", @@ -37,6 +38,35 @@ "node": ">=14" } }, + "node_modules/csv": { + "version": "6.3.6", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.6.tgz", + "integrity": "sha512-jsEsX2HhGp7xiwrJu5srQavKsh+HUJcCi78Ar3m4jlmFKRoTkkMy7ZZPP+LnQChmaztW+uj44oyfMb59daAs/Q==", + "dependencies": { + "csv-generate": "^4.3.1", + "csv-parse": "^5.5.3", + "csv-stringify": "^6.4.5", + "stream-transform": "^3.3.0" + }, + "engines": { + "node": ">= 0.1.90" + } + }, + "node_modules/csv-generate": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.3.1.tgz", + "integrity": "sha512-7YeeJq+44/I/O5N2sr2qBMcHZXhpfe38eh7DOFxyMtYO+Pir7kIfgFkW5MPksqKqqR6+/wX7UGoZm1Ot11151w==" + }, + "node_modules/csv-parse": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz", + "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A==" + }, + "node_modules/csv-stringify": { + "version": "6.4.5", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.5.tgz", + "integrity": "sha512-SPu1Vnh8U5EnzpNOi1NDBL5jU5Rx7DVHr15DNg9LXDTAbQlAVAmEbVt16wZvEW9Fu9Qt4Ji8kmeCJ2B1+4rFTQ==" + }, "node_modules/dayjs": { "version": "1.11.10", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz", @@ -380,6 +410,11 @@ "node": ">= 10.x" } }, + "node_modules/stream-transform": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.0.tgz", + "integrity": "sha512-pG1NeDdmErNYKtvTpFayrEueAmL0xVU5wd22V7InGnatl4Ocq3HY7dcXIKj629kXvYQvglCC7CeDIGAlx1RNGA==" + }, "node_modules/supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", diff --git a/containers/db_cleanup/package.json b/containers/db_cleanup/package.json index ba80ca5..ffa7350 100644 --- a/containers/db_cleanup/package.json +++ b/containers/db_cleanup/package.json @@ -9,6 +9,7 @@ "author": "", "license": "ISC", "dependencies": { + "csv": "^6.3.6", "dayjs": "^1.11.10", "dotenv": "^16.4.1", "knex": "^3.1.0",