Skip to content

Commit

Permalink
Add functionality to copy out old logs and delte them form the db
Browse files Browse the repository at this point in the history
  • Loading branch information
@hannah-macdonald1
hannah-macdonald1 committed Feb 2, 2024
1 parent 1b60a5c commit c8a7471
Showing 1 changed file with 102 additions and 60 deletions.
162 changes: 102 additions & 60 deletions containers/db_cleanup/cleanup.js
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
require('dotenv').config();
const fs = require('fs');
const csv = require('csv');
const path = require('path');
const EOL = require('os').EOL;
const { Client } = require('pg');
const { to } = require('pg-copy-streams');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
const timezone = require('dayjs/plugin/timezone');
Expand All @@ -20,75 +26,111 @@ const pg = require('knex')({
},
});

const pinClient = new Client({
host: process.env.DB_HOST,
port: process.env.DB_PORT,
user: process.env.DB_USERNAME,
database: process.env.DB_NAME,
password: process.env.DB_PASSWORD,
});

const vhersClient = new Client({
host: process.env.DB_HOST,
port: process.env.DB_PORT,
user: process.env.DB_USERNAME,
database: process.env.DB_NAME,
password: process.env.DB_PASSWORD,
});

// Get timestamps for now and deletion interval
const retentionIntervalMonths = Math.abs(parseInt(process.env.RENTENTION_MONTHS));
const now = dayjs();
const retainUntil = now.subtract(retentionIntervalMonths, 'month');
const fileTimeString = `${now.toDate()}-${now.hour().toPrecision(2)}:${now.minute().toPrecision(2)}:${now.second().toPrecision(2)}`;
const fileTimeString = now.format('YYYY-MM-DD-HH-mm-ss');
const retainUntilString = retainUntil.format('YYYY-MM-DD HH:mm:ss.SSS ZZ');

// change to force deploy
// COPY logs that are about to be deleted to csv
// \\g /deleted/vhers-audit-log.csv
// TODO: Automate storing them elsewhere?
// pg.raw(`COPY public.vhers_audit_log TO STDOUT WITH (FORMAT CSV, HEADER)`).then(
// (ret)=>{console.log(ret); process.exit(0);},
// (err) => {console.log(err); process.exit(1);}
// );
// Create directory for entries
const dir = `./deleted/${fileTimeString}`
if (!fs.existsSync(dir)){
fs.mkdirSync(dir, { recursive: true });
}

// const fs = require('fs');
// const csv = require('csv');
// const path = require('path');
// const EOL = require('os').EOL;
// const { Pool } = require('pg');
// const { to } = require('pg-copy-streams');
// Create files
const vhersOutFile = path.join( __dirname, 'deleted', fileTimeString, 'vhers_audit_log.csv');
const pinOutFile = path.join( __dirname, 'deleted', fileTimeString, 'pin_audit_log.csv');
const vhersWriteStream = fs.createWriteStream(vhersOutFile);
const pinWriteStream = fs.createWriteStream(pinOutFile);

// const pool = new Pool({
// host: process.env.DB_HOST,
// port: process.env.DB_PORT,
// user: process.env.DB_USERNAME,
// database: process.env.DB_NAME,
// password: process.env.DB_PASSWORD,
// });
// Csv transforms
const parse = csv.parse();

// const outFile = path.join( __dirname, 'vhers_audit_log.csv');
// const writeStream = fs.createWriteStream(outFile);
const transform = csv.transform((row, cb) => {
result = row.join(',') + EOL;
cb(null, result);
});

const pinParse = csv.parse();

const pinTransform = csv.transform((row, cb) => {
result = row.join(',') + EOL;
cb(null, result);
});

// const parse = csv.parse();
// Copy functions
function async_vhers_output() {
return new Promise(function(resolve, reject) {
const vhersStream = vhersClient.query(to(`COPY (SELECT * FROM public.vhers_audit_log WHERE created_at < '${retainUntilString}') TO STDOUT WITH (FORMAT CSV, HEADER)`));
vhersStream.pipe(parse).pipe(transform).pipe(vhersWriteStream);
vhersStream.on('end', () => { return resolve()});
vhersStream.on('error', (err) => {return reject(err)});
})
}

// const transform = csv.transform((row, cb) => {
// row.push('NEW_COL');
// result = row.join(',') + EOL;
// cb(null, result);
// });
function async_pin_output() {
return new Promise(function(resolve, reject) {
const pinStream = pinClient.query(to(`COPY (SELECT * FROM public.pin_audit_log WHERE log_created_at < '${retainUntilString}') TO STDOUT WITH (FORMAT CSV, HEADER)`));
pinStream.pipe(pinParse).pipe(pinTransform).pipe(pinWriteStream);
pinStream.on('end', () => { return resolve()});
pinStream.on('error', (err) => {return reject(err)});
})
}

// pool.connect(function (err, client, done) {
// const stream = client.query(to(`COPY public.vhers_audit_log TO STDOUT WITH (FORMAT CSV, HEADER)`))
// // var fileStream = fs.createReadStream('/deleted/vhers-audit-log.csv')
// // // fileStream.on('error', done)
// // stream.on('error', done)
// // stream.on('finish', done)
// // // fileStream.pipe(stream)
// stream.pipe(parse).pipe(transform).pipe(writeStream);
// stream.on('end', done)
// stream.on('error', done)
// });
// Copy function IIFE (this gets arounds not allowing async functions outside of modules)
( async() => {
await vhersClient.connect();
await pinClient.connect();
const promises = [];
promises.push(async_vhers_output());
promises.push(async_pin_output());
Promise.all(promises).then(function AcceptHandler() {
delete_entries();
}, function ErrorHandler(error) {
console.log(error);
process.exit(1);
});

})();

// Delete the logs
pg('vhers_audit_log').where('created_at', '<', retainUntilString).delete().then(
() => {
pg('pin_audit_log').where('log_created_at', '<', retainUntilString).delete().then(
() => {
console.log(`Successfully deleted audit log entries prior to ${retainUntilString}`);
process.exit(0);
},
(err) => {
console.log(err);
process.exit(1);
}
);
},
(err) => {
console.log(err);
process.exit(1);
}
);
// Entry deletion function
function delete_entries() {
pg('vhers_audit_log').where('created_at', '<', retainUntilString).delete().then(
() => {
pg('pin_audit_log').where('log_created_at', '<', retainUntilString).delete().then(
() => {
vhersClient.end();
pinClient.end();
console.log(`Successfully deleted audit log entries prior to ${retainUntilString}`);
process.exit(0);
},
(err) => {
console.log(err);
process.exit(1);
}
);
},
(err) => {
console.log(err);
process.exit(1);
}
);
}

0 comments on commit c8a7471

Please sign in to comment.