From 7631d1cecdb6c1237507e359f0dd05f6b7347005 Mon Sep 17 00:00:00 2001 From: Matthew Bystedt Date: Thu, 4 Jan 2024 14:26:56 -0800 Subject: [PATCH] feat: improve database calls and broker/vault integration --- README.md | 11 +- package-lock.json | 30 ++--- package.json | 2 +- setenv-tmpl.sh | 18 ++- src/constants.ts | 15 ++- src/cron/backup.ts | 197 ++++++++++++++++++++----------- src/cron/compress.ts | 12 +- src/cron/janitor.ts | 8 +- src/index.ts | 3 +- src/services/database.service.ts | 123 ++++++++++++------- src/services/minio.ts | 14 +-- 11 files changed, 281 insertions(+), 152 deletions(-) diff --git a/README.md b/README.md index 36e1d95..426e2ce 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,12 @@ Sidecar for rotating log files to objectstore. 1. Copy `setenv-tmpl.sh` to `setenv-local.sh`. 2. Modify cron to run every minute ("*/1 * * * *"). 3. Change LOGROTATE_DIRECTORY to "logs". Add your OBJECT_STORAGE_ secrets. -4. Start sidecar: `npm run start` -5. Create sample log files: `./test/create-log-files.sh` -6. View DB as cron executes: `sqlite3 ./logs/cron.db 'select * from logs'` -7. Use https://min.io/docs/minio/linux/reference/minio-mc.html# to view files -8. Stop and delete test files in objectstore +4. Source env: `source ./setenv-local.sh` +5. Start sidecar: `npm run start` +6. Create sample log files: `./test/create-log-files.sh` +7. View DB as cron executes: `sqlite3 ./logs/cron.db 'select * from logs'` +8. Use https://min.io/docs/minio/linux/reference/minio-mc.html# to view files +9. Stop and delete test files in objectstore # License diff --git a/package-lock.json b/package-lock.json index bb24408..f09827a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "Apache-2.0", "dependencies": { - "axios": "^1.6.3", + "axios": "^1.6.4", "cron": "^3.1.6", "croner": "^8.0.0", "minio": "^7.1.3", @@ -740,11 +740,11 @@ } }, "node_modules/axios": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.3.tgz", - "integrity": "sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==", + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.4.tgz", + "integrity": "sha512-heJnIs6N4aa1eSthhN9M5ioILu8Wi8vmQW9iHQ9NUvfkJb0lEEDUiIdQNAuBtfUt3FxReaKdpQA5DbmMOqzF/A==", "dependencies": { - "follow-redirects": "^1.15.0", + "follow-redirects": "^1.15.4", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } @@ -1470,9 +1470,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", + "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", "funding": [ { "type": "individual", @@ -3977,11 +3977,11 @@ "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==" }, "axios": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.3.tgz", - "integrity": "sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==", + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.4.tgz", + "integrity": "sha512-heJnIs6N4aa1eSthhN9M5ioILu8Wi8vmQW9iHQ9NUvfkJb0lEEDUiIdQNAuBtfUt3FxReaKdpQA5DbmMOqzF/A==", "requires": { - "follow-redirects": "^1.15.0", + "follow-redirects": "^1.15.4", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } @@ -4516,9 +4516,9 @@ "dev": true }, "follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", + "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==" }, "for-each": { "version": "0.3.3", diff --git a/package.json b/package.json index 83251a9..7620022 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "author": "", "license": "Apache-2.0", "dependencies": { - "axios": "^1.6.3", + "axios": "^1.6.4", "cron": "^3.1.6", "croner": "^8.0.0", "minio": "^7.1.3", diff --git a/setenv-tmpl.sh b/setenv-tmpl.sh index cacdbdd..b0fa3bc 100755 --- a/setenv-tmpl.sh +++ b/setenv-tmpl.sh @@ -10,14 +10,26 @@ export JANITOR_COPIES=3 # export LOGROTATE_STATUSFILE="cron.db" # Required +# export OBJECT_STORAGE_ENABLED="true" export OBJECT_STORAGE_END_POINT="" export OBJECT_STORAGE_ACCESS_KEY="" export OBJECT_STORAGE_BUCKET="" -# Required (if not using NR Broker & Vault) export OBJECT_STORAGE_SECRET_KEY="" -# Required (if using NR Broker) -export BROKER_JWT="" +# Set BROKER_JWT to use Broker and Vault +# export BROKER_JWT="" +# export BROKER_URL="" +# export BROKER_USER="" export BROKER_PROJECT="" export BROKER_SERVICE="" export BROKER_ENVIRONMENT="" + +# export VAULT_CRED_PATH="" +# If VAULT_CRED_KEYS_* is set, the value from VAULT_CRED_PATH replaces OBJECT_STORAGE_* +# Example: VAULT_CRED_KEYS_SECRET_KEY="secret_key" would replace OBJECT_STORAGE_SECRET_KEY +# with the value of the key 'secret_key' at the path VAULT_CRED_PATH in Vault. +# export VAULT_CRED_KEYS_END_POINT="" +# export VAULT_CRED_KEYS_ACCESS_KEY="" +# export VAULT_CRED_KEYS_BUCKET="" +# export VAULT_CRED_KEYS_SECRET_KEY="" +# export VAULT_URL="" diff --git a/src/constants.ts b/src/constants.ts index 0f2005b..46acbe4 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -15,6 +15,8 @@ export const JANITOR_COPIES = Number.parseInt( ); // Object storage - required +export const OBJECT_STORAGE_ENABLED = + process.env.OBJECT_STORAGE_ENABLED == 'true' ?? true; export const OBJECT_STORAGE_END_POINT = process.env.OBJECT_STORAGE_END_POINT ?? ''; export const OBJECT_STORAGE_ACCESS_KEY = @@ -38,11 +40,20 @@ export const ENV_LONG_TO_SHORT: { [key: string]: string } = { export const BROKER_PROJECT = process.env.BROKER_PROJECT ?? ''; export const BROKER_SERVICE = process.env.BROKER_SERVICE ?? ''; export const BROKER_ENVIRONMENT = process.env.BROKER_ENVIRONMENT ?? ''; +// Path to the Object storage credentials in Vault export const VAULT_CRED_PATH = process.env.VAULT_CRED_PATH ?? `/apps/${ENV_LONG_TO_SHORT[BROKER_ENVIRONMENT]}/${BROKER_PROJECT}/${BROKER_SERVICE}/rotatebackup`; - -export const VAULT_CRED_KEY = process.env.VAULT_CRED_KEY ?? 'secret_key'; +// If VAULT_CRED_KEYS_* is set, the value from VAULT_CRED_PATH replaces OBJECT_STORAGE_* +// Example: VAULT_CRED_KEYS_SECRET_KEY="secret_key" would replace OBJECT_STORAGE_SECRET_KEY +// with the value of the key 'secret_key' at the path VAULT_CRED_PATH in Vault. +export const VAULT_CRED_KEYS_END_POINT = + process.env.VAULT_CRED_KEYS_END_POINT ?? ''; +export const VAULT_CRED_KEYS_ACCESS_KEY = + process.env.VAULT_CRED_KEYS_ACCESS_KEY ?? ''; +export const VAULT_CRED_KEYS_BUCKET = process.env.VAULT_CRED_KEYS_BUCKET ?? ''; +export const VAULT_CRED_KEYS_SECRET_KEY = + process.env.VAULT_CRED_KEYS_SECRET_KEY ?? ''; export const VAULT_URL = process.env.VAULT_URL ?? 'https://knox.io.nrs.gov.bc.ca'; diff --git a/src/cron/backup.ts b/src/cron/backup.ts index 7d0caf4..72ec526 100644 --- a/src/cron/backup.ts +++ b/src/cron/backup.ts @@ -11,18 +11,40 @@ import { BROKER_SERVICE, BROKER_USER, DB_FILE_STATUS, + OBJECT_STORAGE_ACCESS_KEY, OBJECT_STORAGE_BUCKET, + OBJECT_STORAGE_ENABLED, + OBJECT_STORAGE_END_POINT, OBJECT_STORAGE_SECRET_KEY, - VAULT_CRED_KEY, + VAULT_CRED_KEYS_ACCESS_KEY, + VAULT_CRED_KEYS_BUCKET, + VAULT_CRED_KEYS_END_POINT, + VAULT_CRED_KEYS_SECRET_KEY, VAULT_CRED_PATH, } from '../constants'; import { DatabaseService } from '../services/database.service'; import VaultService from '../broker/vault.service'; import BrokerService from '../broker/broker.service'; +interface LogStatus { + id: number; + basename: string; + path: string; +} + +interface LogArtifact { + id: number; + checksum: string; + name: string; + size: number; + type: string; +} + +type FileUpdateCallback = (id: number) => Promise; + export async function backup(db: DatabaseService) { console.log('backup: start'); - const result = await db.query<{ + const result = await db.all<{ id: number; basename: string; path: string; @@ -41,67 +63,115 @@ export async function backup(db: DatabaseService) { return; } - if (OBJECT_STORAGE_SECRET_KEY) { - await backupWithSecret(db, OBJECT_STORAGE_SECRET_KEY, result); - } else { - const brokerService = new BrokerService(BROKER_JWT); - try { - const openResponse = await brokerService.open({ - event: { - provider: 'nr-objectstore-rotate-backup', - reason: 'Cron triggered', - }, - actions: [ - { - action: 'backup', - id: 'backup', - provision: ['token/self'], - service: { - name: BROKER_SERVICE, - project: BROKER_PROJECT, - environment: BROKER_ENVIRONMENT, - }, - }, - ], - user: { - name: BROKER_USER, - }, - }); - const actionToken = openResponse.actions['backup'].token; - const vaultAccessToken = await brokerService.provisionToken(actionToken); - const vault = new VaultService(vaultAccessToken); - const objectStorageCreds = await vault.read(VAULT_CRED_PATH); - const secretKey = objectStorageCreds[VAULT_CRED_KEY]; - vault.revokeToken(); - const backupFiles = await backupWithSecret(db, secretKey, result); - for (const fileObj of backupFiles) { - await brokerService.attachArtifact(actionToken, fileObj); + const fileUpdateCb: FileUpdateCallback = (id) => { + return db.updatelogStatus(id, DB_FILE_STATUS.CopiedToObjectStore); + }; + + try { + if (!OBJECT_STORAGE_ENABLED) { + // Skip copy to object storage + for (const file of result.rows) { + await db.updatelogStatus(file.id, DB_FILE_STATUS.CopiedToObjectStore); } - brokerService.close(true); - } catch (e: any) { - // Error! - console.log(e); + } else if (BROKER_JWT === '') { + await backupUsingEnv(result.rows, fileUpdateCb); + } else { + await backupUsingBroker(BROKER_JWT, result.rows, fileUpdateCb); } + } catch (e: any) { + // Error! + console.log(e); } } +async function backupUsingEnv( + dbFileRows: LogStatus[], + cb: FileUpdateCallback, +): Promise { + const backupFiles = await backupWithSecret( + dbFileRows, + OBJECT_STORAGE_END_POINT, + OBJECT_STORAGE_ACCESS_KEY, + OBJECT_STORAGE_SECRET_KEY, + OBJECT_STORAGE_BUCKET, + ); + + for (const file of backupFiles) { + await cb(file.id); + } + return backupFiles; +} + +async function backupUsingBroker( + brokerJwt: string, + dbFileRows: LogStatus[], + cb: FileUpdateCallback, +): Promise { + const brokerService = new BrokerService(brokerJwt); + const openResponse = await brokerService.open({ + event: { + provider: 'nr-objectstore-rotate-backup', + reason: 'Cron triggered', + }, + actions: [ + { + action: 'backup', + id: 'backup', + provision: ['token/self'], + service: { + name: BROKER_SERVICE, + project: BROKER_PROJECT, + environment: BROKER_ENVIRONMENT, + }, + }, + ], + user: { + name: BROKER_USER, + }, + }); + const actionToken = openResponse.actions['backup'].token; + const vaultAccessToken = await brokerService.provisionToken(actionToken); + const vault = new VaultService(vaultAccessToken); + const objectStorageCreds = await vault.read(VAULT_CRED_PATH); + vault.revokeToken(); + const backupFiles = await backupWithSecret( + dbFileRows, + VAULT_CRED_KEYS_END_POINT === '' + ? OBJECT_STORAGE_END_POINT + : objectStorageCreds[VAULT_CRED_KEYS_END_POINT], + VAULT_CRED_KEYS_ACCESS_KEY === '' + ? OBJECT_STORAGE_ACCESS_KEY + : objectStorageCreds[VAULT_CRED_KEYS_ACCESS_KEY], + VAULT_CRED_KEYS_BUCKET === '' + ? OBJECT_STORAGE_SECRET_KEY + : objectStorageCreds[VAULT_CRED_KEYS_BUCKET], + VAULT_CRED_KEYS_SECRET_KEY === '' + ? OBJECT_STORAGE_BUCKET + : objectStorageCreds[VAULT_CRED_KEYS_SECRET_KEY], + ); + + for (const file of backupFiles) { + await brokerService.attachArtifact(actionToken, file); + await cb(file.id); + } + brokerService.close(true); + + return backupFiles; +} + async function backupWithSecret( - db: DatabaseService, - secret: string, - dbResult: { - rows: { - id: number; - basename: string; - path: string; - }[]; - }, -): Promise { - const client = getClient(secret); - const files = []; - for (const row of dbResult.rows) { + dbFileRows: LogStatus[], + endPoint: string, + accessKey: string, + secretKey: string, + bucket: string, +): Promise { + const client = getClient(endPoint, accessKey, secretKey); + const files: LogArtifact[] = []; + for (const row of dbFileRows) { try { const response = await client.fPutObject( - OBJECT_STORAGE_BUCKET, + bucket, path.basename(row.path), row.path, ); @@ -111,24 +181,15 @@ async function backupWithSecret( console.log(info); continue; } - db.query<{ - id: number; - basename: string; - path: string; - }>( - ` - UPDATE logs - SET status = ? - WHERE id = ? - `, - [DB_FILE_STATUS.CopiedToObjectStore, row.id], - ); + const checksum = `sha256:${await computeHash(row.path)}`; files.push({ - checksum: `sha256:${await computeHash(row.path)}`, + id: row.id, + checksum, name: path.basename(row.path), size: fs.statSync(row.path).size, type: 'tgz', }); + console.log(`backup: Sent ${row.path} [${checksum}]`); } return files; } diff --git a/src/cron/compress.ts b/src/cron/compress.ts index f8b737b..a9d852c 100644 --- a/src/cron/compress.ts +++ b/src/cron/compress.ts @@ -6,7 +6,7 @@ import { DatabaseService } from '../services/database.service'; export async function compress(db: DatabaseService) { console.log('compress: start'); - const result = await db.query<{ + const result = await db.all<{ id: number; basename: string; path: string; @@ -33,7 +33,9 @@ export async function compress(db: DatabaseService) { } await new Promise((resolve, reject) => { - exec(`tar -zcvf ${row.path}.tgz ${row.path}`, (error, stdout, stderr) => { + const cmd = `tar -zcvf ${row.path}.tgz ${row.path}`; + console.log(`compress: ${cmd}`); + exec(cmd, (error, stdout, stderr) => { if (error) { // node couldn't run the command reject(error); @@ -41,13 +43,13 @@ export async function compress(db: DatabaseService) { } // Using process.stdout.write to prevent double new lines. if (stdout) { - process.stdout.write(`stdout: ${stdout}`); + process.stdout.write(`compress: [stdout] ${stdout}`); } if (stderr) { - process.stdout.write(`stderr: ${stderr}`); + process.stdout.write(`compress: [stderr] ${stderr}`); } - db.query( + db.run( ` UPDATE logs SET status = ?, diff --git a/src/cron/janitor.ts b/src/cron/janitor.ts index a142c41..a5436cf 100644 --- a/src/cron/janitor.ts +++ b/src/cron/janitor.ts @@ -5,7 +5,7 @@ import { DatabaseService } from '../services/database.service'; export async function syncLogsDb(db: DatabaseService) { console.log('janitor: sync database records'); - const result = await db.query<{ + const result = await db.all<{ id: number; basename: string; path: string; @@ -24,7 +24,7 @@ export async function syncLogsDb(db: DatabaseService) { console.log( `janitor: delete database row ${row.id}; file missing: ${row.path}`, ); - db.deleteLog(row.id); + await db.deleteLog(row.id); } } } @@ -32,7 +32,7 @@ export async function syncLogsDb(db: DatabaseService) { export async function removeOldLogs(db: DatabaseService) { console.log('janitor: start'); const nameHash: { [key: string]: number } = {}; - const result = await db.query<{ + const result = await db.all<{ id: number; basename: string; path: string; @@ -54,7 +54,7 @@ export async function removeOldLogs(db: DatabaseService) { if (nameHash[row.basename] > JANITOR_COPIES) { console.log(`Delete: ${row.path}`); - db.deleteLog(row.id); + await db.deleteLog(row.id); fs.rmSync(row.path); } } diff --git a/src/index.ts b/src/index.ts index 18e8eb6..17ec0a0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,8 +28,7 @@ if ( } async function main() { - const db = new DatabaseService(); - await db.init(); + const db = await DatabaseService.create(); const rotateJob = Cron(CRON_ROTATE, async () => { await rotateLogs(db); diff --git a/src/services/database.service.ts b/src/services/database.service.ts index 54b7e6e..3ab6faf 100644 --- a/src/services/database.service.ts +++ b/src/services/database.service.ts @@ -9,61 +9,104 @@ import { const databasePath = path.join(LOGROTATE_DIRECTORY, LOGROTATE_STATUSFILE); export class DatabaseService { - private db: sqlite3.Database; - constructor() { - this.db = new sqlite3.Database(databasePath); + private constructor(private db: sqlite3.Database) {} + + /** + * Factory + * @returns DatabaseService + */ + static async create() { + const service = await new Promise((resolve, reject) => { + const db = new sqlite3.Database(databasePath, (error: Error | null) => { + if (error) reject(error); + else resolve(new DatabaseService(db)); + }); + }); + await service.init(); + return service; } + /** + * Initializes the database + */ async init() { - await this.createDb(); + await this.createTables(); } - async createDb() { - // Create a table - return new Promise((resolve, reject) => { - this.db.serialize(() => { - this.db.run( - ` - CREATE TABLE IF NOT EXISTS logs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - basename TEXT, - path TEXT, - status INT - ) - `, - (error: Error | null) => { - if (error) reject(error); - else resolve(); - }, - ); - }); - }); + /** + * Creates table if it does not exist + * @returns Promise + */ + private async createTables() { + await this.run(` + CREATE TABLE IF NOT EXISTS logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + basename TEXT, + path TEXT, + status INT + ) + `); } - // Insert data into the table - async addLog(basename: string, path: string, status = DB_FILE_STATUS.Moved) { - const statement = this.db.prepare( + + /** + * Add log into table + * @param basename + * @param path + * @param status + * @returns + */ + addLog(basename: string, path: string, status = DB_FILE_STATUS.Moved) { + return this.run( 'INSERT INTO logs (basename, path, status) VALUES (?, ?, ?)', + [basename, path, status], + ); + } + + /** + * Update the status of a log + * @param id + * @param status + * @returns Promise + */ + updatelogStatus(id: number, status: DB_FILE_STATUS) { + return this.run( + ` + UPDATE logs + SET status = ? + WHERE id = ? + `, + [status, id], ); - statement.run(basename, path, status); - statement.finalize(); } + bulkStatusChange(fromStatus: DB_FILE_STATUS, toStatus: DB_FILE_STATUS) { + return this.run('UPDATE logs SET status = ? WHERE status = ?', [ + toStatus, + fromStatus, + ]); + } + + /** + * Delete log + * @param id + * @returns + */ deleteLog(id: number) { - const statement = this.db.prepare('DELETE FROM logs WHERE id = ?'); - statement.run(id); - statement.finalize(); + return this.run('DELETE FROM logs WHERE id = ?', [id]); } - async bulkStatusChange(fromStatus: DB_FILE_STATUS, toStatus: DB_FILE_STATUS) { - const updateStatement = this.db.prepare( - 'UPDATE logs SET status = ? WHERE status = ?', - ); - updateStatement.run(toStatus, fromStatus); - updateStatement.finalize(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + run(sql: string, params: any = []) { + return new Promise((resolve, reject) => { + this.db.run(sql, params, (error) => { + if (error) reject(error); + else resolve(); + }); + }); } // eslint-disable-next-line @typescript-eslint/no-explicit-any - async query(sql: string, params?: any) { + all(sql: string, params: any = []) { return new Promise<{ rows: T[] }>((resolve, reject) => { this.db.all(sql, params, (error, rows) => { if (error) reject(error); @@ -72,7 +115,7 @@ export class DatabaseService { }); } - async close() { + close() { return new Promise((resolve, reject) => { this.db.close((error: Error | null) => { if (error) reject(error); diff --git a/src/services/minio.ts b/src/services/minio.ts index fbc5f06..911e954 100644 --- a/src/services/minio.ts +++ b/src/services/minio.ts @@ -1,15 +1,15 @@ import * as Minio from 'minio'; -import { - OBJECT_STORAGE_ACCESS_KEY, - OBJECT_STORAGE_END_POINT, -} from '../constants'; // Default URL if not defined to avoid startup errors in unit tests, batch, etc. -export function getClient(secretKey: string) { +export function getClient( + endPoint: string, + accessKey: string, + secretKey: string, +) { return new Minio.Client({ - endPoint: OBJECT_STORAGE_END_POINT, + endPoint, useSSL: true, - accessKey: OBJECT_STORAGE_ACCESS_KEY, + accessKey, secretKey, }); }