feat: improve database calls and broker/vault integration

README.md

@@ -7,11 +7,12 @@ Sidecar for rotating log files to objectstore.
 1. Copy `setenv-tmpl.sh` to `setenv-local.sh`.
 2. Modify cron to run every minute ("*/1 * * * *").
 3. Change LOGROTATE_DIRECTORY to "logs". Add your OBJECT_STORAGE_ secrets.
-4. Start sidecar: `npm run start`
-5. Create sample log files: `./test/create-log-files.sh`
-6. View DB as cron executes: `sqlite3 ./logs/cron.db 'select * from logs'`
-7. Use https://min.io/docs/minio/linux/reference/minio-mc.html# to view files
-8. Stop and delete test files in objectstore
+4. Source env: `source ./setenv-local.sh`
+5. Start sidecar: `npm run start`
+6. Create sample log files: `./test/create-log-files.sh`
+7. View DB as cron executes: `sqlite3 ./logs/cron.db 'select * from logs'`
+8. Use https://min.io/docs/minio/linux/reference/minio-mc.html# to view files
+9. Stop and delete test files in objectstore
 
 # License
 

package.json

@@ -13,7 +13,7 @@
   "author": "",
   "license": "Apache-2.0",
   "dependencies": {
-    "axios": "^1.6.3",
+    "axios": "^1.6.4",
     "cron": "^3.1.6",
     "croner": "^8.0.0",
     "minio": "^7.1.3",

setenv-tmpl.sh

@@ -10,14 +10,26 @@ export JANITOR_COPIES=3
 # export LOGROTATE_STATUSFILE="cron.db"
 
 # Required
+# export OBJECT_STORAGE_ENABLED="true"
 export OBJECT_STORAGE_END_POINT=""
 export OBJECT_STORAGE_ACCESS_KEY=""
 export OBJECT_STORAGE_BUCKET=""
-# Required (if not using NR Broker & Vault)
 export OBJECT_STORAGE_SECRET_KEY=""
 
-# Required (if using NR Broker)
-export BROKER_JWT=""
+# Set BROKER_JWT to use Broker and Vault
+# export BROKER_JWT=""
+# export BROKER_URL=""
+# export BROKER_USER=""
 export BROKER_PROJECT=""
 export BROKER_SERVICE=""
 export BROKER_ENVIRONMENT=""
+
+# export VAULT_CRED_PATH=""
+# If VAULT_CRED_KEYS_* is set, the value from VAULT_CRED_PATH replaces OBJECT_STORAGE_*
+# Example: VAULT_CRED_KEYS_SECRET_KEY="secret_key" would replace OBJECT_STORAGE_SECRET_KEY
+#          with the value of the key 'secret_key' at the path VAULT_CRED_PATH in Vault.
+# export VAULT_CRED_KEYS_END_POINT=""
+# export VAULT_CRED_KEYS_ACCESS_KEY=""
+# export VAULT_CRED_KEYS_BUCKET=""
+# export VAULT_CRED_KEYS_SECRET_KEY=""
+# export VAULT_URL=""

src/constants.ts

@@ -15,6 +15,8 @@ export const JANITOR_COPIES = Number.parseInt(
 );
 
 // Object storage - required
+export const OBJECT_STORAGE_ENABLED =
+  process.env.OBJECT_STORAGE_ENABLED == 'true' ?? true;
 export const OBJECT_STORAGE_END_POINT =
   process.env.OBJECT_STORAGE_END_POINT ?? '';
 export const OBJECT_STORAGE_ACCESS_KEY =
@@ -38,11 +40,20 @@ export const ENV_LONG_TO_SHORT: { [key: string]: string } = {
 export const BROKER_PROJECT = process.env.BROKER_PROJECT ?? '';
 export const BROKER_SERVICE = process.env.BROKER_SERVICE ?? '';
 export const BROKER_ENVIRONMENT = process.env.BROKER_ENVIRONMENT ?? '';
+// Path to the Object storage credentials in Vault
 export const VAULT_CRED_PATH =
   process.env.VAULT_CRED_PATH ??
   `/apps/${ENV_LONG_TO_SHORT[BROKER_ENVIRONMENT]}/${BROKER_PROJECT}/${BROKER_SERVICE}/rotatebackup`;
-
-export const VAULT_CRED_KEY = process.env.VAULT_CRED_KEY ?? 'secret_key';
+// If VAULT_CRED_KEYS_* is set, the value from VAULT_CRED_PATH replaces OBJECT_STORAGE_*
+// Example: VAULT_CRED_KEYS_SECRET_KEY="secret_key" would replace OBJECT_STORAGE_SECRET_KEY
+//          with the value of the key 'secret_key' at the path VAULT_CRED_PATH in Vault.
+export const VAULT_CRED_KEYS_END_POINT =
+  process.env.VAULT_CRED_KEYS_END_POINT ?? '';
+export const VAULT_CRED_KEYS_ACCESS_KEY =
+  process.env.VAULT_CRED_KEYS_ACCESS_KEY ?? '';
+export const VAULT_CRED_KEYS_BUCKET = process.env.VAULT_CRED_KEYS_BUCKET ?? '';
+export const VAULT_CRED_KEYS_SECRET_KEY =
+  process.env.VAULT_CRED_KEYS_SECRET_KEY ?? '';
 export const VAULT_URL =
   process.env.VAULT_URL ?? 'https://knox.io.nrs.gov.bc.ca';
 

src/cron/backup.ts

@@ -11,18 +11,40 @@ import {
   BROKER_SERVICE,
   BROKER_USER,
   DB_FILE_STATUS,
+  OBJECT_STORAGE_ACCESS_KEY,
   OBJECT_STORAGE_BUCKET,
+  OBJECT_STORAGE_ENABLED,
+  OBJECT_STORAGE_END_POINT,
   OBJECT_STORAGE_SECRET_KEY,
-  VAULT_CRED_KEY,
+  VAULT_CRED_KEYS_ACCESS_KEY,
+  VAULT_CRED_KEYS_BUCKET,
+  VAULT_CRED_KEYS_END_POINT,
+  VAULT_CRED_KEYS_SECRET_KEY,
   VAULT_CRED_PATH,
 } from '../constants';
 import { DatabaseService } from '../services/database.service';
 import VaultService from '../broker/vault.service';
 import BrokerService from '../broker/broker.service';
 
+interface LogStatus {
+  id: number;
+  basename: string;
+  path: string;
+}
+
+interface LogArtifact {
+  id: number;
+  checksum: string;
+  name: string;
+  size: number;
+  type: string;
+}
+
+type FileUpdateCallback = (id: number) => Promise<any>;
+
 export async function backup(db: DatabaseService) {
   console.log('backup: start');
-  const result = await db.query<{
+  const result = await db.all<{
     id: number;
     basename: string;
     path: string;
@@ -41,67 +63,115 @@ export async function backup(db: DatabaseService) {
     return;
   }
 
-  if (OBJECT_STORAGE_SECRET_KEY) {
-    await backupWithSecret(db, OBJECT_STORAGE_SECRET_KEY, result);
-  } else {
-    const brokerService = new BrokerService(BROKER_JWT);
-    try {
-      const openResponse = await brokerService.open({
-        event: {
-          provider: 'nr-objectstore-rotate-backup',
-          reason: 'Cron triggered',
-        },
-        actions: [
-          {
-            action: 'backup',
-            id: 'backup',
-            provision: ['token/self'],
-            service: {
-              name: BROKER_SERVICE,
-              project: BROKER_PROJECT,
-              environment: BROKER_ENVIRONMENT,
-            },
-          },
-        ],
-        user: {
-          name: BROKER_USER,
-        },
-      });
-      const actionToken = openResponse.actions['backup'].token;
-      const vaultAccessToken = await brokerService.provisionToken(actionToken);
-      const vault = new VaultService(vaultAccessToken);
-      const objectStorageCreds = await vault.read(VAULT_CRED_PATH);
-      const secretKey = objectStorageCreds[VAULT_CRED_KEY];
-      vault.revokeToken();
-      const backupFiles = await backupWithSecret(db, secretKey, result);
-      for (const fileObj of backupFiles) {
-        await brokerService.attachArtifact(actionToken, fileObj);
+  const fileUpdateCb: FileUpdateCallback = (id) => {
+    return db.updatelogStatus(id, DB_FILE_STATUS.CopiedToObjectStore);
+  };
+
+  try {
+    if (!OBJECT_STORAGE_ENABLED) {
+      // Skip copy to object storage
+      for (const file of result.rows) {
+        await db.updatelogStatus(file.id, DB_FILE_STATUS.CopiedToObjectStore);
       }
-      brokerService.close(true);
-    } catch (e: any) {
-      // Error!
-      console.log(e);
+    } else if (BROKER_JWT === '') {
+      await backupUsingEnv(result.rows, fileUpdateCb);
+    } else {
+      await backupUsingBroker(BROKER_JWT, result.rows, fileUpdateCb);
     }
+  } catch (e: any) {
+    // Error!
+    console.log(e);
   }
 }
 
+async function backupUsingEnv(
+  dbFileRows: LogStatus[],
+  cb: FileUpdateCallback,
+): Promise<LogArtifact[]> {
+  const backupFiles = await backupWithSecret(
+    dbFileRows,
+    OBJECT_STORAGE_END_POINT,
+    OBJECT_STORAGE_ACCESS_KEY,
+    OBJECT_STORAGE_SECRET_KEY,
+    OBJECT_STORAGE_BUCKET,
+  );
+
+  for (const file of backupFiles) {
+    await cb(file.id);
+  }
+  return backupFiles;
+}
+
+async function backupUsingBroker(
+  brokerJwt: string,
+  dbFileRows: LogStatus[],
+  cb: FileUpdateCallback,
+): Promise<LogArtifact[]> {
+  const brokerService = new BrokerService(brokerJwt);
+  const openResponse = await brokerService.open({
+    event: {
+      provider: 'nr-objectstore-rotate-backup',
+      reason: 'Cron triggered',
+    },
+    actions: [
+      {
+        action: 'backup',
+        id: 'backup',
+        provision: ['token/self'],
+        service: {
+          name: BROKER_SERVICE,
+          project: BROKER_PROJECT,
+          environment: BROKER_ENVIRONMENT,
+        },
+      },
+    ],
+    user: {
+      name: BROKER_USER,
+    },
+  });
+  const actionToken = openResponse.actions['backup'].token;
+  const vaultAccessToken = await brokerService.provisionToken(actionToken);
+  const vault = new VaultService(vaultAccessToken);
+  const objectStorageCreds = await vault.read(VAULT_CRED_PATH);
+  vault.revokeToken();
+  const backupFiles = await backupWithSecret(
+    dbFileRows,
+    VAULT_CRED_KEYS_END_POINT === ''
+      ? OBJECT_STORAGE_END_POINT
+      : objectStorageCreds[VAULT_CRED_KEYS_END_POINT],
+    VAULT_CRED_KEYS_ACCESS_KEY === ''
+      ? OBJECT_STORAGE_ACCESS_KEY
+      : objectStorageCreds[VAULT_CRED_KEYS_ACCESS_KEY],
+    VAULT_CRED_KEYS_BUCKET === ''
+      ? OBJECT_STORAGE_SECRET_KEY
+      : objectStorageCreds[VAULT_CRED_KEYS_BUCKET],
+    VAULT_CRED_KEYS_SECRET_KEY === ''
+      ? OBJECT_STORAGE_BUCKET
+      : objectStorageCreds[VAULT_CRED_KEYS_SECRET_KEY],
+  );
+
+  for (const file of backupFiles) {
+    await brokerService.attachArtifact(actionToken, file);
+    await cb(file.id);
+  }
+  brokerService.close(true);
+
+  return backupFiles;
+}
+
 async function backupWithSecret(
-  db: DatabaseService,
-  secret: string,
-  dbResult: {
-    rows: {
-      id: number;
-      basename: string;
-      path: string;
-    }[];
-  },
-): Promise<any[]> {
-  const client = getClient(secret);
-  const files = [];
-  for (const row of dbResult.rows) {
+  dbFileRows: LogStatus[],
+  endPoint: string,
+  accessKey: string,
+  secretKey: string,
+  bucket: string,
+): Promise<LogArtifact[]> {
+  const client = getClient(endPoint, accessKey, secretKey);
+  const files: LogArtifact[] = [];
+  for (const row of dbFileRows) {
     try {
       const response = await client.fPutObject(
-        OBJECT_STORAGE_BUCKET,
+        bucket,
         path.basename(row.path),
         row.path,
       );
@@ -111,24 +181,15 @@ async function backupWithSecret(
       console.log(info);
       continue;
     }
-    db.query<{
-      id: number;
-      basename: string;
-      path: string;
-    }>(
-      `
-      UPDATE logs
-      SET status = ?
-      WHERE id = ?
-      `,
-      [DB_FILE_STATUS.CopiedToObjectStore, row.id],
-    );
+    const checksum = `sha256:${await computeHash(row.path)}`;
     files.push({
-      checksum: `sha256:${await computeHash(row.path)}`,
+      id: row.id,
+      checksum,
       name: path.basename(row.path),
       size: fs.statSync(row.path).size,
       type: 'tgz',
     });
+    console.log(`backup: Sent ${row.path} [${checksum}]`);
   }
   return files;
 }