Merge pull request #15 from bcgov/feat/rotationRules

feat: add rotation rules and fix json post

Author: andrwils

README.md

@@ -30,6 +30,18 @@ The environment variable `CRON_BACKUP` is used to schedule the back of the compr
 
 The environment variable `CRON_JANITOR` is used to schedule the janitor which removes files after they have been backed up. The number of log files to retain can be configured by setting `JANITOR_COPIES`.
 
+## Rotation Setups
+
+The default rotates files once every day. If you change the cron to run hourly, then it will rotate hourly. The minimum file size environment variable can be set to skip rotating files until they grow larger enough. The age maximum can ensure files don't remain on the server indefinitely.
+
+#### LOGROTATE_FILESIZE_MIN
+
+The minimum file size (in bytes) before the file is rotated. Empty files are always skipped. If you set the minimum and run cron frequently, you will prevent files from growing much larger than this size. Default: 0
+
+#### LOGROTATE_AGE_MAX
+
+The maximum age (in milliseconds) of a file before it is rotated (even if the minimum file size is not met). Values less than 1 are ignored. Default: 0
+
 ## Local Testing
 
 1. Copy `setenv-tmpl.sh` to `setenv-local.sh`.

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "nr-objectstore-rotate",
-  "version": "1.1.8",
+  "version": "2.0.0",
   "description": "Sidecar for rotating log files to objectstore",
   "main": "index.js",
   "scripts": {

package.json

@@ -22,13 +22,12 @@ export class BrokerService {
 
     const params = ttl ? `?ttl=${ttl}&quickstart=true` : '?quickstart=true';
     const url = `${BROKER_URL}v1/intention/open${params}`;
-    const message = intention;
     const headers = {
       'Content-Type': 'application/json',
       Authorization: `Bearer ${this.brokerJwt}`,
     };
 
-    const response = await sendHttpsRequest(url, 'POST', headers, message);
+    const response = await sendHttpsRequest(url, 'POST', headers, intention);
 
     if (isResponseSuccess(response.statusCode)) {
       this.openResponse = JSON.parse(response.body);
@@ -120,7 +119,7 @@ export class BrokerService {
       [HEADER_BROKER_TOKEN]: actionToken,
     };
 
-    return await sendHttpsRequest(url, 'POST', headers, JSON.stringify(file));
+    return await sendHttpsRequest(url, 'POST', headers, file);
   }
 }
 

src/broker/broker.service.ts

@@ -8,7 +8,7 @@ export function sendHttpsRequest(
   url: string,
   method: string,
   headers: any,
-  body?: string,
+  body?: any,
 ): Promise<any> {
   return new Promise<any>((resolve, reject) => {
     const options: https.RequestOptions = {

src/broker/http-util.ts

@@ -7,6 +7,13 @@ export const LOGROTATE_DIRECTORY = process.env.LOGROTATE_DIRECTORY ?? 'logs';
 export const LOGROTATE_STATUSFILE =
   process.env.LOGROTATE_STATUSFILE ?? 'cron.db';
 
+export const LOGROTATE_FILESIZE_MIN = process.env.LOGROTATE_FILESIZE_MIN
+  ? Number.parseInt(process.env.LOGROTATE_FILESIZE_MIN)
+  : 1;
+export const LOGROTATE_AGE_MAX = process.env.LOGROTATE_AGE_MAX
+  ? Number.parseInt(process.env.LOGROTATE_AGE_MAX)
+  : 0;
+
 export const LOGROTATE_SUFFIX = process.env.LOGROTATE_SUFFIX ?? 'log';
 export const LOGROTATE_POSTROTATE_COMMAND =
   process.env.LOGROTATE_POSTROTATE_COMMAND ?? '';
@@ -26,8 +33,10 @@ export const OBJECT_STORAGE_ACCESS_KEY =
 export const OBJECT_STORAGE_BUCKET = process.env.OBJECT_STORAGE_BUCKET ?? '';
 export const OBJECT_STORAGE_SECRET_KEY =
   process.env.OBJECT_STORAGE_SECRET_KEY ?? '';
-export const OBJECT_STORAGE_FILENAME_PREFIX =
-  process.env.OBJECT_STORAGE_FILENAME_PREFIX ?? '';
+export const OBJECT_STORAGE_FILENAME_PREFIX = process.env
+  .OBJECT_STORAGE_FILENAME_PREFIX
+  ? `${process.env.OBJECT_STORAGE_FILENAME_PREFIX}.`
+  : '';
 // Optional key/value JSON object with metadata
 export const OBJECT_STORAGE_METADATA =
   process.env.OBJECT_STORAGE_METADATA ?? '';

src/constants.ts

@@ -7,13 +7,28 @@ import {
   LOGROTATE_POSTROTATE_COMMAND,
   LOGROTATE_DIRECTORY,
   LOGROTATE_SUFFIX,
+  LOGROTATE_FILESIZE_MIN,
+  LOGROTATE_AGE_MAX,
 } from '../constants';
 import { DatabaseService } from '../services/database.service';
 
 export async function rotateLogs(db: DatabaseService) {
-  const files = fs.readdirSync(LOGROTATE_DIRECTORY);
-  const logFiles = files.filter((file) => file.endsWith(LOGROTATE_SUFFIX));
   console.log('rotate: start');
+  const files = fs.readdirSync(LOGROTATE_DIRECTORY);
+  const now = new Date().getTime();
+  let logFiles = files.filter((file) => file.endsWith(LOGROTATE_SUFFIX));
+  logFiles = files.filter((file) => {
+    const stats = fs.statSync(file);
+    const endTime = stats.ctime.getTime() + LOGROTATE_AGE_MAX;
+    const rotateFile =
+      stats.size > 0 &&
+      ((LOGROTATE_AGE_MAX > 0 && now > endTime) ||
+        stats.size > LOGROTATE_FILESIZE_MIN);
+    if (!rotateFile) {
+      console.log(`rotate: ${file} (skip)`);
+    }
+    return rotateFile;
+  });
   if (logFiles.length > 0) {
     for (const file of logFiles) {
       await rotateLog(db, file);