Merge pull request #17 from bcgov/fix/fileChangeTimeallback

feat: document and use change time instead of birth time with rotated…

Author: mbystedt

README.md

@@ -1,35 +1,76 @@
 # NR Object Storage Rotate
 
-A Sidecar container for rotating, compressing and backing up log files to Object Storage.
+A sidecar container for rotating, compressing and backing up log files to object storage.
 
 ## Architecture
 
 The container is a Typescipt Node.js application that uses a SQLite database to track the files as they are stepped through each stage. The stages run independantely on a configurable cron schedule.
 
-COnfigurable environment variables will be shown like `ENV_VAR` below.
+Configurable environment variables will be shown like `ENV_VAR` below.
 
 ### Stage 0 - Log file generated
 
 The application logs to disk. Files to be rotated must end with `LOGROTATE_SUFFIX`.
 
 ### Stage 1 - Rotate log file
 
-The environment variable `CRON_ROTATE` is used to schedule the rotation of the files. Matching files are rotated by renaming the files to append a timestamp.
+The environment variable `CRON_ROTATE` is used to schedule the rotation of the files. The `LOGROTATE_DIRECTORY` is examined for files with the `LOGROTATE_SUFFIX` (default: log).
 
 If any files are rotated then, optionally, `LOGROTATE_POSTROTATE_COMMAND` is called. It can be necessary to signal the application that the rotation occurred so it can open a new file.
 
+Rotated files are appended with the file's change date and the current UTC timestamp. See: https://nodejs.org/api/fs.html#stat-time-values
+
 ### Stage 2 - Compress log file
 
-The environment variable `CRON_COMPRESS` is used to schedule the compression of the rotated files.
+The environment variable `CRON_COMPRESS` is used to schedule the compression of the rotated files. The each file is compressed into a 'tgz' archive.
+
+This stage can run frequently with little cost.
 
 ### Stage 3 - Backup log file
 
-The environment variable `CRON_BACKUP` is used to schedule the back of the compressed files to Object Storage. To identify the source, a prefix can be configured by setting `OBJECT_STORAGE_FILENAME_PREFIX`. Any arbitrary metadata can be set by setting `OBJECT_STORAGE_METADATA` to be a key/value JSON string.
+The environment variable `CRON_BACKUP` is used to schedule the backup of the compressed files to object storage.
+
+If you have massive files or slow connectivity, increase the cron settings period. Otherwise, this stage can run frequently with little cost.
+
+Any arbitrary metadata can be set by setting `OBJECT_STORAGE_METADATA` to be a key/value JSON string.
+
+If you are sending similarly named files from multiple sources (OpenShift/Kubernetes nodes), then it is recommended that you set `OBJECT_STORAGE_FILENAME_PREFIX` to identify the source and avoid collisions.
+
+If you set `OBJECT_STORAGE_ENABLED` to anything but the default of 'true' then the backup to object storage is skipped.
+
+#### Required Configuration
+
+The following are the environment variables that need to be set for the tool to use object storage.
+
+* `OBJECT_STORAGE_END_POINT`
+* `OBJECT_STORAGE_ACCESS_KEY`
+* `OBJECT_STORAGE_BUCKET`
+* `OBJECT_STORAGE_SECRET_KEY`
 
 ### Stage 4 - Janitor
 
 The environment variable `CRON_JANITOR` is used to schedule the janitor which removes files after they have been backed up. The number of log files to retain can be configured by setting `JANITOR_COPIES`.
 
+This stage can run frequently with little cost.
+
+## SQLite Database
+
+The SQLite database can be viewed by running a command like:
+
+`sqlite3 ./logs/cron.db 'select * from logs'`
+
+### Missing Files
+
+Prior to each stage, the database and the file system are compared. Any file missing from the file system will be logged and deleted from the database.
+
+### Moving the Log Directory
+
+If you are moving the location of the files, you will need to update the path column of the logs table in the SQLite database. As well, you should take care not to trigger the missing files process.
+
+## Object Storage Lifecycle Policies
+
+This tool does not manage the lifecycle policies for the bucket the data is uploaded to. Please refer to the documentation for the object storage service you are using to setup a bucket lifecycle.
+
 ## Rotation Setups
 
 The default rotates files once every day. If you change the cron to run hourly, then it will rotate hourly. The minimum file size environment variable can be set to skip rotating files until they grow larger enough. The age maximum can ensure files don't remain on the server indefinitely.
@@ -42,6 +83,28 @@ The minimum file size (in bytes) before the file is rotated. Empty files are alw
 
 The maximum age (in milliseconds) of a file before it is rotated (even if the minimum file size is not met). Values less than 1 are ignored. Default: 0
 
+### Integration with NR Broker
+
+The backup stage can read credentials from NR Vault and report the backed up files to NR Broker if the NR Broker environment variables (`BROKER_*`) are set.
+
+The required environment variables to set are:
+
+* `BROKER_JWT`
+* `BROKER_PROJECT`
+* `BROKER_SERVICE`
+* `BROKER_ENVIRONMENT`
+
+This will set it up to read secrets from the standard key/value credential location in NR Vault for the service. The `VAULT_CRED_PATH_SUFFIX` variable can be set to include a path from the service's root.
+
+The key/value document read from NR Vault will do nothing by default. The `VAULT_CRED_KEYS_*` variables replace the equivalent `OBJECT_STORAGE_*` with the value of the key read from Vault.
+
+* VaultDoc[`VAULT_CRED_KEYS_END_POINT`] -> `OBJECT_STORAGE_END_POINT`
+* VaultDoc[`VAULT_CRED_KEYS_ACCESS_KEY`] -> `OBJECT_STORAGE_ACCESS_KEY`
+* VaultDoc[`VAULT_CRED_KEYS_SECRET_KEY`] -> `OBJECT_STORAGE_SECRET_KEY`
+* VaultDoc[`VAULT_CRED_KEYS_BUCKET`] -> `OBJECT_STORAGE_BUCKET`
+
+You are free to set as many (or as few) of the `VAULT_CRED_KEYS_*`.
+
 ## Local Testing
 
 1. Copy `setenv-tmpl.sh` to `setenv-local.sh`.

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "nr-objectstore-rotate",
-  "version": "2.0.0",
+  "version": "2.0.2",
   "description": "Sidecar for rotating log files to objectstore",
   "main": "index.js",
   "scripts": {

package.json

@@ -1,28 +1,31 @@
 # export CRON_ROTATE="59 23 * * *"
 # export CRON_COMPRESS="*/10 * * * *"
-# export CRON_BACKUP="*/10 * * * *"
+# export CRON_BACKUP="*/20 * * * *"
 # export CRON_JANITOR="*/10 * * * *"
 
 export LOGROTATE_DIRECTORY="/logs"
-export JANITOR_COPIES=3
-# export LOGROTATE_SUFFIX="log"
-# export LOGROTATE_POSTROTATE_COMMAND=""
 # export LOGROTATE_STATUSFILE="cron.db"
+# export LOGROTATE_FILESIZE_MIN="1024"
+# export LOGROTATE_AGE_MAX="86400"
+# export LOGROTATE_SUFFIX="log"
+# export LOGROTATE_POSTROTATE_COMMAND="echo 'rotated!'"
+export JANITOR_COPIES=3
 
 # Required
 # export OBJECT_STORAGE_ENABLED="true"
 export OBJECT_STORAGE_END_POINT=""
 export OBJECT_STORAGE_ACCESS_KEY=""
-export OBJECT_STORAGE_BUCKET=""
 export OBJECT_STORAGE_SECRET_KEY=""
+export OBJECT_STORAGE_BUCKET=""
+# export OBJECT_STORAGE_FILENAME_PREFIX=""
 
-# Set BROKER_JWT to use Broker and Vault
-# export BROKER_JWT=""
+# Set BROKER_* and VAULT_* values to use NR Broker and Vault
 # export BROKER_URL=""
+# export BROKER_JWT=""
 # export BROKER_USER=""
-export BROKER_PROJECT=""
-export BROKER_SERVICE=""
-export BROKER_ENVIRONMENT=""
+# export BROKER_PROJECT=""
+# export BROKER_SERVICE=""
+# export BROKER_ENVIRONMENT=""
 
 # export VAULT_CRED_PATH=""
 # If VAULT_CRED_KEYS_* is set, the value from VAULT_CRED_PATH replaces OBJECT_STORAGE_*

setenv-tmpl.sh

@@ -1,6 +1,6 @@
 export const CRON_ROTATE = process.env.CRON_ROTATE ?? '59 23 * * *';
 export const CRON_COMPRESS = process.env.CRON_COMPRESS ?? '*/10 * * * *';
-export const CRON_BACKUP = process.env.CRON_BACKUP ?? '*/10 * * * *';
+export const CRON_BACKUP = process.env.CRON_BACKUP ?? '*/20 * * * *';
 export const CRON_JANITOR = process.env.CRON_JANITOR ?? '*/10 * * * *';
 
 export const LOGROTATE_DIRECTORY = process.env.LOGROTATE_DIRECTORY ?? 'logs';
@@ -30,9 +30,9 @@ export const OBJECT_STORAGE_END_POINT =
   process.env.OBJECT_STORAGE_END_POINT ?? '';
 export const OBJECT_STORAGE_ACCESS_KEY =
   process.env.OBJECT_STORAGE_ACCESS_KEY ?? '';
-export const OBJECT_STORAGE_BUCKET = process.env.OBJECT_STORAGE_BUCKET ?? '';
 export const OBJECT_STORAGE_SECRET_KEY =
   process.env.OBJECT_STORAGE_SECRET_KEY ?? '';
+export const OBJECT_STORAGE_BUCKET = process.env.OBJECT_STORAGE_BUCKET ?? '';
 export const OBJECT_STORAGE_FILENAME_PREFIX = process.env
   .OBJECT_STORAGE_FILENAME_PREFIX
   ? `${process.env.OBJECT_STORAGE_FILENAME_PREFIX}.`
@@ -63,7 +63,9 @@ export const VAULT_CRED_PATH =
   `apps/data/${
     ENV_LONG_TO_SHORT[BROKER_ENVIRONMENT]
   }/${BROKER_PROJECT}/${BROKER_SERVICE}${
-    process.env.VAULT_CRED_PATH_SUFFIX ? process.env.VAULT_CRED_PATH_SUFFIX : ''
+    process.env.VAULT_CRED_PATH_SUFFIX
+      ? `/${process.env.VAULT_CRED_PATH_SUFFIX}`
+      : ''
   }`;
 // If VAULT_CRED_KEYS_* is set, the value from VAULT_CRED_PATH replaces OBJECT_STORAGE_*
 // Example: VAULT_CRED_KEYS_SECRET_KEY="secret_key" would replace OBJECT_STORAGE_SECRET_KEY
@@ -72,9 +74,9 @@ export const VAULT_CRED_KEYS_END_POINT =
   process.env.VAULT_CRED_KEYS_END_POINT ?? '';
 export const VAULT_CRED_KEYS_ACCESS_KEY =
   process.env.VAULT_CRED_KEYS_ACCESS_KEY ?? '';
-export const VAULT_CRED_KEYS_BUCKET = process.env.VAULT_CRED_KEYS_BUCKET ?? '';
 export const VAULT_CRED_KEYS_SECRET_KEY =
   process.env.VAULT_CRED_KEYS_SECRET_KEY ?? '';
+export const VAULT_CRED_KEYS_BUCKET = process.env.VAULT_CRED_KEYS_BUCKET ?? '';
 export const VAULT_URL =
   process.env.VAULT_URL ?? 'https://knox.io.nrs.gov.bc.ca/';
 

src/constants.ts

@@ -70,16 +70,16 @@ async function rotateLog(db: DatabaseService, file: string) {
   await db.addLog(file, newPath);
 }
 
-// Append date to logname
+// Append last change time and current timestamp to logname
 export function newLogName(file: string) {
-  const date = createdDate(path.join(LOGROTATE_DIRECTORY, file));
+  const date = changeDate(path.join(LOGROTATE_DIRECTORY, file));
   return `${file}.${date.getUTCFullYear()}${String(date.getUTCMonth()).padStart(
     2,
     '0',
   )}${String(date.getUTCDate()).padStart(2, '0')}.${new Date().valueOf()}`;
 }
 
-function createdDate(filePath: string) {
-  const { birthtime } = fs.statSync(filePath);
-  return birthtime;
+function changeDate(filePath: string) {
+  const { ctime } = fs.statSync(filePath);
+  return ctime;
 }