From aa52fc72f8bd03782b98c175fafa8c2bc6b5b1e8 Mon Sep 17 00:00:00 2001 From: GraceRuan Date: Fri, 9 Feb 2024 11:30:44 -0800 Subject: [PATCH 1/3] feat: add options for compress and run once --- src/constants.ts | 4 ++++ src/index.ts | 62 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index f843063..a95b6b2 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -3,6 +3,10 @@ export const CRON_COMPRESS = process.env.CRON_COMPRESS ?? '*/10 * * * *'; export const CRON_BACKUP = process.env.CRON_BACKUP ?? '*/20 * * * *'; export const CRON_JANITOR = process.env.CRON_JANITOR ?? '*/10 * * * *'; +export const RUN_ONCE = process.env.RUN_ONCE ?? 'false'; + +export const COMPRESS_ENABLED = process.env.COMPRESS_ENABLED ?? 'true'; + export const LOGROTATE_DIRECTORY = process.env.LOGROTATE_DIRECTORY ?? 'logs'; export const LOGROTATE_STATUSFILE = process.env.LOGROTATE_STATUSFILE ?? 'cron.db'; diff --git a/src/index.ts b/src/index.ts index c3348b9..cbf72c4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,30 +28,58 @@ async function main() { 'when launching node to enable forced garbage collection.', ); } + const runOnce = process.env.RUN_ONCE === 'true'; + const compressEnabled = process.env.COMPRESS_ENABLED === 'true'; - const rotateJob = Cron(CRON_ROTATE, async () => { + const combinedJob = async () => { + // Stage 1: Rotate log await rotateLogs(db); - runGarbageCollection(); - }); - const compressJob = Cron(CRON_COMPRESS, async () => { - await syncLogsDb(db); - await compress(db); - runGarbageCollection(); - }); - const backupJob = Cron(CRON_BACKUP, async () => { + + // Stage 2: Compress files - optional + if (compressEnabled) { + await syncLogsDb(db); + await compress(db); + } + + // Stage 3: Backup await syncLogsDb(db); await backup(db); - runGarbageCollection(); - }); - const janitorJob = Cron(CRON_JANITOR, async () => { + + // Stage 4: Janitor await syncLogsDb(db); await removeOldLogs(db); runGarbageCollection(); - }); - console.log(`Rotate job next run: ${rotateJob.nextRun()}`); - console.log(`Compress job next run: ${compressJob.nextRun()}`); - console.log(`Backup job next run: ${backupJob.nextRun()}`); - console.log(`Janitor job next run: ${janitorJob.nextRun()}`); + }; + + if (runOnce) combinedJob(); + else { + const rotateJob = Cron(CRON_ROTATE, async () => { + await rotateLogs(db); + runGarbageCollection(); + }); + + if (compressEnabled) { + const compressJob = Cron(CRON_COMPRESS, async () => { + await syncLogsDb(db); + await compress(db); + runGarbageCollection(); + }); + console.log(`Compress job next run: ${compressJob.nextRun()}`); + } + const backupJob = Cron(CRON_BACKUP, async () => { + await syncLogsDb(db); + await backup(db); + runGarbageCollection(); + }); + const janitorJob = Cron(CRON_JANITOR, async () => { + await syncLogsDb(db); + await removeOldLogs(db); + runGarbageCollection(); + }); + console.log(`Rotate job next run: ${rotateJob.nextRun()}`); + console.log(`Backup job next run: ${backupJob.nextRun()}`); + console.log(`Janitor job next run: ${janitorJob.nextRun()}`); + } } main(); From 15e17a98a73c7ee800cbf4fad9f9582146bf6545 Mon Sep 17 00:00:00 2001 From: GraceRuan Date: Tue, 27 Feb 2024 11:06:21 -0800 Subject: [PATCH 2/3] fix: refactor jobs --- src/index.ts | 68 +++++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/src/index.ts b/src/index.ts index cbf72c4..39dbc0d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,54 +31,58 @@ async function main() { const runOnce = process.env.RUN_ONCE === 'true'; const compressEnabled = process.env.COMPRESS_ENABLED === 'true'; - const combinedJob = async () => { - // Stage 1: Rotate log + const rotateJob = async () => { await rotateLogs(db); + runGarbageCollection(); + }; - // Stage 2: Compress files - optional - if (compressEnabled) { - await syncLogsDb(db); - await compress(db); - } + const compressEnabledJob = async () => { + await syncLogsDb(db); + await compress(db); + runGarbageCollection(); + }; - // Stage 3: Backup + const backupJob = async () => { await syncLogsDb(db); await backup(db); + runGarbageCollection(); + }; - // Stage 4: Janitor + const janitorJob = async () => { await syncLogsDb(db); await removeOldLogs(db); runGarbageCollection(); }; + const combinedJob = async () => { + // Stage 1: Rotate log + rotateJob(); + + // Stage 2: Compress files - optional + // Stage 3: Backup - if compress enabled + if (compressEnabled) { + compressEnabledJob(); + backupJob(); + } + + // Stage 4: Janitor + janitorJob(); + }; + if (runOnce) combinedJob(); else { - const rotateJob = Cron(CRON_ROTATE, async () => { - await rotateLogs(db); - runGarbageCollection(); - }); + const rotateCronJob = Cron(CRON_ROTATE, rotateJob); if (compressEnabled) { - const compressJob = Cron(CRON_COMPRESS, async () => { - await syncLogsDb(db); - await compress(db); - runGarbageCollection(); - }); - console.log(`Compress job next run: ${compressJob.nextRun()}`); + const compressCronJob = Cron(CRON_COMPRESS, compressEnabledJob); + const backupCronJob = Cron(CRON_BACKUP, backupJob); + console.log(`Compress job next run: ${compressCronJob.nextRun()}`); + console.log(`Backup job next run: ${backupCronJob.nextRun()}`); } - const backupJob = Cron(CRON_BACKUP, async () => { - await syncLogsDb(db); - await backup(db); - runGarbageCollection(); - }); - const janitorJob = Cron(CRON_JANITOR, async () => { - await syncLogsDb(db); - await removeOldLogs(db); - runGarbageCollection(); - }); - console.log(`Rotate job next run: ${rotateJob.nextRun()}`); - console.log(`Backup job next run: ${backupJob.nextRun()}`); - console.log(`Janitor job next run: ${janitorJob.nextRun()}`); + + const janitorCronJob = Cron(CRON_JANITOR, janitorJob); + console.log(`Rotate job next run: ${rotateCronJob.nextRun()}`); + console.log(`Janitor job next run: ${janitorCronJob.nextRun()}`); } } From 35694191565ae3c7b3a83685f5796cf674ae08a4 Mon Sep 17 00:00:00 2001 From: GraceRuan Date: Mon, 4 Mar 2024 10:51:35 -0800 Subject: [PATCH 3/3] fix: refactor jobs --- src/cron/backup.ts | 7 +++- src/index.ts | 73 ++------------------------------ src/services/job.service.ts | 84 +++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 71 deletions(-) create mode 100644 src/services/job.service.ts diff --git a/src/cron/backup.ts b/src/cron/backup.ts index 64c94e7..23d9d97 100644 --- a/src/cron/backup.ts +++ b/src/cron/backup.ts @@ -12,6 +12,7 @@ import { BROKER_SERVICE, BROKER_USER, DB_FILE_STATUS, + COMPRESS_ENABLED, OBJECT_STORAGE_ACCESS_KEY, OBJECT_STORAGE_BUCKET, OBJECT_STORAGE_ENABLED, @@ -51,6 +52,10 @@ const objectstorageMetadata: ItemBucketMetadata = : undefined; export async function backup(db: DatabaseService) { + const dbFileStatus = + COMPRESS_ENABLED === 'true' + ? DB_FILE_STATUS.Compressed + : DB_FILE_STATUS.Rotated; console.log('backup: start'); const result = await db.all<{ id: number; @@ -63,7 +68,7 @@ export async function backup(db: DatabaseService) { WHERE status = ? ORDER BY id DESC `, - [DB_FILE_STATUS.Compressed], + [dbFileStatus], ); if (result.rows.length === 0) { diff --git a/src/index.ts b/src/index.ts index 39dbc0d..a8c0f58 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,26 +1,11 @@ -import { Cron } from 'croner'; -import { - CRON_BACKUP, - CRON_COMPRESS, - CRON_JANITOR, - CRON_ROTATE, -} from './constants'; -import { backup } from './cron/backup'; import { DatabaseService } from './services/database.service'; -import { rotateLogs } from './cron/rotate'; -import { removeOldLogs, syncLogsDb } from './cron/janitor'; -import { compress } from './cron/compress'; +import { JobService } from './services/job.service'; console.log('Starting...'); -function runGarbageCollection() { - if (global.gc) { - global.gc(); - } -} - async function main() { const db = await DatabaseService.create(); + const jobService = new JobService(db); if (!global.gc) { console.log( @@ -31,59 +16,7 @@ async function main() { const runOnce = process.env.RUN_ONCE === 'true'; const compressEnabled = process.env.COMPRESS_ENABLED === 'true'; - const rotateJob = async () => { - await rotateLogs(db); - runGarbageCollection(); - }; - - const compressEnabledJob = async () => { - await syncLogsDb(db); - await compress(db); - runGarbageCollection(); - }; - - const backupJob = async () => { - await syncLogsDb(db); - await backup(db); - runGarbageCollection(); - }; - - const janitorJob = async () => { - await syncLogsDb(db); - await removeOldLogs(db); - runGarbageCollection(); - }; - - const combinedJob = async () => { - // Stage 1: Rotate log - rotateJob(); - - // Stage 2: Compress files - optional - // Stage 3: Backup - if compress enabled - if (compressEnabled) { - compressEnabledJob(); - backupJob(); - } - - // Stage 4: Janitor - janitorJob(); - }; - - if (runOnce) combinedJob(); - else { - const rotateCronJob = Cron(CRON_ROTATE, rotateJob); - - if (compressEnabled) { - const compressCronJob = Cron(CRON_COMPRESS, compressEnabledJob); - const backupCronJob = Cron(CRON_BACKUP, backupJob); - console.log(`Compress job next run: ${compressCronJob.nextRun()}`); - console.log(`Backup job next run: ${backupCronJob.nextRun()}`); - } - - const janitorCronJob = Cron(CRON_JANITOR, janitorJob); - console.log(`Rotate job next run: ${rotateCronJob.nextRun()}`); - console.log(`Janitor job next run: ${janitorCronJob.nextRun()}`); - } + jobService.run(runOnce, compressEnabled); } main(); diff --git a/src/services/job.service.ts b/src/services/job.service.ts new file mode 100644 index 0000000..196bf9c --- /dev/null +++ b/src/services/job.service.ts @@ -0,0 +1,84 @@ +import { Cron } from 'croner'; +import { + CRON_BACKUP, + CRON_COMPRESS, + CRON_JANITOR, + CRON_ROTATE, +} from '../constants'; +import { backup } from '../cron/backup'; +import { rotateLogs } from '../cron/rotate'; +import { removeOldLogs, syncLogsDb } from '../cron/janitor'; +import { compress } from '../cron/compress'; +import { DatabaseService } from './database.service'; + +function runGarbageCollection() { + if (global.gc) { + global.gc(); + } +} + +export class JobService { + private db: DatabaseService; + + constructor(db: DatabaseService) { + this.db = db; + } + + rotate = async () => { + await rotateLogs(this.db); + runGarbageCollection(); + }; + + compress = async () => { + console.log('start compress'); + await syncLogsDb(this.db); + await compress(this.db); + runGarbageCollection(); + }; + + backup = async () => { + console.log('start backup'); + await syncLogsDb(this.db); + await backup(this.db); + runGarbageCollection(); + }; + + janitor = async () => { + await syncLogsDb(this.db); + await removeOldLogs(this.db); + runGarbageCollection(); + }; + + private async runOnceJob(compressEnabled: boolean) { + // Stage 1: Rotate log + await this.rotate(); + // Stage 2: Compress files - optional + if (compressEnabled) { + await this.compress(); + } + // Stage 3: Backup + await this.backup(); + // Stage 4: Janitor + await this.janitor(); + } + + private async cronJobs(compressEnabled: boolean) { + const rotateCronJob = Cron(CRON_ROTATE, this.rotate); + + if (compressEnabled) { + const compressCronJob = Cron(CRON_COMPRESS, this.compress); + console.log(`Compress job next run: ${compressCronJob.nextRun()}`); + } + const backupCronJob = Cron(CRON_BACKUP, this.backup); + const janitorCronJob = Cron(CRON_JANITOR, this.janitor); + + console.log(`Backup job next run: ${backupCronJob.nextRun()}`); + console.log(`Rotate job next run: ${rotateCronJob.nextRun()}`); + console.log(`Janitor job next run: ${janitorCronJob.nextRun()}`); + } + + async run(runOnce: boolean, compressEnabled: boolean) { + if (runOnce) this.runOnceJob(compressEnabled); + else this.cronJobs(compressEnabled); + } +}