Merge pull request #939 from 18F/feature/replace_cron_script_with_scheduler

[Tech Debt] Job scheduling with the Bree library

Author: levinmr

README.md

@@ -19,9 +19,9 @@ The process for adding features to this project is described in
 
 ## Local development setup
 
-### Prerequistites
+### Prerequisites
 
-* NodeJS > v20.x
+* NodeJS > v22.x
 * A postgres DB running and/or docker installed
 
 ### Install dependencies
@@ -144,7 +144,7 @@ This file is ignored in the `.gitignore` file and should not be checked in to th
 npm start
 
 # running the app with dotenv-cli
-dotenv -e .env npm start
+npx dotenv -e .env npm start
 ```
 
 ## Configuration

deploy/api.sh

@@ -0,0 +1,46 @@
+if (process.env.NODE_ENV !== "production") {
+  require("dotenv").config();
+}
+
+if (process.env.NEW_RELIC_APP_NAME) {
+  require("newrelic");
+}
+
+const logger = require("../src/logger").initialize();
+logger.info("===================================");
+logger.info("=== STARTING ANALYTICS-REPORTER ===");
+logger.info("    Running /deploy/publisher.js");
+logger.info("===================================");
+
+// Job Scheduler
+const Bree = require("bree");
+const bree = new Bree({
+  logger,
+  jobs: [
+    // Runs `../jobs/realtime.js` 1 millisecond after the process starts and
+    // then every 15 minutes going forward.
+    {
+      name: "realtime",
+      timeout: "1",
+      interval: "15m",
+    },
+    // Runs `../jobs/daily.js` 1 minute after the process starts and then at
+    // 10:01 AM every day going forward.
+    {
+      name: "daily",
+      timeout: "1m",
+      interval: "at 10:01 am",
+    },
+    // Runs `../jobs/api.js` 2 minutes after the process starts and then at
+    // 10:02 AM every day going forward.
+    {
+      name: "api",
+      timeout: "2m",
+      interval: "at 10:02 am",
+    },
+  ],
+});
+
+(async () => {
+  await bree.start();
+})();

deploy/cron.js

@@ -0,0 +1,24 @@
+process.env.ANALYTICS_REPORTS_PATH = "reports/api.json";
+process.env.ANALYTICS_SCRIPT_NAME = "api.js";
+
+const { runQueuePublish } = require("../index.js");
+const options = {
+  frequency: "daily",
+  debug: true,
+  "write-to-database": true,
+  agenciesFile: `${process.env.ANALYTICS_ROOT_PATH}/deploy/agencies.json`,
+};
+const logger = require("../src/logger.js").initialize();
+
+(async () => {
+  logger.info(`Beginning job: ${process.env.ANALYTICS_SCRIPT_NAME}`);
+
+  try {
+    await runQueuePublish(options);
+    logger.info(`Job completed: ${process.env.ANALYTICS_SCRIPT_NAME}`);
+  } catch (e) {
+    logger.error(`Job exited with error: ${process.env.ANALYTICS_SCRIPT_NAME}`);
+    logger.error(e);
+    throw e;
+  }
+})();

deploy/daily.sh

@@ -0,0 +1,27 @@
+process.env.ANALYTICS_REPORTS_PATH = "reports/usa.json";
+process.env.ANALYTICS_SCRIPT_NAME = "daily.js";
+
+const { runQueuePublish } = require("../index.js");
+const options = {
+  publish: true,
+  frequency: "daily",
+  slim: true,
+  debug: true,
+  csv: true,
+  json: true,
+  agenciesFile: `${process.env.ANALYTICS_ROOT_PATH}/deploy/agencies.json`,
+};
+const logger = require("../src/logger.js").initialize();
+
+(async () => {
+  logger.info(`Beginning job: ${process.env.ANALYTICS_SCRIPT_NAME}`);
+
+  try {
+    await runQueuePublish(options);
+    logger.info(`Job completed: ${process.env.ANALYTICS_SCRIPT_NAME}`);
+  } catch (e) {
+    logger.error(`Job exited with error: ${process.env.ANALYTICS_SCRIPT_NAME}`);
+    logger.error(e);
+    throw e;
+  }
+})();

deploy/hourly.sh

@@ -0,0 +1,27 @@
+process.env.ANALYTICS_REPORTS_PATH = "reports/usa.json";
+process.env.ANALYTICS_SCRIPT_NAME = "realtime.js";
+
+const { runQueuePublish } = require("../index.js");
+const options = {
+  publish: true,
+  frequency: "realtime",
+  slim: true,
+  debug: true,
+  csv: true,
+  json: true,
+  agenciesFile: `${process.env.ANALYTICS_ROOT_PATH}/deploy/agencies.json`,
+};
+const logger = require("../src/logger.js").initialize();
+
+(async () => {
+  logger.info(`Beginning job: ${process.env.ANALYTICS_SCRIPT_NAME}`);
+
+  try {
+    await runQueuePublish(options);
+    logger.info(`Job completed: ${process.env.ANALYTICS_SCRIPT_NAME}`);
+  } catch (e) {
+    logger.error(`Job exited with error: ${process.env.ANALYTICS_SCRIPT_NAME}`);
+    logger.error(e);
+    throw e;
+  }
+})();

deploy/publisher.js

@@ -9,7 +9,7 @@ applications:
   health-check-type: process
   buildpacks:
   - nodejs_buildpack
-  command: node deploy/cron.js
+  command: node deploy/publisher.js
   env:
     ANALYTICS_DEBUG: 'true'
     ANALYTICS_LOG_LEVEL: ${ANALYTICS_LOG_LEVEL}