From 9cec046d36b8f715d60450b662b10d546b73866c Mon Sep 17 00:00:00 2001 From: Michael Levin Date: Fri, 3 Jan 2025 12:47:22 -0500 Subject: [PATCH 1/6] [Tech Debt] Don't write to disk for API reports --- src/app_config.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/app_config.js b/src/app_config.js index fdfb0ca6..e757aee7 100644 --- a/src/app_config.js +++ b/src/app_config.js @@ -58,7 +58,11 @@ class AppConfig { } get shouldPublishToDisk() { - return !!this.#options.output && typeof this.#options.output === "string"; + return ( + !!this.#options.output && + !this.shouldWriteToDatabase && + typeof this.#options.output === "string" + ); } get shouldPublishToS3() { From 90b7603ef89fdb8e9229ef837c9b3b84fbd87136 Mon Sep 17 00:00:00 2001 From: Michael Levin Date: Fri, 3 Jan 2025 13:42:14 -0500 Subject: [PATCH 2/6] [Tech Debt] Do not run formatting action if no formats are set --- .../format_processed_analytics_data.js | 10 +++++++ .../format_processed_analytics_data.test.js | 30 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/actions/format_processed_analytics_data.js b/src/actions/format_processed_analytics_data.js index 8df8f38a..733aa9e8 100644 --- a/src/actions/format_processed_analytics_data.js +++ b/src/actions/format_processed_analytics_data.js @@ -5,6 +5,16 @@ const ResultFormatter = require("../process_results/result_formatter"); * Chain of responsibility action for formatting processed analytics data */ class FormatProcessedAnalyticsData extends Action { + /** + * @param {import('../report_processing_context')} context the context for the + * action chain. + * @returns {boolean} true if the application config is set to format + * processed analytics data. + */ + handles(context) { + return context.appConfig.formats.length > 0; + } + /** * Takes the processed analytics data from the context and changes the format * to JSON or CSV based on application and report config options. Writes the diff --git a/test/actions/format_processed_analytics_data.test.js b/test/actions/format_processed_analytics_data.test.js index 0dbe7386..465db03c 100644 --- a/test/actions/format_processed_analytics_data.test.js +++ b/test/actions/format_processed_analytics_data.test.js @@ -14,6 +14,36 @@ describe("FormatProcessedAnalyticsData", () => { let context; let subject; + describe(".handles", () => { + beforeEach(() => { + subject = new FormatProcessedAnalyticsData(); + }); + + describe("when appConfig.formats has values", () => { + beforeEach(() => { + context = { + appConfig: { formats: ["json"] }, + }; + }); + + it("returns true", () => { + expect(subject.handles(context)).to.equal(true); + }); + }); + + describe("when appConfig.formats does not have values", () => { + beforeEach(() => { + context = { + appConfig: { formats: [] }, + }; + }); + + it("returns false", () => { + expect(subject.handles(context)).to.equal(false); + }); + }); + }); + describe(".executeStrategy", () => { const debugLogSpy = sinon.spy(); From f8acbbfc1a812fea18bff48176423f3c4689bc43 Mon Sep 17 00:00:00 2001 From: Michael Levin Date: Wed, 8 Jan 2025 12:14:12 -0500 Subject: [PATCH 3/6] [Feature] Add queue and queue message classes --- index.js | 129 ++++------ package-lock.json | 22 ++ package.json | 1 + src/{ => queue}/pg_boss_knex_adapter.js | 0 src/queue/queue.js | 121 +++++++++ src/queue/queue_message.js | 28 +++ src/queue/report_job_queue_message.js | 109 ++++++++ test/index.test.js | 4 + test/queue/pg_boss_knex_adapter.test.js | 51 ++++ test/queue/queue.test.js | 259 ++++++++++++++++++++ test/queue/queue_message.test.js | 30 +++ test/queue/report_job_queue_message.test.js | 176 +++++++++++++ 12 files changed, 851 insertions(+), 79 deletions(-) rename src/{ => queue}/pg_boss_knex_adapter.js (100%) create mode 100644 src/queue/queue.js create mode 100644 src/queue/queue_message.js create mode 100644 src/queue/report_job_queue_message.js create mode 100644 test/queue/pg_boss_knex_adapter.test.js create mode 100644 test/queue/queue.test.js create mode 100644 test/queue/queue_message.test.js create mode 100644 test/queue/report_job_queue_message.test.js diff --git a/index.js b/index.js index 9a68b7d4..16966683 100644 --- a/index.js +++ b/index.js @@ -1,12 +1,12 @@ const { AsyncLocalStorage } = require("node:async_hooks"); const knex = require("knex"); -const PgBoss = require("pg-boss"); const util = require("util"); const AppConfig = require("./src/app_config"); const ReportProcessingContext = require("./src/report_processing_context"); const Logger = require("./src/logger"); const Processor = require("./src/processor"); -const PgBossKnexAdapter = require("./src/pg_boss_knex_adapter"); +const Queue = require("./src/queue/queue"); +const ReportJobQueueMessage = require("./src/queue/report_job_queue_message"); /** * Gets an array of JSON report objects from the application confing, then runs @@ -37,12 +37,16 @@ async function run(options = {}) { const appConfig = new AppConfig(options); const context = new ReportProcessingContext(new AsyncLocalStorage()); const reportConfigs = appConfig.filteredReportConfigurations; + const knexInstance = appConfig.shouldWriteToDatabase + ? await knex(appConfig.knexConfig) + : undefined; const processor = Processor.buildAnalyticsProcessor( appConfig, Logger.initialize({ agencyName: appConfig.agencyLogName, scriptName: appConfig.scriptName, }), + knexInstance, ); for (const reportConfig of reportConfigs) { @@ -124,7 +128,11 @@ async function runQueuePublish(options = {}) { scriptName: appConfig.scriptName, }); const knexInstance = await knex(appConfig.knexConfig); - const queueClient = await _initQueueClient(knexInstance, appLogger); + const queueClient = await _initQueueClient( + knexInstance, + appConfig.messageQueueName, + appLogger, + ); for (const agency of agencies) { for (const reportConfig of reportConfigs) { @@ -134,47 +142,35 @@ async function runQueuePublish(options = {}) { scriptName: appConfig.scriptName, reportName: reportConfig.name, }); + let messageId; try { - let jobId = await queueClient.send( - appConfig.messageQueueName, - _createQueueMessage( - options, - agency, + messageId = await queueClient.sendMessage( + new ReportJobQueueMessage({ + agencyName: agency.agencyName, + analyticsReportIds: agency.analyticsReportIds, + awsBucketPath: agency.awsBucketPath, + reportOptions: options, reportConfig, - appConfig.scriptName, - ), - { - priority: _messagePriority(reportConfig), - retryLimit: 2, - retryDelay: 10, - retryBackoff: true, - singletonKey: `${appConfig.scriptName}-${agency.agencyName}-${reportConfig.name}`, - }, + scriptName: appConfig.scriptName, + }), ); - if (jobId) { + if (messageId) { reportLogger.info( - `Created job in queue: ${appConfig.messageQueueName} with job ID: ${jobId}`, + `Created message in queue: ${queueClient.name} with message ID: ${messageId}`, ); } else { reportLogger.info( - `Found a duplicate job in queue: ${appConfig.messageQueueName}`, + `Found a duplicate message in queue: ${queueClient.name}`, ); } } catch (e) { - reportLogger.error( - `Error sending to queue: ${appConfig.messageQueueName}`, - ); - reportLogger.error(util.inspect(e)); + // Do nothing so that the remaining messages still process. } } } try { await queueClient.stop(); - appLogger.debug(`Stopping queue client`); - } catch (e) { - appLogger.error("Error stopping queue client"); - appLogger.error(util.inspect(e)); } finally { appLogger.debug(`Destroying database connection pool`); knexInstance.destroy(); @@ -198,41 +194,17 @@ function _initAgencies(agencies_file) { return Array.isArray(agencies) ? agencies : legacyAgencies; } -async function _initQueueClient(knexInstance, logger) { - let queueClient; - try { - queueClient = new PgBoss({ db: new PgBossKnexAdapter(knexInstance) }); - await queueClient.start(); - logger.debug("Starting queue client"); - } catch (e) { - logger.error("Error starting queue client"); - logger.error(util.inspect(e)); - } - +async function _initQueueClient(knexInstance, queueName, logger) { + const queueClient = Queue.buildQueue({ + knexInstance, + queueName, + messageClass: ReportJobQueueMessage, + logger, + }); + await queueClient.start(); return queueClient; } -function _createQueueMessage(options, agency, reportConfig, scriptName) { - return { - ...agency, - options, - reportConfig, - scriptName, - }; -} - -function _messagePriority(reportConfig) { - if (!reportConfig.frequency) { - return 0; - } else if (reportConfig.frequency == "daily") { - return 1; - } else if (reportConfig.frequency == "hourly") { - return 2; - } else if (reportConfig.frequency == "realtime") { - return 3; - } -} - /** * @returns {Promise} when the process ends */ @@ -240,7 +212,11 @@ async function runQueueConsume() { const appConfig = new AppConfig(); const appLogger = Logger.initialize(); const knexInstance = await knex(appConfig.knexConfig); - const queueClient = await _initQueueClient(knexInstance, appLogger); + const queueClient = await _initQueueClient( + knexInstance, + appConfig.messageQueueName, + appLogger, + ); try { const context = new ReportProcessingContext(new AsyncLocalStorage()); @@ -250,24 +226,19 @@ async function runQueueConsume() { knexInstance, ); - await queueClient.work( - appConfig.messageQueueName, - { newJobCheckIntervalSeconds: 1 }, - async (message) => { - appLogger.info("Queue message received"); - process.env.AGENCY_NAME = message.data.agencyName; - process.env.ANALYTICS_REPORT_IDS = message.data.analyticsReportIds; - process.env.AWS_BUCKET_PATH = message.data.awsBucketPath; - process.env.ANALYTICS_SCRIPT_NAME = message.data.scriptName; - - await _processReport( - new AppConfig(message.data.options), - context, - message.data.reportConfig, - processor, - ); - }, - ); + await queueClient.poll(async (message) => { + process.env.AGENCY_NAME = message.agencyName; + process.env.ANALYTICS_REPORT_IDS = message.analyticsReportIds; + process.env.AWS_BUCKET_PATH = message.awsBucketPath; + process.env.ANALYTICS_SCRIPT_NAME = message.scriptName; + + await _processReport( + new AppConfig(message.options), + context, + message.reportConfig, + processor, + ); + }); } catch (e) { appLogger.error("Error polling queue for messages"); appLogger.error(util.inspect(e)); diff --git a/package-lock.json b/package-lock.json index 5131d18b..67f9c51b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "@cucumber/cucumber": "^10.3.1", "@eslint/js": "^8.57.0", "chai": "^4.4.0", + "chai-as-promised": "^8.0.1", "dotenv": "^16.4.5", "dotenv-cli": "^7.4.3", "eslint": "^8.56.0", @@ -3532,6 +3533,27 @@ "node": ">=4" } }, + "node_modules/chai-as-promised": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-8.0.1.tgz", + "integrity": "sha512-OIEJtOL8xxJSH8JJWbIoRjybbzR52iFuDHuF8eb+nTPD6tgXLjRqsgnUGqQfFODxYvq5QdirT0pN9dZ0+Gz6rA==", + "dev": true, + "dependencies": { + "check-error": "^2.0.0" + }, + "peerDependencies": { + "chai": ">= 2.1.2 < 6" + } + }, + "node_modules/chai-as-promised/node_modules/check-error": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.1.tgz", + "integrity": "sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==", + "dev": true, + "engines": { + "node": ">= 16" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", diff --git a/package.json b/package.json index 270a8bcf..ef0a371b 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "@cucumber/cucumber": "^10.3.1", "@eslint/js": "^8.57.0", "chai": "^4.4.0", + "chai-as-promised": "^8.0.1", "dotenv": "^16.4.5", "dotenv-cli": "^7.4.3", "eslint": "^8.56.0", diff --git a/src/pg_boss_knex_adapter.js b/src/queue/pg_boss_knex_adapter.js similarity index 100% rename from src/pg_boss_knex_adapter.js rename to src/queue/pg_boss_knex_adapter.js diff --git a/src/queue/queue.js b/src/queue/queue.js new file mode 100644 index 00000000..f6f54a59 --- /dev/null +++ b/src/queue/queue.js @@ -0,0 +1,121 @@ +const PgBoss = require("pg-boss"); +const PgBossKnexAdapter = require("./pg_boss_knex_adapter"); +const util = require("util"); + +/** + * Implements a message queue using the PgBoss library. + */ +class Queue { + #queueClient; + #queueName; + #messageClass; + #logger; + + /** + * @param {object} params the parameter object + * @param {import('pg-boss')} params.queueClient the queue client instance to + * use for queue operations. + * @param {string} params.queueName the identifier for the queue. + * @param {*} params.messageClass a class which implements the fromMessage + * static method to return an instance of the class from a PgBoss message + * object. This can be omitted if the queue instance only sends messages. + * @param {import('winston').Logger} params.logger an application logger instance. + */ + constructor({ queueClient, queueName, messageClass, logger }) { + this.#queueClient = queueClient; + this.#queueName = queueName; + this.#messageClass = messageClass; + this.#logger = logger; + } + + /** + * @returns {string} the queue name + */ + get name() { + return this.#queueName; + } + + /** + * @returns {Promise} resolves when the PgBoss queue client has started + */ + async start() { + try { + await this.#queueClient.start(); + this.#logger.debug("Starting queue client"); + } catch (e) { + this.#logger.error("Error starting queue client"); + this.#logger.error(util.inspect(e)); + throw e; + } + } + + /** + * @returns {Promise} resolves when the PgBoss queue client has stopped + */ + async stop() { + try { + await this.#queueClient.stop(); + this.#logger.debug(`Stopping queue client`); + } catch (e) { + this.#logger.error("Error stopping queue client"); + this.#logger.error(util.inspect(e)); + throw e; + } + } + + /** + * @param {import('./queue_message')} queueMessage a QueueMessage instance + * @returns {string} a message ID or null if a duplicate message exists on the + * queue. + */ + async sendMessage(queueMessage) { + try { + const messageId = await this.#queueClient.send( + this.#queueName, + queueMessage.toJSON(), + queueMessage.sendOptions(), + ); + return messageId; + } catch (e) { + this.#logger.error(`Error sending to queue: ${this.#queueName}`); + this.#logger.error(util.inspect(e)); + throw e; + } + } + + /** + * @param {Function} callback the function to call for each message + * @param {object} options the options to pass to the PgBoss work function + * @returns {Promise} resolves when the queue poller process stops + */ + poll(callback, options = { newJobCheckIntervalSeconds: 1 }) { + return this.#queueClient.work(this.#queueName, options, async (message) => { + this.#logger.info("Queue message received"); + await callback(this.#messageClass.fromMessage(message).toJSON()); + }); + } + + /** + * @param {object} params the parameter object + * @param {import('knex')} params.knexInstance an initialized instance of the knex + * library which provides a database connection. + * @param {string} params.queueName the name of the queue to use for the + * client. + * @param {*} params.messageClass a class which implements the fromMessage + * static method to return an instance of the class from a PgBoss message + * object. This can be omitted if the queue instance only sends messages. + * @param {import('winston').Logger} params.logger an application logger instance. + * @returns {Queue} the queue instance configured with the PgBoss queue + * client. + */ + static buildQueue({ knexInstance, queueName, messageClass, logger }) { + return new Queue({ + queueClient: new PgBoss({ db: new PgBossKnexAdapter(knexInstance) }), + queueName, + messageClass, + logger, + }); + } +} + +module.exports = Queue; diff --git a/src/queue/queue_message.js b/src/queue/queue_message.js new file mode 100644 index 00000000..1a9dc01a --- /dev/null +++ b/src/queue/queue_message.js @@ -0,0 +1,28 @@ +/** + * Abstract class for a queue message to be sent to a PgBoss queue client. + */ +class QueueMessage { + /** + * @returns {object} the class converted to a JSON object. + */ + toJSON() { + return {}; + } + + /** + * @returns {object} an options object for the PgBoss send method + */ + sendOptions() { + return {}; + } + + /** + * @param {object} message a PgBoss message object from the report job queue. + * @returns {QueueMessage} the built queue message instance. + */ + static fromMessage(message) { + return new QueueMessage(message.data); + } +} + +module.exports = QueueMessage; diff --git a/src/queue/report_job_queue_message.js b/src/queue/report_job_queue_message.js new file mode 100644 index 00000000..5c8b1522 --- /dev/null +++ b/src/queue/report_job_queue_message.js @@ -0,0 +1,109 @@ +const QueueMessage = require("./queue_message"); + +/** + * Data object for a report job queue message to be sent to a PgBoss queue + * client. + */ +class ReportJobQueueMessage extends QueueMessage { + #agencyName; + #analyticsReportIds; + #awsBucketPath; + #reportOptions; + #reportConfig; + #scriptName; + + /** + * @param {object} params the params object. + * @param {string} params.agencyName the name of the agency. + * @param {string} params.analyticsReportIds the google analytics property ids + * for the agency to use when running reports. + * @param {string} params.awsBucketPath the folder in the S3 bucket where + * report data is stored for the agency. + * @param {object} params.reportOptions the options passed to the reporter + * executable. + * @param {object} params.reportConfig the google analytics configuration + * object for the report to run. + * @param {string} params.scriptName the name of the script which was run to + * begin the reporter process. + * @returns {ReportJobQueueMessage} the built queue message instance. + */ + constructor({ + agencyName = "", + analyticsReportIds = "", + awsBucketPath = "", + reportOptions = {}, + reportConfig = {}, + scriptName = "", + }) { + super(); + this.#agencyName = agencyName; + this.#analyticsReportIds = analyticsReportIds; + this.#awsBucketPath = awsBucketPath; + this.#reportOptions = reportOptions; + this.#reportConfig = reportConfig; + this.#scriptName = scriptName; + } + + /** + * @returns {object} the class converted to a JSON object. + */ + toJSON() { + return { + agencyName: this.#agencyName, + analyticsReportIds: this.#analyticsReportIds, + awsBucketPath: this.#awsBucketPath, + options: this.#reportOptions, + reportConfig: this.#reportConfig, + scriptName: this.#scriptName, + }; + } + + /** + * @returns {object} an options object for the PgBoss send method + */ + sendOptions() { + return { + priority: this.#messagePriority(this.#reportConfig.frequency), + retryLimit: 2, + retryDelay: 10, + retryBackoff: true, + singletonKey: `${this.#scriptName}-${this.#agencyName}-${this.#reportConfig.name}`, + }; + } + + #messagePriority(reportFrequency) { + let priority; + switch (reportFrequency) { + case "realtime": + priority = 3; + break; + case "hourly": + priority = 2; + break; + case "daily": + priority = 1; + break; + default: + priority = 0; + } + return priority; + } + + /** + * @param {object} message a PgBoss message object from the report job queue. + * should have a data key with the expected fields. + * @returns {ReportJobQueueMessage} the built queue message instance. + */ + static fromMessage(message = { data: {} }) { + return new ReportJobQueueMessage({ + agencyName: message.data.agencyName, + analyticsReportIds: message.data.analyticsReportIds, + awsBucketPath: message.data.awsBucketPath, + reportOptions: message.data.options, + reportConfig: message.data.reportConfig, + scriptName: message.data.scriptName, + }); + } +} + +module.exports = ReportJobQueueMessage; diff --git a/test/index.test.js b/test/index.test.js index e52ef163..ac4738ee 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -12,6 +12,10 @@ class AppConfig { get filteredReportConfigurations() { return reportConfigs; } + + get shouldWriteToDatabase() { + return true; + } } class ReportProcessingContext { diff --git a/test/queue/pg_boss_knex_adapter.test.js b/test/queue/pg_boss_knex_adapter.test.js new file mode 100644 index 00000000..7a1b1ea1 --- /dev/null +++ b/test/queue/pg_boss_knex_adapter.test.js @@ -0,0 +1,51 @@ +const sinon = require("sinon"); +const PgBossKnexAdapter = require("../../src/queue/pg_boss_knex_adapter"); + +describe("PgBossKnexAdapter", () => { + let subject; + let knexInstance; + + beforeEach(async () => { + knexInstance = { + raw: sinon.spy(), + }; + subject = new PgBossKnexAdapter(knexInstance); + }); + + describe(".executeSql", () => { + describe("when parameters are passed with the SQL statement", () => { + const sql = "SELECT * FROM foobar-table where foo = $1 and bar = $2"; + const parameters = ["foo", "bar"]; + const expectedSql = + "SELECT * FROM foobar-table where foo = :param_1 and bar = :param_2"; + const expectedParameters = { + param_1: parameters[0], + param_2: parameters[1], + }; + + beforeEach(() => { + subject.executeSql(sql, parameters); + }); + + it("calls knex.raw with the changed SQL and the parameters array", () => { + sinon.assert.calledWith( + knexInstance.raw, + expectedSql, + expectedParameters, + ); + }); + }); + + describe("when parameters are not passed with the SQL statement", () => { + const sql = "SELECT * FROM foobar-table"; + + beforeEach(() => { + subject.executeSql(sql); + }); + + it("calls knex.raw with the unchanged SQL and an empty object for parameters", () => { + sinon.assert.calledWith(knexInstance.raw, sql, {}); + }); + }); + }); +}); diff --git a/test/queue/queue.test.js b/test/queue/queue.test.js new file mode 100644 index 00000000..cc2c0c4f --- /dev/null +++ b/test/queue/queue.test.js @@ -0,0 +1,259 @@ +const chai = require("chai"); +const sinon = require("sinon"); +const Queue = require("../../src/queue/queue"); + +let expect; +let messageJSON; +let messageOptions; + +class TestQueueMessage { + toJSON() { + return messageJSON; + } + + sendOptions() { + return messageOptions; + } + + static fromMessage() { + return new TestQueueMessage(); + } +} + +describe("Queue", () => { + const queueName = "foobar-queue"; + let queueClient; + const messageClass = TestQueueMessage; + let logger; + let subject; + + beforeEach(async () => { + const chaiAsPromised = await import("chai-as-promised"); + chai.use(chaiAsPromised.default); + expect = chai.expect; + queueClient = { + start: sinon.spy(), + stop: sinon.spy(), + send: sinon.spy(), + work: sinon.spy(), + }; + logger = { + info: sinon.spy(), + error: sinon.spy(), + debug: sinon.spy(), + }; + subject = new Queue({ queueName, queueClient, messageClass, logger }); + }); + + describe(".name", () => { + it("returns the queue name", () => { + expect(subject.name).to.equal(queueName); + }); + }); + + describe(".start", () => { + describe("when starting the queue client is successful", () => { + beforeEach(async () => { + await subject.start(); + }); + + it("starts the queue client", () => { + expect(queueClient.start.calledWith()).to.equal(true); + }); + }); + + describe("when starting the queue client is throws an error", () => { + beforeEach(async () => { + queueClient.start = () => {}; + sinon + .stub(queueClient, "start") + .throws("Error", "some fake error message"); + }); + + it("throws an error", async () => { + expect(subject.start()).to.eventually.be.rejected; + expect(queueClient.start.calledWith()).to.equal(true); + expect(logger.error.calledWith("Error starting queue client")).to.equal( + true, + ); + }); + }); + }); + + describe(".stop", () => { + describe("when stopping the queue client is successful", () => { + beforeEach(async () => { + await subject.stop(); + }); + + it("stops the queue client", () => { + expect(queueClient.stop.calledWith()).to.equal(true); + }); + }); + + describe("when stopping the queue client is throws an error", () => { + beforeEach(async () => { + queueClient.stop = () => {}; + sinon + .stub(queueClient, "stop") + .throws("Error", "some fake error message"); + }); + + it("throws an error", async () => { + expect(subject.stop()).to.eventually.be.rejected; + expect(queueClient.stop.calledWith()).to.equal(true); + expect(logger.error.calledWith("Error stopping queue client")).to.equal( + true, + ); + }); + }); + }); + + describe(".sendMessage", () => { + let queueMessage; + + beforeEach(() => { + messageJSON = { foo: "bar" }; + messageOptions = { test: 1 }; + queueMessage = new TestQueueMessage(); + }); + + describe("when sending a message to the queue is successful", () => { + let actual; + + describe("and a duplicate message is not found", () => { + const expected = "a fake job id"; + + beforeEach(async () => { + queueClient.send = () => {}; + sinon.stub(queueClient, "send").returns(expected); + actual = await subject.sendMessage(queueMessage); + }); + + it("sends the message to the queue with expected JSON and options", () => { + expect( + queueClient.send.calledWith(queueName, messageJSON, messageOptions), + ).to.equal(true); + }); + + it("returns the message ID", () => { + expect(actual).to.equal(expected); + }); + }); + + describe("and a duplicate message is found", () => { + const expected = null; + + beforeEach(async () => { + queueClient.send = () => {}; + sinon.stub(queueClient, "send").returns(expected); + actual = await subject.sendMessage(queueMessage); + }); + + it("attempts to send the message to the queue with expected JSON and options", () => { + expect( + queueClient.send.calledWith(queueName, messageJSON, messageOptions), + ).to.equal(true); + }); + + it("returns null", () => { + expect(actual).to.equal(expected); + }); + }); + }); + + describe("when sending a message to the queue is not successful", () => { + beforeEach(async () => { + queueClient.send = () => {}; + sinon + .stub(queueClient, "send") + .throws("Error", "some fake error message"); + }); + + it("throws an error", async () => { + expect(subject.sendMessage(queueMessage)).to.eventually.be.rejected; + expect( + queueClient.send.calledWith(queueName, messageJSON, messageOptions), + ).to.equal(true); + expect( + logger.error.calledWith(`Error sending to queue: ${queueName}`), + ).to.equal(true); + }); + }); + }); + + describe(".poll", () => { + describe("when options are not passed", () => { + const callback = () => { + return ""; + }; + + beforeEach(async () => { + await subject.poll(callback); + }); + + it("polls the queue with expected options", () => { + expect(queueClient.work.getCalls()[0].args[0]).to.equal(queueName); + expect(queueClient.work.getCalls()[0].args[1]).to.deep.equal({ + newJobCheckIntervalSeconds: 1, + }); + expect(typeof queueClient.work.getCalls()[0].args[2]).to.equal( + "function", + ); + }); + }); + + describe("when options are passed", () => { + const callback = () => { + return ""; + }; + const options = { foo: "bar" }; + + beforeEach(async () => { + await subject.poll(callback, options); + }); + + it("polls the queue with expected options", () => { + expect(queueClient.work.getCalls()[0].args[0]).to.equal(queueName); + expect(queueClient.work.getCalls()[0].args[1]).to.deep.equal(options); + expect(typeof queueClient.work.getCalls()[0].args[2]).to.equal( + "function", + ); + }); + }); + + describe("polling callback function", () => { + describe("when the polling callback is executed with a message", () => { + let callback; + const message = { foo: "bar" }; + + beforeEach(async () => { + callback = sinon.spy(); + await subject.poll(callback); + queueClient.work.getCalls()[0].args[2](message); + }); + + it("logs that a message was received", () => { + sinon.assert.calledWith(logger.info, "Queue message received"); + }); + + it("executes the callback with the message JSON", () => { + expect(callback.calledWith(messageJSON)).to.equal(true); + }); + }); + }); + }); + + describe(".buildQueue", () => { + it("returns an instance of Queue", () => { + expect( + Queue.buildQueue({ + knexInstance: {}, + queueName, + messageClass: TestQueueMessage, + logger: {}, + }) instanceof Queue, + ).to.equal(true); + }); + }); +}); diff --git a/test/queue/queue_message.test.js b/test/queue/queue_message.test.js new file mode 100644 index 00000000..470a2130 --- /dev/null +++ b/test/queue/queue_message.test.js @@ -0,0 +1,30 @@ +const expect = require("chai").expect; +const QueueMessage = require("../../src/queue/queue_message"); + +describe("QueueMessage", () => { + let subject; + + beforeEach(async () => { + subject = new QueueMessage(); + }); + + describe(".toJSON", () => { + it("returns an empty object", () => { + expect(subject.toJSON()).to.deep.equal({}); + }); + }); + + describe(".sendOptions", () => { + it("returns an empty object", () => { + expect(subject.sendOptions()).to.deep.equal({}); + }); + }); + + describe(".fromMessage", () => { + it("creates a new QueueMessage instance", () => { + expect(QueueMessage.fromMessage({}) instanceof QueueMessage).to.equal( + true, + ); + }); + }); +}); diff --git a/test/queue/report_job_queue_message.test.js b/test/queue/report_job_queue_message.test.js new file mode 100644 index 00000000..17c4f01b --- /dev/null +++ b/test/queue/report_job_queue_message.test.js @@ -0,0 +1,176 @@ +const expect = require("chai").expect; +const ReportJobQueueMessage = require("../../src/queue/report_job_queue_message"); + +describe("ReportJobQueueMessage", () => { + const agencyName = "test-agency"; + const analyticsReportIds = "12343567"; + const awsBucketPath = "/data/test-agency"; + const reportOptions = { foo: "bar" }; + const reportConfig = { query: "get some data", name: "foobar report" }; + const scriptName = "daily.sh"; + let subject; + + beforeEach(async () => { + subject = new ReportJobQueueMessage({}); + }); + + describe(".toJSON", () => { + describe("when no arguments are passed to the constructor", () => { + it("returns an object with default values", () => { + expect(subject.toJSON()).to.deep.equal({ + agencyName: "", + analyticsReportIds: "", + awsBucketPath: "", + reportConfig: {}, + options: {}, + scriptName: "", + }); + }); + }); + + describe("when all arguments are passed to the constructor", () => { + beforeEach(() => { + subject = new ReportJobQueueMessage({ + agencyName, + analyticsReportIds, + awsBucketPath, + reportConfig, + reportOptions, + scriptName, + }); + }); + + it("returns an object with default values", () => { + expect(subject.toJSON()).to.deep.equal({ + agencyName, + analyticsReportIds, + awsBucketPath, + reportConfig, + options: reportOptions, + scriptName, + }); + }); + }); + }); + + describe(".sendOptions", () => { + describe("when report frequency is not set", () => { + const noFrequencyReportConfig = { name: "no frequency" }; + + beforeEach(() => { + subject = new ReportJobQueueMessage({ + agencyName, + analyticsReportIds, + awsBucketPath, + reportConfig: noFrequencyReportConfig, + reportOptions, + scriptName, + }); + }); + + it("returns an options object with priority: 0", () => { + expect(subject.sendOptions()).to.deep.equal({ + priority: 0, + retryLimit: 2, + retryDelay: 10, + retryBackoff: true, + singletonKey: `${scriptName}-${agencyName}-${noFrequencyReportConfig.name}`, + }); + }); + }); + + describe("when report frequency is daily", () => { + const dailyReportConfig = { name: "daily", frequency: "daily" }; + + beforeEach(() => { + subject = new ReportJobQueueMessage({ + agencyName, + analyticsReportIds, + awsBucketPath, + reportConfig: dailyReportConfig, + reportOptions, + scriptName, + }); + }); + + it("returns an options object with priority: 1", () => { + expect(subject.sendOptions()).to.deep.equal({ + priority: 1, + retryLimit: 2, + retryDelay: 10, + retryBackoff: true, + singletonKey: `${scriptName}-${agencyName}-${dailyReportConfig.name}`, + }); + }); + }); + + describe("when report frequency is hourly", () => { + const hourlyReportConfig = { name: "hourly", frequency: "hourly" }; + + beforeEach(() => { + subject = new ReportJobQueueMessage({ + agencyName, + analyticsReportIds, + awsBucketPath, + reportConfig: hourlyReportConfig, + reportOptions, + scriptName, + }); + }); + + it("returns an options object with priority: 2", () => { + expect(subject.sendOptions()).to.deep.equal({ + priority: 2, + retryLimit: 2, + retryDelay: 10, + retryBackoff: true, + singletonKey: `${scriptName}-${agencyName}-${hourlyReportConfig.name}`, + }); + }); + }); + + describe("when report frequency is realtime", () => { + const realtimeReportConfig = { name: "realtime", frequency: "realtime" }; + + beforeEach(() => { + subject = new ReportJobQueueMessage({ + agencyName, + analyticsReportIds, + awsBucketPath, + reportConfig: realtimeReportConfig, + reportOptions, + scriptName, + }); + }); + + it("returns an options object with priority: 3", () => { + expect(subject.sendOptions()).to.deep.equal({ + priority: 3, + retryLimit: 2, + retryDelay: 10, + retryBackoff: true, + singletonKey: `${scriptName}-${agencyName}-${realtimeReportConfig.name}`, + }); + }); + }); + }); + + describe(".fromMessage", () => { + describe("when arguments are passed", () => { + it("creates a new ReportJobQueueMessage instance", () => { + expect( + ReportJobQueueMessage.fromMessage({ data: {} }) instanceof + ReportJobQueueMessage, + ).to.equal(true); + }); + }); + + describe("when no arguments are passed", () => { + it("creates a new ReportJobQueueMessage instance", () => { + expect( + ReportJobQueueMessage.fromMessage() instanceof ReportJobQueueMessage, + ).to.equal(true); + }); + }); + }); +}); From 1d8694f78c151f01bcabaefe67d565515e2df57d Mon Sep 17 00:00:00 2001 From: Michael Levin Date: Thu, 9 Jan 2025 10:21:40 -0500 Subject: [PATCH 4/6] [Chore] Fix typo in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 91971972..96e6cb29 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ The process for adding features to this project is described in ## Local development setup -### Prerequistites +### Prerequisites * NodeJS > v20.x * A postgres DB running and/or docker installed From 069f644c796ed634937c6fe2d5d39823c0eba1e4 Mon Sep 17 00:00:00 2001 From: Michael Levin Date: Thu, 9 Jan 2025 10:50:43 -0500 Subject: [PATCH 5/6] [Tech Debt] Job scheduling with the Bree library --- deploy/api.sh | 6 - deploy/cron.js | 120 ---------------- deploy/daily.sh | 6 - deploy/hourly.sh | 5 - deploy/publisher.js | 46 ++++++ deploy/realtime.sh | 5 - jobs/api.js | 24 ++++ jobs/daily.js | 27 ++++ jobs/realtime.js | 27 ++++ manifest.publisher.yml | 2 +- package-lock.json | 307 ++++++++++++++++++++++++++++++++++++++++- package.json | 1 + 12 files changed, 427 insertions(+), 149 deletions(-) delete mode 100755 deploy/api.sh delete mode 100644 deploy/cron.js delete mode 100755 deploy/daily.sh delete mode 100755 deploy/hourly.sh create mode 100644 deploy/publisher.js delete mode 100755 deploy/realtime.sh create mode 100644 jobs/api.js create mode 100644 jobs/daily.js create mode 100644 jobs/realtime.js diff --git a/deploy/api.sh b/deploy/api.sh deleted file mode 100755 index b107cd4c..00000000 --- a/deploy/api.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -export ANALYTICS_REPORTS_PATH=reports/api.json -export ANALYTICS_SCRIPT_NAME=api.sh - -$ANALYTICS_ROOT_PATH/bin/analytics-publisher --debug --write-to-database --agenciesFile=$ANALYTICS_ROOT_PATH/deploy/agencies.json diff --git a/deploy/cron.js b/deploy/cron.js deleted file mode 100644 index 36e5b758..00000000 --- a/deploy/cron.js +++ /dev/null @@ -1,120 +0,0 @@ -if (process.env.NODE_ENV !== "production") { - require("dotenv").config(); -} - -if (process.env.NEW_RELIC_APP_NAME) { - require("newrelic"); -} - -const maxListenersExceededWarning = require("max-listeners-exceeded-warning"); -maxListenersExceededWarning(); - -const spawn = require("child_process").spawn; -const logger = require("../src/logger").initialize(); - -logger.info("==================================="); -logger.info("=== STARTING ANALYTICS-REPORTER ==="); -logger.info(" Running /deploy/cron.js"); -logger.info("==================================="); - -const scriptRootPath = `${process.env.ANALYTICS_ROOT_PATH}/deploy`; - -const runScriptWithLogName = (scriptPath, scriptLoggingName) => { - logger.info(`Beginning: ${scriptLoggingName}`); - logger.info(`File path: ${scriptPath}`); - const childProcess = spawn(scriptPath); - - childProcess.stdout.on("data", (data) => { - // Writes logging output from child processes to console. - console.log(data.toString().trim()); - }); - - childProcess.stderr.on("data", (data) => { - // Writes error logging output from child processes to console. - console.log(data.toString().trim()); - }); - - childProcess.on("close", (code, signal) => { - logger.info(`${scriptLoggingName} closed with code: ${code}`); - if (signal) { - logger.info(`${scriptLoggingName} received signal: ${signal}`); - } - }); - - childProcess.on("exit", (code, signal) => { - logger.info(`${scriptLoggingName} exitted with code: ${code}`); - if (signal) { - logger.info(`${scriptLoggingName} received signal: ${signal}`); - } - }); - - childProcess.on("error", (err) => { - logger.info(`${scriptLoggingName} errored: ${err}`); - }); -}; - -const api_run = () => { - runScriptWithLogName(`${scriptRootPath}/api.sh`, "api.sh"); -}; - -const daily_run = () => { - runScriptWithLogName(`${scriptRootPath}/daily.sh`, "daily.sh"); -}; - -/*const hourly_run = () => { - runScriptWithLogName(`${scriptRootPath}/hourly.sh`, "hourly.sh"); -};*/ - -const realtime_run = () => { - runScriptWithLogName(`${scriptRootPath}/realtime.sh`, "realtime.sh"); -}; - -/** - * Daily and API reports run every morning at 10 AM UTC. - * This calculates the offset between now and then for the next scheduled run. - */ -const calculateNextDailyRunTimeOffset = () => { - const currentTime = new Date(); - const nextRunTime = new Date( - currentTime.getFullYear(), - currentTime.getMonth(), - currentTime.getDate() + 1, - 10 - currentTime.getTimezoneOffset() / 60, - ); - return (nextRunTime - currentTime) % (1000 * 60 * 60 * 24); -}; - -/** - * All scripts run immediately upon application start (with a 60 second delay - * between each so that they don't all run at once), then run again at intervals - * going forward. - */ -setTimeout(realtime_run, 1000 * 10); -// setTimeout(hourly_run, 1000 * 70); No hourly reports exist at this time. -setTimeout(daily_run, 1000 * 70); -setTimeout(api_run, 1000 * 130); - -// Daily and API recurring script run setup. -// Runs at 10 AM UTC, then every 24 hours afterwards -setTimeout(() => { - // Offset the daily script run by 30 seconds so that it never runs in parallel - // with the realtime script in order to save memory/CPU. - setTimeout(() => { - daily_run(); - setInterval(daily_run, 1000 * 60 * 60 * 24); - }, 1000 * 30); - - // setTimeout(hourly_run, 1000 * 60); - - // Offset the API script run by 90 seconds so that it never runs in parallel - // with the daily or realtime scripts in order to save memory/CPU. - setTimeout(() => { - api_run(); - setInterval(api_run, 1000 * 60 * 60 * 24); - }, 1000 * 90); -}, calculateNextDailyRunTimeOffset()); -// hourly (no hourly reports exist at this time). -// setInterval(hourly_run, 1000 * 60 * 60); - -// Realtime recurring script run setup. Runs every 15 minutes. -setInterval(realtime_run, 1000 * 60 * 15); diff --git a/deploy/daily.sh b/deploy/daily.sh deleted file mode 100755 index cbe9415a..00000000 --- a/deploy/daily.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -export ANALYTICS_SCRIPT_NAME=daily.sh - -$ANALYTICS_ROOT_PATH/bin/analytics-publisher --publish --frequency=daily --slim --debug --csv --json --agenciesFile=$ANALYTICS_ROOT_PATH/deploy/agencies.json - diff --git a/deploy/hourly.sh b/deploy/hourly.sh deleted file mode 100755 index 407e4d24..00000000 --- a/deploy/hourly.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -export ANALYTICS_SCRIPT_NAME=hourly.sh - -$ANALYTICS_ROOT_PATH/bin/analytics-publisher --publish --frequency=hourly --slim --debug --json --agenciesFile=$ANALYTICS_ROOT_PATH/deploy/agencies.json diff --git a/deploy/publisher.js b/deploy/publisher.js new file mode 100644 index 00000000..9a689eef --- /dev/null +++ b/deploy/publisher.js @@ -0,0 +1,46 @@ +if (process.env.NODE_ENV !== "production") { + require("dotenv").config(); +} + +if (process.env.NEW_RELIC_APP_NAME) { + require("newrelic"); +} + +const logger = require("../src/logger").initialize(); +logger.info("==================================="); +logger.info("=== STARTING ANALYTICS-REPORTER ==="); +logger.info(" Running /deploy/publisher.js"); +logger.info("==================================="); + +// Job Scheduler +const Bree = require("bree"); +const bree = new Bree({ + logger, + jobs: [ + // Runs `../jobs/realtime.js` 1 millisecond after the process starts and + // then every 15 minutes going forward. + { + name: "realtime", + timeout: "1", + interval: "15m", + }, + // Runs `../jobs/daily.js` 1 minute after the process starts and then at + // 10:01 AM every day going forward. + { + name: "daily", + timeout: "1m", + interval: "at 10:01 am", + }, + // Runs `../jobs/api.js` 2 minutes after the process starts and then at + // 10:02 AM every day going forward. + { + name: "api", + timeout: "2m", + interval: "at 10:02 am", + }, + ], +}); + +(async () => { + await bree.start(); +})(); diff --git a/deploy/realtime.sh b/deploy/realtime.sh deleted file mode 100755 index 10ca0f8c..00000000 --- a/deploy/realtime.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -export ANALYTICS_SCRIPT_NAME=realtime.sh - -$ANALYTICS_ROOT_PATH/bin/analytics-publisher --publish --frequency=realtime --slim --debug --csv --json --agenciesFile=$ANALYTICS_ROOT_PATH/deploy/agencies.json diff --git a/jobs/api.js b/jobs/api.js new file mode 100644 index 00000000..cec86261 --- /dev/null +++ b/jobs/api.js @@ -0,0 +1,24 @@ +process.env.ANALYTICS_REPORTS_PATH = "reports/api.json"; +process.env.ANALYTICS_SCRIPT_NAME = "api.js"; + +const { runQueuePublish } = require("../index.js"); +const options = { + frequency: "daily", + debug: true, + "write-to-database": true, + agenciesFile: `${process.env.ANALYTICS_ROOT_PATH}/deploy/agencies.json`, +}; +const logger = require("../src/logger.js").initialize(); + +(async () => { + logger.info(`Beginning job: ${process.env.ANALYTICS_SCRIPT_NAME}`); + + try { + await runQueuePublish(options); + logger.info(`Job completed: ${process.env.ANALYTICS_SCRIPT_NAME}`); + } catch (e) { + logger.error(`Job exited with error: ${process.env.ANALYTICS_SCRIPT_NAME}`); + logger.error(e); + throw e; + } +})(); diff --git a/jobs/daily.js b/jobs/daily.js new file mode 100644 index 00000000..1a7429a0 --- /dev/null +++ b/jobs/daily.js @@ -0,0 +1,27 @@ +process.env.ANALYTICS_REPORTS_PATH = "reports/usa.json"; +process.env.ANALYTICS_SCRIPT_NAME = "daily.js"; + +const { runQueuePublish } = require("../index.js"); +const options = { + publish: true, + frequency: "daily", + slim: true, + debug: true, + csv: true, + json: true, + agenciesFile: `${process.env.ANALYTICS_ROOT_PATH}/deploy/agencies.json`, +}; +const logger = require("../src/logger.js").initialize(); + +(async () => { + logger.info(`Beginning job: ${process.env.ANALYTICS_SCRIPT_NAME}`); + + try { + await runQueuePublish(options); + logger.info(`Job completed: ${process.env.ANALYTICS_SCRIPT_NAME}`); + } catch (e) { + logger.error(`Job exited with error: ${process.env.ANALYTICS_SCRIPT_NAME}`); + logger.error(e); + throw e; + } +})(); diff --git a/jobs/realtime.js b/jobs/realtime.js new file mode 100644 index 00000000..bb0425d8 --- /dev/null +++ b/jobs/realtime.js @@ -0,0 +1,27 @@ +process.env.ANALYTICS_REPORTS_PATH = "reports/usa.json"; +process.env.ANALYTICS_SCRIPT_NAME = "realtime.js"; + +const { runQueuePublish } = require("../index.js"); +const options = { + publish: true, + frequency: "realtime", + slim: true, + debug: true, + csv: true, + json: true, + agenciesFile: `${process.env.ANALYTICS_ROOT_PATH}/deploy/agencies.json`, +}; +const logger = require("../src/logger.js").initialize(); + +(async () => { + logger.info(`Beginning job: ${process.env.ANALYTICS_SCRIPT_NAME}`); + + try { + await runQueuePublish(options); + logger.info(`Job completed: ${process.env.ANALYTICS_SCRIPT_NAME}`); + } catch (e) { + logger.error(`Job exited with error: ${process.env.ANALYTICS_SCRIPT_NAME}`); + logger.error(e); + throw e; + } +})(); diff --git a/manifest.publisher.yml b/manifest.publisher.yml index f916a61a..88ecd9aa 100644 --- a/manifest.publisher.yml +++ b/manifest.publisher.yml @@ -9,7 +9,7 @@ applications: health-check-type: process buildpacks: - nodejs_buildpack - command: node deploy/cron.js + command: node deploy/publisher.js env: ANALYTICS_DEBUG: 'true' ANALYTICS_LOG_LEVEL: ${ANALYTICS_LOG_LEVEL} diff --git a/package-lock.json b/package-lock.json index 67f9c51b..db662bd3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@google-analytics/data": "^4.12.0", "@smithy/node-http-handler": "^3.0.0", "@snyk/protect": "^1.1269.0", + "bree": "^9.2.4", "fast-csv": "^4.3.6", "googleapis": "^140.0.0", "max-listeners-exceeded-warning": "^0.0.1", @@ -1235,6 +1236,18 @@ "node": ">=6.0.0" } }, + "node_modules/@babel/runtime": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz", + "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==", + "license": "MIT", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.25.0", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.0.tgz", @@ -1290,6 +1303,15 @@ "node": ">=6.9.0" } }, + "node_modules/@breejs/later": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@breejs/later/-/later-4.2.0.tgz", + "integrity": "sha512-EVMD0SgJtOuFeg0lAVbCwa+qeTKILb87jqvLyUtQswGD9+ce2nB52Y5zbTF1Hc0MDFfbydcMcxb47jSdhikVHA==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, "node_modules/@colors/colors": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", @@ -2930,6 +2952,12 @@ "@types/node": "*" } }, + "node_modules/@types/lodash": { + "version": "4.17.14", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.14.tgz", + "integrity": "sha512-jsxagdikDiDBeIRaPYtArcT8my4tN1og7MtMRquFT3XNA6axxyHDRUemqDz/taRDdOUn0GnGHRCuff4q48sW9A==", + "license": "MIT" + }, "node_modules/@types/long": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", @@ -3304,6 +3332,13 @@ "readable-stream": "^3.4.0" } }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT" + }, "node_modules/bowser": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", @@ -3331,6 +3366,33 @@ "node": ">=8" } }, + "node_modules/bree": { + "version": "9.2.4", + "resolved": "https://registry.npmjs.org/bree/-/bree-9.2.4.tgz", + "integrity": "sha512-3GDVYbRYxPIIKgqu00FlIDD//q/0XkMC+zq74sp/qRRQQUWdc39lsFkdHW2g2lTlhaxbqkHd97p8oRMm/YeSJw==", + "license": "MIT", + "dependencies": { + "@breejs/later": "^4.2.0", + "boolean": "^3.2.0", + "combine-errors": "^3.0.3", + "cron-validate": "^1.4.5", + "human-interval": "^2.0.1", + "is-string-and-not-blank": "^0.0.2", + "is-valid-path": "^0.1.1", + "ms": "^2.1.3", + "p-wait-for": "3", + "safe-timers": "^1.1.0" + }, + "engines": { + "node": ">=12.17.0 <13.0.0-0||>=13.2.0" + } + }, + "node_modules/bree/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, "node_modules/browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", @@ -3774,6 +3836,15 @@ "text-hex": "1.0.x" } }, + "node_modules/combine-errors": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/combine-errors/-/combine-errors-3.0.3.tgz", + "integrity": "sha512-C8ikRNRMygCwaTx+Ek3Yr+OuZzgZjduCOfSQBjbM8V3MfgcjSTeto/GXP6PAwKvJz/v15b7GHZvx5rOlczFw/Q==", + "dependencies": { + "custom-error-instance": "2.1.1", + "lodash.uniqby": "4.5.0" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -3862,6 +3933,33 @@ "node": ">=12.0.0" } }, + "node_modules/cron-validate": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/cron-validate/-/cron-validate-1.4.5.tgz", + "integrity": "sha512-nKlOJEnYKudMn/aNyNH8xxWczlfpaazfWV32Pcx/2St51r2bxWbGhZD7uwzMcRhunA/ZNL+Htm/i0792Z59UMQ==", + "license": "MIT", + "dependencies": { + "yup": "0.32.9" + } + }, + "node_modules/cron-validate/node_modules/yup": { + "version": "0.32.9", + "resolved": "https://registry.npmjs.org/yup/-/yup-0.32.9.tgz", + "integrity": "sha512-Ci1qN+i2H0XpY7syDQ0k5zKQ/DoxO0LzPg8PAR/X4Mpj6DqaeCoIYEEjDJwhArh3Fa7GWbQQVDZKeXYlSH4JMg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.10.5", + "@types/lodash": "^4.14.165", + "lodash": "^4.17.20", + "lodash-es": "^4.17.15", + "nanoclone": "^0.2.1", + "property-expr": "^2.0.4", + "toposort": "^2.0.2" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -3877,6 +3975,12 @@ "node": ">= 8" } }, + "node_modules/custom-error-instance": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/custom-error-instance/-/custom-error-instance-2.1.1.tgz", + "integrity": "sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg==", + "license": "ISC" + }, "node_modules/data-uri-to-buffer": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", @@ -5436,6 +5540,15 @@ "node": ">= 14" } }, + "node_modules/human-interval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/human-interval/-/human-interval-2.0.1.tgz", + "integrity": "sha512-r4Aotzf+OtKIGQCB3odUowy4GfUDTy3aTWTfLd7ZF2gBCy3XW3v/dJLRefZnOFFnjqs5B1TypvS8WarpBkYUNQ==", + "license": "MIT", + "dependencies": { + "numbered": "^1.1.0" + } + }, "node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", @@ -5728,6 +5841,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-string-and-not-blank": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/is-string-and-not-blank/-/is-string-and-not-blank-0.0.2.tgz", + "integrity": "sha512-FyPGAbNVyZpTeDCTXnzuwbu9/WpNXbCfbHXLpCRpN4GANhS00eEIP5Ef+k5HYSNIzIhdN9zRDoBj6unscECvtQ==", + "license": "MIT", + "dependencies": { + "is-string-blank": "^1.0.1" + }, + "engines": { + "node": ">=6.4.0" + } + }, + "node_modules/is-string-blank": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-string-blank/-/is-string-blank-1.0.1.tgz", + "integrity": "sha512-9H+ZBCVs3L9OYqv8nuUAzpcT9OTgMD1yAWrG7ihlnibdkbtB850heAmYWxHuXc4CHy4lKeK69tN+ny1K7gBIrw==", + "license": "MIT" + }, "node_modules/is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -5746,6 +5877,51 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-valid-path": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-valid-path/-/is-valid-path-0.1.1.tgz", + "integrity": "sha512-+kwPrVDu9Ms03L90Qaml+79+6DZHqHyRoANI6IsZJ/g8frhnfchDOBCa0RbQ6/kdHt5CS5OeIEyrYznNuVN+8A==", + "license": "MIT", + "dependencies": { + "is-invalid-path": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-valid-path/node_modules/is-extglob": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-1.0.0.tgz", + "integrity": "sha512-7Q+VbVafe6x2T+Tu6NcOf6sRklazEPmBoB3IWk3WdGZM2iGUwU/Oe3Wtq5lSEkDTTlpp8yx+5t4pzO/i9Ty1ww==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-valid-path/node_modules/is-glob": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz", + "integrity": "sha512-a1dBeB19NXsf/E0+FHqkagizel/LQw2DjSQpvQrj3zT+jYPpaUCryPnrQajXKFLCMuf4I6FhRpaGtw4lPrG6Eg==", + "license": "MIT", + "dependencies": { + "is-extglob": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-valid-path/node_modules/is-invalid-path": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-invalid-path/-/is-invalid-path-0.1.0.tgz", + "integrity": "sha512-aZMG0T3F34mTg4eTdszcGXx54oiZ4NtHSft3hWNJMGJXUUqdIj3cOZuHcU0nCWWcY3jd7yRe/3AEm3vSNTpBGQ==", + "license": "MIT", + "dependencies": { + "is-glob": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", @@ -6209,8 +6385,59 @@ "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "optional": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/lodash-es": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", + "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==", + "license": "MIT" + }, + "node_modules/lodash._baseiteratee": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash._baseiteratee/-/lodash._baseiteratee-4.7.0.tgz", + "integrity": "sha512-nqB9M+wITz0BX/Q2xg6fQ8mLkyfF7MU7eE+MNBNjTHFKeKaZAPEzEg+E8LWxKWf1DQVflNEn9N49yAuqKh2mWQ==", + "license": "MIT", + "dependencies": { + "lodash._stringtopath": "~4.8.0" + } + }, + "node_modules/lodash._basetostring": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-4.12.0.tgz", + "integrity": "sha512-SwcRIbyxnN6CFEEK4K1y+zuApvWdpQdBHM/swxP962s8HIxPO3alBH5t3m/dl+f4CMUug6sJb7Pww8d13/9WSw==", + "license": "MIT" + }, + "node_modules/lodash._baseuniq": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz", + "integrity": "sha512-Ja1YevpHZctlI5beLA7oc5KNDhGcPixFhcqSiORHNsp/1QTv7amAXzw+gu4YOvErqVlMVyIJGgtzeepCnnur0A==", + "license": "MIT", + "dependencies": { + "lodash._createset": "~4.0.0", + "lodash._root": "~3.0.0" + } + }, + "node_modules/lodash._createset": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/lodash._createset/-/lodash._createset-4.0.3.tgz", + "integrity": "sha512-GTkC6YMprrJZCYU3zcqZj+jkXkrXzq3IPBcF/fIPpNEAB4hZEtXU8zp/RwKOvZl43NUmwDbyRk3+ZTbeRdEBXA==", + "license": "MIT" + }, + "node_modules/lodash._root": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash._root/-/lodash._root-3.0.1.tgz", + "integrity": "sha512-O0pWuFSK6x4EXhM1dhZ8gchNtG7JMqBtrHdoUFUWXD7dJnNSUze1GuyQr5sOs0aCvgGeI3o/OJW8f4ca7FDxmQ==", + "license": "MIT" + }, + "node_modules/lodash._stringtopath": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/lodash._stringtopath/-/lodash._stringtopath-4.8.0.tgz", + "integrity": "sha512-SXL66C731p0xPDC5LZg4wI5H+dJo/EO4KTqOMwLYCH3+FmmfAKJEZCm6ohGpI+T1xwsDsJCfL4OnhorllvlTPQ==", + "license": "MIT", + "dependencies": { + "lodash._basetostring": "~4.12.0" + } }, "node_modules/lodash.camelcase": { "version": "4.3.0", @@ -6286,6 +6513,16 @@ "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==" }, + "node_modules/lodash.uniqby": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.5.0.tgz", + "integrity": "sha512-IRt7cfTtHy6f1aRVA5n7kT8rgN3N1nH6MOWLcHfpWG2SH19E3JksLK38MktLxZDhlAjCP9jpIXkOnRXlu6oByQ==", + "license": "MIT", + "dependencies": { + "lodash._baseiteratee": "~4.7.0", + "lodash._baseuniq": "~4.6.0" + } + }, "node_modules/log-symbols": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", @@ -6815,6 +7052,12 @@ "integrity": "sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==", "optional": true }, + "node_modules/nanoclone": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/nanoclone/-/nanoclone-0.2.1.tgz", + "integrity": "sha512-wynEP02LmIbLpcYw8uBKpcfF6dmg2vcpKqxeH5UcoKEYdExslsdUA4ugFauuaeYdTB76ez6gJW8XAZ6CgkXYxA==", + "license": "MIT" + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -7060,6 +7303,12 @@ "node": ">=8" } }, + "node_modules/numbered": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/numbered/-/numbered-1.1.0.tgz", + "integrity": "sha512-pv/ue2Odr7IfYOO0byC1KgBI10wo5YDauLhxY6/saNzAdAs0r1SotGCPzzCLNPL0xtrAwWRialLu23AAu9xO1g==", + "license": "MIT" + }, "node_modules/nyc": { "version": "15.1.0", "resolved": "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz", @@ -7341,6 +7590,15 @@ "node": ">= 0.8.0" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -7401,6 +7659,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -7410,6 +7680,21 @@ "node": ">=6" } }, + "node_modules/p-wait-for": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-wait-for/-/p-wait-for-3.2.0.tgz", + "integrity": "sha512-wpgERjNkLrBiFmkMEjuZJEWKKDrNfHCKA1OhyN1wg1FrLkULbviEy6py1AyJUgZ72YWFbZ38FIpnqvVqAlDUwA==", + "license": "MIT", + "dependencies": { + "p-timeout": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/pac-proxy-agent": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz", @@ -7943,8 +8228,7 @@ "node_modules/property-expr": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/property-expr/-/property-expr-2.0.6.tgz", - "integrity": "sha512-SVtmxhRE/CGkn3eZY1T6pC8Nln6Fr/lu1mKSgRud0eC73whjGfoAogbn78LkD8aFL0zz3bAFerKSnOl7NlErBA==", - "dev": true + "integrity": "sha512-SVtmxhRE/CGkn3eZY1T6pC8Nln6Fr/lu1mKSgRud0eC73whjGfoAogbn78LkD8aFL0zz3bAFerKSnOl7NlErBA==" }, "node_modules/proto3-json-serializer": { "version": "2.0.2", @@ -8230,6 +8514,12 @@ "deprecated": "This version has a critical bug in fallback handling. Please upgrade to reflect-metadata@0.2.2 or newer.", "dev": true }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", + "license": "MIT" + }, "node_modules/regexp-match-indices": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regexp-match-indices/-/regexp-match-indices-1.0.2.tgz", @@ -8466,6 +8756,12 @@ "node": ">=10" } }, + "node_modules/safe-timers": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-timers/-/safe-timers-1.1.0.tgz", + "integrity": "sha512-9aqY+v5eMvmRaluUEtdRThV1EjlSElzO7HuCj0sTW9xvp++8iJ9t/RWGNWV6/WHcUJLHpyT2SNf/apoKTU2EpA==", + "license": "MIT" + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -9446,8 +9742,7 @@ "node_modules/toposort": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/toposort/-/toposort-2.0.2.tgz", - "integrity": "sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg==", - "dev": true + "integrity": "sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg==" }, "node_modules/tr46": { "version": "0.0.3", diff --git a/package.json b/package.json index ef0a371b..c99c2b1a 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "@google-analytics/data": "^4.12.0", "@smithy/node-http-handler": "^3.0.0", "@snyk/protect": "^1.1269.0", + "bree": "^9.2.4", "fast-csv": "^4.3.6", "googleapis": "^140.0.0", "max-listeners-exceeded-warning": "^0.0.1", From 929e2429ecffcca1756c592fa4f697a13c21606a Mon Sep 17 00:00:00 2001 From: Michael Levin Date: Thu, 9 Jan 2025 12:18:17 -0500 Subject: [PATCH 6/6] [Chore] Update minor README details --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 96e6cb29..e7c0f108 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ The process for adding features to this project is described in ### Prerequisites -* NodeJS > v20.x +* NodeJS > v22.x * A postgres DB running and/or docker installed ### Install dependencies @@ -144,7 +144,7 @@ This file is ignored in the `.gitignore` file and should not be checked in to th npm start # running the app with dotenv-cli -dotenv -e .env npm start +npx dotenv -e .env npm start ``` ## Configuration