Merge pull request #63 from nhsuk/feature/enable-disable

✋ expose env var to disable scheduler

Author: st3v3nhunt

README.md

@@ -27,6 +27,8 @@ i.e. `export ETL_SCHEDULE='25 15 * * *'` to start the processing a 3:25pm. Note:
 
 Further details available [here](https://www.npmjs.com/package/node-schedule)
 
+The scheduler can be completely disabled by setting the `DISABLE_SCHEDULER` variable to `true`. This sets the run date to run once in the future on Jan 1st, 2100.
+
 A successful scrape will result in the file `gp-data.json` being written to the `output` folder.
 Two files are also written to Azure storage: `gp-data.json` and a time stamped version of the same data named `gp-data-YYYYMMDD.json` where `YYYYMMDD` is the current year, day and month. Summary JSON is available at `output/summary.json`.
 
@@ -53,6 +55,23 @@ No facility is provided to interrogate the contents of the blob storage. However
 all uploaded files are available at the address `https://nhsukgpdataetl.blob.core.windows.net/etl-output`,
 i.e. [https://nhsukgpdataetl.blob.core.windows.net/etl-output/gp-data.json](https://nhsukgpdataetl.blob.core.windows.net/etl-output/gp-data.json).
 
+## Environment variables
+
+Environment variables are expected to be managed by the environment in which
+the application is being run. This is best practice as described by
+[twelve-factor](https://12factor.net/config).
+
+| Variable                           | Description                                                          | Default                 | Required   |
+| :--------------------------------- | :------------------------------------------------------------------- | ----------------------- | :--------- |
+| `NODE_ENV`                         | node environment                                                     | development             |            |
+| `LOG_LEVEL`                        | [log level](https://github.com/trentm/node-bunyan#levels)            | Depends on `NODE_ENV`   |            |
+| `SYNDICATION_API_KEY`              | API key to access syndication                                        |                         | yes        |
+| `AZURE_STORAGE_CONNECTION_STRING`  | Azure storage connection string                                      |                         | yes        |
+| `AZURE_TIMEOUT_MINUTES`            | Maximum wait time when uploading file to Azure                       | 5                       |            |
+| `CONTAINER_NAME`                   | Azure storage container name                                         | etl-output              |            |
+| `UPDATE_SCHEDULE`                  | time of day to run the upgrade                                       | * 23 * * * (11:00 pm)   |            |
+| `DISABLE_SCHEDULER`                | set to 'true' to disable the scheduler                               | false                   |            |
+
 ## Architecture Decision Records
 
 This repo uses

app/lib/scheduleConfig.js

@@ -0,0 +1,16 @@
+const defaultSchedule = '0 23 * * *';
+const farFutureDate = new Date(2100, 0, 1, 0, 0);
+
+function schedulerDisabled() {
+  return process.env.DISABLE_SCHEDULER === 'true';
+}
+
+function getSchedule() {
+  return schedulerDisabled() ? farFutureDate : process.env.ETL_SCHEDULE || defaultSchedule;
+}
+
+module.exports = {
+  defaultSchedule,
+  farFutureDate,
+  getSchedule
+};

docker-compose.yml

@@ -8,6 +8,7 @@ services:
     environment:
       SYNDICATION_API_KEY: ${SYNDICATION_API_KEY}
       ETL_SCHEDULE: ${ETL_SCHEDULE}
+      DISABLE_SCHEDULER: ${DISABLE_SCHEDULER}
       AZURE_STORAGE_CONNECTION_STRING: '${AZURE_STORAGE_CONNECTION_STRING}'
       AZURE_TIMEOUT_MINUTES: ${AZURE_TIMEOUT_MINUTES}
       CONTAINER_NAME: ${CONTAINER_NAME}

rancher-config/docker-compose.yml

@@ -7,6 +7,7 @@ services:
     environment:
       NODE_ENV: production
       SYNDICATION_API_KEY: ${SYNDICATION_API_KEY}
+      DISABLE_SCHEDULER: ${DISABLE_SCHEDULER}
       ETL_SCHEDULE: ${ETL_SCHEDULE}
       AZURE_STORAGE_CONNECTION_STRING: '${AZURE_STORAGE_CONNECTION_STRING}'
       AZURE_TIMEOUT_MINUTES: ${AZURE_TIMEOUT_MINUTES}

schedule.js

@@ -1,11 +1,9 @@
 const schedule = require('node-schedule');
-
+const scheduleConfig = require('./app/lib/scheduleConfig');
 const etl = require('./app/lib/etl');
 const log = require('./app/lib/utils/logger');
 
-const rule = process.env.ETL_SCHEDULE || '0 23 * * *';
-
-log.info(`Scheduling job with rule '${rule}'`);
-schedule.scheduleJob(rule, () => {
+log.info(`Scheduling job with rule '${scheduleConfig.getSchedule()}'`);
+schedule.scheduleJob(scheduleConfig.getSchedule(), () => {
   etl.start();
 });

scripts/README.md

@@ -7,12 +7,16 @@ Below is a list of scripts available, along with a simple description of
 what each one does. The details of what they are doing is available within the
 script.
 
-[`pre-bootstrap`](pre-bootstrap)
-Directs towards base development machine setup.
+[`deploy`](deploy)
+Clone [ci-deployment](https://github.com/nhsuk/ci-deployment.git) repo and
+execute `deploy` script.
 
 [`bootstrap`](bootstrap)
 Installs project's direct dependencies e.g. npm packages.
 
+[`pre-bootstrap`](pre-bootstrap)
+Directs towards base development machine setup.
+
 [`start`](start)
 Starts the application a Docker container.
 
@@ -25,6 +29,3 @@ Starts a Docker container specifically for continually running tests.
 [`test-ci`](test-ci)
 Runs the tests in a Docker container once so that an exit code is reported and
 can be used by the CI server.
-
-[`ci-deployment`](ci-deployment)
-Infrastructure related work.

test/unit/scheduleConfig.js

@@ -0,0 +1,38 @@
+const chai = require('chai');
+const config = require('../../app/lib/scheduleConfig');
+
+const expect = chai.expect;
+
+const disableScheduler = process.env.DISABLE_SCHEDULER;
+const etlSchedule = process.env.ETL_SCHEDULE;
+
+function resetEnv() {
+  process.env.disableScheduler = disableScheduler;
+  process.env.ETL_SCHEDULE = etlSchedule;
+}
+
+describe('getSchedule', () => {
+  it('should return a date far in the future when scheduler disabled', () => {
+    process.env.DISABLE_SCHEDULER = 'true';
+    expect(config.getSchedule()).to.be.a('Date');
+    expect(config.getSchedule()).to.equal(config.farFutureDate);
+    expect(config.getSchedule().getFullYear()).to.equal(2100);
+  });
+
+  it('should return default cron string when scheduler enabled and no ETL_SCHEDULE set', () => {
+    process.env.DISABLE_SCHEDULER = 'false';
+    delete process.env.ETL_SCHEDULE;
+    expect(config.getSchedule()).to.equal(config.defaultSchedule);
+  });
+
+  it('should return the ETL_SCHEDULE cron string when scheduler enabled and ETL_SCHEDULE set', () => {
+    const customSchedule = '0 7 * * *';
+    process.env.DISABLE_SCHEDULER = 'false';
+    process.env.ETL_SCHEDULE = customSchedule;
+    expect(config.getSchedule()).to.equal(customSchedule);
+  });
+
+  afterEach(() => {
+    resetEnv();
+  });
+});