Skip to content

Commit

Permalink
adding report locking cron job (#363)
Browse files Browse the repository at this point in the history
  • Loading branch information
goemen authored Apr 3, 2024
1 parent 109eb8e commit e402c2a
Show file tree
Hide file tree
Showing 19 changed files with 341 additions and 390 deletions.
409 changes: 99 additions & 310 deletions backend/package-lock.json

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions backend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,12 @@
"passport-jwt": "^4.0.1",
"passport-openidconnect-keycloak-idp": "^0.1.6",
"pg": "^8.11.3",
"prisma-mock": "^0.10.0",
"prom-client": "^15.1.0",
"reflect-metadata": "^0.2.1",
"session-file-store": "^1.5.0",
"uuid": "^9.0.1",
"wait-for-expect": "^3.0.2",
"winston": "^3.11.0",
"xml2js": "^0.6.2"
},
Expand Down
3 changes: 2 additions & 1 deletion backend/src/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ import { reportRouter } from './v1/routes/report-routes';
import userRouter from './v1/routes/user-info-routes';
import { auth } from './v1/services/auth-service';
import { utils } from './v1/services/utils-service';
require('./schedulers/delete-draft-service-scheduler');

import './schedulers';

const MAX_CSV_FILE_SIZE_ON_DISK_BYTES =
config.get('server:uploadFileMaxSizeBytes') || 8388608;
Expand Down
5 changes: 3 additions & 2 deletions backend/src/config/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,14 @@ config.defaults({
templatePath: process.env.TEMPLATE_PATH || './src/templates',
uploadFileMaxSizeBytes: parseFloat(process.env.UPLOAD_FILE_MAX_SIZE),
schedulerDeleteDraftCronTime: process.env.DELETE_DRAFT_REPORT_CRON_CRONTIME,
schedulerDeleteDraftTimeZone: process.env.DELETE_DRAFT_REPORT_CRON_TIMEZONE,
schedulerLockReportCronTime: process.env.LOCK_REPORT_CRON_CRONTIME,
schedulerTimeZone: process.env.REPORTS_SCHEDULER_CRON_TIMEZONE,
databaseUrl: datasourceUrl,
reportEditDurationInDays: parseInt(
process.env.REPORT_EDIT_DURATION_IN_DAYS || '30',
),
reportUnlockDurationInDays: parseInt(
process.env.REPORT_UNLOCK_DURATION_IN_DAYS || '1',
process.env.REPORT_UNLOCK_DURATION_IN_DAYS || '2',
),
rateLimit: {
enabled: process.env.IS_RATE_LIMIT_ENABLED || false, // Disable if rate limiting is not required
Expand Down
2 changes: 1 addition & 1 deletion backend/src/schedulers/delete-draft-service-scheduler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ try {
'delete_draft_reports',
);
const crontime = config.get('server:schedulerDeleteDraftCronTime');
const timezone = config.get('server:schedulerDeleteDraftTimeZone');
const timezone = config.get('server:schedulerTimeZone');

const job = new CronJob(
crontime, // cronTime
Expand Down
2 changes: 2 additions & 0 deletions backend/src/schedulers/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
import './delete-draft-service-scheduler';
import './lock-reports-scheduler';
135 changes: 135 additions & 0 deletions backend/src/schedulers/lock-reports-scheduler.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
import waitFor from 'wait-for-expect';
import { LocalDateTime, ZoneId, convert } from '@js-joda/core';
import createPrismaMock from 'prisma-mock';
import { Prisma, PrismaClient } from '@prisma/client';

let prismaClient: PrismaClient<
Prisma.PrismaClientOptions,
'query' | 'info' | 'warn' | 'error'
>;

const mock_prismaClient = () => prismaClient;

jest.mock('../v1/prisma/prisma-client', () => ({
__esModule: true,
...jest.requireActual('../v1/prisma/prisma-client'),
default: mock_prismaClient(),
}));

jest.mock('cron', () => ({
CronJob: class MockCron {
constructor(
public expression: string,
public cb: Function,
) {}
async start() {
return this.cb();
}
},
}));

jest.mock('../config', () => ({
config: {
get: (key: string) => {
const settings = {
'server:reportEditDurationInDays': 30,
'server:reportUnlockDurationInDays': 2
}

return settings[key];
}
}
}))

const mock_tryLock = jest.fn();
const mock_unlock = jest.fn();
jest.mock('advisory-lock', () => ({
...jest.requireActual('advisory-lock'),
default: () => {
return () => ({
tryLock: () => mock_tryLock(),
});
},
}));

describe('lock-report-scheduler', () => {
beforeEach(async () => {
prismaClient = createPrismaMock(
{
pay_transparency_report: [
// Cannot be locked
{
report_id: '1',
is_unlocked: true,
report_status: 'Published',
create_date: convert(LocalDateTime.now(ZoneId.UTC)).toDate(),
report_unlock_date: null,
},
// Can be locked, older than default editable period (30 days)
{
report_id: '2',
is_unlocked: true,
report_status: 'Published',
create_date: convert(
LocalDateTime.now(ZoneId.UTC).minusDays(40),
).toDate(),
report_unlock_date: null,
},
// Cannot be locked, before report_unlock_date
{
report_id: '3',
is_unlocked: true,
report_status: 'Published',
create_date: convert(
LocalDateTime.now(ZoneId.UTC).minusDays(40),
).toDate(),
report_unlock_date: convert(
LocalDateTime.now(ZoneId.UTC)).toDate(),
},
// Can be locked, past report_unlock_date
{
report_id: '4',
is_unlocked: true,
report_status: 'Published',
create_date: convert(
LocalDateTime.now(ZoneId.UTC).minusDays(40),
).toDate(),
report_unlock_date: convert(
LocalDateTime.now(ZoneId.UTC).minusDays(3),
).toDate(),
},
// Cannot be locked, not published
{
report_id: '5',
is_unlocked: true,
report_status: 'Draft',
create_date: convert(
LocalDateTime.now(ZoneId.UTC).minusDays(40),
).toDate(),
report_unlock_date: convert(
LocalDateTime.now(ZoneId.UTC).plusDays(3),
).toDate(),
},
],
},
Prisma.dmmf.datamodel,
);
});

it('should lock all lockable reports', async () => {
mock_tryLock.mockReturnValue(mock_unlock);
await require('./lock-reports-scheduler');
await waitFor(async () => {
expect(mock_tryLock).toHaveBeenCalledTimes(1);
expect(mock_unlock).toHaveBeenCalledTimes(1);
const reports = await prismaClient.pay_transparency_report.findMany({
select: { report_id: true, is_unlocked: true },
});
expect(reports.find(r => r.report_id === '1').is_unlocked).toBeTruthy();
expect(reports.find(r => r.report_id === '2').is_unlocked).toBeFalsy();
expect(reports.find(r => r.report_id === '3').is_unlocked).toBeTruthy();
expect(reports.find(r => r.report_id === '4').is_unlocked).toBeFalsy();
expect(reports.find(r => r.report_id === '5').is_unlocked).toBeTruthy();
});
});
});
80 changes: 80 additions & 0 deletions backend/src/schedulers/lock-reports-scheduler.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import { CronJob } from 'cron';
import { config } from '../config';
import { logger as log } from '../logger';
import advisoryLock from 'advisory-lock';
import prisma from '../v1/prisma/prisma-client';
import { LocalDateTime, ZoneId, convert } from '@js-joda/core';

try {
const mutex = advisoryLock(config.get('server:databaseUrl'))('lock_reports');
const crontime = config.get('server:schedulerLockReportCronTime');
const timezone = config.get('server:schedulerTimeZone');
const reportEditDurationInDays = config.get(
'server:reportEditDurationInDays',
);
const reportUnlockDurationInDays = config.get(
'server:reportUnlockDurationInDays',
);

const job = new CronJob(
crontime, // cronTime
async function () {
try {
const unlock = await mutex.tryLock();
if (unlock) {
log.info('Starting report locking Schedule Job.');
await prisma.$transaction(async (tx) => {
await tx.pay_transparency_report.updateMany({
data: { is_unlocked: false },
where: {
AND: [
{ is_unlocked: true },
{ report_status: 'Published' },
{
OR: [
{
report_unlock_date: null,
create_date: {
lt: convert(
LocalDateTime.now(ZoneId.UTC).minusDays(
reportEditDurationInDays,
),
).toDate(),
},
},
{
report_unlock_date: {
not: null,
lt: convert(
LocalDateTime.now(ZoneId.UTC).minusDays(
reportUnlockDurationInDays,
),
).toDate(),
},
},
],
},
],
},
});
});

log.info('Report locking Schedule Job completed.');
await unlock();
}
} catch (e) {
/* istanbul ignore next */
log.error('Error in lockReports.');
/* istanbul ignore next */
log.error(e);
}
}, // onTick
null, // onComplete
true, // start
timezone, // timeZone
);
job.start();
} catch (e) {
/* istanbul ignore next */
log.error(e);
}
3 changes: 2 additions & 1 deletion charts/fin-pay-transparency/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,5 @@ data:
REPORT_EDIT_DURATION_IN_DAYS: {{ .Values.global.config.report_edit_duration_in_days | quote }}
REPORT_UNLOCK_DURATION_IN_DAYS: {{ .Values.global.config.report_unlock_duration_in_days | quote }}
DELETE_DRAFT_REPORT_CRON_CRONTIME: {{ .Values.global.config.delete_draft_report_cron_crontime | quote }}
DELETE_DRAFT_REPORT_CRON_TIMEZONE: {{ .Values.global.config.delete_draft_report_cron_timezone | quote }}
LOCK_REPORT_CRON_CRONTIME: {{ .Values.global.config.lock_report_cron_crontime | quote }}
REPORTS_SCHEDULER_CRON_TIMEZONE: {{ .Values.global.config.reports_scheduler_cron_timezone | quote }}
5 changes: 3 additions & 2 deletions charts/fin-pay-transparency/values-dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ global:
config:
upload_file_max_size: "8388608"
report_edit_duration_in_days: "30"
report_unlock_duration_in_days: "1"
report_unlock_duration_in_days: "2"
template_path: "./dist/templates"
delete_draft_report_cron_crontime: "0 0 0 * * *" # 12:00 AM PST/PDT
delete_draft_report_cron_timezone: "America/Vancouver"
lock_report_cron_crontime: "0 15 0 * * *" # 12:15 AM PST/PDT
reports_scheduler_cron_timezone: "America/Vancouver"
crunchyEnabled: true
backend:
enabled: true
Expand Down
5 changes: 3 additions & 2 deletions charts/fin-pay-transparency/values-prod.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ global:
config:
upload_file_max_size: "8388608"
report_edit_duration_in_days: "30"
report_unlock_duration_in_days: "1"
report_unlock_duration_in_days: "2"
template_path: "./dist/templates"
delete_draft_report_cron_crontime: "0 0 0 * * *" # 12:00 AM PST/PDT
delete_draft_report_cron_timezone: "America/Vancouver"
lock_report_cron_crontime: "0 15 0 * * *" # 12:15 AM PST/PDT
reports_scheduler_cron_timezone: "America/Vancouver"
crunchyEnabled: true
backend:
enabled: true
Expand Down
5 changes: 3 additions & 2 deletions charts/fin-pay-transparency/values-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ global:
config:
upload_file_max_size: "8388608"
report_edit_duration_in_days: "30"
report_unlock_duration_in_days: "1"
report_unlock_duration_in_days: "2"
template_path: "./dist/templates"
delete_draft_report_cron_crontime: "0 0 0 * * *" # 12:00 AM PST/PDT
delete_draft_report_cron_timezone: "America/Vancouver"
lock_report_cron_crontime: "0 15 0 * * *" # 12:15 AM PST/PDT
reports_scheduler_cron_timezone: "America/Vancouver"
crunchyEnabled: true
backend:
enabled: true
Expand Down
5 changes: 3 additions & 2 deletions charts/fin-pay-transparency/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ global:
config:
upload_file_max_size: "8388608"
report_edit_duration_in_days: "30"
report_unlock_duration_in_days: "1"
report_unlock_duration_in_days: "2"
template_path: "./dist/templates"
delete_draft_report_cron_crontime: "0 0 0 * * *" # 12:00 AM PST/PDT
delete_draft_report_cron_timezone: "America/Vancouver"
lock_report_cron_crontime: "0 15 0 * * *" # 12:15 AM PST/PDT
reports_scheduler_cron_timezone: "America/Vancouver"
crunchyEnabled: true
backend:
enabled: true
Expand Down
49 changes: 0 additions & 49 deletions frontend/src/common/helpers/__tests__/report.spec.ts

This file was deleted.

1 change: 0 additions & 1 deletion frontend/src/common/helpers/index.ts

This file was deleted.

10 changes: 0 additions & 10 deletions frontend/src/common/helpers/report.ts

This file was deleted.

Loading

0 comments on commit e402c2a

Please sign in to comment.