From 793372e7860fa730535c81f0a8843527d3abf5b5 Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Wed, 2 Oct 2024 18:23:52 -0400 Subject: [PATCH 01/16] feat: send emails for milestone when due in 30 days --- .../templates/notifyMilestoneReportDue.ts | 78 ++++++++++ app/backend/lib/milestoneDueDate.ts | 135 ++++++++++++++++++ app/server.ts | 2 + 3 files changed, 215 insertions(+) create mode 100644 app/backend/lib/emails/templates/notifyMilestoneReportDue.ts create mode 100644 app/backend/lib/milestoneDueDate.ts diff --git a/app/backend/lib/emails/templates/notifyMilestoneReportDue.ts b/app/backend/lib/emails/templates/notifyMilestoneReportDue.ts new file mode 100644 index 000000000..790aac19b --- /dev/null +++ b/app/backend/lib/emails/templates/notifyMilestoneReportDue.ts @@ -0,0 +1,78 @@ +import { performQuery } from '../../graphql'; +import { Context } from '../../ches/sendEmailMerge'; +import { + EmailTemplate, + EmailTemplateProvider, + replaceEmailsInNonProd, +} from '../handleEmailNotification'; + +const getAnalystInfoByUserIds = ` + query getAnalystNameByUserIds($_rowIds: [Int!]!) { + allAnalysts(filter: {rowId: {in: $_rowIds}}) { + nodes { + email + givenName + } + } + } +`; +const getEmails = async (ids: number[], req: any) => { + const results = await performQuery( + getAnalystInfoByUserIds, + { _rowIds: ids }, + req + ); + return results?.data?.allAnalysts.nodes; +}; + +const notifyMilestoneReportDue: EmailTemplateProvider = async ( + applicationId: string, + url: string, + initiator: any, + params: any, + req +): Promise => { + const { milestoneReportData } = params; + const recipients = [70, 71]; + + const emails = await getEmails(recipients, req); + + const contexts = emails.map((email) => { + const { givenName, email: recipientEmail } = email; + const emailTo = replaceEmailsInNonProd([recipientEmail]); + const emailCC = replaceEmailsInNonProd([]); + return { + to: emailTo, + cc: emailCC, + context: { + recipientName: givenName, + milestones: milestoneReportData, + }, + delayTS: 0, + tag: 'milestone-due', + } as Context; + }); + + const subject = `Reminder: Milestone Report${milestoneReportData.length > 1 ? 's' : ''} ${milestoneReportData.length > 1 ? 'are' : 'is'} coming Due`; + + return { + emailTo: [], + emailCC: [], + tag: 'milestone-due', + subject, + body: ` +

Hi {{ recipientName | trim }}

+

This is a notification to let you know that one or more Milestone Reports are coming due in 30 days:

+

+

To unsubscribe from these email notifications, email meherzad.romer@gov.bc.ca

+ `, + contexts, + params: { milestoneReportData }, + }; +}; + +export default notifyMilestoneReportDue; diff --git a/app/backend/lib/milestoneDueDate.ts b/app/backend/lib/milestoneDueDate.ts new file mode 100644 index 000000000..5b7010be9 --- /dev/null +++ b/app/backend/lib/milestoneDueDate.ts @@ -0,0 +1,135 @@ +import { Router } from 'express'; +import getAuthRole from '../../utils/getAuthRole'; +import { performQuery } from './graphql'; +import handleEmailNotification from './emails/handleEmailNotification'; +import notifyMilestoneReportDue from './emails/templates/notifyMilestoneReportDue'; +import validateKeycloakToken from './keycloakValidate'; + +const milestonesRouter = Router(); + +const processMilestones = async (req, res) => { + // GraphQL query to get all milestones with archivedAt: null + const sowMilestoneQuery = ` + query MilestoneDatesQuery { + allApplicationSowData( + orderBy: AMENDMENT_NUMBER_DESC + filter: {archivedAt: {isNull: true}} + ) { + nodes { + amendmentNumber + applicationId + applicationByApplicationId { + ccbcNumber + organizationName + projectName + } + sowTab2SBySowId( + first: 1 + orderBy: ID_DESC + filter: {archivedAt: {isNull: true}} + ) { + nodes { + jsonData + } + } + } + } + } + `; + let result; + const applicationRowIdsVisited = new Set(); + + try { + result = await performQuery(sowMilestoneQuery, {}, req); + } catch (error) { + return res.status(500).json({ error: result.error }).end(); + } + + const today = new Date(); + // Function to check if a given due date string is within 30 to 31 days from today. + const isWithin30To31Days = (dueDateStr) => { + const dueDate = new Date(dueDateStr); + const timeDiff = dueDate.getTime() - today.getTime(); + const daysDiff = timeDiff / (1000 * 3600 * 24); + return daysDiff >= 30 && daysDiff <= 31; + }; + + // Traverse the result, if there is a milestone due date within 30 to 31 days from today, + // add the application row ID, CCBC number, and whether it is a milestone 1 or 2 to a list. + const milestoneReportData = result.data.allApplicationSowData.nodes.reduce( + (acc, node) => { + const { applicationId, applicationByApplicationId, sowTab2SBySowId } = + node; + if (applicationRowIdsVisited.has(applicationId)) { + return acc; + } + const { ccbcNumber, organizationName, projectName } = + applicationByApplicationId; + const milestoneData: Array = sowTab2SBySowId.nodes[0] + ?.jsonData as Array; + const milestoneDue = milestoneData.find( + (milestone) => + isWithin30To31Days(milestone.milestone1) || + isWithin30To31Days(milestone.milestone2) + ); + if (milestoneDue) { + const applicationRowId = applicationId; + if (!applicationRowIdsVisited.has(applicationRowId)) { + acc.push({ + applicationRowId, + ccbcNumber, + organizationName, + projectName, + milestoneNumber: isWithin30To31Days(milestoneDue.milestone1) + ? '1' + : '2', + milestoneDate: isWithin30To31Days(milestoneDue.milestone1) + ? new Date(milestoneDue.milestone1).toLocaleDateString() + : new Date(milestoneDue.milestone2).toLocaleDateString(), + }); + applicationRowIdsVisited.add(applicationRowId); + } + } + return acc; + }, + [] + ); + + if (milestoneReportData.length > 0) { + // Send an email to the analyst with the list of applications that have milestones due within 30 to 31 days. + return handleEmailNotification( + req, + res, + notifyMilestoneReportDue, + { milestoneReportData }, + true + ); + } + + return res + .status(200) + .json({ message: 'No milestones due in 30 days' }) + .end(); +}; + +milestonesRouter.get('/api/analyst/milestones/upcoming', (req, res) => { + const authRole = getAuthRole(req); + const isRoleAuthorized = + authRole?.pgRole === 'cbc_admin' || authRole?.pgRole === 'super_admin'; + + if (!isRoleAuthorized) { + return res.status(404).end(); + } + return processMilestones(req, res); +}); + +milestonesRouter.get( + '/api/analyst/cron-milestones', + validateKeycloakToken, + (req, res) => { + req.claims.identity_provider = 'serviceaccount'; + processMilestones(req, res); + } +); + +export default milestonesRouter; diff --git a/app/server.ts b/app/server.ts index 36be0ba3b..f35750aaa 100644 --- a/app/server.ts +++ b/app/server.ts @@ -36,6 +36,7 @@ import sharepoint from './backend/lib/sharepoint'; import templateUpload from './backend/lib/template-upload'; import s3upload from './backend/lib/s3upload'; import templateNine from './backend/lib/excel_import/template_nine'; +import milestoneDue from './backend/lib/milestoneDueDate'; // Function to exclude middleware from certain routes // The paths argument takes an array of strings containing routes to exclude from the middleware @@ -151,6 +152,7 @@ app.prepare().then(async () => { server.use('/', reporting); server.use('/', templateNine); server.use('/', validation); + server.use('/', milestoneDue); server.all('*', async (req, res) => handle(req, res)); From 5e1b65fcd21b469c3a231f01c3cbc398cec5391f Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Wed, 2 Oct 2024 22:49:13 -0400 Subject: [PATCH 02/16] feat: milestone cron --- .../trigger-milestone-notification.yaml | 50 +++++++++++++++++++ helm/app/values.yaml | 4 ++ 2 files changed, 54 insertions(+) create mode 100644 helm/app/templates/cronJobs/trigger-milestone-notification.yaml diff --git a/helm/app/templates/cronJobs/trigger-milestone-notification.yaml b/helm/app/templates/cronJobs/trigger-milestone-notification.yaml new file mode 100644 index 000000000..b92c656ca --- /dev/null +++ b/helm/app/templates/cronJobs/trigger-milestone-notification.yaml @@ -0,0 +1,50 @@ +{{- if .Values.deploy.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ template "ccbc.fullname" . }}-cron-milestone + labels: {{ include "ccbc.labels" . | nindent 4 }} +spec: + suspend: true + schedule: {{ .Values.cronmilestone.schedule }} + timeZone: "America/Vancouver" + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 600 + template: + metadata: + labels: {{ include "ccbc.labels" . | nindent 14 }} + spec: + restartPolicy: Never + containers: + - env: + - name: KEYCLOAK_HOST + value: {{ .Values.cronsp.keycloakHost | quote }} + - name: CCBC_SERVER_PATH + value: {{ .Values.cronmilestone.path | quote }} + - name: CCBC_SERVER_HOST + value: {{ template "ccbc.fullname" . }} + - name: CCBC_SERVER_PORT + value: {{ .Values.cronsp.port | quote }} + - name: SA_CLIENT_ID + valueFrom: + secretKeyRef: + key: client-id + name: ccbc-cron-sp-sso + - name: SA_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: client-secret + name: ccbc-cron-sp-sso + name: {{ template "ccbc.fullname" . }}-cron-sp + image: {{ .Values.image.cronsp.repository }}:{{ .Values.image.cronsp.tag }} + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 100m + memory: 64Mi +{{- end }} diff --git a/helm/app/values.yaml b/helm/app/values.yaml index d88b5348d..1735c0650 100644 --- a/helm/app/values.yaml +++ b/helm/app/values.yaml @@ -46,6 +46,10 @@ cronshp: rdFile: '' # The value must be passed in via the deploy script coveragesFile: '' # The value must be passed in via the deploy script +cronmilestone: + path: '/api/analyst/cron-milestones' + schedule: '0 7 * * 1-5' # Trigger 7AM on weekdays + app: port: '3000' probesPort: '9000' From f63bdc51a4108f41cba64a0301febef72c7fc784 Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Thu, 3 Oct 2024 11:06:49 -0400 Subject: [PATCH 03/16] chore: update for correct permissions --- app/backend/lib/milestoneDueDate.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/backend/lib/milestoneDueDate.ts b/app/backend/lib/milestoneDueDate.ts index 5b7010be9..b974cd8a5 100644 --- a/app/backend/lib/milestoneDueDate.ts +++ b/app/backend/lib/milestoneDueDate.ts @@ -112,10 +112,10 @@ const processMilestones = async (req, res) => { .end(); }; -milestonesRouter.get('/api/analyst/milestones/upcoming', (req, res) => { +milestonesRouter.get('/api/analyst/milestone/upcoming', (req, res) => { const authRole = getAuthRole(req); const isRoleAuthorized = - authRole?.pgRole === 'cbc_admin' || authRole?.pgRole === 'super_admin'; + authRole?.pgRole === 'ccbc_admin' || authRole?.pgRole === 'super_admin'; if (!isRoleAuthorized) { return res.status(404).end(); From b6997baf3577d253cf477cb5e03e067eac94f866 Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Thu, 3 Oct 2024 11:07:20 -0400 Subject: [PATCH 04/16] chore: update the cron with better container name --- helm/app/templates/cronJobs/trigger-milestone-notification.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/app/templates/cronJobs/trigger-milestone-notification.yaml b/helm/app/templates/cronJobs/trigger-milestone-notification.yaml index b92c656ca..5cc359036 100644 --- a/helm/app/templates/cronJobs/trigger-milestone-notification.yaml +++ b/helm/app/templates/cronJobs/trigger-milestone-notification.yaml @@ -37,7 +37,7 @@ spec: secretKeyRef: key: client-secret name: ccbc-cron-sp-sso - name: {{ template "ccbc.fullname" . }}-cron-sp + name: {{ template "ccbc.fullname" . }}-cron-milestone image: {{ .Values.image.cronsp.repository }}:{{ .Values.image.cronsp.tag }} imagePullPolicy: IfNotPresent resources: From a0de0736e7af9f77b4407e4f66ac3dca9d096b6f Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Thu, 3 Oct 2024 11:07:57 -0400 Subject: [PATCH 05/16] test: add tests api route and notify --- .../notifyMilestoneReportDue.test.ts | 98 +++++++++++ .../backend/lib/milestoneDueDate.test.ts | 162 ++++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 app/tests/backend/lib/emails/templates/notifyMilestoneReportDue.test.ts create mode 100644 app/tests/backend/lib/milestoneDueDate.test.ts diff --git a/app/tests/backend/lib/emails/templates/notifyMilestoneReportDue.test.ts b/app/tests/backend/lib/emails/templates/notifyMilestoneReportDue.test.ts new file mode 100644 index 000000000..71f87c02a --- /dev/null +++ b/app/tests/backend/lib/emails/templates/notifyMilestoneReportDue.test.ts @@ -0,0 +1,98 @@ +import { mocked } from 'jest-mock'; +import notifyMilestoneReportDue from '../../../../../backend/lib/emails/templates/notifyMilestoneReportDue'; +import { performQuery } from '../../../../../backend/lib/graphql'; + +jest.mock('../../../../../backend/lib/graphql', () => ({ + performQuery: jest.fn(), +})); + +jest.mock('../../../../../backend/lib/emails/handleEmailNotification', () => ({ + replaceEmailsInNonProd: jest.fn((emails) => emails), +})); + +describe('notifyMilestoneReportDue template', () => { + it('should return an email template with correct properties', async () => { + const req = { + body: { applicationId: '1', host: 'http://mock_host.ca' }, + }; + + // Mock the performQuery function to return analyst data + mocked(performQuery).mockResolvedValue({ + data: { + allAnalysts: { + nodes: [ + { + email: 'analyst1@example.com', + givenName: 'Analyst One', + }, + { + email: 'analyst2@example.com', + givenName: 'Analyst Two', + }, + ], + }, + }, + }); + + const applicationId = '1'; + const url = 'http://mock_host.ca'; + const milestoneReportData = [ + { + milestoneNumber: 1, + organizationName: 'Organization A', + ccbcNumber: 'CCBC-000001', + milestoneDate: '2023-11-01', + }, + { + milestoneNumber: 2, + organizationName: 'Organization B', + ccbcNumber: 'CCBC-000002', + milestoneDate: '2023-12-01', + }, + ]; + + const params = { milestoneReportData }; + const emailTemplate = await notifyMilestoneReportDue( + applicationId, + url, + {}, + params, + req + ); + + // Check that the subject is correct + expect(emailTemplate.subject).toEqual( + 'Reminder: Milestone Reports are coming Due' + ); + + // Ensure contexts are created for each analyst + expect(emailTemplate.contexts.length).toBe(2); + + // Validate the first context + expect(emailTemplate.contexts[0].to).toEqual(['analyst1@example.com']); + expect(emailTemplate.contexts[0].context.recipientName).toEqual( + 'Analyst One' + ); + expect(emailTemplate.contexts[0].context.milestones).toEqual( + milestoneReportData + ); + + // Validate the second context + expect(emailTemplate.contexts[1].to).toEqual(['analyst2@example.com']); + expect(emailTemplate.contexts[1].context.recipientName).toEqual( + 'Analyst Two' + ); + expect(emailTemplate.contexts[1].context.milestones).toEqual( + milestoneReportData + ); + + // Check other properties + expect(emailTemplate.tag).toEqual('milestone-due'); + expect(emailTemplate.params).toEqual({ milestoneReportData }); + + // Optionally, you can check the body content if needed + expect(emailTemplate.body).toContain( + '

Hi {{ recipientName | trim }}

' + ); + }); +}); diff --git a/app/tests/backend/lib/milestoneDueDate.test.ts b/app/tests/backend/lib/milestoneDueDate.test.ts new file mode 100644 index 000000000..0c8adf524 --- /dev/null +++ b/app/tests/backend/lib/milestoneDueDate.test.ts @@ -0,0 +1,162 @@ +/** + * @jest-environment node + */ +import { mocked } from 'jest-mock'; +import request from 'supertest'; +import express from 'express'; +import session from 'express-session'; +import crypto from 'crypto'; +import milestoneDueDate from '../../../backend/lib/milestoneDueDate'; +import { performQuery } from '../../../backend/lib/graphql'; +import handleEmailNotification from '../../../backend/lib/emails/handleEmailNotification'; +import getAuthRole from '../../../utils/getAuthRole'; + +jest.mock('../../../backend/lib/graphql'); +jest.mock('../../../utils/getAuthRole'); +jest.mock('../../../backend/lib/emails/handleEmailNotification'); + +jest.setTimeout(100000); + +describe('The Milestone excel import api route', () => { + let app; + + beforeEach(async () => { + app = express(); + app.use(session({ secret: crypto.randomUUID(), cookie: { secure: true } })); + app.use('/', milestoneDueDate); + }); + + it('should receive the correct response for unauthorized user', async () => { + mocked(getAuthRole).mockImplementation(() => { + return { + pgRole: 'ccbc_guest', + landingRoute: '/', + }; + }); + + const response = await request(app).get('/api/analyst/milestone/upcoming'); + expect(response.status).toBe(404); + }); + + it('should process milestones for authorized user', async () => { + mocked(getAuthRole).mockImplementation(() => { + return { + pgRole: 'ccbc_admin', + landingRoute: '/', + }; + }); + + mocked(performQuery).mockImplementation(async () => { + return { + data: { + allApplicationSowData: { + nodes: [ + { + applicationId: 1, + applicationByApplicationId: { + ccbcNumber: 'CCBC-010001', + organizationName: 'Organization Name', + projectName: 'Project Name', + }, + sowTab2SBySowId: { + nodes: [ + { + jsonData: [ + { + milestone1: '2022-01-01', + milestone2: '2022-01-02', + }, + ], + }, + { + jsonData: [ + { + milestone1: '2022-01-02', + milestone2: '2022-01-01', + }, + ], + }, + ], + }, + }, + ], + }, + }, + }; + }); + + const response = await request(app) + .get('/api/analyst/milestone/upcoming') + .expect(200); + + expect(response.status).toBe(200); + expect(response.body).toEqual({ message: 'No milestones due in 30 days' }); + }); + + it('should process milestones for authorized user and send email', async () => { + mocked(getAuthRole).mockImplementation(() => { + return { + pgRole: 'ccbc_admin', + landingRoute: '/', + }; + }); + + const mockDate = new Date('2021-12-03T00:00:00Z'); + const OriginalDate = Date; + + global.Date = jest.fn((dateString) => { + if (dateString === undefined) { + return mockDate; + } + return new OriginalDate(dateString); + }) as unknown as DateConstructor; + + mocked(handleEmailNotification).mockImplementation(async (req, res) => { + return res.status(200).json({ emails: 'sent' }).end(); + }); + + mocked(performQuery).mockImplementation(async () => { + return { + data: { + allApplicationSowData: { + nodes: [ + { + applicationId: 1, + applicationByApplicationId: { + ccbcNumber: 'CCBC-010001', + organizationName: 'Organization Name', + projectName: 'Project Name', + }, + sowTab2SBySowId: { + nodes: [ + { + jsonData: [ + { + milestone1: '2022-01-01', + milestone2: '2022-01-02', + }, + ], + }, + ], + }, + }, + ], + }, + }, + }; + }); + + const response = await request(app) + .get('/api/analyst/milestone/upcoming') + .expect(200); + + expect(response.status).toBe(200); + expect(response.body).toEqual({ emails: 'sent' }); + }); + + afterEach(async () => { + // eslint-disable-next-line no-promise-executor-return + await new Promise((resolve) => setTimeout(() => resolve(), 500)); // avoid jest open handle error + }); + jest.resetAllMocks(); +}); From 4168e232bdb14879bb2b103569718cb38a54d4db Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Thu, 3 Oct 2024 11:27:20 -0400 Subject: [PATCH 06/16] chore: add limiter --- app/backend/lib/milestoneDueDate.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/backend/lib/milestoneDueDate.ts b/app/backend/lib/milestoneDueDate.ts index b974cd8a5..cd6ff3bf9 100644 --- a/app/backend/lib/milestoneDueDate.ts +++ b/app/backend/lib/milestoneDueDate.ts @@ -4,6 +4,7 @@ import { performQuery } from './graphql'; import handleEmailNotification from './emails/handleEmailNotification'; import notifyMilestoneReportDue from './emails/templates/notifyMilestoneReportDue'; import validateKeycloakToken from './keycloakValidate'; +import limiter from './excel_import/excel-limiter'; const milestonesRouter = Router(); @@ -112,7 +113,7 @@ const processMilestones = async (req, res) => { .end(); }; -milestonesRouter.get('/api/analyst/milestone/upcoming', (req, res) => { +milestonesRouter.get('/api/analyst/milestone/upcoming', limiter, (req, res) => { const authRole = getAuthRole(req); const isRoleAuthorized = authRole?.pgRole === 'ccbc_admin' || authRole?.pgRole === 'super_admin'; From facd129bd5a674343ec23d01351e5bc74d06ea96 Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Fri, 4 Oct 2024 15:22:44 -0400 Subject: [PATCH 07/16] chore: fix failing test and add limiter --- app/backend/lib/milestoneDueDate.ts | 8 ++++++-- app/tests/backend/lib/milestoneDueDate.test.ts | 5 ++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/backend/lib/milestoneDueDate.ts b/app/backend/lib/milestoneDueDate.ts index cd6ff3bf9..bb5f8218d 100644 --- a/app/backend/lib/milestoneDueDate.ts +++ b/app/backend/lib/milestoneDueDate.ts @@ -1,10 +1,15 @@ import { Router } from 'express'; +import RateLimit from 'express-rate-limit'; import getAuthRole from '../../utils/getAuthRole'; import { performQuery } from './graphql'; import handleEmailNotification from './emails/handleEmailNotification'; import notifyMilestoneReportDue from './emails/templates/notifyMilestoneReportDue'; import validateKeycloakToken from './keycloakValidate'; -import limiter from './excel_import/excel-limiter'; + +const limiter = RateLimit({ + windowMs: 1 * 60 * 1000, + max: 30, +}); const milestonesRouter = Router(); @@ -17,7 +22,6 @@ const processMilestones = async (req, res) => { filter: {archivedAt: {isNull: true}} ) { nodes { - amendmentNumber applicationId applicationByApplicationId { ccbcNumber diff --git a/app/tests/backend/lib/milestoneDueDate.test.ts b/app/tests/backend/lib/milestoneDueDate.test.ts index 0c8adf524..43c50936c 100644 --- a/app/tests/backend/lib/milestoneDueDate.test.ts +++ b/app/tests/backend/lib/milestoneDueDate.test.ts @@ -110,6 +110,7 @@ describe('The Milestone excel import api route', () => { } return new OriginalDate(dateString); }) as unknown as DateConstructor; + global.Date.now = OriginalDate.now; mocked(handleEmailNotification).mockImplementation(async (req, res) => { return res.status(200).json({ emails: 'sent' }).end(); @@ -146,9 +147,7 @@ describe('The Milestone excel import api route', () => { }; }); - const response = await request(app) - .get('/api/analyst/milestone/upcoming') - .expect(200); + const response = await request(app).get('/api/analyst/milestone/upcoming'); expect(response.status).toBe(200); expect(response.body).toEqual({ emails: 'sent' }); From 41d84c5ec98dea4bc066b6c6bd75fdbb286b8b78 Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Fri, 4 Oct 2024 15:23:43 -0400 Subject: [PATCH 08/16] feat: service account permissions and get scope in request --- .../lib/emails/handleEmailNotification.ts | 1 + cron/sp/app.mjs | 8 +++-- .../grant_read_access_to_service_account.sql | 30 +++++++++++++++++++ .../grant_read_access_to_service_account.sql | 7 +++++ db/sqitch.plan | 3 +- 5 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 db/deploy/grant_read_access_to_service_account.sql create mode 100644 db/revert/grant_read_access_to_service_account.sql diff --git a/app/backend/lib/emails/handleEmailNotification.ts b/app/backend/lib/emails/handleEmailNotification.ts index e9dd16cbe..f004d5c87 100644 --- a/app/backend/lib/emails/handleEmailNotification.ts +++ b/app/backend/lib/emails/handleEmailNotification.ts @@ -46,6 +46,7 @@ const isAuthorized = (authRole: any) => { 'ccbc_auth_user', 'super_admin', 'cbc_admin', + 'ccbc_service_account', ]; return authRole && authorizedRoles.includes(authRole.pgRole); }; diff --git a/cron/sp/app.mjs b/cron/sp/app.mjs index 19a99c750..14dab82b2 100644 --- a/cron/sp/app.mjs +++ b/cron/sp/app.mjs @@ -14,6 +14,7 @@ const clientSecret = process.env.SA_CLIENT_SECRET || ''; function fetchAccessToken() { const data = stringify({ grant_type: 'client_credentials', + scope: 'openid', }); const authHeader = `Basic ${Buffer.from( @@ -26,21 +27,22 @@ function fetchAccessToken() { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded', + 'Content-Length': Buffer.byteLength(data), Authorization: authHeader, }, }; return new Promise((resolve, reject) => { const req = https.request(options, (res) => { - let data = ''; + let responseData = ''; res.on('data', (chunk) => { - data += chunk; + responseData += chunk; }); res.on('end', () => { try { - const tokenData = JSON.parse(data); + const tokenData = JSON.parse(responseData); const accessToken = tokenData.access_token; resolve(accessToken); } catch (error) { diff --git a/db/deploy/grant_read_access_to_service_account.sql b/db/deploy/grant_read_access_to_service_account.sql new file mode 100644 index 000000000..b858d72a7 --- /dev/null +++ b/db/deploy/grant_read_access_to_service_account.sql @@ -0,0 +1,30 @@ +-- Deploy ccbc:grant_read_access_to_service_account to pg + +BEGIN; + +grant execute on function ccbc_public.application_organization_name to ccbc_service_account; +grant execute on function ccbc_public.application_project_name to ccbc_service_account; +grant execute on function ccbc_public.application_form_data to ccbc_service_account; + +do +$grant_service$ +begin + +-- table grants +perform ccbc_private.grant_permissions('select', 'application', 'ccbc_service_account'); +perform ccbc_private.grant_permissions('select', 'application_form_data', 'ccbc_service_account'); +perform ccbc_private.grant_permissions('select', 'form_data', 'ccbc_service_account'); +perform ccbc_private.grant_permissions('select', 'form', 'ccbc_service_account'); +perform ccbc_private.grant_permissions('select', 'analyst', 'ccbc_service_account'); +perform ccbc_private.grant_permissions('select', 'application_sow_data', 'ccbc_service_account'); +perform ccbc_private.grant_permissions('select', 'sow_tab_2', 'ccbc_service_account'); + +-- RLS +perform ccbc_private.upsert_policy('ccbc_service_account_select_application', 'application', 'select', 'ccbc_service_account', 'true'); +perform ccbc_private.upsert_policy('ccbc_service_account_select_application_form_data', 'application_form_data', 'select', 'ccbc_service_account', 'true'); +perform ccbc_private.upsert_policy('ccbc_service_account_select_form_data', 'application_form_data', 'select', 'ccbc_service_account', 'true'); + +end +$grant_service$; + +COMMIT; diff --git a/db/revert/grant_read_access_to_service_account.sql b/db/revert/grant_read_access_to_service_account.sql new file mode 100644 index 000000000..130b5da79 --- /dev/null +++ b/db/revert/grant_read_access_to_service_account.sql @@ -0,0 +1,7 @@ +-- Revert ccbc:grant_read_access_to_service_account from pg + +BEGIN; + +-- XXX Add DDLs here. + +COMMIT; diff --git a/db/sqitch.plan b/db/sqitch.plan index 065cbc67b..62364840f 100644 --- a/db/sqitch.plan +++ b/db/sqitch.plan @@ -718,6 +718,7 @@ tables/application_form_template_9_data 2024-09-25T22:50:17Z Rafael Solorzano <6 @1.203.0 2024-10-25T16:54:48Z CCBC Service Account # release v1.203.0 @1.203.1 2024-10-28T19:01:53Z CCBC Service Account # release v1.203.1 @1.203.2 2024-10-29T17:37:47Z CCBC Service Account # release v1.203.2 -mutations/archive_application_sow [mutations/archive_application_sow@1.201.0] 2024-10-24T19:57:11Z Anthony Bushara # Add application ID to prevent archiving other applications @1.203.3 2024-10-29T21:17:39Z CCBC Service Account # release v1.203.3 @1.203.4 2024-11-01T15:43:03Z CCBC Service Account # release v1.203.4 +mutations/archive_application_sow [mutations/archive_application_sow@1.201.0] 2024-10-24T19:57:11Z Anthony Bushara # Add application ID to prevent archiving other applications +grant_read_access_to_service_account 2024-10-04T18:45:20Z Anthony Bushara # Add read access to service account From e080e7b80ffd60bef5bfaa89e13e9c11d3312182 Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Fri, 4 Oct 2024 15:26:02 -0400 Subject: [PATCH 09/16] chore: schedule is now every day not just weekdays --- helm/app/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/app/values.yaml b/helm/app/values.yaml index 1735c0650..a02808cc8 100644 --- a/helm/app/values.yaml +++ b/helm/app/values.yaml @@ -48,7 +48,7 @@ cronshp: cronmilestone: path: '/api/analyst/cron-milestones' - schedule: '0 7 * * 1-5' # Trigger 7AM on weekdays + schedule: '0 7 * * *' # Trigger 7AM every day app: port: '3000' From eb2ce651d45f33c622d1c8212f3e6488fea01605 Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Fri, 4 Oct 2024 15:44:34 -0400 Subject: [PATCH 10/16] chore: add limiter to serviceaccount route --- app/backend/lib/milestoneDueDate.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/app/backend/lib/milestoneDueDate.ts b/app/backend/lib/milestoneDueDate.ts index bb5f8218d..46406b31c 100644 --- a/app/backend/lib/milestoneDueDate.ts +++ b/app/backend/lib/milestoneDueDate.ts @@ -130,6 +130,7 @@ milestonesRouter.get('/api/analyst/milestone/upcoming', limiter, (req, res) => { milestonesRouter.get( '/api/analyst/cron-milestones', + limiter, validateKeycloakToken, (req, res) => { req.claims.identity_provider = 'serviceaccount'; From 7c7654a496e89428a147c8a987f5ffe22ef78b72 Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Thu, 17 Oct 2024 18:30:04 -0400 Subject: [PATCH 11/16] chore: add console log --- app/backend/lib/emails/handleEmailNotification.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/app/backend/lib/emails/handleEmailNotification.ts b/app/backend/lib/emails/handleEmailNotification.ts index f004d5c87..726e1586e 100644 --- a/app/backend/lib/emails/handleEmailNotification.ts +++ b/app/backend/lib/emails/handleEmailNotification.ts @@ -90,6 +90,7 @@ const handleEmailBatch = async ( try { const token = await getAccessToken(); const emailResult = await sendEmailMerge(token, body, subject, contexts); + console.log(emailResult, contexts, details); if (emailResult) { const emailRecordResults = emailResult.messages.map( (message: any, i: number) => { From c01acae959210841c7a6b526e0906a6ada43c0a7 Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Wed, 23 Oct 2024 10:46:45 -0400 Subject: [PATCH 12/16] chore: update to handle multiple milestones due on the same day --- app/backend/lib/milestoneDueDate.ts | 56 +++++++++++++++---- .../backend/lib/milestoneDueDate.test.ts | 4 +- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/app/backend/lib/milestoneDueDate.ts b/app/backend/lib/milestoneDueDate.ts index 46406b31c..892c7ad57 100644 --- a/app/backend/lib/milestoneDueDate.ts +++ b/app/backend/lib/milestoneDueDate.ts @@ -72,12 +72,48 @@ const processMilestones = async (req, res) => { applicationByApplicationId; const milestoneData: Array = sowTab2SBySowId.nodes[0] ?.jsonData as Array; - const milestoneDue = milestoneData.find( - (milestone) => - isWithin30To31Days(milestone.milestone1) || - isWithin30To31Days(milestone.milestone2) + const milestoneOneDue = milestoneData.find((milestone) => + isWithin30To31Days(milestone.milestone1) ); - if (milestoneDue) { + const milestoneTwoDue = milestoneData.find((milestone) => + isWithin30To31Days(milestone.milestone2) + ); + const milestoneThreeDue = milestoneData.find((milestone) => + isWithin30To31Days(milestone.milestone3) + ); + if (milestoneOneDue) { + const applicationRowId = applicationId; + if (!applicationRowIdsVisited.has(applicationRowId)) { + acc.push({ + applicationRowId, + ccbcNumber, + organizationName, + projectName, + milestoneNumber: '1', + milestoneDate: new Date( + milestoneOneDue.milestone1 + ).toLocaleDateString(), + }); + applicationRowIdsVisited.add(applicationRowId); + } + } + if (milestoneTwoDue) { + const applicationRowId = applicationId; + if (!applicationRowIdsVisited.has(applicationRowId)) { + acc.push({ + applicationRowId, + ccbcNumber, + organizationName, + projectName, + milestoneNumber: '2', + milestoneDate: new Date( + milestoneTwoDue.milestone2 + ).toLocaleDateString(), + }); + applicationRowIdsVisited.add(applicationRowId); + } + } + if (milestoneThreeDue) { const applicationRowId = applicationId; if (!applicationRowIdsVisited.has(applicationRowId)) { acc.push({ @@ -85,12 +121,10 @@ const processMilestones = async (req, res) => { ccbcNumber, organizationName, projectName, - milestoneNumber: isWithin30To31Days(milestoneDue.milestone1) - ? '1' - : '2', - milestoneDate: isWithin30To31Days(milestoneDue.milestone1) - ? new Date(milestoneDue.milestone1).toLocaleDateString() - : new Date(milestoneDue.milestone2).toLocaleDateString(), + milestoneNumber: '3', + milestoneDate: new Date( + milestoneThreeDue.milestone3 + ).toLocaleDateString(), }); applicationRowIdsVisited.add(applicationRowId); } diff --git a/app/tests/backend/lib/milestoneDueDate.test.ts b/app/tests/backend/lib/milestoneDueDate.test.ts index 43c50936c..714c96d9d 100644 --- a/app/tests/backend/lib/milestoneDueDate.test.ts +++ b/app/tests/backend/lib/milestoneDueDate.test.ts @@ -73,6 +73,7 @@ describe('The Milestone excel import api route', () => { { milestone1: '2022-01-02', milestone2: '2022-01-01', + milestone3: '2022-01-03', }, ], }, @@ -133,8 +134,9 @@ describe('The Milestone excel import api route', () => { { jsonData: [ { - milestone1: '2022-01-01', + milestone1: '2022-01-02', milestone2: '2022-01-02', + milestone3: '2022-01-02', }, ], }, From 524ed0cee66a9a9ab9d4975bb96042f49a0cb8e0 Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Fri, 25 Oct 2024 14:41:24 -0400 Subject: [PATCH 13/16] chore: fix the applications visited logic --- .../lib/emails/handleEmailNotification.ts | 1 - app/backend/lib/milestoneDueDate.ts | 24 +++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/app/backend/lib/emails/handleEmailNotification.ts b/app/backend/lib/emails/handleEmailNotification.ts index 726e1586e..f004d5c87 100644 --- a/app/backend/lib/emails/handleEmailNotification.ts +++ b/app/backend/lib/emails/handleEmailNotification.ts @@ -90,7 +90,6 @@ const handleEmailBatch = async ( try { const token = await getAccessToken(); const emailResult = await sendEmailMerge(token, body, subject, contexts); - console.log(emailResult, contexts, details); if (emailResult) { const emailRecordResults = emailResult.messages.map( (message: any, i: number) => { diff --git a/app/backend/lib/milestoneDueDate.ts b/app/backend/lib/milestoneDueDate.ts index 892c7ad57..455438c48 100644 --- a/app/backend/lib/milestoneDueDate.ts +++ b/app/backend/lib/milestoneDueDate.ts @@ -72,15 +72,15 @@ const processMilestones = async (req, res) => { applicationByApplicationId; const milestoneData: Array = sowTab2SBySowId.nodes[0] ?.jsonData as Array; - const milestoneOneDue = milestoneData.find((milestone) => - isWithin30To31Days(milestone.milestone1) - ); - const milestoneTwoDue = milestoneData.find((milestone) => - isWithin30To31Days(milestone.milestone2) - ); - const milestoneThreeDue = milestoneData.find((milestone) => - isWithin30To31Days(milestone.milestone3) - ); + const milestoneOneDue = milestoneData.find((milestone) => { + return isWithin30To31Days(milestone.milestone1); + }); + const milestoneTwoDue = milestoneData.find((milestone) => { + return isWithin30To31Days(milestone.milestone2); + }); + const milestoneThreeDue = milestoneData.find((milestone) => { + return isWithin30To31Days(milestone.milestone3); + }); if (milestoneOneDue) { const applicationRowId = applicationId; if (!applicationRowIdsVisited.has(applicationRowId)) { @@ -94,7 +94,6 @@ const processMilestones = async (req, res) => { milestoneOneDue.milestone1 ).toLocaleDateString(), }); - applicationRowIdsVisited.add(applicationRowId); } } if (milestoneTwoDue) { @@ -110,7 +109,6 @@ const processMilestones = async (req, res) => { milestoneTwoDue.milestone2 ).toLocaleDateString(), }); - applicationRowIdsVisited.add(applicationRowId); } } if (milestoneThreeDue) { @@ -126,9 +124,11 @@ const processMilestones = async (req, res) => { milestoneThreeDue.milestone3 ).toLocaleDateString(), }); - applicationRowIdsVisited.add(applicationRowId); } } + if (milestoneThreeDue || milestoneOneDue || milestoneTwoDue) { + applicationRowIdsVisited.add(applicationId); + } return acc; }, [] From ec3ea86fe278065b51cb47bfbf17f085b3fc7634 Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Wed, 30 Oct 2024 13:03:11 -0400 Subject: [PATCH 14/16] chore: update timestamp logic using rounding and hours offset --- app/backend/lib/milestoneDueDate.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/backend/lib/milestoneDueDate.ts b/app/backend/lib/milestoneDueDate.ts index 455438c48..b3b311159 100644 --- a/app/backend/lib/milestoneDueDate.ts +++ b/app/backend/lib/milestoneDueDate.ts @@ -54,9 +54,10 @@ const processMilestones = async (req, res) => { // Function to check if a given due date string is within 30 to 31 days from today. const isWithin30To31Days = (dueDateStr) => { const dueDate = new Date(dueDateStr); + today.setHours(0, 0, 0, 0); const timeDiff = dueDate.getTime() - today.getTime(); - const daysDiff = timeDiff / (1000 * 3600 * 24); - return daysDiff >= 30 && daysDiff <= 31; + const daysDiff = Math.round(timeDiff / (1000 * 3600 * 24)); + return daysDiff === 30; }; // Traverse the result, if there is a milestone due date within 30 to 31 days from today, From acbce7ec5ac95bd99f57ea1a6bdb22ae16fecf52 Mon Sep 17 00:00:00 2001 From: Anthony Bushara Date: Tue, 5 Nov 2024 12:38:14 -0500 Subject: [PATCH 15/16] test: update tests for new rounding logic --- .../backend/lib/milestoneDueDate.test.ts | 65 ++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/app/tests/backend/lib/milestoneDueDate.test.ts b/app/tests/backend/lib/milestoneDueDate.test.ts index 714c96d9d..1cebe07fd 100644 --- a/app/tests/backend/lib/milestoneDueDate.test.ts +++ b/app/tests/backend/lib/milestoneDueDate.test.ts @@ -134,8 +134,50 @@ describe('The Milestone excel import api route', () => { { jsonData: [ { - milestone1: '2022-01-02', + milestone1: '2022-01-03', milestone2: '2022-01-02', + milestone3: '2022-01-01', + }, + ], + }, + ], + }, + }, + { + applicationId: 2, + applicationByApplicationId: { + ccbcNumber: 'CCBC-010001', + organizationName: 'Organization Name', + projectName: 'Project Name', + }, + sowTab2SBySowId: { + nodes: [ + { + jsonData: [ + { + milestone1: '2022-01-01', + milestone2: '2022-01-02', + milestone3: '2022-01-03', + }, + ], + }, + ], + }, + }, + { + applicationId: 3, + applicationByApplicationId: { + ccbcNumber: 'CCBC-010001', + organizationName: 'Organization Name', + projectName: 'Project Name', + }, + sowTab2SBySowId: { + nodes: [ + { + jsonData: [ + { + milestone1: '2022-01-03', + milestone2: '2022-01-01', milestone3: '2022-01-02', }, ], @@ -143,6 +185,27 @@ describe('The Milestone excel import api route', () => { ], }, }, + { + applicationId: 1, + applicationByApplicationId: { + ccbcNumber: 'CCBC-010001', + organizationName: 'Organization Name', + projectName: 'Project Name', + }, + sowTab2SBySowId: { + nodes: [ + { + jsonData: [ + { + milestone1: '2022-01-03', + milestone2: '2022-01-02', + milestone3: '2022-01-01', + }, + ], + }, + ], + }, + }, ], }, }, From 21e66534ab2ae94c375c6083acb348bc1ed0b040 Mon Sep 17 00:00:00 2001 From: CCBC Service Account <116113628+ccbc-service-account@users.noreply.github.com> Date: Tue, 5 Nov 2024 20:41:08 +0000 Subject: [PATCH 16/16] chore: release v1.204.0 --- CHANGELOG.md | 8 ++++++++ db/sqitch.plan | 1 + package.json | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dac6cc48..bcdbf9c5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# [1.204.0](https://github.com/bcgov/CONN-CCBC-portal/compare/v1.203.4...v1.204.0) (2024-11-05) + +### Features + +- milestone cron ([5e1b65f](https://github.com/bcgov/CONN-CCBC-portal/commit/5e1b65fcd21b469c3a231f01c3cbc398cec5391f)) +- send emails for milestone when due in 30 days ([793372e](https://github.com/bcgov/CONN-CCBC-portal/commit/793372e7860fa730535c81f0a8843527d3abf5b5)) +- service account permissions and get scope in request ([41d84c5](https://github.com/bcgov/CONN-CCBC-portal/commit/41d84c5ec98dea4bc066b6c6bd75fdbb286b8b78)) + ## [1.203.4](https://github.com/bcgov/CONN-CCBC-portal/compare/v1.203.3...v1.203.4) (2024-11-01) ## [1.203.3](https://github.com/bcgov/CONN-CCBC-portal/compare/v1.203.2...v1.203.3) (2024-10-29) diff --git a/db/sqitch.plan b/db/sqitch.plan index 62364840f..4700f5821 100644 --- a/db/sqitch.plan +++ b/db/sqitch.plan @@ -722,3 +722,4 @@ tables/application_form_template_9_data 2024-09-25T22:50:17Z Rafael Solorzano <6 @1.203.4 2024-11-01T15:43:03Z CCBC Service Account # release v1.203.4 mutations/archive_application_sow [mutations/archive_application_sow@1.201.0] 2024-10-24T19:57:11Z Anthony Bushara # Add application ID to prevent archiving other applications grant_read_access_to_service_account 2024-10-04T18:45:20Z Anthony Bushara # Add read access to service account +@1.204.0 2024-11-05T20:41:06Z CCBC Service Account # release v1.204.0 diff --git a/package.json b/package.json index 941f9ff57..12d1a16c1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "CONN-CCBC-portal", - "version": "1.203.4", + "version": "1.204.0", "main": "index.js", "repository": "https://github.com/bcgov/CONN-CCBC-portal.git", "author": "Romer, Meherzad CITZ:EX ",