CHI-1384 SQS integration for hrm services (#251)

* chore: removed unused no-lambda stack

* feat: remove SNS and send messages to SQS in service

* chore: update test docker container instructions

* feat: poll sqs queue and fix test mocks

* fix: revert mock for SQS.sendMessage

* fix: import mocks correctly

* chore: move back to date-fns

* CHI-1384 local e2e testing improvements (#253)

* chore: documentation for local e2e tests

* fix: run hrm-service on port 8080 by default in local dev

* fix: don't migrate on hrm-service start

* fix: start:localstack naming nit

* chore: download .env for hrm-service from ssm

* fix: set dockerfile in hrm-service action build

* fix: correctly transpile TS in dockerfiles

* fix: issues based on testing deployment via actions

* CHI-1384: improve tests for jobs and remove unused npm package (#257)

* test: improve tests for jobs and remove unused npm package

* fix: set prototype to fix TS built-in extension issue

.dockerignore

@@ -6,6 +6,8 @@
 # Build dependencies
 node_modules
 
+dist
+
 # Environment (contains sensitive data)
 .env
 *task-definitions*

.github/workflows/hrm-ecs-ci-multiregion.yml

@@ -100,7 +100,7 @@ jobs:
           # Build a docker container and
           # push it to ECR so that it can
           # be deployed to ECS.
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_1 -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_2 ../
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_1 -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_2 -f Dockerfile ../
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_1
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_2
           echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_2"

.prettierrc

@@ -1,5 +1,6 @@
 {
+  "arrowParens": "avoid",
   "printWidth": 100,
   "singleQuote": true,
   "trailingComma": "all"
-}
+}

README.md

@@ -15,6 +15,11 @@ In order to prevent sensitive credentials to be leaked, please follow this instr
 
 ## Local Development
 
+### Requirements
+
+[nvm](https://github.com/nvm-sh/nvm) or [nvm-windows](https://github.com/coreybutler/nvm-windows)
+[docker](https://docs.docker.com/get-docker/)
+
 ### Initial setup
 
 #### NVM (install/use)
@@ -53,7 +58,11 @@ This can be handy for some use cases like deterministic testing, but would get a
 
 #### Starting HRM service
 
-This requires an `./hrm-service/dist/.env` file to be present. Contents of that file are outside the scope of this documentation currently.
+Build the HRM service, packages, and jobs by running `npm run build` from the root directory.
+
+Run `npm run ssm:local` from the root directory to download required secrets for local development from AWS SSM Parameter Store.
+
+Run `npm run start:service` from the root directory to start the HRM service.
 
 You can run the full stack quickly by running `npm run build-and-start` from the root directory (after running `npm ci`). This will start the hrm-service and hrm-jobs packages.
 
@@ -79,3 +88,4 @@ Package management is handled at the root level for sub modules. Since the packa
 
 - [Debugging Service Jest](./docs/debugging-service-jest.md)
 - [Testing Service Docker Build](./docs/test-service-docker.md)
+- [Import Dev Db to local postgres](./docs/import-dev-db-locally.md)

cdk/contact-complete-stack.ts

@@ -3,6 +3,7 @@ import * as cdk from '@aws-cdk/core';
 import * as lambda from '@aws-cdk/aws-lambda';
 import * as lambdaNode from '@aws-cdk/aws-lambda-nodejs';
 import * as sqs from '@aws-cdk/aws-sqs';
+import * as ssm from '@aws-cdk/aws-ssm';
 import { SqsEventSource } from '@aws-cdk/aws-lambda-event-sources';
 
 export default class ContactCompleteStack extends cdk.Stack {
@@ -26,6 +27,17 @@ export default class ContactCompleteStack extends cdk.Stack {
     super(scope, id, props);
     this.completeQueue = new sqs.Queue(this, id);
 
+    new ssm.StringParameter(this, `complete-queue-url`, {
+      parameterName: `/local/sqs/jobs/contact/queue-url-complete`,
+      stringValue: this.completeQueue.queueUrl,
+    });
+
+    // duplicated for test env
+    new ssm.StringParameter(this, `complete-queue-url-test`, {
+      parameterName: `/test/sqs/jobs/contact/queue-url-complete`,
+      stringValue: this.completeQueue.queueUrl,
+    });
+
     new cdk.CfnOutput(this, 'queueUrl', {
       value: this.completeQueue.queueUrl,
       description: 'The url of the complete queue',

docker-compose.yml

@@ -5,7 +5,7 @@ services:
     ports:
       - '127.0.0.1:4566:4566' # LocalStack Gateway
       - '127.0.0.1:4510-4559:4510-4559' # external services port range
-      - '${PORT_WEB_UI-8080}:${PORT_WEB_UI-8080}'
+      - '${PORT_WEB_UI-8088}:8080'
     environment:
       - HOSTNAME_EXTERNAL=localstack
       - DEBUG=1

docs/import-dev-db-locally.md

@@ -0,0 +1,17 @@
+# Import development db from RDS
+
+## Requirements
+
+- Aselo VPN Access to development environment
+- psql/pg_dump/pg_restore installed locally
+- credentials for the development database
+
+## Steps
+
+1. Connect to the VPN
+
+2. Run `pg_dump -U hrm -h development-postgres-rds.c9itqcbmhgxo.us-east-1.rds.amazonaws.com -W -F t hrmdb -f hrmdb.sql` to dump the database to a file after replacing the values in the angle brackets with the correct values.
+
+3. Clear out the local db with: `psql -U rdsadmin -h 127.0.0.1 -d hrmdb -c "drop schema public cascade; create schema public;"`
+
+4. Import the dev database with `pg_restore -U rdsadmin -h 127.0.0.1 -x -d hrmdb -1 hrmdb.sql`

docs/test-service-docker.md

@@ -1,8 +1,18 @@
 # Run flex-plugins e2e tests against locally running docker version of hrm-service
 
-1. Build the docker contianer locally with: `npm run -w hrm-service docker:build`
+## Requirements
+
+Working aws credentials with access to local ssm parameters configured in your terminal
+
+_This probably doesn't work on windows yet_
+
+## Steps
+
+1. Build the docker container locally with: `npm run -w hrm-service docker:build`
 2. Start the hrm db with: `npm run docker:compose:db:up`
 3. Optionally run localstack for AWS resources with: `npm run localstack:init`
 4. Bring up the hrm service container with: `npm run docker:compose:service:up`
 5. Get logs from `npm run docker:compose:logs`
-6. Run tests with: `npm run test:docker:service` - this is only tested on nix and requires aws-cli to be installed locally
+6. Follow the instructions for importing the current dev db into your local postgres server [here](./import-dev-db-locally.md).
+7. In another terminal, follow the instructions in the flex-plugins/plugin-hrm-form to get the flex plugin running locally against the local docker service with the correct configuration files for e2e testing
+8. Follow the instructions in the flex-plugins/e2e-tests readme to run the e2e tests locally

hrm-service/Dockerfile

@@ -6,11 +6,11 @@ FROM node:16-alpine as build
 # dependency in package.json can be resolved
 COPY packages /tmp/build/packages
 COPY hrm-service /tmp/build/hrm-service
-COPY package.json package-lock.json tsconfig.build.service.json /tmp/build/
+COPY package.json package-lock.json tsconfig.base.json tsconfig.build.service.json /tmp/build/
 
 RUN cd /tmp/build \
     && npm ci -w=hrm-service -w packages/* \
-    && npx tsc --project tsconfig.build.service.json \
+    && npx tsc -b tsconfig.build.service.json \
     && cp -r hrm-service/* /home/node/ \
     && cp -r packages /home/node/ \
     && cp -r node_modules /home/node/ \

hrm-service/docker-compose.yml

@@ -12,3 +12,4 @@ services:
 
     env_file:
       - ../.env.localstack
+      - dist/.env

hrm-service/package.json

@@ -3,9 +3,7 @@
   "version": "0.0.0",
   "description": "The easiest way to run the tests locally is to run the test DB in docker.",
   "scripts": {
-    "start": "run-s test:service:migrate start:run",
-    "start:run": "cd dist && node ./bin/www",
-    "build-and-start": "npm run build && npm run start",
+    "start": "cd dist && node ./bin/www",
     "docker:build": "docker build -t hrm-service -f Dockerfile ../",
     "docker:compose:test:up": "docker compose -f ./docker-compose-test.yml up -d",
     "docker:compose:test:down": "docker compose -f ./docker-compose-test.yml down",
@@ -23,7 +21,9 @@
     "db:create-migration": "sequelize migration:generate --name ",
     "db:undo-last": "sequelize db:migrate:undo",
     "lint": "eslint --ext js,ts .",
-    "lint:fix": "npm run lint -- --fix"
+    "lint:fix": "npm run lint -- --fix",
+    "ssm:local": "run-s ssm:local:*",
+    "ssm:local:env": "docker run --rm -v ~/.aws:/home/aws/.aws -e AWS_DEFAULT_REGION -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY interrobangc/awscli2 ssm get-parameter --name \"/local/hrm/hrm-service/.env\" --with-decryption --query \"Parameter.Value\" --output text > ./dist/.env"
   },
   "repository": {
     "type": "git",

hrm-service/service-tests/contact-job/contact-job-processor.test.ts

@@ -1,8 +1,7 @@
 import supertest from 'supertest';
 import timers from 'timers';
 
-jest.mock('../../src/contact-job/client-sns');
-jest.mock('../../src/contact-job/client-sqs');
+require('./mocks');
 
 let server;
 let createService: typeof import('../../src/app').createService;
@@ -25,7 +24,7 @@ const stopServer = async () => {
   server = null;
 };
 
-beforeEach(() => {
+beforeEach(async () => {
   jest.isolateModules(() => {
     createService = require('../../src/app').createService;
     contactJobComplete = require('../../src/contact-job/contact-job-complete');

hrm-service/service-tests/contact-job/jobTypes/retrieve-transcript.test.ts

@@ -1,18 +1,20 @@
-import { isAfter } from 'date-fns';
 import each from 'jest-each';
+import { isAfter } from 'date-fns';
 import timers from 'timers';
 
 import { withTaskId, accountSid, workerSid } from '../../mocks';
 import * as contactJobApi from '../../../src/contact-job/contact-job-data-access';
 import { db } from '../../../src/connection-pool';
 import '../../case-validation';
-import { CompletedContactJobBody } from '../../../src/contact-job/contact-job-messages';
 import { ContactMediaType } from '../../../src/contact/contact-json';
 import { Contact } from '../../../src/contact/contact';
 import { chatChannels } from '../../../src/contact/channelTypes';
 import { JOB_MAX_ATTEMPTS } from '../../../src/contact-job/contact-job-processor';
 
-jest.mock('../../../src/contact-job/client-sns');
+// eslint-disable-next-line prettier/prettier
+import type { CompletedContactJobBody } from '@tech-matters/hrm-types/ContactJob';
+
+require('../mocks');
 
 // eslint-disable-next-line @typescript-eslint/no-shadow
 const selectJobsByContactId = (contactId: number, accountSid: string) =>
@@ -129,9 +131,8 @@ describe('publish retrieve-transcript job type', () => {
       return callback as any;
     });
 
-    const processorIntervalCallback = (contactJobProcessor.processContactJobs() as unknown) as () => Promise<
-      void
-    >;
+    const processorIntervalCallback =
+      contactJobProcessor.processContactJobs() as unknown as () => Promise<void>;
 
     await processorIntervalCallback();
 
@@ -161,9 +162,8 @@ describe('publish retrieve-transcript job type', () => {
       throw new Error('updatedRetrieveContactTranscriptJob is null!');
 
     expect(updatedRetrieveContactTranscriptJob.completed).toBeNull();
-    expect(
-      isAfter(updatedRetrieveContactTranscriptJob.lastAttempt!, startedTimestamp),
-    ).toBeTruthy();
+
+    expect(isAfter(updatedRetrieveContactTranscriptJob.lastAttempt!, startedTimestamp)).toBeTruthy();
     expect(updatedRetrieveContactTranscriptJob.numberOfAttempts).toBe(1);
   });
 
@@ -192,9 +192,8 @@ describe('publish retrieve-transcript job type', () => {
         return callback as any;
       });
 
-      const processorIntervalCallback = (contactJobProcessor.processContactJobs() as unknown) as () => Promise<
-        void
-      >;
+      const processorIntervalCallback =
+        contactJobProcessor.processContactJobs() as unknown as () => Promise<void>;
 
       await processorIntervalCallback();
       await processorIntervalCallback();
@@ -225,9 +224,7 @@ describe('publish retrieve-transcript job type', () => {
         throw new Error('updatedRetrieveContactTranscriptJob is null!');
 
       expect(updatedRetrieveContactTranscriptJob.completed).toBeNull();
-      expect(
-        isAfter(updatedRetrieveContactTranscriptJob.lastAttempt!, startedTimestamp),
-      ).toBeTruthy();
+      expect(isAfter(updatedRetrieveContactTranscriptJob.lastAttempt!, startedTimestamp)).toBeTruthy();
       expect(updatedRetrieveContactTranscriptJob.numberOfAttempts).toBe(1);
     },
   );
@@ -292,9 +289,8 @@ describe('complete retrieve-transcript job type', () => {
         return callback as any;
       });
 
-      const processorIntervalCallback = (contactJobProcessor.processContactJobs() as unknown) as () => Promise<
-        void
-      >;
+      const processorIntervalCallback =
+        contactJobProcessor.processContactJobs() as unknown as () => Promise<void>;
 
       await processorIntervalCallback();
 
@@ -327,9 +323,8 @@ describe('complete retrieve-transcript job type', () => {
       if (!updatedRetrieveContactTranscriptJob)
         throw new Error('updatedRetrieveContactTranscriptJob is null!');
 
-      expect(
-        isAfter(updatedRetrieveContactTranscriptJob.completed!, startedTimestamp),
-      ).toBeTruthy();
+
+      expect(isAfter(updatedRetrieveContactTranscriptJob.completed!, startedTimestamp)).toBeTruthy();
       expect(updatedRetrieveContactTranscriptJob.completionPayload).toMatchObject({
         message: 'Job processed successfully',
         value: 'some-url-here',
@@ -413,9 +408,8 @@ describe('complete retrieve-transcript job type', () => {
         return callback as any;
       });
 
-      const processorIntervalCallback = (contactJobProcessor.processContactJobs() as unknown) as () => Promise<
-        void
-      >;
+      const processorIntervalCallback =
+        contactJobProcessor.processContactJobs() as unknown as () => Promise<void>;
 
       await processorIntervalCallback();
 
@@ -442,9 +436,8 @@ describe('complete retrieve-transcript job type', () => {
       if (expectMarkedAsComplete) {
         // And previous job is not completed hence retrieved as due
         // expect(publishRetrieveContactTranscriptSpy).toHaveBeenCalledTimes(1);
-        expect(
-          isAfter(updatedRetrieveContactTranscriptJob.completed!, startedTimestamp),
-        ).toBeTruthy();
+
+        expect(isAfter(updatedRetrieveContactTranscriptJob.completed!, startedTimestamp)).toBeTruthy();
         expect(updatedRetrieveContactTranscriptJob.completionPayload).toMatchObject({
           message: 'Attempts limit reached',
         });