chore: attempt to mock SQS in hrmservice

cdk/hrm-search-index-stack.ts

@@ -0,0 +1,154 @@
+/**
+ * Copyright (C) 2021-2023 Technology Matters
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/.
+ */
+
+import * as cdk from 'aws-cdk-lib';
+import * as lambdaNode from 'aws-cdk-lib/aws-lambda-nodejs';
+import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources';
+import * as sqs from 'aws-cdk-lib/aws-sqs';
+import * as s3 from 'aws-cdk-lib/aws-s3';
+
+export default class ContactRetrieveStack extends cdk.Stack {
+  constructor({
+    scope,
+    id,
+    params = {
+      skipLambda: false,
+    },
+    props,
+  }: {
+    scope: cdk.Construct;
+    id: string;
+    params: {
+      completeQueue: sqs.Queue;
+      docsBucket: s3.Bucket;
+      skipLambda?: boolean;
+    };
+    props?: cdk.StackProps;
+  }) {
+    super(scope, id, props);
+
+    const queue = new cdk.aws_sqs.Queue(this, id, {
+      deadLetterQueue: { maxReceiveCount: 1, queue: params.completeQueue },
+    });
+
+    new cdk.CfnOutput(this, `queueUrl`, {
+      value: queue.queueUrl,
+      description: `The url of the ${id} queue`,
+    });
+
+    new cdk.aws_ssm.StringParameter(this, `queue-url-consumer`, {
+      parameterName: `/local/us-east-1/sqs/jobs/hrm-search-index/queue-url-consumer`,
+      stringValue: queue.queueUrl,
+    });
+
+    // duplicated for test env
+    new cdk.aws_ssm.StringParameter(this, `queue-url-consumer-test`, {
+      parameterName: `/local/us-east-1/sqs/jobs/hrm-search-index/queue-url-consumer-test`,
+      stringValue: queue.queueUrl,
+    });
+
+    if (params.skipLambda) return;
+
+    /*
+      Here, there be dragons.
+
+      To use the queue urls from inside of a lambda, we have to replace
+      'localhost' with 'localstack' so that container to container dns
+      lookups resolve correctly on the docker network.
+
+      This is WAY more complicated than it should be and took way too long
+      to figure out. But here goes:
+
+      CDK passes around tokens that are used inside of the final CloudFormation
+      template that is generated and deployed, not the actual string values for
+      things like queue urls that aren't known until the deployment is partially
+      complete.
+
+      Tokens are unresolvable until they are applied in the produced CF template.
+      So, you can't just do normal string replace operations. You have to use
+      native cloudformation functions to manipulate the string.
+
+      BUUUT. There is no "replace" function in cloudformation. So you have
+      to use split/join to do a janky replace.
+
+      Also... for some reason we can't use "Fn::split" inside of a "Fn::Join"
+      function directly. We have to use the select to iterate over items in the
+      split or else we just get a string of "Fn::split" as our url. I have no idea
+      why, but i discovered this working pattern by trial and error mixed with reviewing
+      generate cloudformation templates from the CDK that used the join/split replace
+      pattern.
+
+      This is pretty fragile since we can't arbitrarily split/join if there are
+      multiple instances of the needle in the haystack. But it works for this
+      simple case.
+      (rbd 08/10/22)
+    */
+    const splitCompleteQueueUrl = cdk.Fn.split(
+      'localhost',
+      params.completeQueue.queueUrl,
+    );
+    const completedQueueUrl = cdk.Fn.join('localstack', [
+      cdk.Fn.select(0, splitCompleteQueueUrl),
+      cdk.Fn.select(1, splitCompleteQueueUrl),
+    ]);
+
+    const fn = new lambdaNode.NodejsFunction(this, 'search-index-consumer', {
+      // TODO: change this back to 18 once it isn't broken upstream
+      runtime: cdk.aws_lambda.Runtime.NODEJS_16_X,
+      memorySize: 512,
+      timeout: cdk.Duration.seconds(10),
+      handler: 'handler',
+      entry: `./hrm-domain/lambdas/search-index-consumer/index.ts`,
+      environment: {
+        NODE_OPTIONS: '--enable-source-maps',
+        S3_ENDPOINT: 'http://localstack:4566',
+        S3_FORCE_PATH_STYLE: 'true',
+        S3_REGION: 'us-east-1',
+        SSM_ENDPOINT: 'http://localstack:4566',
+        SQS_ENDPOINT: 'http://localstack:4566',
+        NODE_ENV: 'local',
+        completed_sqs_queue_url: completedQueueUrl,
+      },
+      bundling: { sourceMap: true },
+      deadLetterQueueEnabled: true,
+      deadLetterQueue: params.completeQueue,
+    });
+
+    fn.addEventSource(
+      new SqsEventSource(queue, { batchSize: 10, reportBatchItemFailures: true }),
+    );
+
+    fn.addToRolePolicy(
+      new cdk.aws_iam.PolicyStatement({
+        actions: ['ssm:GetParametersByPath'],
+        resources: [`arn:aws:ssm:${this.region}:*:parameter/local/*`],
+      }),
+    );
+
+    fn.addToRolePolicy(
+      new cdk.aws_iam.PolicyStatement({
+        actions: [
+          's3:PutObject',
+          's3:PutObjectAcl',
+          's3:GetObject',
+          's3:GetObjectAcl',
+          's3:DeleteObject',
+        ],
+        resources: [params.docsBucket.bucketArn],
+      }),
+    );
+  }
+}

cdk/init-stack.ts

@@ -22,6 +22,7 @@ import * as dotenv from 'dotenv';
 import ContactCompleteStack from './contact-complete-stack';
 import ContactCoreStack from './contact-core-stack';
 import ContactRetrieveStack from './contact-retrieve-stack';
+import HrmSearchIndexStack from './hrm-search-index-stack';
 import HrmMicoservicesStack from './hrm-micoroservices-stack';
 import LocalCoreStack from './local-core-stack';
 import ResourcesCoreStack from './resources-core-stack';
@@ -100,6 +101,18 @@ async function main() {
     },
   });
 
+  new HrmSearchIndexStack({
+    scope: app,
+    id: 'hrm-search-index',
+    params: {
+      completeQueue: contactComplete.completeQueue,
+      docsBucket: localCore.docsBucket,
+    },
+    props: {
+      env: { region: app.node.tryGetContext('region') },
+    },
+  });
+
   new ResourcesCoreStack({
     scope: app,
     id: 'resources-core',

hrm-domain/hrm-core/case/caseDataAccess.ts

@@ -29,10 +29,11 @@ import { Contact } from '../contact/contactDataAccess';
 import { DateFilter, OrderByDirectionType } from '../sql';
 import { TKConditionsSets } from '../permissions/rulesMap';
 import { TwilioUser } from '@tech-matters/twilio-worker-auth';
-import { TwilioUserIdentifier } from '@tech-matters/types';
+import { AccountSID, TwilioUserIdentifier } from '@tech-matters/types';
 import {
   PrecalculatedCasePermissionConditions,
   CaseRecordCommon,
+  CaseService,
 } from '@tech-matters/hrm-types';
 import { CaseSectionRecord } from './caseSection/types';
 import { pick } from 'lodash';
@@ -249,8 +250,8 @@ export const searchByProfileId = generalizedSearchQueryFunction<{
   }),
 );
 
-export const deleteById = async (id, accountSid) => {
-  return db.oneOrNone(DELETE_BY_ID, [accountSid, id]);
+export const deleteById = async (id: CaseService['id'], accountSid: AccountSID) => {
+  return db.oneOrNone<CaseRecord>(DELETE_BY_ID, [accountSid, id]);
 };
 
 export const updateStatus = async (

hrm-domain/hrm-core/case/caseRoutesV0.ts

@@ -15,7 +15,6 @@
  */
 
 import createError from 'http-errors';
-import * as casesDb from './caseDataAccess';
 import * as caseApi from './caseService';
 import { publicEndpoint, SafeRouter } from '../permissions';
 import {
@@ -93,7 +92,7 @@ casesRouter.expressRouter.use('/:caseId/sections', caseSectionRoutesV0);
 casesRouter.delete('/:id', publicEndpoint, async (req, res) => {
   const { hrmAccountId } = req;
   const { id } = req.params;
-  const deleted = await casesDb.deleteById(id, hrmAccountId);
+  const deleted = await caseApi.deleteCaseById({ accountSid: hrmAccountId, caseId: id });
   if (!deleted) {
     throw createError(404);
   }

hrm-domain/hrm-core/case/caseSection/caseSectionService.ts

@@ -36,6 +36,7 @@ import { TwilioUser } from '@tech-matters/twilio-worker-auth';
 import { RulesFile, TKConditionsSets } from '../../permissions/rulesMap';
 import { ListConfiguration } from '../caseDataAccess';
 import { HrmAccountId } from '@tech-matters/types';
+import { indexCaseInSearchIndex } from '../caseService';
 
 const sectionRecordToSection = (
   sectionRecord: CaseSectionRecord | undefined,
@@ -65,7 +66,13 @@ export const createCaseSection = async (
     createdAt: nowISO,
     accountSid,
   };
-  return sectionRecordToSection(await create()(record));
+
+  const created = await create()(record);
+
+  // trigger index operation but don't await for it
+  indexCaseInSearchIndex({ accountSid, caseId: parseInt(caseId, 10) });
+
+  return sectionRecordToSection(created);
 };
 
 export const replaceCaseSection = async (
@@ -83,15 +90,19 @@ export const replaceCaseSection = async (
     updatedBy: workerSid,
     updatedAt: nowISO,
   };
-  return sectionRecordToSection(
-    await updateById()(
-      accountSid,
-      Number.parseInt(caseId),
-      sectionType,
-      sectionId,
-      record,
-    ),
+
+  const updated = await updateById()(
+    accountSid,
+    Number.parseInt(caseId),
+    sectionType,
+    sectionId,
+    record,
   );
+
+  // trigger index operation but don't await for it
+  indexCaseInSearchIndex({ accountSid, caseId: parseInt(caseId, 10) });
+
+  return sectionRecordToSection(updated);
 };
 
 export const getCaseSection = async (
@@ -155,13 +166,16 @@ export const deleteCaseSection = async (
   sectionId: string,
   { user }: { user: TwilioUser },
 ): Promise<CaseSection | undefined> => {
-  return sectionRecordToSection(
-    await deleteById()(
-      accountSid,
-      Number.parseInt(caseId),
-      sectionType,
-      sectionId,
-      user.workerSid,
-    ),
+  const deleted = await deleteById()(
+    accountSid,
+    Number.parseInt(caseId),
+    sectionType,
+    sectionId,
+    user.workerSid,
   );
+
+  // trigger index operation but don't await for it
+  indexCaseInSearchIndex({ accountSid, caseId: parseInt(caseId, 10) });
+
+  return sectionRecordToSection(deleted);
 };