CMDCT-3960: creating new dynamo table for measures with PKs that can …

…be queried instead of scan all (#2431)

serverless-compose.yml

@@ -11,6 +11,7 @@ services:
     params:
       CoreSetTableName: ${database.CoreSetTableName}
       MeasureTableName: ${database.MeasureTableName}
+      MeasureTable: ${database.MeasureTable}
       RateTableName: ${database.RateTableName}
 
   app-api:
@@ -19,6 +20,7 @@ services:
       CoreSetTableName: ${database.CoreSetTableName}
       CoreSetTableStreamArn: ${database.CoreSetTableStreamArn}
       MeasureTableName: ${database.MeasureTableName}
+      MeasureTable: ${database.MeasureTable}
       MeasureTableStreamArn: ${database.MeasureTableStreamArn}
       RateTableName: ${database.RateTableName}
       RateTableStreamArn: ${database.RateTableStreamArn}

services/app-api/handlers/coreSets/create.ts

@@ -1,7 +1,6 @@
 import handler from "../../libs/handler-lib";
 import dynamoDb from "../../libs/dynamodb-lib";
 import { getCoreSet } from "./get";
-import { createCoreSetKey } from "../dynamoUtils/createCompoundKey";
 import { MeasureMetaData, measures } from "../dynamoUtils/measureList";
 import {
   hasRolePermissions,
@@ -42,18 +41,17 @@ export const createCoreSet = handler(async (event, context) => {
       body: Errors.CORESET_ALREADY_EXISTS,
     };
   }
-  const dynamoKey = createCoreSetKey({ state, year, coreSet });
 
   await createDependentMeasures(
     state,
-    parseInt(year),
+    year,
     coreSet as Types.CoreSetAbbr,
     type
   );
 
   // filter out qualifier and account for autocomplete measures on creation
   let autoCompletedMeasures = 0;
-  const measuresLengthWithoutQualifiers = measures[parseInt(year)].filter(
+  const measuresLengthWithoutQualifiers = measures[year].filter(
     (measure: MeasureMetaData) => {
       if (measure.autocompleteOnCreation && measure.type === type) {
         autoCompletedMeasures++;
@@ -70,9 +68,9 @@ export const createCoreSet = handler(async (event, context) => {
   const params = {
     TableName: process.env.coreSetTableName!,
     Item: {
-      compoundKey: dynamoKey,
+      compoundKey: `${state}${year}${coreSet}`,
       state: state,
-      year: parseInt(year),
+      year: year,
       coreSet: coreSet,
       createdAt: Date.now(),
       lastAltered: Date.now(),
@@ -104,12 +102,10 @@ const createDependentMeasures = async (
   for await (const measure of filteredMeasures) {
     // The State Year and ID are all part of the path
     const measureId = measure["measure"];
-    // Dynamo only accepts one row as a key, so we are using a combination for the dynamoKey
-    const dynamoKey = `${state}${year}${coreSet}${measureId}`;
     const params = {
-      TableName: process.env.measureTableName!,
+      TableName: process.env.measureTable!,
       Item: {
-        compoundKey: dynamoKey,
+        compoundKey: `${state}${year}${coreSet}`,
         state: state,
         year: year,
         coreSet: coreSet,

services/app-api/handlers/coreSets/delete.ts

@@ -1,7 +1,5 @@
 import handler from "../../libs/handler-lib";
 import dynamoDb from "../../libs/dynamodb-lib";
-import { createCoreSetKey } from "../dynamoUtils/createCompoundKey";
-import { convertToDynamoExpression } from "../dynamoUtils/convertToDynamoExpressionVars";
 import { hasStatePermissions } from "../../libs/authorization";
 import { Measure } from "../../types";
 import { Errors, StatusCodes } from "../../utils/constants/constants";
@@ -24,17 +22,16 @@ export const deleteCoreSet = handler(async (event, context) => {
     };
   }
 
-  const dynamoKey = createCoreSetKey({ state, year, coreSet });
   const params = {
     TableName: process.env.coreSetTableName!,
     Key: {
-      compoundKey: dynamoKey,
+      compoundKey: `${state}${year}${coreSet}`,
       coreSet: coreSet,
     },
   };
 
   await dynamoDb.delete(params);
-  await deleteDependentMeasures(state, parseInt(year), coreSet);
+  await deleteDependentMeasures(state, year, coreSet);
 
   return {
     status: StatusCodes.SUCCESS,
@@ -51,12 +48,11 @@ const deleteDependentMeasures = async (
   const Items = measures;
 
   for await (const { measure } of Items) {
-    const dynamoKey = `${state}${year}${coreSet}${measure}`;
     const params = {
-      TableName: process.env.measureTableName!,
+      TableName: process.env.measureTable!,
       Key: {
-        compoundKey: dynamoKey,
-        coreSet: coreSet,
+        compoundKey: `${state}${year}${coreSet}`,
+        measure: measure,
       },
     };
 
@@ -66,12 +62,12 @@ const deleteDependentMeasures = async (
 
 const getMeasures = async (state: string, year: number, coreSet: string) => {
   const params = {
-    TableName: process.env.measureTableName!,
-    ...convertToDynamoExpression(
-      { state: state, year: year, coreSet: coreSet },
-      "list"
-    ),
+    TableName: process.env.measureTable!,
+    KeyConditionExpression: "compoundKey = :compoundKey",
+    ExpressionAttributeValues: {
+      ":compoundKey": `${state}${year}${coreSet}`,
+    },
   };
-  const queryValue = await dynamoDb.scanAll<Measure>(params);
+  const queryValue = await dynamoDb.queryAll<Measure>(params);
   return queryValue;
 };

services/app-api/handlers/coreSets/get.ts

@@ -2,7 +2,6 @@ import handler from "../../libs/handler-lib";
 import dynamoDb from "../../libs/dynamodb-lib";
 import { updateCoreSetProgress } from "../../libs/updateCoreProgress";
 import { convertToDynamoExpression } from "../dynamoUtils/convertToDynamoExpressionVars";
-import { createCoreSetKey } from "../dynamoUtils/createCompoundKey";
 import { createCoreSet } from "./create";
 import {
   hasRolePermissions,
@@ -41,7 +40,7 @@ export const coreSetList = handler(async (event, context) => {
     ...convertToDynamoExpression(
       {
         state: state,
-        year: parseInt(year),
+        year: year,
       },
       "list"
     ),
@@ -121,11 +120,10 @@ export const getCoreSet = handler(async (event, context) => {
     }
   } // if not state user, can safely assume admin type user due to baseline handler protections
 
-  const dynamoKey = createCoreSetKey({ state, year, coreSet });
   const params = {
     TableName: process.env.coreSetTableName!,
     Key: {
-      compoundKey: dynamoKey,
+      compoundKey: `${state}${year}${coreSet}`,
       coreSet: coreSet,
     },
   };

services/app-api/handlers/coreSets/tests/create.test.ts

@@ -23,11 +23,6 @@ jest.mock("../../../libs/authorization", () => ({
   hasStatePermissions: () => mockHasStatePermissions(),
 }));
 
-jest.mock("../../dynamoUtils/createCompoundKey", () => ({
-  __esModule: true,
-  createCoreSetKey: jest.fn().mockReturnValue("FL2021ACSFUA-AD"),
-}));
-
 jest.mock("../get", () => ({
   __esModule: true,
   getCoreSet: jest.fn(),

services/app-api/handlers/coreSets/tests/delete.test.ts

@@ -6,7 +6,7 @@ import { CoreSetAbbr } from "../../../types";
 
 jest.mock("../../../libs/dynamodb-lib", () => ({
   delete: jest.fn(),
-  scanAll: jest.fn(),
+  queryAll: jest.fn(),
 }));
 
 const mockHasStatePermissions = jest.fn();
@@ -15,11 +15,6 @@ jest.mock("../../../libs/authorization", () => ({
   hasStatePermissions: () => mockHasStatePermissions(),
 }));
 
-jest.mock("../../dynamoUtils/createCompoundKey", () => ({
-  __esModule: true,
-  createCoreSetKey: jest.fn().mockReturnValue("FL2020ACSFUA-AD"),
-}));
-
 jest.mock("../../../libs/updateCoreProgress", () => ({
   __esModule: true,
   updateCoreSetProgress: jest.fn(),
@@ -29,7 +24,7 @@ const event = { ...testEvent };
 
 describe("Testing Delete Core Set Functions", () => {
   beforeEach(() => {
-    (db.scanAll as jest.Mock).mockReset();
+    (db.queryAll as jest.Mock).mockReset();
     (db.delete as jest.Mock).mockReset();
     mockHasStatePermissions.mockImplementation(() => true);
     event.pathParameters = {
@@ -67,24 +62,24 @@ describe("Testing Delete Core Set Functions", () => {
   });
 
   test("Test deleteCoreSet with associated measures", async () => {
-    (db.scanAll as jest.Mock).mockReturnValue([
+    (db.queryAll as jest.Mock).mockReturnValue([
       testMeasure,
       testMeasure,
       testMeasure,
     ]);
 
     await deleteCoreSet(event, null);
 
-    expect(db.scanAll).toHaveBeenCalled();
+    expect(db.queryAll).toHaveBeenCalled();
     expect(db.delete).toHaveBeenCalledTimes(4);
   });
 
   test("Test deleteCoreSet with no associated measures", async () => {
-    (db.scanAll as jest.Mock).mockReturnValue([]);
+    (db.queryAll as jest.Mock).mockReturnValue([]);
 
     await deleteCoreSet(event, null);
 
-    expect(db.scanAll).toHaveBeenCalled();
+    expect(db.queryAll).toHaveBeenCalled();
     expect(db.delete).toHaveBeenCalled();
   });
 });

services/app-api/handlers/coreSets/tests/get.test.ts

@@ -4,7 +4,6 @@ import { testEvent } from "../../../test-util/testEvents";
 
 import dynamodbLib from "../../../libs/dynamodb-lib";
 import { updateCoreSetProgress } from "../../../libs/updateCoreProgress";
-import { createCoreSetKey } from "../../dynamoUtils/createCompoundKey";
 import { CoreSetAbbr } from "../../../types";
 import { createCoreSet } from "../create";
 import { Errors, StatusCodes } from "../../../utils/constants/constants";
@@ -35,11 +34,6 @@ jest.mock("../../../libs/updateCoreProgress", () => ({
   updateCoreSetProgress: jest.fn(),
 }));
 
-jest.mock("../../dynamoUtils/createCompoundKey", () => ({
-  __esModule: true,
-  createCoreSetKey: jest.fn().mockReturnValue("FL2020ACSFUA-AD"),
-}));
-
 jest.mock("../../dynamoUtils/convertToDynamoExpressionVars", () => ({
   __esModule: true,
   convertToDynamoExpression: jest.fn().mockReturnValue({ testValue: "test" }),
@@ -96,12 +90,6 @@ describe("Test Get Core Set Functions", () => {
     await getCoreSet(event, null);
 
     expect(dynamodbLib.get).toHaveBeenCalled();
-    expect(createCoreSetKey).toHaveBeenCalled();
-    expect(createCoreSetKey).toHaveBeenCalledWith({
-      state: "AL",
-      year: "2021",
-      coreSet: "ACS",
-    });
   });
 
   test("Test coreSetList unauthorized user attempt (incorrect state)", async () => {

services/app-api/handlers/coreSets/tests/update.test.ts

@@ -1,7 +1,6 @@
 import dynamodbLib from "../../../libs/dynamodb-lib";
 import { testEvent } from "../../../test-util/testEvents";
 import { convertToDynamoExpression } from "../../dynamoUtils/convertToDynamoExpressionVars";
-import { createCoreSetKey } from "../../dynamoUtils/createCompoundKey";
 import { editCoreSet } from "../update";
 import { Errors, StatusCodes } from "../../../utils/constants/constants";
 import { CoreSetAbbr } from "../../../types";
@@ -17,11 +16,6 @@ jest.mock("../../../libs/authorization", () => ({
   hasStatePermissions: () => mockHasStatePermissions(),
 }));
 
-jest.mock("../../dynamoUtils/createCompoundKey", () => ({
-  __esModule: true,
-  createCoreSetKey: jest.fn().mockReturnValue("FL2020ACSFUA-AD"),
-}));
-
 jest.mock("../../dynamoUtils/convertToDynamoExpressionVars", () => ({
   __esModule: true,
   convertToDynamoExpression: jest.fn().mockReturnValue({ testValue: "test" }),
@@ -73,7 +67,6 @@ describe("Testing Updating Core Set Functions", () => {
     Date.now = jest.fn(() => 20);
     const res = await editCoreSet(event, null);
 
-    expect(createCoreSetKey).toHaveBeenCalled();
     expect(dynamodbLib.update).toHaveBeenCalled();
     expect(convertToDynamoExpression).toHaveBeenCalledWith(
       {
@@ -84,14 +77,13 @@ describe("Testing Updating Core Set Functions", () => {
       "post"
     );
     expect(res.statusCode).toBe(StatusCodes.SUCCESS);
-    expect(res.body).toContain("FL2020ACSFUA-AD");
+    expect(res.body).toContain("WA2021ACS");
   });
 
   test("Test with no user id", async () => {
     Date.now = jest.fn(() => 20);
     const res = await editCoreSet(event, null);
 
-    expect(createCoreSetKey).toHaveBeenCalled();
     expect(dynamodbLib.update).toHaveBeenCalled();
     expect(convertToDynamoExpression).toHaveBeenCalledWith(
       {
@@ -102,6 +94,6 @@ describe("Testing Updating Core Set Functions", () => {
       "post"
     );
     expect(res.statusCode).toBe(StatusCodes.SUCCESS);
-    expect(res.body).toContain("FL2020ACSFUA-AD");
+    expect(res.body).toContain("WA2021ACS");
   });
 });

services/app-api/handlers/coreSets/update.ts

@@ -1,7 +1,6 @@
 import handler from "../../libs/handler-lib";
 import dynamoDb from "../../libs/dynamodb-lib";
 import { convertToDynamoExpression } from "../dynamoUtils/convertToDynamoExpressionVars";
-import { createCoreSetKey } from "../dynamoUtils/createCompoundKey";
 import {
   getUserNameFromJwt,
   hasStatePermissions,
@@ -27,12 +26,11 @@ export const editCoreSet = handler(async (event, context) => {
   }
 
   const { submitted, status } = JSON.parse(event!.body!);
-  const dynamoKey = createCoreSetKey({ state, year, coreSet });
   const lastAlteredBy = getUserNameFromJwt(event);
   const params = {
     TableName: process.env.coreSetTableName!,
     Key: {
-      compoundKey: dynamoKey,
+      compoundKey: `${state}${year}${coreSet}`,
       coreSet: coreSet,
     },
     ...convertToDynamoExpression(