From 8740adb655891fca7350d71ba9a011349189f122 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Thu, 12 Sep 2024 08:32:34 -0400 Subject: [PATCH 01/23] CMDCT-3960: pushing new table to deploy to dev --- .../database/scripts/transformMeasureTable.ts | 62 +++++++++++++++++++ services/database/serverless.yml | 22 ++++++- 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 services/database/scripts/transformMeasureTable.ts diff --git a/services/database/scripts/transformMeasureTable.ts b/services/database/scripts/transformMeasureTable.ts new file mode 100644 index 0000000000..542035ccad --- /dev/null +++ b/services/database/scripts/transformMeasureTable.ts @@ -0,0 +1,62 @@ +import { DynamoDBClient, paginateScan } from "@aws-sdk/client-dynamodb"; +import { DynamoDBDocumentClient, PutCommand } from "@aws-sdk/lib-dynamodb"; + +const transformMeasureTable = async () => { + const dbClient = buildClient(!!process.env.DYNAMODB_URL); + const tableName = "local-measures"; + const newTableName = "local-cs-measures"; + console.log(`Processing table ${tableName}`); + for await (let entry of scan(dbClient, tableName)) { + add(dbClient, newTableName, entry); + } +}; + +async function* scan(client: DynamoDBDocumentClient, table: string) { + const query = { + TableName: table, + }; + for await (const result of paginateScan({ client }, query)) { + yield* result.Items ?? []; + } +} + +async function add(client: DynamoDBDocumentClient, table: string, entry: any) { + const newCompoundKey = `${entry.state}${entry.year}${entry.coreSet}`; + const params = { + TableName: table, + Item: { + compoundKey: newCompoundKey, + coreSet: entry.coreSet, + }, + }; + await client.send(new PutCommand(params)); +} + +function buildClient(isLocal: boolean) { + if (isLocal) { + return DynamoDBDocumentClient.from( + new DynamoDBClient({ + region: "localhost", + endpoint: "http://localhost:8000", + credentials: { + accessKeyId: "LOCALFAKEKEY", // pragma: allowlist secret + secretAccessKey: "LOCALFAKESECRET", // pragma: allowlist secret + }, + }) + ); + } else { + return DynamoDBDocumentClient.from( + new DynamoDBClient({ + region: "us-east-1", + logger: { + debug: () => { + /* Dynamo's debug logs are extremely noisy */ + }, + info: console.info, + warn: console.warn, + error: console.error, + }, + }) + ); + } +} diff --git a/services/database/serverless.yml b/services/database/serverless.yml index 0d5f5dec8e..b8a09cafd1 100644 --- a/services/database/serverless.yml +++ b/services/database/serverless.yml @@ -20,6 +20,7 @@ custom: measureTableName: ${self:custom.stage}-measures bannerTableName: ${self:custom.stage}-banners rateTableName: ${self:custom.stage}-rates + measureTableName2: ${self:custom.stage}-cs-measures dynamodb: stages: - local @@ -34,7 +35,7 @@ provider: region: us-east-1 stackTags: PROJECT: ${self:custom.project} - SERVICE: ${self:service} + SERVICE: ${self:service} resources: Resources: @@ -95,6 +96,25 @@ resources: - AttributeName: coreSet KeyType: RANGE BillingMode: PAY_PER_REQUEST # Set the capacity to auto-scale + MeasureTable2: + Type: AWS::DynamoDB::Table + Properties: + TableName: ${self:custom.measureTableName2} + PointInTimeRecoverySpecification: + PointInTimeRecoveryEnabled: true + StreamSpecification: + StreamViewType: NEW_AND_OLD_IMAGES + AttributeDefinitions: + - AttributeName: compoundKey + AttributeType: S + - AttributeName: coreSet + AttributeType: S + KeySchema: + - AttributeName: compoundKey + KeyType: HASH + - AttributeName: coreSet + KeyType: RANGE + BillingMode: PAY_PER_REQUEST # Set the capacity to auto-scale BannerTable: Type: AWS::DynamoDB::Table Properties: From 08117da2dbaaa4db2004916a49f95f42122b27cd Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Thu, 12 Sep 2024 14:42:41 -0400 Subject: [PATCH 02/23] stuff --- .../database/scripts/transformMeasureTable.ts | 22 +++++++++++++------ services/database/serverless.yml | 4 ++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/services/database/scripts/transformMeasureTable.ts b/services/database/scripts/transformMeasureTable.ts index 542035ccad..58061531ca 100644 --- a/services/database/scripts/transformMeasureTable.ts +++ b/services/database/scripts/transformMeasureTable.ts @@ -1,10 +1,14 @@ -import { DynamoDBClient, paginateScan } from "@aws-sdk/client-dynamodb"; -import { DynamoDBDocumentClient, PutCommand } from "@aws-sdk/lib-dynamodb"; +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { + DynamoDBDocumentClient, + paginateScan, + PutCommand, +} from "@aws-sdk/lib-dynamodb"; const transformMeasureTable = async () => { - const dbClient = buildClient(!!process.env.DYNAMODB_URL); - const tableName = "local-measures"; - const newTableName = "local-cs-measures"; + const dbClient = buildClient(false); + const tableName = "cmdct-3960-measures"; + const newTableName = "cmdct-3960-cs-measures"; console.log(`Processing table ${tableName}`); for await (let entry of scan(dbClient, tableName)) { add(dbClient, newTableName, entry); @@ -25,11 +29,13 @@ async function add(client: DynamoDBDocumentClient, table: string, entry: any) { const params = { TableName: table, Item: { + ...entry, compoundKey: newCompoundKey, - coreSet: entry.coreSet, }, }; - await client.send(new PutCommand(params)); + console.log("PARAMS!!!!!!!!", params.Item); + console.log("ENTRY!!!!!!!", entry); + // await client.send(new PutCommand(params)); } function buildClient(isLocal: boolean) { @@ -60,3 +66,5 @@ function buildClient(isLocal: boolean) { ); } } + +transformMeasureTable(); diff --git a/services/database/serverless.yml b/services/database/serverless.yml index b8a09cafd1..45615fff1b 100644 --- a/services/database/serverless.yml +++ b/services/database/serverless.yml @@ -107,12 +107,12 @@ resources: AttributeDefinitions: - AttributeName: compoundKey AttributeType: S - - AttributeName: coreSet + - AttributeName: measure AttributeType: S KeySchema: - AttributeName: compoundKey KeyType: HASH - - AttributeName: coreSet + - AttributeName: measure KeyType: RANGE BillingMode: PAY_PER_REQUEST # Set the capacity to auto-scale BannerTable: From ea92872adc4a5775cb64e3875bacf1fd198de397 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Thu, 12 Sep 2024 14:58:42 -0400 Subject: [PATCH 03/23] deleting new table to make a new one --- services/database/serverless.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/services/database/serverless.yml b/services/database/serverless.yml index 45615fff1b..f8337b5895 100644 --- a/services/database/serverless.yml +++ b/services/database/serverless.yml @@ -96,25 +96,6 @@ resources: - AttributeName: coreSet KeyType: RANGE BillingMode: PAY_PER_REQUEST # Set the capacity to auto-scale - MeasureTable2: - Type: AWS::DynamoDB::Table - Properties: - TableName: ${self:custom.measureTableName2} - PointInTimeRecoverySpecification: - PointInTimeRecoveryEnabled: true - StreamSpecification: - StreamViewType: NEW_AND_OLD_IMAGES - AttributeDefinitions: - - AttributeName: compoundKey - AttributeType: S - - AttributeName: measure - AttributeType: S - KeySchema: - - AttributeName: compoundKey - KeyType: HASH - - AttributeName: measure - KeyType: RANGE - BillingMode: PAY_PER_REQUEST # Set the capacity to auto-scale BannerTable: Type: AWS::DynamoDB::Table Properties: From 2d836d698af5e4dd72b043459d5ae8a6feb24ed3 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Thu, 12 Sep 2024 15:03:30 -0400 Subject: [PATCH 04/23] pushing another new measure table --- services/database/serverless.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/services/database/serverless.yml b/services/database/serverless.yml index f8337b5895..05bdf8e5c1 100644 --- a/services/database/serverless.yml +++ b/services/database/serverless.yml @@ -20,7 +20,7 @@ custom: measureTableName: ${self:custom.stage}-measures bannerTableName: ${self:custom.stage}-banners rateTableName: ${self:custom.stage}-rates - measureTableName2: ${self:custom.stage}-cs-measures + measureTable: ${self:custom.stage}-measure dynamodb: stages: - local @@ -109,6 +109,25 @@ resources: - AttributeName: key KeyType: HASH BillingMode: PAY_PER_REQUEST # Set the capacity to auto-scale + CSMeasureTable: + Type: AWS::DynamoDB::Table + Properties: + TableName: ${self:custom.measureTable} + PointInTimeRecoverySpecification: + PointInTimeRecoveryEnabled: true + StreamSpecification: + StreamViewType: NEW_AND_OLD_IMAGES + AttributeDefinitions: + - AttributeName: compoundKey + AttributeType: S + - AttributeName: measure + AttributeType: S + KeySchema: + - AttributeName: compoundKey + KeyType: HASH + - AttributeName: measure + KeyType: RANGE + BillingMode: PAY_PER_REQUEST # Set the capacity to auto-scale Outputs: CoreSetTableName: Value: !Ref CoreSetTable From 3191e9f542c1096ea9aaa01545a1bfc073ee1dee Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Fri, 13 Sep 2024 15:07:05 -0400 Subject: [PATCH 05/23] changing scanAll to queryAll and hoping it works --- services/app-api/handlers/measures/get.ts | 2 +- services/app-api/handlers/measures/tests/get.test.ts | 4 ++-- services/database/scripts/transformMeasureTable.ts | 10 ++++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/services/app-api/handlers/measures/get.ts b/services/app-api/handlers/measures/get.ts index 812cfdbae3..f192f73e70 100644 --- a/services/app-api/handlers/measures/get.ts +++ b/services/app-api/handlers/measures/get.ts @@ -42,7 +42,7 @@ export const listMeasures = handler(async (event, context) => { ), }; - let queriedMeasures = await dynamoDb.scanAll(params); + let queriedMeasures = await dynamoDb.queryAll(params); for (let v of queriedMeasures) { const measure = measures[parseInt(year as string)]?.filter( (m) => m.measure === (v as Measure)?.measure diff --git a/services/app-api/handlers/measures/tests/get.test.ts b/services/app-api/handlers/measures/tests/get.test.ts index b2d521f58e..12ad8df77e 100644 --- a/services/app-api/handlers/measures/tests/get.test.ts +++ b/services/app-api/handlers/measures/tests/get.test.ts @@ -12,7 +12,7 @@ import { Errors, StatusCodes } from "../../../utils/constants/constants"; jest.mock("../../../libs/dynamodb-lib", () => ({ get: jest.fn().mockResolvedValue("single measure"), - scanAll: jest + queryAll: jest .fn() .mockResolvedValue([{ measure: "CSQ" }, { measure: "LBW-CH" }]), })); @@ -118,7 +118,7 @@ describe("Test Get Measure Handlers", () => { expect(res.statusCode).toBe(StatusCodes.SUCCESS); expect(res.body).toContain("CSQ"); expect(res.body).toContain("LBW-CH"); - expect(dbLib.scanAll).toHaveBeenCalledWith({ + expect(dbLib.queryAll).toHaveBeenCalledWith({ TableName: "SAMPLE TABLE", testValue: "test", }); diff --git a/services/database/scripts/transformMeasureTable.ts b/services/database/scripts/transformMeasureTable.ts index 58061531ca..9ff14c81ec 100644 --- a/services/database/scripts/transformMeasureTable.ts +++ b/services/database/scripts/transformMeasureTable.ts @@ -6,9 +6,9 @@ import { } from "@aws-sdk/lib-dynamodb"; const transformMeasureTable = async () => { - const dbClient = buildClient(false); - const tableName = "cmdct-3960-measures"; - const newTableName = "cmdct-3960-cs-measures"; + const dbClient = buildClient(true); + const tableName = "local-measures"; + const newTableName = "local-measure"; console.log(`Processing table ${tableName}`); for await (let entry of scan(dbClient, tableName)) { add(dbClient, newTableName, entry); @@ -33,9 +33,7 @@ async function add(client: DynamoDBDocumentClient, table: string, entry: any) { compoundKey: newCompoundKey, }, }; - console.log("PARAMS!!!!!!!!", params.Item); - console.log("ENTRY!!!!!!!", entry); - // await client.send(new PutCommand(params)); + await client.send(new PutCommand(params)); } function buildClient(isLocal: boolean) { From 68743b79dd6ede384e2e5b38c1cab906426ea1a0 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Fri, 13 Sep 2024 15:16:01 -0400 Subject: [PATCH 06/23] experiment --- services/app-api/handlers/measures/get.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/app-api/handlers/measures/get.ts b/services/app-api/handlers/measures/get.ts index f192f73e70..22f9852800 100644 --- a/services/app-api/handlers/measures/get.ts +++ b/services/app-api/handlers/measures/get.ts @@ -35,7 +35,7 @@ export const listMeasures = handler(async (event, context) => { } } // if not state user, can safely assume admin type user due to baseline handler protections const params = { - TableName: process.env.measureTableName!, + TableName: "cmdct-3960-measure", // TODO: change this back to use env variable ...convertToDynamoExpression( { state: state, year: parseInt(year), coreSet: coreSet }, "list" From ca6d60e4e4245e7198d83171e16850a670078ae2 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Fri, 13 Sep 2024 15:47:13 -0400 Subject: [PATCH 07/23] more experimenting --- services/app-api/handlers/measures/get.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/services/app-api/handlers/measures/get.ts b/services/app-api/handlers/measures/get.ts index 22f9852800..b812e4d515 100644 --- a/services/app-api/handlers/measures/get.ts +++ b/services/app-api/handlers/measures/get.ts @@ -40,6 +40,7 @@ export const listMeasures = handler(async (event, context) => { { state: state, year: parseInt(year), coreSet: coreSet }, "list" ), + KeyConditionExpression: "compoundKey = :compoundKey", }; let queriedMeasures = await dynamoDb.queryAll(params); From f762013dcc57553ddb713e6ae8ed75ae8eaf4681 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Fri, 13 Sep 2024 16:44:06 -0400 Subject: [PATCH 08/23] more more experiment --- services/app-api/handlers/measures/get.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/app-api/handlers/measures/get.ts b/services/app-api/handlers/measures/get.ts index b812e4d515..c8f9ed30e0 100644 --- a/services/app-api/handlers/measures/get.ts +++ b/services/app-api/handlers/measures/get.ts @@ -41,6 +41,9 @@ export const listMeasures = handler(async (event, context) => { "list" ), KeyConditionExpression: "compoundKey = :compoundKey", + ExpressionAttributeValues: { + ":compoundKey": `${state}${year}${coreSet}`, + }, }; let queriedMeasures = await dynamoDb.queryAll(params); From f74849b18e91b0a626a0f232329491eb48e4b73e Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Fri, 13 Sep 2024 16:59:42 -0400 Subject: [PATCH 09/23] k --- services/app-api/handlers/measures/get.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/app-api/handlers/measures/get.ts b/services/app-api/handlers/measures/get.ts index c8f9ed30e0..3e43b72ef0 100644 --- a/services/app-api/handlers/measures/get.ts +++ b/services/app-api/handlers/measures/get.ts @@ -36,10 +36,10 @@ export const listMeasures = handler(async (event, context) => { } // if not state user, can safely assume admin type user due to baseline handler protections const params = { TableName: "cmdct-3960-measure", // TODO: change this back to use env variable - ...convertToDynamoExpression( - { state: state, year: parseInt(year), coreSet: coreSet }, - "list" - ), + // ...convertToDynamoExpression( + // { state: state, year: parseInt(year), coreSet: coreSet }, + // "list" + // ), KeyConditionExpression: "compoundKey = :compoundKey", ExpressionAttributeValues: { ":compoundKey": `${state}${year}${coreSet}`, From 614de6939e2491412f3356f08f4116272d9b63dc Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Mon, 16 Sep 2024 10:09:05 -0400 Subject: [PATCH 10/23] trying things with year int converstion --- services/app-api/handlers/measures/get.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/app-api/handlers/measures/get.ts b/services/app-api/handlers/measures/get.ts index 3e43b72ef0..2c25b36ca8 100644 --- a/services/app-api/handlers/measures/get.ts +++ b/services/app-api/handlers/measures/get.ts @@ -42,7 +42,7 @@ export const listMeasures = handler(async (event, context) => { // ), KeyConditionExpression: "compoundKey = :compoundKey", ExpressionAttributeValues: { - ":compoundKey": `${state}${year}${coreSet}`, + ":compoundKey": `${state}${parseInt(year)}${coreSet}`, }, }; From 33734244ae7330e70a002895d26af6f63df851b9 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Mon, 16 Sep 2024 12:22:08 -0400 Subject: [PATCH 11/23] console logs --- services/app-api/handlers/measures/get.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/app-api/handlers/measures/get.ts b/services/app-api/handlers/measures/get.ts index 2c25b36ca8..d48f7e1bc4 100644 --- a/services/app-api/handlers/measures/get.ts +++ b/services/app-api/handlers/measures/get.ts @@ -46,7 +46,9 @@ export const listMeasures = handler(async (event, context) => { }, }; - let queriedMeasures = await dynamoDb.queryAll(params); + console.log("PARAMS", params); + const queriedMeasures = await dynamoDb.queryAll(params); + console.log("QUERIED MEASURES", queriedMeasures); for (let v of queriedMeasures) { const measure = measures[parseInt(year as string)]?.filter( (m) => m.measure === (v as Measure)?.measure From e65e8cdd6286dcdaa2f49261fb8dab871a240cea Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Mon, 16 Sep 2024 13:51:04 -0400 Subject: [PATCH 12/23] updating configs and hoping it just works --- services/app-api/handlers/measures/get.ts | 2 +- services/app-api/serverless.yml | 2 ++ services/database/scripts/transformMeasureTable.ts | 1 + services/uploads/serverless.yml | 4 +++- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/services/app-api/handlers/measures/get.ts b/services/app-api/handlers/measures/get.ts index d48f7e1bc4..64294ca57d 100644 --- a/services/app-api/handlers/measures/get.ts +++ b/services/app-api/handlers/measures/get.ts @@ -35,7 +35,7 @@ export const listMeasures = handler(async (event, context) => { } } // if not state user, can safely assume admin type user due to baseline handler protections const params = { - TableName: "cmdct-3960-measure", // TODO: change this back to use env variable + TableName: process.env.measureTable, // TODO: change this back to use env variable // ...convertToDynamoExpression( // { state: state, year: parseInt(year), coreSet: coreSet }, // "list" diff --git a/services/app-api/serverless.yml b/services/app-api/serverless.yml index ef980255af..7f60f954b9 100644 --- a/services/app-api/serverless.yml +++ b/services/app-api/serverless.yml @@ -46,6 +46,7 @@ custom: coreSetTableName: ${env:coreSetTableName, param:CoreSetTableName} coreSetTableStreamArn: ${env:DYNAMO_TABLE_ARN, param:CoreSetTableStreamArn} measureTableName: ${env:measureTableName, param:MeasureTableName} + measureTable: ${env:measureTable, param:MeasureTable} measureTableStreamArn: ${env:DYNAMO_TABLE_ARN, param:MeasureTableStreamArn} rateTableName: ${env:rateTableName, param:RateTableName} rateTableStreamArn: ${env:DYNAMO_TABLE_ARN, param:RateTableStreamArn} @@ -101,6 +102,7 @@ provider: environment: coreSetTableName: ${self:custom.coreSetTableName} measureTableName: ${self:custom.measureTableName} + measureTable: ${self:custom.measureTable} rateTableName: ${self:custom.rateTableName} bannerTableName: ${self:custom.bannerTableName} stage: ${opt:stage, self:provider.stage} diff --git a/services/database/scripts/transformMeasureTable.ts b/services/database/scripts/transformMeasureTable.ts index 9ff14c81ec..6c56bcf284 100644 --- a/services/database/scripts/transformMeasureTable.ts +++ b/services/database/scripts/transformMeasureTable.ts @@ -33,6 +33,7 @@ async function add(client: DynamoDBDocumentClient, table: string, entry: any) { compoundKey: newCompoundKey, }, }; + console.log("ADDING:", entry); await client.send(new PutCommand(params)); } diff --git a/services/uploads/serverless.yml b/services/uploads/serverless.yml index 0a9f9aff32..15eec41a8c 100644 --- a/services/uploads/serverless.yml +++ b/services/uploads/serverless.yml @@ -17,7 +17,7 @@ provider: region: us-east-1 stackTags: PROJECT: ${self:custom.project} - SERVICE: ${self:service} + SERVICE: ${self:service} iam: role: path: ${ssm:/configuration/${self:custom.stage}/iam/path, ssm:/configuration/default/iam/path, "/"} @@ -53,6 +53,7 @@ custom: iamPermissionsBoundaryPolicy: ${ssm:/configuration/${self:custom.stage}/iam/permissionsBoundaryPolicy, ssm:/configuration/default/iam/permissionsBoundaryPolicy, ""} coreSetTableName: ${env:coreSetTableName, param:CoreSetTableName} measureTableName: ${env:measureTableName, param:MeasureTableName} + measureTable: ${env:measureTable, param:MeasureTable} rateTableName: ${env:rateTableName, param:RateTableName} serverlessTerminationProtection: stages: @@ -91,6 +92,7 @@ functions: environment: coreSetTableName: ${self:custom.coreSetTableName} measureTableName: ${self:custom.measureTableName} + measureTable: ${self:custom.measureTable} rateTableName: ${self:custom.rateTableName} dynamoSnapshotS3BucketName: !Ref DynamoSnapshotBucket avScan: From 843f2a5805862c596b106cf8ba152f93c53bcd3b Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Mon, 16 Sep 2024 13:59:29 -0400 Subject: [PATCH 13/23] fixing config --- serverless-compose.yml | 2 ++ services/database/serverless.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/serverless-compose.yml b/serverless-compose.yml index 105c7a6704..dbd587575b 100644 --- a/serverless-compose.yml +++ b/serverless-compose.yml @@ -11,6 +11,7 @@ services: params: CoreSetTableName: ${database.CoreSetTableName} MeasureTableName: ${database.MeasureTableName} + MeasureTable: ${database.MeasureTable} RateTableName: ${database.RateTableName} app-api: @@ -19,6 +20,7 @@ services: CoreSetTableName: ${database.CoreSetTableName} CoreSetTableStreamArn: ${database.CoreSetTableStreamArn} MeasureTableName: ${database.MeasureTableName} + MeasureTable: ${database.MeasureTable} MeasureTableStreamArn: ${database.MeasureTableStreamArn} RateTableName: ${database.RateTableName} RateTableStreamArn: ${database.RateTableStreamArn} diff --git a/services/database/serverless.yml b/services/database/serverless.yml index 05bdf8e5c1..88b0441cbd 100644 --- a/services/database/serverless.yml +++ b/services/database/serverless.yml @@ -137,6 +137,8 @@ resources: Value: !GetAtt CoreSetTable.StreamArn MeasureTableName: Value: !Ref MeasureTable + MeasureTable: + Value: !Ref CSMeasureTable MeasureTableArn: Value: !GetAtt MeasureTable.Arn MeasureTableStreamArn: From c58a3e12e8c7f7cdd9280869ac3556019541c081 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Mon, 16 Sep 2024 16:41:20 -0400 Subject: [PATCH 14/23] fixing tests --- services/app-api/handlers/measures/tests/get.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/app-api/handlers/measures/tests/get.test.ts b/services/app-api/handlers/measures/tests/get.test.ts index 12ad8df77e..ad52f93c2f 100644 --- a/services/app-api/handlers/measures/tests/get.test.ts +++ b/services/app-api/handlers/measures/tests/get.test.ts @@ -37,6 +37,7 @@ jest.mock("../../dynamoUtils/convertToDynamoExpressionVars", () => ({ const event = { ...testEvent }; process.env.measureTableName = "SAMPLE TABLE"; +process.env.measureTable = "SAMPLE TABLE"; describe("Test Get Measure Handlers", () => { beforeEach(() => { @@ -120,7 +121,10 @@ describe("Test Get Measure Handlers", () => { expect(res.body).toContain("LBW-CH"); expect(dbLib.queryAll).toHaveBeenCalledWith({ TableName: "SAMPLE TABLE", - testValue: "test", + KeyConditionExpression: "compoundKey = :compoundKey", + ExpressionAttributeValues: { + ":compoundKey": "FL2021ACS", + }, }); }); From 65b7bb2e657fd8b2f8419ec81e0394c630d9e6ba Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Tue, 17 Sep 2024 08:31:32 -0400 Subject: [PATCH 15/23] cleaning up script --- services/app-api/handlers/measures/get.ts | 8 +------ .../database/scripts/transformMeasureTable.ts | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/services/app-api/handlers/measures/get.ts b/services/app-api/handlers/measures/get.ts index 64294ca57d..c90f65593b 100644 --- a/services/app-api/handlers/measures/get.ts +++ b/services/app-api/handlers/measures/get.ts @@ -35,20 +35,14 @@ export const listMeasures = handler(async (event, context) => { } } // if not state user, can safely assume admin type user due to baseline handler protections const params = { - TableName: process.env.measureTable, // TODO: change this back to use env variable - // ...convertToDynamoExpression( - // { state: state, year: parseInt(year), coreSet: coreSet }, - // "list" - // ), + TableName: process.env.measureTable, KeyConditionExpression: "compoundKey = :compoundKey", ExpressionAttributeValues: { ":compoundKey": `${state}${parseInt(year)}${coreSet}`, }, }; - console.log("PARAMS", params); const queriedMeasures = await dynamoDb.queryAll(params); - console.log("QUERIED MEASURES", queriedMeasures); for (let v of queriedMeasures) { const measure = measures[parseInt(year as string)]?.filter( (m) => m.measure === (v as Measure)?.measure diff --git a/services/database/scripts/transformMeasureTable.ts b/services/database/scripts/transformMeasureTable.ts index 6c56bcf284..e78b705b79 100644 --- a/services/database/scripts/transformMeasureTable.ts +++ b/services/database/scripts/transformMeasureTable.ts @@ -4,14 +4,22 @@ import { paginateScan, PutCommand, } from "@aws-sdk/lib-dynamodb"; +import prompt from "prompt-sync"; const transformMeasureTable = async () => { - const dbClient = buildClient(true); - const tableName = "local-measures"; - const newTableName = "local-measure"; - console.log(`Processing table ${tableName}`); - for await (let entry of scan(dbClient, tableName)) { - add(dbClient, newTableName, entry); + let stage = "local"; + const isLocal = !!process.env.DYNAMODB_URL; + const dbClient = buildClient(isLocal); + const p = prompt(); + if (!isLocal) { + stage = p("What environment would you like to modify: "); + } + + const oldTable = `${stage}-measures`; + const newTable = `${stage}-measure`; + console.log(`Processing table ${oldTable}`); + for await (let entry of scan(dbClient, oldTable)) { + add(dbClient, newTable, entry); } }; @@ -33,7 +41,6 @@ async function add(client: DynamoDBDocumentClient, table: string, entry: any) { compoundKey: newCompoundKey, }, }; - console.log("ADDING:", entry); await client.send(new PutCommand(params)); } From 3df7020ea16bae872486af05231dcdaf9fa841e4 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Tue, 17 Sep 2024 12:26:12 -0400 Subject: [PATCH 16/23] updating all measure queries to use new measure table --- services/app-api/handlers/coreSets/create.ts | 16 ++++++----- services/app-api/handlers/coreSets/delete.ts | 28 +++++++++++-------- .../handlers/dynamoUtils/createCompoundKey.ts | 4 +-- services/app-api/handlers/measures/create.ts | 4 +-- services/app-api/handlers/measures/delete.ts | 6 ++-- services/app-api/handlers/measures/get.ts | 12 ++++---- services/app-api/handlers/measures/update.ts | 6 ++-- services/app-api/handlers/rate/get.ts | 2 +- .../handlers/rate/rateCalculations.test.ts | 12 ++++---- .../app-api/handlers/rate/rateCalculations.ts | 6 ++-- services/app-api/storage/table.ts | 8 +++--- services/app-api/types.ts | 8 +++++- 12 files changed, 61 insertions(+), 51 deletions(-) diff --git a/services/app-api/handlers/coreSets/create.ts b/services/app-api/handlers/coreSets/create.ts index 519f5ff30f..60dd55ce33 100644 --- a/services/app-api/handlers/coreSets/create.ts +++ b/services/app-api/handlers/coreSets/create.ts @@ -1,7 +1,10 @@ import handler from "../../libs/handler-lib"; import dynamoDb from "../../libs/dynamodb-lib"; import { getCoreSet } from "./get"; -import { createCoreSetKey } from "../dynamoUtils/createCompoundKey"; +import { + createCoreSetKey, + createMeasureKey, +} from "../dynamoUtils/createCompoundKey"; import { MeasureMetaData, measures } from "../dynamoUtils/measureList"; import { hasRolePermissions, @@ -46,7 +49,7 @@ export const createCoreSet = handler(async (event, context) => { await createDependentMeasures( state, - parseInt(year), + year, coreSet as Types.CoreSetAbbr, type ); @@ -91,11 +94,11 @@ export const createCoreSet = handler(async (event, context) => { const createDependentMeasures = async ( state: string, - year: number, + year: string, coreSet: Types.CoreSetAbbr, type: string ) => { - const filteredMeasures = measures[year].filter( + const filteredMeasures = measures[parseInt(year)].filter( (measure: MeasureMetaData) => measure.type === type ); @@ -104,10 +107,9 @@ const createDependentMeasures = async ( for await (const measure of filteredMeasures) { // The State Year and ID are all part of the path const measureId = measure["measure"]; - // Dynamo only accepts one row as a key, so we are using a combination for the dynamoKey - const dynamoKey = `${state}${year}${coreSet}${measureId}`; + const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { - TableName: process.env.measureTableName!, + TableName: process.env.measureTable!, Item: { compoundKey: dynamoKey, state: state, diff --git a/services/app-api/handlers/coreSets/delete.ts b/services/app-api/handlers/coreSets/delete.ts index dbb7fde4e9..4c33cdd1b7 100644 --- a/services/app-api/handlers/coreSets/delete.ts +++ b/services/app-api/handlers/coreSets/delete.ts @@ -1,6 +1,9 @@ import handler from "../../libs/handler-lib"; import dynamoDb from "../../libs/dynamodb-lib"; -import { createCoreSetKey } from "../dynamoUtils/createCompoundKey"; +import { + createCoreSetKey, + createMeasureKey, +} from "../dynamoUtils/createCompoundKey"; import { convertToDynamoExpression } from "../dynamoUtils/convertToDynamoExpressionVars"; import { hasStatePermissions } from "../../libs/authorization"; import { Measure } from "../../types"; @@ -34,7 +37,7 @@ export const deleteCoreSet = handler(async (event, context) => { }; await dynamoDb.delete(params); - await deleteDependentMeasures(state, parseInt(year), coreSet); + await deleteDependentMeasures(state, year, coreSet); return { status: StatusCodes.SUCCESS, @@ -44,16 +47,16 @@ export const deleteCoreSet = handler(async (event, context) => { const deleteDependentMeasures = async ( state: string, - year: number, + year: string, coreSet: string ) => { const measures = await getMeasures(state, year, coreSet); const Items = measures; for await (const { measure } of Items) { - const dynamoKey = `${state}${year}${coreSet}${measure}`; + const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { - TableName: process.env.measureTableName!, + TableName: process.env.measureTable!, Key: { compoundKey: dynamoKey, coreSet: coreSet, @@ -64,14 +67,15 @@ const deleteDependentMeasures = async ( } }; -const getMeasures = async (state: string, year: number, coreSet: string) => { +const getMeasures = async (state: string, year: string, coreSet: string) => { + const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { - TableName: process.env.measureTableName!, - ...convertToDynamoExpression( - { state: state, year: year, coreSet: coreSet }, - "list" - ), + TableName: process.env.measureTable!, + KeyConditionExpression: "compoundKey = :compoundKey", + ExpressionAttributeValues: { + ":compoundKey": dynamoKey, + }, }; - const queryValue = await dynamoDb.scanAll(params); + const queryValue = await dynamoDb.queryAll(params); return queryValue; }; diff --git a/services/app-api/handlers/dynamoUtils/createCompoundKey.ts b/services/app-api/handlers/dynamoUtils/createCompoundKey.ts index 53d5950a4e..3ff05ac94a 100644 --- a/services/app-api/handlers/dynamoUtils/createCompoundKey.ts +++ b/services/app-api/handlers/dynamoUtils/createCompoundKey.ts @@ -1,9 +1,9 @@ import { CoreSetParameters, MeasureParameters } from "../../types"; export const createMeasureKey = (measureParams: MeasureParameters) => { - const { state, year, coreSet, measure } = measureParams; + const { state, year, coreSet } = measureParams; - return `${state}${year}${coreSet}${measure}`; + return `${state}${year}${coreSet}`; }; export const createCoreSetKey = (coreSetParams: CoreSetParameters) => { diff --git a/services/app-api/handlers/measures/create.ts b/services/app-api/handlers/measures/create.ts index 681a14441c..9eee5c1438 100644 --- a/services/app-api/handlers/measures/create.ts +++ b/services/app-api/handlers/measures/create.ts @@ -31,9 +31,9 @@ export const createMeasure = handler(async (event, context) => { } // if not state user, can safely assume admin type user due to baseline handler protections const body = JSON.parse(event!.body!); - const dynamoKey = createMeasureKey({ state, year, coreSet, measure }); + const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { - TableName: process.env.measureTableName!, + TableName: process.env.measureTable!, Item: { compoundKey: dynamoKey, state: state, diff --git a/services/app-api/handlers/measures/delete.ts b/services/app-api/handlers/measures/delete.ts index ecfb2ceea4..865d3ea053 100644 --- a/services/app-api/handlers/measures/delete.ts +++ b/services/app-api/handlers/measures/delete.ts @@ -22,12 +22,12 @@ export const deleteMeasure = handler(async (event, context) => { }; } - const dynamoKey = createMeasureKey({ state, year, coreSet, measure }); + const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { - TableName: process.env.measureTableName!, + TableName: process.env.measureTable!, Key: { compoundKey: dynamoKey, - coreSet: coreSet, + measure: measure, }, }; diff --git a/services/app-api/handlers/measures/get.ts b/services/app-api/handlers/measures/get.ts index c90f65593b..43ff3e1037 100644 --- a/services/app-api/handlers/measures/get.ts +++ b/services/app-api/handlers/measures/get.ts @@ -1,6 +1,5 @@ import handler from "../../libs/handler-lib"; import dynamoDb from "../../libs/dynamodb-lib"; -import { convertToDynamoExpression } from "../dynamoUtils/convertToDynamoExpressionVars"; import { createMeasureKey } from "../dynamoUtils/createCompoundKey"; import { measures } from "../dynamoUtils/measureList"; import { @@ -34,11 +33,12 @@ export const listMeasures = handler(async (event, context) => { }; } } // if not state user, can safely assume admin type user due to baseline handler protections + const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { - TableName: process.env.measureTable, + TableName: process.env.measureTable!, KeyConditionExpression: "compoundKey = :compoundKey", ExpressionAttributeValues: { - ":compoundKey": `${state}${parseInt(year)}${coreSet}`, + ":compoundKey": dynamoKey, }, }; @@ -81,12 +81,12 @@ export const getMeasure = handler(async (event, context) => { } } // if not state user, can safely assume admin type user due to baseline handler protections - const dynamoKey = createMeasureKey({ state, year, coreSet, measure }); + const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { - TableName: process.env.measureTableName!, + TableName: process.env.measureTable!, Key: { compoundKey: dynamoKey, - coreSet: coreSet, + measure: measure, }, }; const queryValue = await dynamoDb.get(params); diff --git a/services/app-api/handlers/measures/update.ts b/services/app-api/handlers/measures/update.ts index f04fb39327..c258682b6a 100644 --- a/services/app-api/handlers/measures/update.ts +++ b/services/app-api/handlers/measures/update.ts @@ -35,7 +35,7 @@ export const editMeasure = handler(async (event, context) => { detailedDescription, } = JSON.parse(event!.body!); - const dynamoKey = createMeasureKey({ state, year, coreSet, measure }); + const dynamoKey = createMeasureKey({ state, year, coreSet }); const lastAlteredBy = getUserNameFromJwt(event); const descriptionParams = @@ -46,10 +46,10 @@ export const editMeasure = handler(async (event, context) => { } : {}; const params = { - TableName: process.env.measureTableName!, + TableName: process.env.measureTable!, Key: { compoundKey: dynamoKey, - coreSet: coreSet, + measure: measure, }, ...convertToDynamoExpression( { diff --git a/services/app-api/handlers/rate/get.ts b/services/app-api/handlers/rate/get.ts index cf8a3c908e..9f9e1a9deb 100644 --- a/services/app-api/handlers/rate/get.ts +++ b/services/app-api/handlers/rate/get.ts @@ -4,7 +4,7 @@ import { } from "../../libs/authorization"; import handler from "../../libs/handler-lib"; import { getCombinedRatesFromTable } from "../../storage/table"; -import { MeasureParameters, UserRoles } from "../../types"; +import { UserRoles } from "../../types"; import { Errors, StatusCodes } from "../../utils/constants/constants"; import { parseMeasureParameters } from "../../utils/parseParameters"; diff --git a/services/app-api/handlers/rate/rateCalculations.test.ts b/services/app-api/handlers/rate/rateCalculations.test.ts index ff3124e9bc..3a6b5fe0f0 100644 --- a/services/app-api/handlers/rate/rateCalculations.test.ts +++ b/services/app-api/handlers/rate/rateCalculations.test.ts @@ -2,7 +2,7 @@ import { getMeasureFromTable, putCombinedRatesToTable, } from "../../storage/table"; -import { MeasureParameters } from "../../types"; +import { RateParameters } from "../../types"; import { calculateAndPutRate } from "./rateCalculations"; jest.mock("../../storage/table", () => ({ @@ -21,15 +21,15 @@ describe("Combined Rate Calculations", () => { }); it("Should not calculate a combined rate for core sets that are already combined", async () => { - await calculateAndPutRate({ coreSet: "ACS" } as MeasureParameters); - await calculateAndPutRate({ coreSet: "CCS" } as MeasureParameters); - await calculateAndPutRate({ coreSet: "HHCS" } as MeasureParameters); + await calculateAndPutRate({ coreSet: "ACS" } as RateParameters); + await calculateAndPutRate({ coreSet: "CCS" } as RateParameters); + await calculateAndPutRate({ coreSet: "HHCS" } as RateParameters); expect(getMeasureFromTable).not.toHaveBeenCalled(); }); it("Should calculate combined rates for separated child core sets", async () => { - await calculateAndPutRate({ coreSet: "CCSM" } as MeasureParameters); + await calculateAndPutRate({ coreSet: "CCSM" } as RateParameters); expect(getMeasureFromTable).toHaveBeenCalledWith( expect.objectContaining({ coreSet: "CCSM" }) @@ -44,7 +44,7 @@ describe("Combined Rate Calculations", () => { }); it("Should calculate combined rates for separated asult core sets", async () => { - await calculateAndPutRate({ coreSet: "ACSC" } as MeasureParameters); + await calculateAndPutRate({ coreSet: "ACSC" } as RateParameters); expect(getMeasureFromTable).toHaveBeenCalledWith( expect.objectContaining({ coreSet: "ACSM" }) diff --git a/services/app-api/handlers/rate/rateCalculations.ts b/services/app-api/handlers/rate/rateCalculations.ts index 8af9687438..99f2d4fa75 100644 --- a/services/app-api/handlers/rate/rateCalculations.ts +++ b/services/app-api/handlers/rate/rateCalculations.ts @@ -6,7 +6,7 @@ import { CoreSetAbbr, isCoreSetAbbr, Measure, - MeasureParameters, + RateParameters, } from "../../types"; import { collectDataSources } from "./dataSourceAnalysis"; import { combineRates } from "./rateNDRCalculations"; @@ -19,9 +19,7 @@ import { calculateAdditionalValues } from "./rateValueCalculations"; * 3. Perform all calculations * 4. Store the results to the Rates table */ -export const calculateAndPutRate = async ( - pathParameters: MeasureParameters -) => { +export const calculateAndPutRate = async (pathParameters: RateParameters) => { const { coreSet, measure } = pathParameters; if (!isCoreSetAbbr(coreSet)) { return; diff --git a/services/app-api/storage/table.ts b/services/app-api/storage/table.ts index 8d4af36e1a..9c50c50fb0 100644 --- a/services/app-api/storage/table.ts +++ b/services/app-api/storage/table.ts @@ -1,8 +1,8 @@ import dynamoDb from "../libs/dynamodb-lib"; import * as Types from "../types"; -import { MeasureParameters, CombinedRatesPayload } from "../types"; +import { RateParameters, CombinedRatesPayload } from "../types"; -export const getMeasureFromTable = async (parameters: MeasureParameters) => { +export const getMeasureFromTable = async (parameters: RateParameters) => { const { state, year, coreSet, measure } = parameters; return await dynamoDb.get({ TableName: process.env.measureTableName, @@ -14,7 +14,7 @@ export const getMeasureFromTable = async (parameters: MeasureParameters) => { }; export const putCombinedRatesToTable = async ( - parameters: MeasureParameters, + parameters: RateParameters, combinedRates: CombinedRatesPayload ) => { const { year, state, coreSet, measure } = parameters; @@ -39,7 +39,7 @@ export const putCombinedRatesToTable = async ( }; export const getCombinedRatesFromTable = async ( - parameters: MeasureParameters + parameters: RateParameters ): Promise => { const { year, state, coreSet, measure } = parameters; const queryValue = await dynamoDb.get({ diff --git a/services/app-api/types.ts b/services/app-api/types.ts index 0f25b4af04..116669cab3 100644 --- a/services/app-api/types.ts +++ b/services/app-api/types.ts @@ -172,13 +172,19 @@ export enum MeasurementSpecificationType { TheJointCommission = "TheJointCommission", } -export interface MeasureParameters { +export interface RateParameters { state: string; year: string; coreSet: string; measure: string; } +export interface MeasureParameters { + state: string; + year: string; + coreSet: string; +} + export interface CoreSetParameters { state: string; year: string; From 5e2020ea561b3c3afdf27a13e326789289241506 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Tue, 17 Sep 2024 14:44:23 -0400 Subject: [PATCH 17/23] CMDCT-3960: fixing tests, making year an int from the start --- services/app-api/handlers/coreSets/create.ts | 18 +++++---------- services/app-api/handlers/coreSets/delete.ts | 18 ++++----------- services/app-api/handlers/coreSets/get.ts | 6 ++--- .../handlers/coreSets/tests/create.test.ts | 5 ---- .../handlers/coreSets/tests/delete.test.ts | 17 +++++--------- .../handlers/coreSets/tests/get.test.ts | 12 ---------- .../handlers/coreSets/tests/update.test.ts | 12 ++-------- services/app-api/handlers/coreSets/update.ts | 4 +--- .../handlers/dynamoUtils/createCompoundKey.ts | 13 ----------- .../tests/createCompoundKey.test.ts | 20 ---------------- services/app-api/handlers/measures/create.ts | 6 ++--- services/app-api/handlers/measures/delete.ts | 4 +--- services/app-api/handlers/measures/get.ts | 9 +++----- .../handlers/measures/tests/create.test.ts | 9 ++------ .../handlers/measures/tests/delete.test.ts | 18 ++++----------- .../handlers/measures/tests/get.test.ts | 9 ++------ .../handlers/measures/tests/update.test.ts | 23 ++++++++----------- services/app-api/handlers/measures/update.ts | 8 +++---- services/app-api/utils/parseParameters.ts | 17 +++++++++++--- 19 files changed, 63 insertions(+), 165 deletions(-) delete mode 100644 services/app-api/handlers/dynamoUtils/createCompoundKey.ts delete mode 100644 services/app-api/handlers/dynamoUtils/tests/createCompoundKey.test.ts diff --git a/services/app-api/handlers/coreSets/create.ts b/services/app-api/handlers/coreSets/create.ts index 60dd55ce33..6972a5481a 100644 --- a/services/app-api/handlers/coreSets/create.ts +++ b/services/app-api/handlers/coreSets/create.ts @@ -1,10 +1,6 @@ import handler from "../../libs/handler-lib"; import dynamoDb from "../../libs/dynamodb-lib"; import { getCoreSet } from "./get"; -import { - createCoreSetKey, - createMeasureKey, -} from "../dynamoUtils/createCompoundKey"; import { MeasureMetaData, measures } from "../dynamoUtils/measureList"; import { hasRolePermissions, @@ -45,7 +41,6 @@ export const createCoreSet = handler(async (event, context) => { body: Errors.CORESET_ALREADY_EXISTS, }; } - const dynamoKey = createCoreSetKey({ state, year, coreSet }); await createDependentMeasures( state, @@ -56,7 +51,7 @@ export const createCoreSet = handler(async (event, context) => { // filter out qualifier and account for autocomplete measures on creation let autoCompletedMeasures = 0; - const measuresLengthWithoutQualifiers = measures[parseInt(year)].filter( + const measuresLengthWithoutQualifiers = measures[year].filter( (measure: MeasureMetaData) => { if (measure.autocompleteOnCreation && measure.type === type) { autoCompletedMeasures++; @@ -73,9 +68,9 @@ export const createCoreSet = handler(async (event, context) => { const params = { TableName: process.env.coreSetTableName!, Item: { - compoundKey: dynamoKey, + compoundKey: `${state}${year}${coreSet}`, state: state, - year: parseInt(year), + year: year, coreSet: coreSet, createdAt: Date.now(), lastAltered: Date.now(), @@ -94,11 +89,11 @@ export const createCoreSet = handler(async (event, context) => { const createDependentMeasures = async ( state: string, - year: string, + year: number, coreSet: Types.CoreSetAbbr, type: string ) => { - const filteredMeasures = measures[parseInt(year)].filter( + const filteredMeasures = measures[year].filter( (measure: MeasureMetaData) => measure.type === type ); @@ -107,11 +102,10 @@ const createDependentMeasures = async ( for await (const measure of filteredMeasures) { // The State Year and ID are all part of the path const measureId = measure["measure"]; - const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { TableName: process.env.measureTable!, Item: { - compoundKey: dynamoKey, + compoundKey: `${state}${year}${coreSet}`, state: state, year: year, coreSet: coreSet, diff --git a/services/app-api/handlers/coreSets/delete.ts b/services/app-api/handlers/coreSets/delete.ts index 4c33cdd1b7..2632a37351 100644 --- a/services/app-api/handlers/coreSets/delete.ts +++ b/services/app-api/handlers/coreSets/delete.ts @@ -1,10 +1,5 @@ import handler from "../../libs/handler-lib"; import dynamoDb from "../../libs/dynamodb-lib"; -import { - createCoreSetKey, - createMeasureKey, -} from "../dynamoUtils/createCompoundKey"; -import { convertToDynamoExpression } from "../dynamoUtils/convertToDynamoExpressionVars"; import { hasStatePermissions } from "../../libs/authorization"; import { Measure } from "../../types"; import { Errors, StatusCodes } from "../../utils/constants/constants"; @@ -27,11 +22,10 @@ export const deleteCoreSet = handler(async (event, context) => { }; } - const dynamoKey = createCoreSetKey({ state, year, coreSet }); const params = { TableName: process.env.coreSetTableName!, Key: { - compoundKey: dynamoKey, + compoundKey: `${state}${year}${coreSet}`, coreSet: coreSet, }, }; @@ -47,18 +41,17 @@ export const deleteCoreSet = handler(async (event, context) => { const deleteDependentMeasures = async ( state: string, - year: string, + year: number, coreSet: string ) => { const measures = await getMeasures(state, year, coreSet); const Items = measures; for await (const { measure } of Items) { - const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { TableName: process.env.measureTable!, Key: { - compoundKey: dynamoKey, + compoundKey: `${state}${year}${coreSet}`, coreSet: coreSet, }, }; @@ -67,13 +60,12 @@ const deleteDependentMeasures = async ( } }; -const getMeasures = async (state: string, year: string, coreSet: string) => { - const dynamoKey = createMeasureKey({ state, year, coreSet }); +const getMeasures = async (state: string, year: number, coreSet: string) => { const params = { TableName: process.env.measureTable!, KeyConditionExpression: "compoundKey = :compoundKey", ExpressionAttributeValues: { - ":compoundKey": dynamoKey, + ":compoundKey": `${state}${year}${coreSet}`, }, }; const queryValue = await dynamoDb.queryAll(params); diff --git a/services/app-api/handlers/coreSets/get.ts b/services/app-api/handlers/coreSets/get.ts index 379555c7aa..203b0e8d59 100644 --- a/services/app-api/handlers/coreSets/get.ts +++ b/services/app-api/handlers/coreSets/get.ts @@ -2,7 +2,6 @@ import handler from "../../libs/handler-lib"; import dynamoDb from "../../libs/dynamodb-lib"; import { updateCoreSetProgress } from "../../libs/updateCoreProgress"; import { convertToDynamoExpression } from "../dynamoUtils/convertToDynamoExpressionVars"; -import { createCoreSetKey } from "../dynamoUtils/createCompoundKey"; import { createCoreSet } from "./create"; import { hasRolePermissions, @@ -41,7 +40,7 @@ export const coreSetList = handler(async (event, context) => { ...convertToDynamoExpression( { state: state, - year: parseInt(year), + year: year, }, "list" ), @@ -121,11 +120,10 @@ export const getCoreSet = handler(async (event, context) => { } } // if not state user, can safely assume admin type user due to baseline handler protections - const dynamoKey = createCoreSetKey({ state, year, coreSet }); const params = { TableName: process.env.coreSetTableName!, Key: { - compoundKey: dynamoKey, + compoundKey: `${state}${year}${coreSet}`, coreSet: coreSet, }, }; diff --git a/services/app-api/handlers/coreSets/tests/create.test.ts b/services/app-api/handlers/coreSets/tests/create.test.ts index e9b62d56a4..62b04f29f7 100644 --- a/services/app-api/handlers/coreSets/tests/create.test.ts +++ b/services/app-api/handlers/coreSets/tests/create.test.ts @@ -23,11 +23,6 @@ jest.mock("../../../libs/authorization", () => ({ hasStatePermissions: () => mockHasStatePermissions(), })); -jest.mock("../../dynamoUtils/createCompoundKey", () => ({ - __esModule: true, - createCoreSetKey: jest.fn().mockReturnValue("FL2021ACSFUA-AD"), -})); - jest.mock("../get", () => ({ __esModule: true, getCoreSet: jest.fn(), diff --git a/services/app-api/handlers/coreSets/tests/delete.test.ts b/services/app-api/handlers/coreSets/tests/delete.test.ts index b34f9b8810..3b0130f9bc 100644 --- a/services/app-api/handlers/coreSets/tests/delete.test.ts +++ b/services/app-api/handlers/coreSets/tests/delete.test.ts @@ -6,7 +6,7 @@ import { CoreSetAbbr } from "../../../types"; jest.mock("../../../libs/dynamodb-lib", () => ({ delete: jest.fn(), - scanAll: jest.fn(), + queryAll: jest.fn(), })); const mockHasStatePermissions = jest.fn(); @@ -15,11 +15,6 @@ jest.mock("../../../libs/authorization", () => ({ hasStatePermissions: () => mockHasStatePermissions(), })); -jest.mock("../../dynamoUtils/createCompoundKey", () => ({ - __esModule: true, - createCoreSetKey: jest.fn().mockReturnValue("FL2020ACSFUA-AD"), -})); - jest.mock("../../../libs/updateCoreProgress", () => ({ __esModule: true, updateCoreSetProgress: jest.fn(), @@ -29,7 +24,7 @@ const event = { ...testEvent }; describe("Testing Delete Core Set Functions", () => { beforeEach(() => { - (db.scanAll as jest.Mock).mockReset(); + (db.queryAll as jest.Mock).mockReset(); (db.delete as jest.Mock).mockReset(); mockHasStatePermissions.mockImplementation(() => true); event.pathParameters = { @@ -67,7 +62,7 @@ describe("Testing Delete Core Set Functions", () => { }); test("Test deleteCoreSet with associated measures", async () => { - (db.scanAll as jest.Mock).mockReturnValue([ + (db.queryAll as jest.Mock).mockReturnValue([ testMeasure, testMeasure, testMeasure, @@ -75,16 +70,16 @@ describe("Testing Delete Core Set Functions", () => { await deleteCoreSet(event, null); - expect(db.scanAll).toHaveBeenCalled(); + expect(db.queryAll).toHaveBeenCalled(); expect(db.delete).toHaveBeenCalledTimes(4); }); test("Test deleteCoreSet with no associated measures", async () => { - (db.scanAll as jest.Mock).mockReturnValue([]); + (db.queryAll as jest.Mock).mockReturnValue([]); await deleteCoreSet(event, null); - expect(db.scanAll).toHaveBeenCalled(); + expect(db.queryAll).toHaveBeenCalled(); expect(db.delete).toHaveBeenCalled(); }); }); diff --git a/services/app-api/handlers/coreSets/tests/get.test.ts b/services/app-api/handlers/coreSets/tests/get.test.ts index 6479f39f92..9f2df1f8c3 100644 --- a/services/app-api/handlers/coreSets/tests/get.test.ts +++ b/services/app-api/handlers/coreSets/tests/get.test.ts @@ -4,7 +4,6 @@ import { testEvent } from "../../../test-util/testEvents"; import dynamodbLib from "../../../libs/dynamodb-lib"; import { updateCoreSetProgress } from "../../../libs/updateCoreProgress"; -import { createCoreSetKey } from "../../dynamoUtils/createCompoundKey"; import { CoreSetAbbr } from "../../../types"; import { createCoreSet } from "../create"; import { Errors, StatusCodes } from "../../../utils/constants/constants"; @@ -35,11 +34,6 @@ jest.mock("../../../libs/updateCoreProgress", () => ({ updateCoreSetProgress: jest.fn(), })); -jest.mock("../../dynamoUtils/createCompoundKey", () => ({ - __esModule: true, - createCoreSetKey: jest.fn().mockReturnValue("FL2020ACSFUA-AD"), -})); - jest.mock("../../dynamoUtils/convertToDynamoExpressionVars", () => ({ __esModule: true, convertToDynamoExpression: jest.fn().mockReturnValue({ testValue: "test" }), @@ -96,12 +90,6 @@ describe("Test Get Core Set Functions", () => { await getCoreSet(event, null); expect(dynamodbLib.get).toHaveBeenCalled(); - expect(createCoreSetKey).toHaveBeenCalled(); - expect(createCoreSetKey).toHaveBeenCalledWith({ - state: "AL", - year: "2021", - coreSet: "ACS", - }); }); test("Test coreSetList unauthorized user attempt (incorrect state)", async () => { diff --git a/services/app-api/handlers/coreSets/tests/update.test.ts b/services/app-api/handlers/coreSets/tests/update.test.ts index d4638cd7bd..f2fe786ad4 100644 --- a/services/app-api/handlers/coreSets/tests/update.test.ts +++ b/services/app-api/handlers/coreSets/tests/update.test.ts @@ -1,7 +1,6 @@ import dynamodbLib from "../../../libs/dynamodb-lib"; import { testEvent } from "../../../test-util/testEvents"; import { convertToDynamoExpression } from "../../dynamoUtils/convertToDynamoExpressionVars"; -import { createCoreSetKey } from "../../dynamoUtils/createCompoundKey"; import { editCoreSet } from "../update"; import { Errors, StatusCodes } from "../../../utils/constants/constants"; import { CoreSetAbbr } from "../../../types"; @@ -17,11 +16,6 @@ jest.mock("../../../libs/authorization", () => ({ hasStatePermissions: () => mockHasStatePermissions(), })); -jest.mock("../../dynamoUtils/createCompoundKey", () => ({ - __esModule: true, - createCoreSetKey: jest.fn().mockReturnValue("FL2020ACSFUA-AD"), -})); - jest.mock("../../dynamoUtils/convertToDynamoExpressionVars", () => ({ __esModule: true, convertToDynamoExpression: jest.fn().mockReturnValue({ testValue: "test" }), @@ -73,7 +67,6 @@ describe("Testing Updating Core Set Functions", () => { Date.now = jest.fn(() => 20); const res = await editCoreSet(event, null); - expect(createCoreSetKey).toHaveBeenCalled(); expect(dynamodbLib.update).toHaveBeenCalled(); expect(convertToDynamoExpression).toHaveBeenCalledWith( { @@ -84,14 +77,13 @@ describe("Testing Updating Core Set Functions", () => { "post" ); expect(res.statusCode).toBe(StatusCodes.SUCCESS); - expect(res.body).toContain("FL2020ACSFUA-AD"); + expect(res.body).toContain("WA2021ACS"); }); test("Test with no user id", async () => { Date.now = jest.fn(() => 20); const res = await editCoreSet(event, null); - expect(createCoreSetKey).toHaveBeenCalled(); expect(dynamodbLib.update).toHaveBeenCalled(); expect(convertToDynamoExpression).toHaveBeenCalledWith( { @@ -102,6 +94,6 @@ describe("Testing Updating Core Set Functions", () => { "post" ); expect(res.statusCode).toBe(StatusCodes.SUCCESS); - expect(res.body).toContain("FL2020ACSFUA-AD"); + expect(res.body).toContain("WA2021ACS"); }); }); diff --git a/services/app-api/handlers/coreSets/update.ts b/services/app-api/handlers/coreSets/update.ts index a5e8c456f0..85b341c900 100644 --- a/services/app-api/handlers/coreSets/update.ts +++ b/services/app-api/handlers/coreSets/update.ts @@ -1,7 +1,6 @@ import handler from "../../libs/handler-lib"; import dynamoDb from "../../libs/dynamodb-lib"; import { convertToDynamoExpression } from "../dynamoUtils/convertToDynamoExpressionVars"; -import { createCoreSetKey } from "../dynamoUtils/createCompoundKey"; import { getUserNameFromJwt, hasStatePermissions, @@ -27,12 +26,11 @@ export const editCoreSet = handler(async (event, context) => { } const { submitted, status } = JSON.parse(event!.body!); - const dynamoKey = createCoreSetKey({ state, year, coreSet }); const lastAlteredBy = getUserNameFromJwt(event); const params = { TableName: process.env.coreSetTableName!, Key: { - compoundKey: dynamoKey, + compoundKey: `${state}${year}${coreSet}`, coreSet: coreSet, }, ...convertToDynamoExpression( diff --git a/services/app-api/handlers/dynamoUtils/createCompoundKey.ts b/services/app-api/handlers/dynamoUtils/createCompoundKey.ts deleted file mode 100644 index 3ff05ac94a..0000000000 --- a/services/app-api/handlers/dynamoUtils/createCompoundKey.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { CoreSetParameters, MeasureParameters } from "../../types"; - -export const createMeasureKey = (measureParams: MeasureParameters) => { - const { state, year, coreSet } = measureParams; - - return `${state}${year}${coreSet}`; -}; - -export const createCoreSetKey = (coreSetParams: CoreSetParameters) => { - const { state, year, coreSet } = coreSetParams; - - return `${state}${year}${coreSet}`; -}; diff --git a/services/app-api/handlers/dynamoUtils/tests/createCompoundKey.test.ts b/services/app-api/handlers/dynamoUtils/tests/createCompoundKey.test.ts deleted file mode 100644 index b120f2a8e4..0000000000 --- a/services/app-api/handlers/dynamoUtils/tests/createCompoundKey.test.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { createCoreSetKey, createMeasureKey } from "../createCompoundKey"; - -describe("Testing createCoreSetKey", () => { - test("Successful key creation", () => { - const key = createCoreSetKey({ year: "2022", state: "FL", coreSet: "ACS" }); - expect(key).toEqual("FL2022ACS"); - }); -}); - -describe("Testing createMeasureKey", () => { - test("Successful key creation", () => { - const key = createMeasureKey({ - year: "2022", - state: "FL", - coreSet: "ACS", - measure: "FUA-AD", - }); - expect(key).toEqual("FL2022ACSFUA-AD"); - }); -}); diff --git a/services/app-api/handlers/measures/create.ts b/services/app-api/handlers/measures/create.ts index 9eee5c1438..ac63624668 100644 --- a/services/app-api/handlers/measures/create.ts +++ b/services/app-api/handlers/measures/create.ts @@ -1,6 +1,5 @@ import handler from "../../libs/handler-lib"; import dynamoDb from "../../libs/dynamodb-lib"; -import { createMeasureKey } from "../dynamoUtils/createCompoundKey"; import { hasRolePermissions, hasStatePermissions, @@ -31,13 +30,12 @@ export const createMeasure = handler(async (event, context) => { } // if not state user, can safely assume admin type user due to baseline handler protections const body = JSON.parse(event!.body!); - const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { TableName: process.env.measureTable!, Item: { - compoundKey: dynamoKey, + compoundKey: `${state}${year}${coreSet}`, state: state, - year: parseInt(year), + year: year, coreSet: coreSet, measure: measure, createdAt: Date.now(), diff --git a/services/app-api/handlers/measures/delete.ts b/services/app-api/handlers/measures/delete.ts index 865d3ea053..d8fbda0219 100644 --- a/services/app-api/handlers/measures/delete.ts +++ b/services/app-api/handlers/measures/delete.ts @@ -1,6 +1,5 @@ import handler from "../../libs/handler-lib"; import dynamoDb from "../../libs/dynamodb-lib"; -import { createMeasureKey } from "../dynamoUtils/createCompoundKey"; import { hasStatePermissions } from "../../libs/authorization"; import { Errors, StatusCodes } from "../../utils/constants/constants"; import { parseMeasureParameters } from "../../utils/parseParameters"; @@ -22,11 +21,10 @@ export const deleteMeasure = handler(async (event, context) => { }; } - const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { TableName: process.env.measureTable!, Key: { - compoundKey: dynamoKey, + compoundKey: `${state}${year}${coreSet}`, measure: measure, }, }; diff --git a/services/app-api/handlers/measures/get.ts b/services/app-api/handlers/measures/get.ts index 43ff3e1037..a8325a0a0b 100644 --- a/services/app-api/handlers/measures/get.ts +++ b/services/app-api/handlers/measures/get.ts @@ -1,6 +1,5 @@ import handler from "../../libs/handler-lib"; import dynamoDb from "../../libs/dynamodb-lib"; -import { createMeasureKey } from "../dynamoUtils/createCompoundKey"; import { measures } from "../dynamoUtils/measureList"; import { hasRolePermissions, @@ -33,18 +32,17 @@ export const listMeasures = handler(async (event, context) => { }; } } // if not state user, can safely assume admin type user due to baseline handler protections - const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { TableName: process.env.measureTable!, KeyConditionExpression: "compoundKey = :compoundKey", ExpressionAttributeValues: { - ":compoundKey": dynamoKey, + ":compoundKey": `${state}${year}${coreSet}`, }, }; const queriedMeasures = await dynamoDb.queryAll(params); for (let v of queriedMeasures) { - const measure = measures[parseInt(year as string)]?.filter( + const measure = measures[year as number]?.filter( (m) => m.measure === (v as Measure)?.measure )[0]; @@ -81,11 +79,10 @@ export const getMeasure = handler(async (event, context) => { } } // if not state user, can safely assume admin type user due to baseline handler protections - const dynamoKey = createMeasureKey({ state, year, coreSet }); const params = { TableName: process.env.measureTable!, Key: { - compoundKey: dynamoKey, + compoundKey: `${state}${year}${coreSet}`, measure: measure, }, }; diff --git a/services/app-api/handlers/measures/tests/create.test.ts b/services/app-api/handlers/measures/tests/create.test.ts index 21dbee992e..722fa49377 100644 --- a/services/app-api/handlers/measures/tests/create.test.ts +++ b/services/app-api/handlers/measures/tests/create.test.ts @@ -14,11 +14,6 @@ jest.mock("../../../libs/authorization", () => ({ hasStatePermissions: () => mockHasStatePermissions(), })); -jest.mock("../../dynamoUtils/createCompoundKey", () => ({ - __esModule: true, - createMeasureKey: jest.fn().mockReturnValue("FL2020ACSFUA-AD"), -})); - const event = { ...testEvent }; process.env.measureTableName = "SAMPLE TABLE"; @@ -52,7 +47,7 @@ describe("Test Create Measure Handler", () => { expect(res.statusCode).toBe(StatusCodes.SUCCESS); expect(res.body).toContain("sample desc"); expect(res.body).toContain("sample detailed desc"); - expect(res.body).toContain("FL2020ACSFUA-AD"); + expect(res.body).toContain("IN2022ACS"); }); test("Test Successful Run of Measure Creation without description", async () => { @@ -62,7 +57,7 @@ describe("Test Create Measure Handler", () => { expect(res.statusCode).toBe(StatusCodes.SUCCESS); expect(res.body).toContain("test"); - expect(res.body).toContain("FL2020ACSFUA-AD"); + expect(res.body).toContain("IN2022ACS"); }); test("Fails with bad request when path params are missing", async () => { diff --git a/services/app-api/handlers/measures/tests/delete.test.ts b/services/app-api/handlers/measures/tests/delete.test.ts index fc7e766080..b0886fdfde 100644 --- a/services/app-api/handlers/measures/tests/delete.test.ts +++ b/services/app-api/handlers/measures/tests/delete.test.ts @@ -1,8 +1,5 @@ import { deleteMeasure } from "../delete"; - import dbLib from "../../../libs/dynamodb-lib"; - -import { APIGatewayProxyEvent } from "../../../types"; import { testEvent } from "../../../test-util/testEvents"; import { StatusCodes, Errors } from "../../../utils/constants/constants"; @@ -16,13 +13,8 @@ jest.mock("../../../libs/authorization", () => ({ hasStatePermissions: () => mockHasStatePermissions(), })); -jest.mock("../../dynamoUtils/createCompoundKey", () => ({ - __esModule: true, - createMeasureKey: jest.fn().mockReturnValue("FL2020ACSFUA-AD"), -})); - const event = { ...testEvent }; -process.env.measureTableName = "SAMPLE TABLE"; +process.env.measureTable = "SAMPLE TABLE"; describe("Test Delete Measure Handler", () => { beforeEach(() => { @@ -69,13 +61,13 @@ describe("Test Delete Measure Handler", () => { const res = await deleteMeasure(event, null); expect(res.statusCode).toBe(StatusCodes.SUCCESS); - expect(res.body).toContain("FL2020ACSFUA-AD"); - expect(res.body).toContain('"coreSet":"ACS"'); + expect(res.body).toContain("IN2022ACS"); + expect(res.body).toContain('"measure":"AAB-AD"'); expect(dbLib.delete).toHaveBeenCalledWith({ TableName: "SAMPLE TABLE", Key: { - compoundKey: "FL2020ACSFUA-AD", - coreSet: "ACS", + compoundKey: "IN2022ACS", + measure: "AAB-AD", }, }); }); diff --git a/services/app-api/handlers/measures/tests/get.test.ts b/services/app-api/handlers/measures/tests/get.test.ts index ad52f93c2f..a3b7c16769 100644 --- a/services/app-api/handlers/measures/tests/get.test.ts +++ b/services/app-api/handlers/measures/tests/get.test.ts @@ -25,11 +25,6 @@ jest.mock("../../../libs/authorization", () => ({ hasStatePermissions: () => mockHasStatePermissions(), })); -jest.mock("../../dynamoUtils/createCompoundKey", () => ({ - __esModule: true, - createMeasureKey: jest.fn().mockReturnValue("FL2020ACSFUA-AD"), -})); - jest.mock("../../dynamoUtils/convertToDynamoExpressionVars", () => ({ __esModule: true, convertToDynamoExpression: jest.fn().mockReturnValue({ testValue: "test" }), @@ -68,8 +63,8 @@ describe("Test Get Measure Handlers", () => { expect(dbLib.get).toHaveBeenCalledWith({ TableName: "SAMPLE TABLE", Key: { - compoundKey: "FL2020ACSFUA-AD", - coreSet: "ACS", + compoundKey: "IN2022ACS", + measure: "AAB-AD", }, }); }); diff --git a/services/app-api/handlers/measures/tests/update.test.ts b/services/app-api/handlers/measures/tests/update.test.ts index 5c3123c4d9..5f2cea2817 100644 --- a/services/app-api/handlers/measures/tests/update.test.ts +++ b/services/app-api/handlers/measures/tests/update.test.ts @@ -18,18 +18,13 @@ jest.mock("../../../libs/authorization", () => ({ hasStatePermissions: () => mockHasStatePermissions(), })); -jest.mock("../../dynamoUtils/createCompoundKey", () => ({ - __esModule: true, - createMeasureKey: jest.fn().mockReturnValue("FL2020ACSFUA-AD"), -})); - jest.mock("../../dynamoUtils/convertToDynamoExpressionVars", () => ({ __esModule: true, convertToDynamoExpression: jest.fn().mockReturnValue({ testValue: "test" }), })); const event = { ...testEvent }; -process.env.measureTableName = "SAMPLE TABLE"; +process.env.measureTable = "SAMPLE TABLE"; describe("Test Update Measure Handler", () => { beforeEach(() => { @@ -78,8 +73,8 @@ describe("Test Update Measure Handler", () => { const res = await editMeasure(event, null); expect(res.statusCode).toBe(StatusCodes.SUCCESS); - expect(res.body).toContain("FL2020ACSFUA-AD"); - expect(res.body).toContain('"coreSet":"ACS"'); + expect(res.body).toContain("IN2022ACS"); + expect(res.body).toContain('"measure":"AAB-AD"'); expect(convertToDynamoExpression).toHaveBeenCalledWith( { status: "status", @@ -93,8 +88,8 @@ describe("Test Update Measure Handler", () => { expect(dbLib.update).toHaveBeenCalledWith({ TableName: "SAMPLE TABLE", Key: { - compoundKey: "FL2020ACSFUA-AD", - coreSet: "ACS", + compoundKey: "IN2022ACS", + measure: "AAB-AD", }, testValue: "test", }); @@ -107,8 +102,8 @@ describe("Test Update Measure Handler", () => { const res = await editMeasure(event, null); expect(res.statusCode).toBe(StatusCodes.SUCCESS); - expect(res.body).toContain("FL2020ACSFUA-AD"); - expect(res.body).toContain('"coreSet":"ACS"'); + expect(res.body).toContain("IN2022ACS"); + expect(res.body).toContain('"measure":"AAB-AD"'); expect(convertToDynamoExpression).toHaveBeenCalledWith( { status: "status", @@ -122,8 +117,8 @@ describe("Test Update Measure Handler", () => { expect(dbLib.update).toHaveBeenCalledWith({ TableName: "SAMPLE TABLE", Key: { - compoundKey: "FL2020ACSFUA-AD", - coreSet: "ACS", + compoundKey: "IN2022ACS", + measure: "AAB-AD", }, testValue: "test", }); diff --git a/services/app-api/handlers/measures/update.ts b/services/app-api/handlers/measures/update.ts index c258682b6a..cae549cf7e 100644 --- a/services/app-api/handlers/measures/update.ts +++ b/services/app-api/handlers/measures/update.ts @@ -1,7 +1,6 @@ import handler from "../../libs/handler-lib"; import dynamoDb from "../../libs/dynamodb-lib"; import { convertToDynamoExpression } from "../dynamoUtils/convertToDynamoExpressionVars"; -import { createMeasureKey } from "../dynamoUtils/createCompoundKey"; import { getUserNameFromJwt, hasStatePermissions, @@ -35,7 +34,6 @@ export const editMeasure = handler(async (event, context) => { detailedDescription, } = JSON.parse(event!.body!); - const dynamoKey = createMeasureKey({ state, year, coreSet }); const lastAlteredBy = getUserNameFromJwt(event); const descriptionParams = @@ -48,7 +46,7 @@ export const editMeasure = handler(async (event, context) => { const params = { TableName: process.env.measureTable!, Key: { - compoundKey: dynamoKey, + compoundKey: `${state}${year}${coreSet}`, measure: measure, }, ...convertToDynamoExpression( @@ -66,11 +64,11 @@ export const editMeasure = handler(async (event, context) => { await dynamoDb.update(params); //in 2024 and onward, we added a new feature called combined rates which requires rate calculations to the rates table - if (parseInt(year) >= 2024) { + if (year >= 2024) { //after updating the database with the latest values for the measure, we run the combine rates calculations for said measure await calculateAndPutRate({ state, - year, + year: year.toString(), coreSet, measure, }); diff --git a/services/app-api/utils/parseParameters.ts b/services/app-api/utils/parseParameters.ts index 7ed6b10404..c61df5a3fc 100644 --- a/services/app-api/utils/parseParameters.ts +++ b/services/app-api/utils/parseParameters.ts @@ -29,7 +29,7 @@ export const parseStateAndYearParameters = (event: APIGatewayProxyEvent) => { logger.warn("Invalid or missing year in path"); return { allParamsValid: false as const }; } - return { allParamsValid: true as const, state, year }; + return { allParamsValid: true as const, state, year: parseInt(year) }; }; export const parseCoreSetParameters = (event: APIGatewayProxyEvent) => { @@ -47,7 +47,12 @@ export const parseCoreSetParameters = (event: APIGatewayProxyEvent) => { logger.warn("Invalid or missing coreset in path"); return { allParamsValid: false as const }; } - return { allParamsValid: true as const, state, year, coreSet }; + return { + allParamsValid: true as const, + state, + year: parseInt(year), + coreSet, + }; }; export const parseMeasureParameters = (event: APIGatewayProxyEvent) => { @@ -69,5 +74,11 @@ export const parseMeasureParameters = (event: APIGatewayProxyEvent) => { logger.warn("Invalid or missing measure in path"); return { allParamsValid: false as const }; } - return { allParamsValid: true as const, state, year, coreSet, measure }; + return { + allParamsValid: true as const, + state, + year: parseInt(year), + coreSet, + measure, + }; }; From ee41519d9d4560ead089c41ecfc92a5b64c36bd8 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Tue, 17 Sep 2024 15:13:48 -0400 Subject: [PATCH 18/23] doing stupid stuff --- services/app-api/handlers/measures/update.ts | 2 +- services/app-api/storage/table.test.ts | 2 +- services/app-api/types.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/app-api/handlers/measures/update.ts b/services/app-api/handlers/measures/update.ts index cae549cf7e..59ab618c5b 100644 --- a/services/app-api/handlers/measures/update.ts +++ b/services/app-api/handlers/measures/update.ts @@ -68,7 +68,7 @@ export const editMeasure = handler(async (event, context) => { //after updating the database with the latest values for the measure, we run the combine rates calculations for said measure await calculateAndPutRate({ state, - year: year.toString(), + year, coreSet, measure, }); diff --git a/services/app-api/storage/table.test.ts b/services/app-api/storage/table.test.ts index af814b96f6..a0644e1c85 100644 --- a/services/app-api/storage/table.test.ts +++ b/services/app-api/storage/table.test.ts @@ -15,7 +15,7 @@ const mockMeasureParameters = { state: "CO", measure: "ZZZ-AD", coreSet: "ACS", - year: "2024", + year: 2024, }; const mockMeasure = { diff --git a/services/app-api/types.ts b/services/app-api/types.ts index 116669cab3..f5913f2160 100644 --- a/services/app-api/types.ts +++ b/services/app-api/types.ts @@ -174,7 +174,7 @@ export enum MeasurementSpecificationType { export interface RateParameters { state: string; - year: string; + year: number; coreSet: string; measure: string; } From af6c51674b807381dd27df71a46d611ae17317ca Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Fri, 20 Sep 2024 08:30:13 -0400 Subject: [PATCH 19/23] removing all instances of old measure table --- services/app-api/handlers/coreSets/delete.ts | 2 +- .../app-api/handlers/measures/tests/create.test.ts | 1 - services/app-api/handlers/measures/tests/get.test.ts | 1 - services/app-api/storage/table.test.ts | 8 ++++---- services/app-api/storage/table.ts | 6 +++--- services/database/scripts/transformMeasureTable.ts | 11 ++++++++--- services/uploads/src/dynamoSync/handler.js | 2 +- 7 files changed, 17 insertions(+), 14 deletions(-) diff --git a/services/app-api/handlers/coreSets/delete.ts b/services/app-api/handlers/coreSets/delete.ts index 2632a37351..07c765a26a 100644 --- a/services/app-api/handlers/coreSets/delete.ts +++ b/services/app-api/handlers/coreSets/delete.ts @@ -52,7 +52,7 @@ const deleteDependentMeasures = async ( TableName: process.env.measureTable!, Key: { compoundKey: `${state}${year}${coreSet}`, - coreSet: coreSet, + measure: measure, }, }; diff --git a/services/app-api/handlers/measures/tests/create.test.ts b/services/app-api/handlers/measures/tests/create.test.ts index 722fa49377..f5c6ba5458 100644 --- a/services/app-api/handlers/measures/tests/create.test.ts +++ b/services/app-api/handlers/measures/tests/create.test.ts @@ -15,7 +15,6 @@ jest.mock("../../../libs/authorization", () => ({ })); const event = { ...testEvent }; -process.env.measureTableName = "SAMPLE TABLE"; describe("Test Create Measure Handler", () => { beforeEach(() => { diff --git a/services/app-api/handlers/measures/tests/get.test.ts b/services/app-api/handlers/measures/tests/get.test.ts index a3b7c16769..dbc2ddbf51 100644 --- a/services/app-api/handlers/measures/tests/get.test.ts +++ b/services/app-api/handlers/measures/tests/get.test.ts @@ -31,7 +31,6 @@ jest.mock("../../dynamoUtils/convertToDynamoExpressionVars", () => ({ })); const event = { ...testEvent }; -process.env.measureTableName = "SAMPLE TABLE"; process.env.measureTable = "SAMPLE TABLE"; describe("Test Get Measure Handlers", () => { diff --git a/services/app-api/storage/table.test.ts b/services/app-api/storage/table.test.ts index a0644e1c85..d18f8a22be 100644 --- a/services/app-api/storage/table.test.ts +++ b/services/app-api/storage/table.test.ts @@ -61,7 +61,7 @@ const mockCombinedRate = { describe("Test database helper functions", () => { beforeAll(() => { - process.env.measureTableName = "local-measures"; + process.env.measureTable = "local-measure"; process.env.rateTableName = "local-rates"; }); @@ -72,10 +72,10 @@ describe("Test database helper functions", () => { expect(result).toBe(mockMeasure); expect(dynamodbLib.get).toHaveBeenCalledWith({ - TableName: "local-measures", + TableName: "local-measure", Key: { - compoundKey: "CO2024ACSZZZ-AD", - coreSet: "ACS", + compoundKey: "CO2024ACS", + measure: "ZZZ-AD", }, }); }); diff --git a/services/app-api/storage/table.ts b/services/app-api/storage/table.ts index 9c50c50fb0..54726126be 100644 --- a/services/app-api/storage/table.ts +++ b/services/app-api/storage/table.ts @@ -5,10 +5,10 @@ import { RateParameters, CombinedRatesPayload } from "../types"; export const getMeasureFromTable = async (parameters: RateParameters) => { const { state, year, coreSet, measure } = parameters; return await dynamoDb.get({ - TableName: process.env.measureTableName, + TableName: process.env.measureTable, Key: { - compoundKey: `${state}${year}${coreSet}${measure}`, - coreSet: coreSet, + compoundKey: `${state}${year}${coreSet}`, + measure: measure, }, }); }; diff --git a/services/database/scripts/transformMeasureTable.ts b/services/database/scripts/transformMeasureTable.ts index e78b705b79..0dc97f20f4 100644 --- a/services/database/scripts/transformMeasureTable.ts +++ b/services/database/scripts/transformMeasureTable.ts @@ -6,15 +6,20 @@ import { } from "@aws-sdk/lib-dynamodb"; import prompt from "prompt-sync"; +/*** + * Run with `npx tsx transformMeasureTable.ts` + */ const transformMeasureTable = async () => { let stage = "local"; - const isLocal = !!process.env.DYNAMODB_URL; - const dbClient = buildClient(isLocal); const p = prompt(); + const runLocally = p("Do you want to run this script locally? Y/N: "); + const isLocal = runLocally === "Y" ? true : false; if (!isLocal) { - stage = p("What environment would you like to modify: "); + stage = p("What environment are we running on (e.g. master, val, prod)? "); } + const dbClient = buildClient(isLocal); + const oldTable = `${stage}-measures`; const newTable = `${stage}-measure`; console.log(`Processing table ${oldTable}`); diff --git a/services/uploads/src/dynamoSync/handler.js b/services/uploads/src/dynamoSync/handler.js index 1be0b1368c..58d64c5e5b 100644 --- a/services/uploads/src/dynamoSync/handler.js +++ b/services/uploads/src/dynamoSync/handler.js @@ -27,7 +27,7 @@ const uploadFileToS3 = async (filePath, scanResult) => { const syncDynamoToS3 = handler(async (_event, _context) => { console.log("Syncing Dynamo to Uploads"); - const measureResults = await scanAll(process.env.measureTableName); + const measureResults = await scanAll(process.env.measureTable); const coreSetResults = await scanAll(process.env.coreSetTableName); const rateResults = await scanAll(process.env.rateTableName); From a0aca0668ab5e9b2dcb2fdf8ac0a8f01734ff7d1 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Mon, 23 Sep 2024 10:44:59 -0400 Subject: [PATCH 20/23] pushing new serverless table name and seeing what happens --- services/database/serverless.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/database/serverless.yml b/services/database/serverless.yml index 88b0441cbd..103cce5ea1 100644 --- a/services/database/serverless.yml +++ b/services/database/serverless.yml @@ -109,7 +109,7 @@ resources: - AttributeName: key KeyType: HASH BillingMode: PAY_PER_REQUEST # Set the capacity to auto-scale - CSMeasureTable: + QualityMeasureTable: Type: AWS::DynamoDB::Table Properties: TableName: ${self:custom.measureTable} From 6b267d1b51c17bdbd45c69f2a67dd8b6c24e5f4b Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Mon, 23 Sep 2024 11:16:11 -0400 Subject: [PATCH 21/23] forgot to change ref to measure table --- services/database/serverless.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/database/serverless.yml b/services/database/serverless.yml index 103cce5ea1..d34f51e24c 100644 --- a/services/database/serverless.yml +++ b/services/database/serverless.yml @@ -138,7 +138,7 @@ resources: MeasureTableName: Value: !Ref MeasureTable MeasureTable: - Value: !Ref CSMeasureTable + Value: !Ref QualityMeasureTable MeasureTableArn: Value: !GetAtt MeasureTable.Arn MeasureTableStreamArn: From 046b44ca23ac0a8b252922ec118efdbd2ed28df8 Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Mon, 23 Sep 2024 12:24:19 -0400 Subject: [PATCH 22/23] pushing dumb readme update to redeploy table --- services/database/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/database/README.md b/services/database/README.md index 3be9984d05..c8e64b7ee1 100644 --- a/services/database/README.md +++ b/services/database/README.md @@ -1 +1,6 @@ # database + +Run these commands to spin up local dynamo database instance: +`./run local` +`DYNAMO_ENDPOINT=http://localhost:8000 dynamodb-admin` +Navigate to http://localhost:8001/ From b01c160683096a8de54bce04a98b5c70d0bd92ed Mon Sep 17 00:00:00 2001 From: Angela Cooney Date: Mon, 23 Sep 2024 13:04:03 -0400 Subject: [PATCH 23/23] redeploying --- services/database/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/database/README.md b/services/database/README.md index c8e64b7ee1..e0182119fb 100644 --- a/services/database/README.md +++ b/services/database/README.md @@ -3,4 +3,4 @@ Run these commands to spin up local dynamo database instance: `./run local` `DYNAMO_ENDPOINT=http://localhost:8000 dynamodb-admin` -Navigate to http://localhost:8001/ +Navigate to http://localhost:8001