CMDCT-4019: commiting new coreSet table infrastructure setup

serverless-compose.yml

@@ -12,6 +12,7 @@ services:
       CoreSetTableName: ${database.CoreSetTableName}
       MeasureTableName: ${database.MeasureTableName}
       MeasureTable: ${database.MeasureTable}
+      CoreSetTable: ${database.CoreSetTable}
       RateTableName: ${database.RateTableName}
 
   app-api:
@@ -21,6 +22,7 @@ services:
       CoreSetTableStreamArn: ${database.CoreSetTableStreamArn}
       MeasureTableName: ${database.MeasureTableName}
       MeasureTable: ${database.MeasureTable}
+      CoreSetTable: ${database.CoreSetTable}
       MeasureTableStreamArn: ${database.MeasureTableStreamArn}
       RateTableName: ${database.RateTableName}
       RateTableStreamArn: ${database.RateTableStreamArn}

services/app-api/serverless.yml

@@ -47,6 +47,7 @@ custom:
   coreSetTableStreamArn: ${env:DYNAMO_TABLE_ARN, param:CoreSetTableStreamArn}
   measureTableName: ${env:measureTableName, param:MeasureTableName}
   measureTable: ${env:measureTable, param:MeasureTable}
+  coreSetTable: ${env:coreSetTable, param:CoreSetTable}
   measureTableStreamArn: ${env:DYNAMO_TABLE_ARN, param:MeasureTableStreamArn}
   rateTableName: ${env:rateTableName, param:RateTableName}
   rateTableStreamArn: ${env:DYNAMO_TABLE_ARN, param:RateTableStreamArn}
@@ -103,6 +104,7 @@ provider:
     coreSetTableName: ${self:custom.coreSetTableName}
     measureTableName: ${self:custom.measureTableName}
     measureTable: ${self:custom.measureTable}
+    coreSetTable: ${self:custom.coreSetTable}
     rateTableName: ${self:custom.rateTableName}
     bannerTableName: ${self:custom.bannerTableName}
     stage: ${opt:stage, self:provider.stage}

services/database/scripts/transformCoreSetTable.ts

@@ -0,0 +1,81 @@
+import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
+import {
+  DynamoDBDocumentClient,
+  paginateScan,
+  PutCommand,
+} from "@aws-sdk/lib-dynamodb";
+import prompt from "prompt-sync";
+
+/***
+ * Run with `npx tsx transformCoreSetTable.ts`
+ */
+const transformMeasureTable = async () => {
+  let stage = "local";
+  const p = prompt();
+  const runLocally = p("Do you want to run this script locally? Y/N: ");
+  const isLocal = runLocally === "Y" ? true : false;
+  if (!isLocal) {
+    stage = p("What environment are we running on (e.g. master, val, prod)? ");
+  }
+
+  const dbClient = buildClient(isLocal);
+
+  const oldTable = `${stage}-coreSets`;
+  const newTable = `${stage}-coreSet`;
+  console.log(`Processing table ${oldTable}`);
+  for await (let entry of scan(dbClient, oldTable)) {
+    add(dbClient, newTable, entry);
+  }
+};
+
+async function* scan(client: DynamoDBDocumentClient, table: string) {
+  const query = {
+    TableName: table,
+  };
+  for await (const result of paginateScan({ client }, query)) {
+    yield* result.Items ?? [];
+  }
+}
+
+async function add(client: DynamoDBDocumentClient, table: string, entry: any) {
+  const newCompoundKey = `${entry.state}${entry.year}`;
+  const params = {
+    TableName: table,
+    Item: {
+      ...entry,
+      compoundKey: newCompoundKey,
+    },
+  };
+  await client.send(new PutCommand(params));
+}
+
+function buildClient(isLocal: boolean) {
+  if (isLocal) {
+    return DynamoDBDocumentClient.from(
+      new DynamoDBClient({
+        region: "localhost",
+        endpoint: "http://localhost:8000",
+        credentials: {
+          accessKeyId: "LOCALFAKEKEY", // pragma: allowlist secret
+          secretAccessKey: "LOCALFAKESECRET", // pragma: allowlist secret
+        },
+      })
+    );
+  } else {
+    return DynamoDBDocumentClient.from(
+      new DynamoDBClient({
+        region: "us-east-1",
+        logger: {
+          debug: () => {
+            /* Dynamo's debug logs are extremely noisy */
+          },
+          info: console.info,
+          warn: console.warn,
+          error: console.error,
+        },
+      })
+    );
+  }
+}
+
+transformMeasureTable();

services/database/serverless.yml

@@ -21,6 +21,7 @@ custom:
   bannerTableName: ${self:custom.stage}-banners
   rateTableName: ${self:custom.stage}-rates
   measureTable: ${self:custom.stage}-measure
+  coreSetTable: ${self:custom.stage}-coreSet
   dynamodb:
     stages:
       - local
@@ -128,6 +129,25 @@ resources:
           - AttributeName: measure
             KeyType: RANGE
         BillingMode: PAY_PER_REQUEST # Set the capacity to auto-scale
+    QualityCoreSetTable:
+      Type: AWS::DynamoDB::Table
+      Properties:
+        TableName: ${self:custom.coreSetTable}
+        PointInTimeRecoverySpecification:
+          PointInTimeRecoveryEnabled: true
+        StreamSpecification:
+          StreamViewType: NEW_AND_OLD_IMAGES
+        AttributeDefinitions:
+          - AttributeName: compoundKey
+            AttributeType: S
+          - AttributeName: coreSet
+            AttributeType: S
+        KeySchema:
+          - AttributeName: compoundKey
+            KeyType: HASH
+          - AttributeName: coreSet
+            KeyType: RANGE
+        BillingMode: PAY_PER_REQUEST # Set the capacity to auto-scale
   Outputs:
     CoreSetTableName:
       Value: !Ref CoreSetTable
@@ -143,6 +163,8 @@ resources:
       Value: !GetAtt MeasureTable.Arn
     MeasureTableStreamArn:
       Value: !GetAtt MeasureTable.StreamArn
+    CoreSetTable:
+      Value: !Ref QualityCoreSetTable
     RateTableName:
       Value: !Ref RateTable
     RateTableArn:

services/uploads/serverless.yml

@@ -54,6 +54,7 @@ custom:
   coreSetTableName: ${env:coreSetTableName, param:CoreSetTableName}
   measureTableName: ${env:measureTableName, param:MeasureTableName}
   measureTable: ${env:measureTable, param:MeasureTable}
+  coreSetTable: ${env:coreSetTable, param:CoreSetTable}
   rateTableName: ${env:rateTableName, param:RateTableName}
   serverlessTerminationProtection:
     stages:
@@ -93,6 +94,7 @@ functions:
       coreSetTableName: ${self:custom.coreSetTableName}
       measureTableName: ${self:custom.measureTableName}
       measureTable: ${self:custom.measureTable}
+      coreSetTable: ${self:custom.coreSetTable}
       rateTableName: ${self:custom.rateTableName}
       dynamoSnapshotS3BucketName: !Ref DynamoSnapshotBucket
   avScan: