CB2-12965: Refactor test result and technical record logs (#115)

* feat(CB2-12695): Added addiitonal logs * feat(CB2-12695): fixed linting issue * feat(CB2-12695): fixed linting issue * feat(CB2-12965): fixed linting issue * feat(CB2-12965): added semi colons * feat(CB2-12965): added semi colons * feat(CB2-12965): unit test to cover console log * feat(CB2-12965): unit test to cover console log * feat(CB2-12965): change let ot const * feat(CB2-12965): Added operation type to the log * feat(CB2-12965): Formatting in logger unit test * feat(cb2-12965): linting * feat(cb2-12965): refactor logging * feat(cb2-12965): remove type definitions * feat(CB2-12965): format imports and linting
dvsa · Jul 25, 2024 · c131241 · c131241
1 parent 56c20f6
commit c131241
Show file tree

Hide file tree

Showing 6 changed files with 134 additions and 91 deletions.
diff --git a/package.json b/package.json
@@ -55,7 +55,6 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@dvsa/cvs-type-definitions": "^7.2.0",
     "@aws-sdk/client-dynamodb": "3.577.0",
     "@aws-sdk/client-dynamodb-streams": "3.577.0",
     "@aws-sdk/client-secrets-manager": "3.565.0",

diff --git a/src/functions/process-stream-event.ts b/src/functions/process-stream-event.ts
@@ -7,38 +7,36 @@ import type {
   SQSEvent,
   StreamRecord,
 } from 'aws-lambda';
+import { unmarshall } from '@aws-sdk/util-dynamodb';
 import { BatchItemFailuresResponse } from '../models/batch-item-failure-response';
 import { destroyConnectionPool } from '../services/connection-pool';
 import { DynamoDbImage } from '../services/dynamodb-images';
 import { convert } from '../services/entity-conversion';
-import { debugLog } from '../services/logger';
+import {
+  addToLogManager, clearLogs, createLogEntry, debugLog, printLogs, updateLogEntry,
+} from '../services/logger';
 import { SqlOperation, deriveSqlOperation } from '../services/sql-operations';
 import { transformTechRecord } from '../utils/transform-tech-record';
-import { unmarshall } from "@aws-sdk/util-dynamodb";
-
-let logManager: ILog[] = [];
 
 /**
  * λ function: convert a DynamoDB document to Aurora RDS rows
  * @param event - DynamoDB stream event, containing DynamoDB document image
  * @param context - λ context
  */
 export const processStreamEvent: Handler = async (
-    event: SQSEvent,
-    context: Context,
+  event: SQSEvent,
+  context: Context,
 ): Promise<any> => {
   const res: BatchItemFailuresResponse = {
     batchItemFailures: [],
   };
   try {
-    const processStartTime: Date = new Date();
     debugLog('Received SQS event: ', JSON.stringify(event));
-    let iLog: ILog = { changeType: "", identifier: "", operationType: "" };
 
+    const currentLog = createLogEntry();
     validateEvent(event);
 
     const region = process.env.AWS_REGION;
-
     if (!region) {
       console.error('AWS_REGION envvar not available');
       return;
@@ -56,69 +54,75 @@ export const processStreamEvent: Handler = async (
 
       // parse source ARN
       const tableName: string = getTableNameFromArn(
-          dynamoRecord.eventSourceARN!,
+        dynamoRecord.eventSourceARN!,
       );
-
       if (tableName.includes('flat-tech-records')) {
         transformTechRecord(dynamoRecord as _Record);
         debugLog(`Dynamo Record after transformation: ${dynamoRecord}`);
+
         const technicalRecord: any = dynamoRecord.dynamodb?.NewImage;
         const unmarshalledTechnicalRecord = unmarshall(technicalRecord);
-        iLog.statusCode = unmarshalledTechnicalRecord.statusCode;
-        iLog.changeType = "Technical Record Change";
-        iLog.identifier = unmarshalledTechnicalRecord.vehicleType === 'trl'
+        updateLogEntry(currentLog, {
+          changeType: 'Technical Record Change',
+          identifier: unmarshalledTechnicalRecord.vehicleType === 'trl'
             ? unmarshalledTechnicalRecord.trailerId
-            : unmarshalledTechnicalRecord.primaryVrm;
-      } else if (tableName.includes('test-results')) {
+            : unmarshalledTechnicalRecord.primaryVrm,
+          statusCode: unmarshalledTechnicalRecord.techRecord[0]?.statusCode,
+        });
+      }
+      if (tableName.includes('test-result')) {
         const testResult: any = dynamoRecord.dynamodb?.NewImage;
         const unmarshalledTestResult = unmarshall(testResult);
-        iLog.changeType = 'Test record change';
-        iLog.testResultId = unmarshalledTestResult.testResultId;
-        iLog.identifier = unmarshalledTestResult.vehicleType === 'trl'
-            ? unmarshalledTestResult.trailerId :
-            unmarshalledTestResult.primaryVrm;
+        updateLogEntry(currentLog, {
+          changeType: 'Test Record Change',
+          testResultId: unmarshalledTestResult.testResultId,
+          identifier: unmarshalledTestResult.vehicleType === 'trl'
+            ? unmarshalledTestResult.trailerId
+            : unmarshalledTestResult.vrm,
+        });
       }
+
       // is this an INSERT, UPDATE, or DELETE?
       const operationType: SqlOperation = deriveSqlOperation(
-          dynamoRecord.eventName!,
+        dynamoRecord.eventName!,
       );
-
-      iLog.operationType = operationType;
-      addToILog(iLog);
+      updateLogEntry(currentLog, { operationType });
+      addToLogManager(currentLog);
 
       // parse native DynamoDB format to usable TS map
       const image: DynamoDbImage = selectImage(
-          operationType,
-          dynamoRecord.dynamodb!,
+        operationType,
+        dynamoRecord.dynamodb!,
       );
 
       debugLog('Dynamo image dump:', image);
 
       try {
         debugLog(
-            `DynamoDB ---> Aurora | START (event ID: ${dynamoRecord.eventID})`,
+          `DynamoDB ---> Aurora | START (event ID: ${dynamoRecord.eventID})`,
         );
 
         await convert(tableName, operationType, image);
 
+        printLogs();
+        clearLogs();
+
         debugLog(
-            `DynamoDB ---> Aurora | END   (event ID: ${dynamoRecord.eventID})`,
+          `DynamoDB ---> Aurora | END   (event ID: ${dynamoRecord.eventID})`,
         );
-        console.log(`** RESULTS **\nProcess start time is: ${processStartTime.toISOString()} \n${JSON.stringify(logManager)}`,
-        )
       } catch (err) {
         console.error(
-            "Couldn't convert DynamoDB entity to Aurora, will return record to SQS for retry",
-            [`messageId: ${id}`, err],
+          "Couldn't convert DynamoDB entity to Aurora, will return record to SQS for retry",
+          [`messageId: ${id}`, err],
         );
-        res.batchItemFailures.push({itemIdentifier: id});
+        res.batchItemFailures.push({ itemIdentifier: id });
         dumpArguments(event, context);
       }
     }
   } catch (err) {
     console.error(
-        'An error unrelated to Dynamo-to-Aurora conversion has occurred, event will not be retried',
-        err,
+      'An error unrelated to Dynamo-to-Aurora conversion has occurred, event will not be retried',
+      err,
     );
     dumpArguments(event, context);
     await destroyConnectionPool();
@@ -130,9 +134,9 @@ export const processStreamEvent: Handler = async (
 export const getTableNameFromArn = (eventSourceArn: string): string => eventSourceArn.split(':')[5].split('/')[1];
 
 const selectImage = (
-    operationType: SqlOperation,
-    streamRecord: StreamRecord,
-    // eslint-disable-next-line consistent-return
+  operationType: SqlOperation,
+  streamRecord: StreamRecord,
+  // eslint-disable-next-line consistent-return
 ): DynamoDbImage => {
   // eslint-disable-next-line default-case
   switch (operationType) {
@@ -188,7 +192,3 @@ const dumpArguments = (event: DynamoDBStreamEvent, context: Context): void => {
   console.error('Event dump  : ', JSON.stringify(event));
   console.error('Context dump: ', JSON.stringify(context));
 };
-
-const addToILog = (iLog: ILog) => {
-  if (iLog.identifier && iLog.changeType) logManager.push(iLog);
-};
diff --git a/src/models/ILog.ts b/src/models/ILog.ts
@@ -1,7 +1,8 @@
 interface ILog {
-    changeType: string;
-    identifier: string;
-    operationType: string;
-    statusCode?: string;
-    testResultId?: string;
+  timestamp: string;
+  changeType: string;
+  identifier: string;
+  operationType: string;
+  statusCode?: string;
+  testResultId?: string;
 }
diff --git a/src/services/logger.ts b/src/services/logger.ts
@@ -3,3 +3,17 @@ export const debugLog = (message?: any, ...optionalParams: any[]) => {
     console.info(message, optionalParams);
   }
 };
+
+let logManager: ILog[] = [];
+
+export const createLogEntry = (): Partial<ILog> => ({
+  timestamp: new Date().toISOString(),
+});
+
+export const updateLogEntry = (entry: Partial<ILog>, updates: Partial<ILog>) => Object.assign(entry, updates);
+
+export const addToLogManager = (entry: Partial<ILog>) => logManager.push(<ILog>entry);
+
+export const printLogs = () => console.log(JSON.stringify(logManager));
+
+export const clearLogs = () => logManager = [];
diff --git a/tests/unit/functions/process-stream-event.unitTest.ts b/tests/unit/functions/process-stream-event.unitTest.ts
@@ -8,14 +8,14 @@ import { exampleContext } from '../../utils';
 import testResultWithTestType from '../../resources/dynamodb-image-test-results-with-testtypes.json';
 import techRecordV3 from '../../resources/dynamodb-image-technical-record-V3.json';
 
-
 jest.mock('../../../src/services/entity-conversion', () => ({
   convert: jest.fn(),
 }));
 
 describe('processStreamEvent()', () => {
   beforeEach(() => {
     jest.resetAllMocks();
+    jest.restoreAllMocks();
     mocked(convert).mockResolvedValueOnce({});
   });
 
@@ -28,7 +28,7 @@ describe('processStreamEvent()', () => {
               body: JSON.stringify({
                 eventName: 'INSERT',
                 dynamodb: {
-                  NewImage: techRecordV3
+                  NewImage: techRecordV3,
                 },
                 eventSourceARN:
                   'arn:aws:dynamodb:eu-west-1:1:table/flat-tech-records/stream/2020-01-01T00:00:00.000',
@@ -47,26 +47,26 @@ describe('processStreamEvent()', () => {
 
   it('should allow valid events to reach the entity conversion procedure test RECORD TRL', async () => {
     await expect(
-        processStreamEvent(
+      processStreamEvent(
+        {
+          Records: [
             {
-              Records: [
-                {
-                  body: JSON.stringify({
-                    eventName: 'INSERT',
-                    dynamodb: {
-                      NewImage: testResultWithTestType,
-                    },
-                    eventSourceARN:
-                        'arn:aws:dynamodb:eu-west-1:1:table/test-result/stream/2020-01-01T00:00:00.000',
-                  }),
+              body: JSON.stringify({
+                eventName: 'INSERT',
+                dynamodb: {
+                  NewImage: testResultWithTestType,
                 },
-              ],
+                eventSourceARN:
+                        'arn:aws:dynamodb:eu-west-1:1:table/test-result/stream/2020-01-01T00:00:00.000',
+              }),
             },
-            exampleContext(),
-            () => {
+          ],
+        },
+        exampleContext(),
+        () => {
 
-            },
-        ),
+        },
+      ),
     ).resolves.not.toThrow();
     expect(convert).toHaveBeenCalledTimes(1);
   });
@@ -75,28 +75,57 @@ describe('processStreamEvent()', () => {
     const consoleSpy = jest.spyOn(console, 'log');
 
     await expect(
-        processStreamEvent(
+      processStreamEvent(
+        {
+          Records: [
             {
-              Records: [
-                {
-                  body: JSON.stringify({
-                    eventName: 'INSERT',
-                    dynamodb: {
-                      NewImage: testResultWithTestType,
-                    },
-                    eventSourceARN:
-                        'arn:aws:dynamodb:eu-west-1:1:table/test-result/stream/2020-01-01T00:00:00.000',
-                  }),
+              body: JSON.stringify({
+                eventName: 'INSERT',
+                dynamodb: {
+                  NewImage: testResultWithTestType,
                 },
-              ],
+                eventSourceARN:
+                        'arn:aws:dynamodb:eu-west-1:1:table/test-results/stream/2020-01-01T00:00:00.000',
+              }),
             },
-            exampleContext(),
-            () => {
+          ],
+        },
+        exampleContext(),
+        () => {
+        },
+      ),
+    ).resolves.not.toThrow();
+    expect(convert).toHaveBeenCalledTimes(1);
+    expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('changeType\":\"Test Record Change\",\"testResultId\":\"TEST-RESULT-ID-3\",\"identifier\":\"VRM-3\",\"operationType\":\"INSERT\"}]'));
+  });
+
+  it('should allow valid events to reach the entity conversion procedure tech RECORD TRL and produce result log', async () => {
+    const consoleSpy = jest.spyOn(console, 'log');
+
+    await expect(
+      processStreamEvent(
+        {
+          Records: [
+            {
+              body: JSON.stringify({
+                eventName: 'INSERT',
+                dynamodb: {
+                  NewImage: techRecordV3,
+                },
+                eventSourceARN:
+                        'arn:aws:dynamodb:eu-west-1:1:table/flat-tech-records/stream/2020-01-01T00:00:00.000',
+              }),
             },
-        ),
+          ],
+        },
+        exampleContext(),
+        () => {
+        },
+      ),
     ).resolves.not.toThrow();
     expect(convert).toHaveBeenCalledTimes(1);
-    expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('[{\"changeType\":\"Technical Record Change\",\"identifier\":\"VRM-1\",\"operationType\":\"INSERT\"}]'));
+    expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('"changeType\":\"Technical Record Change\",\"identifier\":\"VRM-1\",\"statusCode\":\"STATUS-CODE\",\"operationType\":\"INSERT\"}]'));
+    consoleSpy.mockRestore();
   });
 
   it('should fail on null event', async () => {

diff --git a/tests/unit/services/logger.unitTest.ts b/tests/unit/services/logger.unitTest.ts
@@ -1,10 +1,10 @@
-import { debugLog } from "../../../src/services/logger";
+import { debugLog } from '../../../src/services/logger';
 
 describe('logger service', () => {
-    it('Should be able to get a logger instance', () => {
-        process.env.DEBUG = 'true'
-        let consoleSpy = jest.spyOn(console, 'info');
-        debugLog("Test");
-        expect(consoleSpy).toHaveBeenCalledWith('Test', []);
-    })
-})
+  it('Should be able to get a logger instance', () => {
+    process.env.DEBUG = 'true';
+    const consoleSpy = jest.spyOn(console, 'info');
+    debugLog('Test');
+    expect(consoleSpy).toHaveBeenCalledWith('Test', []);
+  });
+});