chore(cb2-11267): upgrade aws-sdk v2 to v3 (#81)

* chore(cb2-11267): upgrade some packages

* chore(cb2-11267): fix lint and test

* chore(cb2-11267): update caniuse

package.json

@@ -47,8 +47,9 @@
     "tools-setup": "echo 'pipeline requires this'"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "3.554.0",
+    "@aws-sdk/client-secrets-manager": "3.554.0",
     "aws-lambda": "1.0.6",
-    "aws-sdk": "2.857.0",
     "express": "4.18.2",
     "moment": "2.29.4",
     "mysql2": "^2.2.5",
@@ -70,6 +71,7 @@
     "@types/supertest": "2.0.10",
     "@typescript-eslint/eslint-plugin": "4.12.0",
     "@typescript-eslint/parser": "4.12.0",
+    "aws-sdk-client-mock": "4.0.0",
     "clean-webpack-plugin": "3.0.0",
     "commitlint-plugin-function-rules": "1.1.20",
     "concurrently": "5.3.0",

src/app/databaseService.ts

@@ -56,9 +56,9 @@ async function getVehicleDetails(vehicleDetailsQueryResult: QueryOutput, databas
   const vehicleDetailsResult = vehicleDetailsQueryResult[0][0] as VehicleQueryResult;
 
   if (
-    vehicleDetailsResult === undefined ||
-    vehicleDetailsResult.id === undefined ||
-    vehicleDetailsResult.result === undefined
+    vehicleDetailsResult === undefined
+    || vehicleDetailsResult.id === undefined
+    || vehicleDetailsResult.result === undefined
   ) {
     throw new NotFoundError('Vehicle was not found');
   }
@@ -137,9 +137,9 @@ async function getTestResultDetails(
   const testResultQueryResult = queryResult[0][0] as TestResultQueryResult;
 
   if (
-    testResultQueryResult === undefined ||
-    testResultQueryResult.id === undefined ||
-    testResultQueryResult.result === undefined
+    testResultQueryResult === undefined
+    || testResultQueryResult.id === undefined
+    || testResultQueryResult.result === undefined
   ) {
     throw new NotFoundError('Test not found');
   }
@@ -221,8 +221,7 @@ function getEvlFeedByVrmDetails(queryResult: QueryOutput): EvlFeedData {
 }
 
 function getFeedDetails(queryResult: QueryOutput, feedName: FeedName): EvlFeedData[] | TflFeedData[] {
-  const feedQueryResults: EvlFeedData[] | TflFeedData[] =
-    feedName === FeedName.EVL ? (queryResult[0][1] as EvlFeedData[]) : (queryResult[0] as TflFeedData[]);
+  const feedQueryResults: EvlFeedData[] | TflFeedData[] = feedName === FeedName.EVL ? (queryResult[0][1] as EvlFeedData[]) : (queryResult[0] as TflFeedData[]);
   if (feedQueryResults === undefined || feedQueryResults.length === 0) {
     throw new NotFoundError('No tests found');
   }

src/app/evlFeedQueryFunctionFactory.ts

@@ -9,7 +9,8 @@ export default (
   event: EvlEvent,
 ):
   | ((databaseService: DatabaseService, event: EvlEvent) => Promise<EvlFeedData[]>)
-  | ((databaseService: DatabaseService, feedName: FeedName, event: EvlEvent) => Promise<EvlFeedData[]>) => {
+  | ((databaseService: DatabaseService, feedName: FeedName, event: EvlEvent) => Promise<EvlFeedData[]>
+  ) => {
   if (event.vrm_trm) {
     logger.debug('redirecting to getEVLFeedByVrm using evl factory');
     return getEvlFeedByVrm;

src/infrastructure/api/index.ts

@@ -1,6 +1,6 @@
 /* eslint-disable @typescript-eslint/no-unsafe-call */
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
-import AWS from 'aws-sdk';
+import { SecretsManager } from '@aws-sdk/client-secrets-manager';
 import express, { Request, Router } from 'express';
 import mysql from 'mysql2/promise';
 import moment from 'moment';
@@ -53,7 +53,7 @@ router.get(
       if (process.env.IS_OFFLINE === 'true') {
         secretsManager = new LocalSecretsManagerService();
       } else {
-        secretsManager = new SecretsManagerService(new AWS.SecretsManager());
+        secretsManager = new SecretsManagerService(new SecretsManager());
       }
     } catch (e) {
       if (e instanceof Error) {
@@ -91,7 +91,7 @@ router.get(
     if (process.env.IS_OFFLINE === 'true') {
       secretsManager = new LocalSecretsManagerService();
     } else {
-      secretsManager = new SecretsManagerService(new AWS.SecretsManager());
+      secretsManager = new SecretsManagerService(new SecretsManager());
     }
 
     DatabaseService.build(secretsManager, mysql)
@@ -125,23 +125,22 @@ router.get(
       secretsManager = new LocalSecretsManagerService();
     } else {
       logger.debug('configuring aws secret manager');
-      secretsManager = new SecretsManagerService(new AWS.SecretsManager());
+      secretsManager = new SecretsManagerService(new SecretsManager());
     }
     const fileName = `EVL_GVT_${moment(Date.now()).format('YYYYMMDD')}.csv`;
     logger.debug(`creating file for EVL feed called: ${fileName}`);
     DatabaseService.build(secretsManager, mysql)
       .then((dbService) => getFeedDetails(evlFeedQueryFunctionFactory, FeedName.EVL, dbService, request.query))
-      .then((result: EvlFeedData[]) => {
+      .then(async (result: EvlFeedData[]) => {
         logger.info('Generating EVL File Data');
         const evlFeedProcessedData: string = result
           .map(
-            (entry) =>
-              `${entry.vrm_trm},${entry.certificateNumber},${moment(entry.testExpiryDate).format('DD-MMM-YYYY')}`,
+            (entry) => `${entry.vrm_trm},${entry.certificateNumber},${moment(entry.testExpiryDate).format('DD-MMM-YYYY')}`,
           )
           .join('\n');
         logger.debug(`\nData captured for file generation: ${evlFeedProcessedData} \n\n`);
 
-        uploadToS3(evlFeedProcessedData, fileName, () => {
+        await uploadToS3(evlFeedProcessedData, fileName, () => {
           logger.info(`Successfully uploaded ${fileName} to S3`);
           res.status(200);
           res.contentType('json').send();
@@ -168,36 +167,35 @@ router.get('/tfl', (_req, res) => {
     secretsManager = new LocalSecretsManagerService();
   } else {
     logger.debug('configuring aws secret manager');
-    secretsManager = new SecretsManagerService(new AWS.SecretsManager());
+    secretsManager = new SecretsManagerService(new SecretsManager());
   }
   DatabaseService.build(secretsManager, mysql)
     .then((dbService) => getFeedDetails(tflFeedQueryFunctionFactory, FeedName.TFL, dbService))
-    .then((result: TflFeedData[]) => {
+    .then(async (result: TflFeedData[]) => {
       const numberOfRows = result.length;
       const fileName = `VOSA-${moment(Date.now()).format('YYYY-MM-DD')}-G1-${numberOfRows}-01-01.csv`;
       logger.debug(`creating file for TFL feed called: ${fileName}`);
       logger.info('Generating TFL File Data');
       const processedResult = result.map((entry) => processTFLFeedData(entry));
       const tflFeedProcessedData: string = processedResult
         .map(
-          (entry) =>
-            `${entry.VRM},${entry.VIN},${entry.SerialNumberOfCertificate},${entry.CertificationModificationType},${entry.TestStatus},${entry.PMEuropeanEmissionClassificationCode},${entry.ValidFromDate},${entry.ExpiryDate},${entry.IssuedBy},${entry.IssueDate}`,
+          (entry) => `${entry.VRM},${entry.VIN},${entry.SerialNumberOfCertificate},${entry.CertificationModificationType},${entry.TestStatus},${entry.PMEuropeanEmissionClassificationCode},${entry.ValidFromDate},${entry.ExpiryDate},${entry.IssuedBy},${entry.IssueDate}`,
         )
         .join('\n');
       logger.debug(`\nData captured for file generation: ${tflFeedProcessedData} \n\n`);
-      uploadToS3(tflFeedProcessedData, fileName, () => {
+      await uploadToS3(tflFeedProcessedData, fileName, () => {
         logger.info(`Successfully uploaded ${fileName} to S3`);
         res.status(200);
         res.contentType('json').send();
       });
     })
-    .catch((e: Error) => {
+    .catch(async (e: Error) => {
       if (e instanceof ParametersError) {
         res.status(400);
         res.send(`Error Generating TFL Feed Data: ${e.message}`);
       } else if (e instanceof NotFoundError) {
         const fileName = `VOSA-${moment(Date.now()).format('YYYY-MM-DD')}-G1-0-01-01.csv`;
-        uploadToS3(' , ,', fileName, () => {
+        await uploadToS3(' , ,', fileName, () => {
           logger.info(`Successfully uploaded ${fileName} to S3`);
           res.status(200);
           res.contentType('json').send();

src/infrastructure/s3BucketService.ts

@@ -1,25 +1,29 @@
-import AWS from 'aws-sdk';
+import {
+  GetObjectCommand,
+  GetObjectCommandInput,
+  PutObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
 import logger from '../utils/logger';
 
-export function uploadToS3(processedData: string, fileName: string, callback: () => void): void {
-  const s3 = configureS3();
+export async function uploadToS3(processedData: string, fileName: string, callback: () => void): Promise<void> {
+  const s3: S3Client = configureS3();
   const params = { Bucket: process.env.AWS_S3_BUCKET_NAME ?? '', Key: fileName, Body: processedData };
-
-  logger.info(`uploading ${fileName} to S3`);
-  s3.upload(params, (err: unknown) => {
-    if (err) {
-      logger.error(err);
-    }
-    callback();
-  });
+  try {
+    logger.info(`uploading ${fileName} to S3`);
+    await s3.send(new PutObjectCommand(params));
+  } catch (err) {
+    logger.error(err);
+  }
+  callback();
 }
 
 export async function getItemFromS3(key: string): Promise<string | undefined> {
   logger.info(`Reading contents of file ${key}`);
   const s3 = configureS3();
-  const params: AWS.S3.GetObjectRequest = { Bucket: process.env.AWS_S3_BUCKET_NAME ?? '', Key: key };
+  const params: GetObjectCommandInput = { Bucket: process.env.AWS_S3_BUCKET_NAME ?? '', Key: key };
   try {
-    const body = (await s3.getObject(params).promise()).Body?.toString();
+    const body = (await s3.send(new GetObjectCommand(params))).Body?.toString();
     logger.info(`File contents retrieved: ${body}`);
     return body;
   } catch (err) {
@@ -35,7 +39,7 @@ export async function readAndUpsert(key: string, body: string, valueIfNotFound?:
   };
   try {
     const contents = await getItemFromS3(key);
-    uploadToS3(body, key, cb);
+    await uploadToS3(body, key, cb);
     return contents ?? '';
   } catch (err) {
     // the "not found" status code depends on if the lambda has the s3:ListObjects permission, adding both to be safe
@@ -44,7 +48,7 @@ export async function readAndUpsert(key: string, body: string, valueIfNotFound?:
     // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
     if (notFoundStatusCode.includes(err.statusCode)) {
       logger.debug('Creating missing file');
-      uploadToS3(body, key, cb);
+      await uploadToS3(body, key, cb);
       return valueIfNotFound ?? body;
     }
     logger.error(`Error occured when upserting file ${JSON.stringify(err)}`);
@@ -55,12 +59,22 @@ export async function readAndUpsert(key: string, body: string, valueIfNotFound?:
 function configureS3() {
   if (process.env.IS_OFFLINE === 'true') {
     logger.debug('configuring s3 using serverless');
-    return new AWS.S3({
-      s3ForcePathStyle: true,
-      accessKeyId: 'S3RVER', // This specific key is required when working offline
-      secretAccessKey: 'S3RVER',
+    return new S3Client({
+      // The key s3ForcePathStyle is renamed to forcePathStyle.
+      forcePathStyle: true,
+
+      credentials: {
+        // This specific key is required when working offline
+        accessKeyId: 'S3RVER',
+
+        secretAccessKey: 'S3RVER',
+      },
+
+      // The transformation for endpoint is not implemented.
+      // Refer to UPGRADING.md on aws-sdk-js-v3 for changes needed.
+      // Please create/upvote feature request on aws-sdk-js-codemod for endpoint.
       endpoint: 'http://localhost:4569',
     });
   }
-  return new AWS.S3();
+  return new S3Client({});
 }

src/infrastructure/secretsManagerService.ts

@@ -1,4 +1,4 @@
-import { SecretsManager } from 'aws-sdk';
+import { SecretsManager } from '@aws-sdk/client-secrets-manager';
 import MissingSecretError from '../errors/MissingSecretError';
 import SecretsManagerServiceInterface from '../interfaces/SecretsManagerService';
 
@@ -10,13 +10,13 @@ export default class SecretsManagerService implements SecretsManagerServiceInter
   }
 
   async getSecret(secretName: string): Promise<string> {
-    const data = await this.secretsManager.getSecretValue({ SecretId: secretName }).promise();
+    const data = await this.secretsManager.getSecretValue({ SecretId: secretName });
 
     if (data.SecretString) {
       return data.SecretString;
     }
     if (data.SecretBinary) {
-      return data.SecretBinary.toString('utf-8');
+      return Buffer.from(data.SecretBinary).toString();
     }
 
     throw new MissingSecretError();

tests/unit/infrastructure/api/index.test.ts

@@ -191,7 +191,7 @@ describe('API', () => {
           vrm_trm: '123',
         };
         DatabaseService.build = jest.fn().mockResolvedValue({} as DatabaseServiceInterface);
-        jest.spyOn(upload, 'uploadToS3').mockImplementation((_data, _fileName, callback) => callback());
+        jest.spyOn(upload, 'uploadToS3').mockImplementation((_data, _fileName, callback) => new Promise((resolve) => resolve(callback())));
         jest.spyOn(enquiryService, 'getFeedDetails').mockResolvedValue([evlFeedData]);
         const result = await supertest(app).get('/v1/enquiry/evl');
         expect(result.status).toEqual(200);
@@ -245,7 +245,7 @@ describe('API', () => {
         IssuedBy: 'some person',
       };
       DatabaseService.build = jest.fn().mockResolvedValue({} as DatabaseServiceInterface);
-      jest.spyOn(upload, 'uploadToS3').mockImplementation((_data, _fileName, callback) => callback());
+      jest.spyOn(upload, 'uploadToS3').mockImplementation((_data, _fileName, callback) => new Promise((resolve) => resolve(callback())));
       jest.spyOn(enquiryService, 'getFeedDetails').mockResolvedValue([tflFeedData]);
       const result = await supertest(app).get('/v1/enquiry/tfl');
       expect(result.status).toEqual(200);
@@ -269,7 +269,7 @@ describe('API', () => {
 
     it('sets the status to 404 for a not found error', async () => {
       DatabaseService.build = jest.fn().mockResolvedValue({} as DatabaseServiceInterface);
-      jest.spyOn(upload, 'uploadToS3').mockImplementation((_data, _fileName, callback) => callback());
+      jest.spyOn(upload, 'uploadToS3').mockImplementation((_data, _fileName, callback) => new Promise((resolve) => resolve(callback())));
       jest.spyOn(enquiryService, 'getFeedDetails').mockRejectedValue(new NotFoundError('This is an error'));
       const result = await supertest(app).get('/v1/enquiry/tfl');
 

tests/unit/infrastructure/s3BucketService.test.ts

@@ -9,22 +9,31 @@ const mockS3 = jest.fn(() => ({
   upload: mockUpload,
   getObject: mockGetObject,
 }));
+import {
+  S3Client,
+  GetObjectCommand,
+  GetObjectCommandOutput,
+  PutObjectCommand,
+} from '@aws-sdk/client-s3';
+import { mockClient } from 'aws-sdk-client-mock';
 import { readAndUpsert } from '../../../src/infrastructure/s3BucketService';
 
 jest.mock('aws-sdk', () => ({
   S3: mockS3,
 }));
-
+const client = mockClient(S3Client);
 describe('readAndUpsert', () => {
   beforeEach(() => {
     jest.clearAllMocks();
+    client.reset();
   });
   it('should read the file, return the value in the file and update the stored value', async () => {
     const fileName = 'a_file_with_a_date.txt';
     const originalFileContents = 'the original content of the file';
     const newFileContents = 'new content for the file';
 
-    mockPromise.mockResolvedValueOnce({ Body: Buffer.from(originalFileContents) });
+    client.on(GetObjectCommand).resolves({ Body: Buffer.from(originalFileContents) } as unknown as GetObjectCommandOutput);
+    client.on(PutObjectCommand).callsFake(mockUpload);
 
     const contents = await readAndUpsert(fileName, newFileContents);
     expect(contents).toEqual(originalFileContents);
@@ -38,7 +47,8 @@ describe('readAndUpsert', () => {
     const newFileContents = 'new content for the file';
     const valueIfNotFound = 'a week ago';
 
-    mockPromise.mockRejectedValueOnce({ statusCode });
+    client.on(GetObjectCommand).rejects({ statusCode } as unknown as GetObjectCommandOutput);
+    client.on(PutObjectCommand).callsFake(mockUpload);
 
     const contents = await readAndUpsert(fileName, newFileContents, valueIfNotFound);
     expect(contents).toEqual(valueIfNotFound);

tests/unit/infrastructure/secretsManagerService.test.ts

@@ -1,4 +1,4 @@
-import { SecretsManager } from 'aws-sdk';
+import { SecretsManager } from '@aws-sdk/client-secrets-manager';
 import MissingSecretError from '../../../src/errors/MissingSecretError';
 import SecretsManagerService from '../../../src/infrastructure/secretsManagerService';
 
@@ -24,34 +24,28 @@ describe('Secrets service', () => {
   it('should return the secret string', async () => {
     const secretValue = 'this is a secret';
     const mockSecretsManager = ({} as unknown) as SecretsManager;
-    const mockPromise = Promise.resolve({ SecretString: secretValue });
-    const mockPromiseFunc = jest.fn().mockReturnValue(mockPromise);
     const service = new SecretsManagerService(mockSecretsManager);
 
-    mockSecretsManager.getSecretValue = jest.fn().mockReturnValue({ promise: mockPromiseFunc });
+    mockSecretsManager.getSecretValue = jest.fn().mockReturnValue({ SecretString: secretValue });
 
     expect(await service.getSecret('secret')).toEqual('this is a secret');
   });
 
   it('should convert a binary secret to a string', async () => {
     const secretValue = Buffer.from('this is a secret');
     const mockSecretsManager = ({} as unknown) as SecretsManager;
-    const mockPromise = Promise.resolve({ SecretBinary: secretValue });
-    const mockPromiseFunc = jest.fn().mockReturnValue(mockPromise);
     const service = new SecretsManagerService(mockSecretsManager);
 
-    mockSecretsManager.getSecretValue = jest.fn().mockReturnValue({ promise: mockPromiseFunc });
+    mockSecretsManager.getSecretValue = jest.fn().mockReturnValue({ SecretBinary: secretValue });
 
     expect(await service.getSecret('secret')).toEqual('this is a secret');
   });
 
   it('should throw if there is no secret available in the response from the secret manager', async () => {
     const mockSecretsManager = ({} as unknown) as SecretsManager;
-    const mockPromise = Promise.resolve({});
-    const mockPromiseFunc = jest.fn().mockReturnValue(mockPromise);
     const service = new SecretsManagerService(mockSecretsManager);
 
-    mockSecretsManager.getSecretValue = jest.fn().mockReturnValue({ promise: mockPromiseFunc });
+    mockSecretsManager.getSecretValue = jest.fn().mockReturnValue({});
 
     await expect(service.getSecret('secret')).rejects.toThrow(MissingSecretError);
   });