Skip to content

chore(cb2-12276): upgrade packages #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23,526 changes: 12,148 additions & 11,378 deletions package-lock.json

Large diffs are not rendered by default.

87 changes: 40 additions & 47 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
"version": "1.1.0",
"description": "Gets, packages, and sends the EVL data file.",
"main": "index.js",
"engines": {
"node": "^18.*"
},
"scripts": {
"start": "sls offline start",
"build": "tsc --rootDir ./ --outDir .build --sourceMap false && npm run build:copy",
Expand All @@ -27,67 +30,57 @@
"predeploy": "npm install && npm run prepush",
"prepush": "npm run test && npm run build && npm run test-i"
},
"contributors": [
{
"name": "Shaun Hare",
"email": "shaun.hare@dvsa.gov.uk"
}
],
"license": "MIT",
"dependencies": {
"@aws-sdk/client-s3": "^3.565.0",
"@aws-sdk/client-secrets-manager": "^3.565.0",
"@types/md5": "^2.3.2",
"@types/ssh2-sftp-client": "^7.0.1",
"@types/tar": "^6.1.1",
"@aws-sdk/client-s3": "^3.577.0",
"@aws-sdk/client-secrets-manager": "^3.580.0",
"@types/md5": "^2.3.5",
"@types/ssh2-sftp-client": "^9.0.3",
"@types/tar": "^6.1.13",
"md5": "^2.3.0",
"ssh2-sftp-client": "^9.0.0",
"tar": "^6.1.11",
"winston": "^3.7.2"
"ssh2-sftp-client": "^10.0.3",
"tar": "^7.1.0",
"winston": "^3.13.0"
},
"devDependencies": {
"@commitlint/cli": "^17.0.2",
"@commitlint/config-conventional": "^17.0.2",
"@dvsa/eslint-config-ts": "^2.4.0",
"@types/aws-lambda": "^8.10.78",
"@types/jest": "^27.0.2",
"@types/node": "^16.10.2",
"@types/uuid": "^8.3.1",
"@typescript-eslint/eslint-plugin": "^5.27.1",
"@typescript-eslint/parser": "^5.27.1",
"archiver": "^5.3.0",
"@commitlint/cli": "^19.3.0",
"@commitlint/config-conventional": "^19.2.2",
"@dvsa/eslint-config-ts": "^3.0.1",
"@types/aws-lambda": "^8.10.138",
"@types/jest": "^29.5.12",
"@types/node": "^20.12.12",
"@types/uuid": "^9.0.8",
"@typescript-eslint/eslint-plugin": "^7.10.0",
"@typescript-eslint/parser": "^7.10.0",
"archiver": "^7.0.1",
"aws-sdk-client-mock": "^4.0.0",
"commitlint-plugin-function-rules": "^1.7.1",
"commitlint-plugin-function-rules": "^4.0.0",
"current-git-branch": "^1.1.0",
"dotenv": "^16.0.0",
"esbuild": "^0.14.43",
"eslint": "^7.32.0",
"eslint-config-airbnb-typescript": "^17.0.0",
"eslint-import-resolver-typescript": "^2.5.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-jest": "^26.5.3",
"eslint-plugin-security": "^1.5.0",
"fs-extra": "^10.0.0",
"husky": "^8.0.1",
"jest": "^27.2.4",
"dotenv": "^16.4.5",
"esbuild": "^0.21.3",
"eslint-config-airbnb-typescript": "^18.0.0",
"eslint-import-resolver-typescript": "^3.6.1",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-jest": "^28.5.0",
"eslint-plugin-security": "^3.0.0",
"fs-extra": "^11.2.0",
"husky": "^9.0.11",
"jest": "^29.7.0",
"jest-sonar-reporter": "^2.0.0",
"prettier": "^2.6.2",
"serverless": "^3.19.0",
"serverless-offline": "^8.8.0",
"serverless-plugin-typescript": "^2.1.2",
"prettier": "^3.2.5",
"serverless": "^3.38.0",
"serverless-offline": "^13.6.0",
"serverless-plugin-typescript": "^2.1.5",
"serverless-s3-local": "^0.8.4",
"sonar-scanner": "^3.1.0",
"source-map-support": "^0.5.19",
"ts-jest": "^27.0.5",
"ts-loader": "^9.2.6",
"typescript": "^4.7.4"
"source-map-support": "^0.5.21",
"ts-jest": "^29.1.3",
"ts-loader": "^9.5.1",
"typescript": "^5.4.5"
},
"overrides": {
"got": "^11.8.5"
},
"engines": {
"node": "^18.*"
},
"jestSonar": {
"reportPath": ".reports",
"reportFile": "test-report.xml",
Expand Down
20 changes: 8 additions & 12 deletions src/fileConvert/fileConvert.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
/* eslint-disable security/detect-non-literal-fs-filename */
import md5 from 'md5';
import tar from 'tar';
import * as tar from 'tar';
import * as fs from 'fs';
import logger from '../util/logger';

Expand All @@ -26,23 +25,20 @@ export const configureEvlFile = async (
const textFilenamePrefix = 'crc32_';
const archiveNamePrefix = 'EVL_GVT_';
const dateFromFilename = filename.split('_')[2].split('.')[0];
const csvFilename = archiveNamePrefix + dateFromFilename + '.csv';
const textFilename = textFilenamePrefix + dateFromFilename + '.txt';
const archiveName =
workingDir + archiveNamePrefix + dateFromFilename + '.tar.gz';
const csvFilename = `${archiveNamePrefix + dateFromFilename}.csv`;
const textFilename = `${textFilenamePrefix + dateFromFilename}.txt`;
const archiveName = `${workingDir + archiveNamePrefix + dateFromFilename}.tar.gz`;

try {
const data = bufferInput.toString().split('\n');

if (data.length == 1 && data[0] === '') {
if (data.length === 1 && data[0] === '') {
throw new Error('No data provided');
}

const numberOfRecords = data.length.toString();
const headerLine =
recordInfoHeader + description + dateFileCreation + ', , ';
const trailerLine =
recordInfoTrailer + numberOfRecords + ' ' + vrmsExtracted + ', , ';
const headerLine = `${recordInfoHeader + description + dateFileCreation}, , `;
const trailerLine = `${recordInfoTrailer + numberOfRecords} ${vrmsExtracted}, , `;

data.splice(0, 0, headerLine);
data.splice(data.length, 0, trailerLine);
Expand All @@ -53,7 +49,7 @@ export const configureEvlFile = async (
logger.info('Written csv file');

const md5sum = md5(csvData);
const md5string = 'MD5 (' + csvFilename + ') = ' + md5sum;
const md5string = `MD5 (${csvFilename}) = ${md5sum}`;
fs.writeFileSync(workingDir + textFilename, md5string);
logger.info('Written txt checksum file');

Expand Down
3 changes: 2 additions & 1 deletion src/filePull/fromS3.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import logger from '../util/logger';
/* eslint-disable no-restricted-syntax */
import {
GetObjectCommand,
GetObjectCommandOutput,
Expand All @@ -10,6 +10,7 @@ import {
import { S3EventRecord } from 'aws-lambda';
import { Stream } from 'stream';
import { IncomingMessage } from 'http';
import logger from '../util/logger';

const s3Client = new S3Client(
(process.env.IS_LOCAL || process.env.IS_OFFLINE) && {
Expand Down
6 changes: 2 additions & 4 deletions src/filePush/filePush.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
/* eslint-disable security/detect-object-injection */
import Client from 'ssh2-sftp-client';
import logger from '../util/logger';
import path from 'path';
import logger from '../util/logger';
import { getSecret } from '../util/getSecret';

export interface Config {
Expand Down Expand Up @@ -45,8 +44,7 @@ export const filePush = async (filepath: string, eventType: Feeds) => {
evl: process.env.EVL_SFTP_PATH,
tfl: process.env.TFL_SFTP_PATH,
};
const remoteFileLocation =
(sftpPath[eventType] ?? '') + path.basename(filepath);
const remoteFileLocation = (sftpPath[eventType] ?? '') + path.basename(filepath);

try {
logger.info('Attempt connection');
Expand Down
10 changes: 6 additions & 4 deletions src/handler/s3Event.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
/* eslint-disable security/detect-non-literal-fs-filename */
/* eslint-disable prefer-promise-reject-errors */
/* eslint-disable no-await-in-loop */
/* eslint-disable no-restricted-syntax */
import * as fs from 'fs';
import type { S3Event, S3EventRecord } from 'aws-lambda';
import { randomUUID } from 'crypto';
import { configureEvlFile } from '../fileConvert/fileConvert';
import { filePull } from '../filePull/fromS3';
import { filePush } from '../filePush/filePush';
import { randomUUID } from 'crypto';
import logger from '../util/logger';

const handleEvlEvent = async (record: S3EventRecord) => {
Expand Down Expand Up @@ -52,8 +54,8 @@ export const handler = async (event: S3Event): Promise<string> => {
if (fileName.startsWith('EVL_') && process.env.EVL_SFTP_SEND === 'true') {
await handleEvlEvent(record);
} else if (
fileName.startsWith('VOSA') &&
process.env.TFL_SFTP_SEND === 'true'
fileName.startsWith('VOSA')
&& process.env.TFL_SFTP_SEND === 'true'
) {
await handleTflEvent(record);
} else {
Expand Down
7 changes: 1 addition & 6 deletions src/util/getSecret.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,7 @@
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { SecretsManager } from '@aws-sdk/client-secrets-manager';

async function getSecret(secretName: string): Promise<string> {
const envVariable: string | undefined =
process.env[secretName.replace(/\//g, '-')];
const envVariable: string | undefined = process.env[secretName.replace(/\//g, '-')];
if (envVariable !== undefined) {
return envVariable.replace(/\\n/g, '\n');
}
Expand Down
10 changes: 5 additions & 5 deletions tests/fileConvert/fileConvert.test.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
/* eslint-disable security/detect-non-literal-fs-filename */
import { configureEvlFile } from '../../src/fileConvert/fileConvert';
import * as fs from 'fs';
import md5 from 'md5';
import { configureEvlFile } from '../../src/fileConvert/fileConvert';

describe('test the file config', () => {
const testFilename = 'tests/resources/EVL_GVT_20220621.csv';
const dateFromFilename = testFilename
.split('/')[2]
.split('_')[2]
.split('.')[0];
const txtFilename = 'crc32_' + dateFromFilename + '.txt';
const csvFilename = 'EVL_GVT_' + dateFromFilename + '.csv';
const finalFilename = 'EVL_GVT_' + dateFromFilename + '.tar.gz';
const txtFilename = `crc32_${dateFromFilename}.txt`;
const csvFilename = `EVL_GVT_${dateFromFilename}.csv`;
const finalFilename = `EVL_GVT_${dateFromFilename}.tar.gz`;

let buffer: Buffer;

Expand Down Expand Up @@ -55,7 +55,7 @@ describe('test the file config', () => {
await configureEvlFile('', buffer, 'EVL_GVT_20220621.csv');
const csvFile = fs.readFileSync(csvFilename);
const textFile = fs.readFileSync(txtFilename);
const hash = 'MD5 (EVL_GVT_20220621.csv) = ' + md5(csvFile);
const hash = `MD5 (EVL_GVT_20220621.csv) = ${md5(csvFile)}`;
expect(textFile.toString()).toBe(hash);
});
});
Expand Down
2 changes: 0 additions & 2 deletions tests/filePull/fromS3.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-explicit-any */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { S3Event } from 'aws-lambda';
Expand Down
33 changes: 14 additions & 19 deletions tests/filePush/filePush.test.ts
Original file line number Diff line number Diff line change
@@ -1,30 +1,25 @@
/* eslint-disable no-void */
import { filePush, createConfig, Config } from '../../src/filePush/filePush';

const mockConnect = jest.fn();
const mockFastPut = jest.fn();
const mockEnd = jest.fn();

jest.mock('ssh2-sftp-client', () => {
return {
__esModule: true,
default: jest.fn().mockImplementation(() => {
return {
connect: mockConnect,
fastPut: mockFastPut,
end: mockEnd,
};
}),
};
});
jest.mock('ssh2-sftp-client', () => ({
__esModule: true,
default: jest.fn().mockImplementation(() => ({
connect: mockConnect,
fastPut: mockFastPut,
end: mockEnd,
})),
}));

jest.mock('../../src/util/getSecret', () => ({
getSecret: jest.fn(() =>
Promise.resolve(
'{ "host" : "test", "username" : "user", "retries" : 3, "password": "psswrd" }',
),
),
getSecret: jest.fn(() => Promise.resolve(
'{ "host" : "test", "username" : "user", "retries" : 3, "password": "psswrd" }',
)),
}));

import { filePush, createConfig, Config } from '../../src/filePush/filePush';

describe('test the push to SFTP server', () => {
process.env.EVL_SFTP_CONFIG = 'mockString';
process.env.TFL_SFTP_CONFIG = 'mockString';
Expand Down
1 change: 1 addition & 0 deletions tests/handler/s3Event.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable no-void */
import type { S3Event } from 'aws-lambda';
import { mockClient } from 'aws-sdk-client-mock';
import { S3Client } from '@aws-sdk/client-s3';
Expand Down
1 change: 0 additions & 1 deletion tests/util/getSecrets.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
/* eslint-disable @typescript-eslint/no-unsafe-argument */
import {
GetSecretValueCommand,
SecretsManager,
Expand Down
9 changes: 5 additions & 4 deletions tests/util/logger.test.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
/* eslint-disable import/first */
/* eslint-disable no-underscore-dangle */
/* eslint-disable @typescript-eslint/ban-ts-comment */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { EOL } from 'os';

process.env.LOG_LEVEL = 'debug';
// eslint-disable-next-line import/first
import logger from '../../src/util/logger';

describe('logger functions', () => {
it('GIVEN a logger WHEN an info is logged THEN the console message is correct.', () => {
// @ts-ignore
// @ts-expect-error
const consoleSpy = jest.spyOn(console._stdout, 'write');
logger.info('I am an info message!');
expect(consoleSpy).toHaveBeenCalledWith(
Expand All @@ -17,7 +18,7 @@ describe('logger functions', () => {
});

it('GIVEN a logger WHEN a debug is logged THEN the console message is correct.', () => {
// @ts-ignore
// @ts-expect-error
const consoleSpy = jest.spyOn(console._stdout, 'write');
logger.debug('I am a debug message!');
expect(consoleSpy).toHaveBeenCalledWith(
Expand All @@ -26,7 +27,7 @@ describe('logger functions', () => {
});

it('GIVEN a logger WHEN an error is logged THEN the console message is correct.', () => {
// @ts-ignore
// @ts-expect-error
const consoleSpy = jest.spyOn(console._stdout, 'write');
const error = new Error('AN ERROR');
logger.error('', error);
Expand Down