Skip to content

Commit

Permalink
ci(custom-checks): fix depcheck not detecting missing dependencies
Browse files Browse the repository at this point in the history 
1. The depcheck tool we use have not correctly discovered some of the
missing dependencies that we have because it only verifies that the
imported dependency is present SOMEwhere in the package.json file, not that
it is specifically present in the production dependencies section which
leads to crashes and broken packages due to the API server not installing
dev dependencies when instantiating a plugin and therefore missing a few
of the dependencies that are otherwise very much needed at runtime in
production.
2. The solution to the problem was to implement our own typescript parsing
with babel and then double check the work of depcheck to make sure that
the dependencies that it marks as "no issues" are actually OK and have no
issues.
3. The hardest edge case was type imports e.g. `import type { Express } from "express";`
because the import was there, but we did not actually need that dependency
in the production dependencies as long as ALL of the imports to it in the
given package were type imports. To robustly verify this being the case or not
we had to pull out the big guns and parse all the typescript code per package
to make sure that we've looked at every single import of the dependency in
question at every single code file of the package in question.

Depends on #3345

Signed-off-by: Peter Somogyvari <peter.somogyvari@accenture.com>
  • Loading branch information
@petermetz
petermetz committed Jun 27, 2024
1 parent d9d5904 commit 5abbbff
Show file tree
Hide file tree
Showing 3 changed files with 261 additions and 20 deletions.
2 changes: 2 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@
"zod": ">=3.22.3"
},
"devDependencies": {
"@babel/parser": "7.24.7",
"@babel/types": "7.24.7",
"@bufbuild/buf": "1.30.0",
"@bufbuild/protobuf": "1.8.0",
"@bufbuild/protoc-gen-es": "1.8.0",
Expand Down
243 changes: 223 additions & 20 deletions tools/custom-checks/check-missing-node-deps.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import { RuntimeError } from "run-time-error";
import fs from "fs-extra";
import depcheck from "depcheck";
import { getAllPkgDirs } from "./get-all-pkg-dirs";
import { ParseResult, parse } from "@babel/parser";
import { File as BabelFile, ImportDeclaration } from "@babel/types";

export interface ICheckMissingNodeDepsRequest {
readonly pkgDirsToCheck: Readonly<Array<string>>;
Expand All @@ -28,12 +30,14 @@ export async function checkMissingNodeDeps(
throw new RuntimeError(`req.pkgDirsToCheck parameter cannot be falsy.`);
}

if (req.verbose) {
const verbose = req.verbose === true;
if (verbose) {
console.log(`${TAG} SCRIPT_DIR=${SCRIPT_DIR}`);
console.log(`${TAG} PROJECT_DIR=${PROJECT_DIR}`);
console.log("%s Package directories to check: %o", TAG, req.pkgDirsToCheck);
}

const depCheckOptions = {
const depCheckOptions: depcheck.Options = {
ignoreBinPackage: false, // ignore the packages with bin entry
skipMissing: false, // skip calculation of missing dependencies
ignorePatterns: [
Expand All @@ -42,6 +46,16 @@ export async function checkMissingNodeDeps(
"dist",
"bower_components",
"node_modules",
"docs/",
"src/main/kotlin",
"src/test/",
"Dockerfile",
"*.md",
"*.json",
"*.test.ts",
"*.tsx",
"*.mts",
"*.scss",
],
ignoreMatches: [
// ignore dependencies that matches these globs
Expand All @@ -52,46 +66,54 @@ export async function checkMissingNodeDeps(
"tap",
"@ionic-native/*",
],
// parsers: {
// // the target parsers
// '**/*.js': depcheck.parser.es6,
// '**/*.jsx': depcheck.parser.jsx,
// },
detectors: [
// the target detectors
depcheck.detector.requireCallExpression,
depcheck.detector.importDeclaration,
depcheck.detector.importCallExpression,
],
specials: [
// the target special parsers
// depcheck.special.eslint,
// depcheck.special.webpack,
],
};

const tasks = req.pkgDirsToCheck.map(async (pkgDir) => {
const manifest = path.join(pkgDir, "package.json");
const manifestExists = await fs.pathExists(manifest);
const manifestPath = path.join(pkgDir, "package.json");
const manifestExists = await fs.pathExists(manifestPath);
if (!manifestExists) {
if (req.verbose) {
console.log("%s %s has no package.json. Skipping.", TAG, manifest);
if (verbose) {
console.log("%s %s has no package.json. Skipping.", TAG, manifestPath);
}
return;
}
const { missing } = await depcheck(pkgDir, depCheckOptions);

const depCheckResult = await depcheck(pkgDir, depCheckOptions);
const { missing } = depCheckResult;
const missingDepNames = Object.keys(missing);

if (verbose) {
console.log("%s DepCheck result %s: %o", TAG, pkgDir, depCheckResult);
}

const prodCodePathPrefixes = [path.join(pkgDir, "src/main/")];

const prodDepMissingErrors = await findMissingPrdDeps({
depCheckResult,
manifestPath,
verbose,
TAG,
prodCodePathPrefixes,
ignoredPkgNameRegExps: [new RegExp("@types/.*")],
});

prodDepMissingErrors.forEach((e) => errors.push(e));

missingDepNames.forEach((depName) => {
const filesUsingDep = missing[depName].join("\n\t");
const anError = `"${depName}" is a missing dependency from ${manifest} with the following files using them:\n\t${filesUsingDep}`;
const anError = `"${depName}" is a missing dependency from ${manifestPath} with the following files using them:\n\t${filesUsingDep}`;
errors.push(anError);
if (req.verbose) {
console.log(
"%s Missing dep %s in %s",
TAG,
depName,
manifest,
manifestPath,
missing[depName],
);
}
Expand All @@ -103,6 +125,186 @@ export async function checkMissingNodeDeps(
return [errors.length === 0, errors];
}

async function findMissingPrdDeps(req: {
readonly depCheckResult: depcheck.Results;
readonly manifestPath: Readonly<string>;
readonly TAG: Readonly<string>;
readonly verbose: boolean;
readonly prodCodePathPrefixes: Readonly<Array<string>>;
readonly ignoredPkgNameRegExps: Readonly<Array<RegExp>>;
}): Promise<string[]> {
const fn =
"[tools/custom-checks/check-missing-node-deps.ts]#filterUsingToSrcMainDirs()";
if (!req) {
throw new Error(fn + "Expected arg req to be truthy");
}
if (!req.depCheckResult) {
throw new Error(fn + "Expected arg req.depCheckResult to be truthy");
}
if (!req.depCheckResult.using) {
throw new Error(fn + "Expected req.depCheckResult.using to be truthy");
}
if (!req.manifestPath) {
throw new Error(fn + "Expected arg req.manifestPath to be truthy");
}
if (req.verbose !== true && req.verbose !== false) {
throw new Error(fn + "Expected arg req.verbose to be strictly bool");
}
if (!req.TAG) {
throw new Error(fn + "Expected arg req.TAG to be truthy");
}
if (!req.prodCodePathPrefixes) {
throw new Error(fn + "Expected arg req.prodCodePathPrefixes to be truthy");
}
if (!Array.isArray(req.prodCodePathPrefixes)) {
throw new Error(fn + "Expected arg req.prodCodePathPrefixes to be Array");
}
if (!req.ignoredPkgNameRegExps) {
throw new Error(fn + "Expected arg req.ignoredPkgNameRegExps to be truthy");
}
if (!Array.isArray(req.ignoredPkgNameRegExps)) {
throw new Error(fn + "Expected arg req.ignoredPkgNameRegExps to be Array");
}

const {
manifestPath,
depCheckResult,
TAG,
verbose,
prodCodePathPrefixes,
ignoredPkgNameRegExps,
} = req;

const prodDepMissingErrors: string[] = [];

const pkgPojo = await fs.readJson(req.manifestPath);

if (!pkgPojo) {
prodDepMissingErrors.push(`${manifestPath} was parsed into a falsy POJO.`);
}

const { dependencies } = pkgPojo;
if (!dependencies || typeof dependencies !== "object") {
if (verbose) {
console.log("%s Skipping %s: no 'dependencies' key", fn, manifestPath);
}
return prodDepMissingErrors;
}
if (verbose) {
console.log("%s prod dependencies %s: %o", TAG, manifestPath, dependencies);
}

const depEntries = Object.entries(dependencies);
if (depEntries.length <= 0) {
if (verbose) {
console.log("%s Skipping %s: empty 'dependencies'", fn, manifestPath);
}
return prodDepMissingErrors;
}

const prodDepPkgs = depEntries.map(([k]) => k);

const { using } = depCheckResult;

const parserCache = new Map<string, ParseResult<BabelFile>>();

const seenAndIgnoredExt = new Set<string>();

const fileToAstMap = new Map<string, Array<ParseResult<BabelFile>>>();

const parseTasks = Object.entries(using).map(async ([k, v]) => {
const astsOfK = [];
for (let i = 0; i < v.length; i++) {
const filePath = v[i];
const supportedFileExtensions = [".ts", ".js"];
const fileExtension = path.extname(filePath);
if (!supportedFileExtensions.includes(fileExtension)) {
seenAndIgnoredExt.add(fileExtension);
continue;
}
const contentBuffer = await fs.readFile(filePath);
const contentStr = contentBuffer.toString("utf-8");

const cachedAst = parserCache.get(filePath);

const ast =
cachedAst ||
(await parse(contentStr, {
sourceType: "module",
sourceFilename: filePath,
attachComment: false,
plugins: [
"typescript",
"decorators-legacy",
"dynamicImport",
"importAttributes",
"importMeta",
"importReflection",
"deferredImportEvaluation",
"sourcePhaseImports",
],
}));

parserCache.set(filePath, ast as ParseResult<BabelFile>);

astsOfK.push(ast);
}
fileToAstMap.set(k, astsOfK as Array<ParseResult<BabelFile>>);
});

if (seenAndIgnoredExt.size > 0) {
console.log("%s Seen+Ignored file extensions: %o", fn, seenAndIgnoredExt);
}

await Promise.all(parseTasks);

const missingProdDeps = Object.entries(using).filter(([k, v]) => {
// Is it being imported by code that is in a directory indicating prod use?
const pathPrefixMatch = v.some((x) =>
prodCodePathPrefixes.some((prefix) => x.startsWith(prefix)),
);
// Is it actually missing from the package.json#dependencies hash?
const isMissing = !prodDepPkgs.includes(k);

// We choose to ignore dependencies like @types/express or @types/* because
// we know that at runtime the Typescript compiler will delete those imports
const notIgnored = ignoredPkgNameRegExps.every((r: RegExp) => !r.test(k));

// If both are true we caught ourselves a missing production dependency that
// will crash at runtime if not rectified.
const isMissingProdDep = pathPrefixMatch && isMissing && notIgnored;

const asts = fileToAstMap.get(k);
if (!asts) {
const errorMessage = `${fn} Expected presence of parsed AST in map for dependency=${k}`;
throw new Error(errorMessage);
}

const andNotOnlyTypeImports = asts.some((ast) =>
ast.program.body
.filter((n): n is ImportDeclaration => {
return n.type === "ImportDeclaration";
})
.filter((n) => n.source.value === k)
.some((n) => n.importKind !== "type"),
);

return isMissingProdDep && andNotOnlyTypeImports;
});

if (!Array.isArray(missingProdDeps)) {
throw new Error(fn + "Expected missingProdDeps to be Array");
}

missingProdDeps.forEach(([k, v]) => {
const usageLocations = v.join("\n\t");
const errorMsg = `${TAG} ERROR - MISSING production dependency ${k} from ${manifestPath}. Found usage in \n\t${usageLocations}`;
prodDepMissingErrors.push(errorMsg);
});

return prodDepMissingErrors;
}

const nodePath = path.resolve(process.argv[1]);
const modulePath = path.resolve(fileURLToPath(import.meta.url));
const isRunningDirectlyViaCLI = nodePath === modulePath;
Expand All @@ -111,6 +313,7 @@ if (isRunningDirectlyViaCLI) {
const { absolutePaths: pkgDirsToCheck } = await getAllPkgDirs();

const req: ICheckMissingNodeDepsRequest = {
verbose: false,
pkgDirsToCheck,
};
const [success, errors] = await checkMissingNodeDeps(req);
Expand Down
36 changes: 36 additions & 0 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2074,6 +2074,13 @@ __metadata:
languageName: node
linkType: hard

"@babel/helper-string-parser@npm:^7.24.7":
version: 7.24.7
resolution: "@babel/helper-string-parser@npm:7.24.7"
checksum: 10/603d8d962bbe89907aa99a8f19a006759ab7b2464615f20a6a22e3e2e8375af37ddd0e5175c9e622e1c4b2d83607ffb41055a59d0ce34404502af30fde573a5c
languageName: node
linkType: hard

"@babel/helper-validator-identifier@npm:^7.16.7":
version: 7.16.7
resolution: "@babel/helper-validator-identifier@npm:7.16.7"
Expand Down Expand Up @@ -2116,6 +2123,13 @@ __metadata:
languageName: node
linkType: hard

"@babel/helper-validator-identifier@npm:^7.24.7":
version: 7.24.7
resolution: "@babel/helper-validator-identifier@npm:7.24.7"
checksum: 10/86875063f57361471b531dbc2ea10bbf5406e12b06d249b03827d361db4cad2388c6f00936bcd9dc86479f7e2c69ea21412c2228d4b3672588b754b70a449d4b
languageName: node
linkType: hard

"@babel/helper-validator-option@npm:^7.16.7":
version: 7.16.7
resolution: "@babel/helper-validator-option@npm:7.16.7"
Expand Down Expand Up @@ -2348,6 +2362,15 @@ __metadata:
languageName: node
linkType: hard

"@babel/parser@npm:7.24.7":
version: 7.24.7
resolution: "@babel/parser@npm:7.24.7"
bin:
parser: ./bin/babel-parser.js
checksum: 10/ef9ebce60e13db560ccc7af9235d460f6726bb7e23ae2d675098c1fc43d5249067be60d4118889dad33b1d4f85162cf66baf554719e1669f29bb20e71322568e
languageName: node
linkType: hard

"@babel/parser@npm:^7.1.0, @babel/parser@npm:^7.14.7, @babel/parser@npm:^7.16.7, @babel/parser@npm:^7.17.3":
version: 7.17.3
resolution: "@babel/parser@npm:7.17.3"
Expand Down Expand Up @@ -4089,6 +4112,17 @@ __metadata:
languageName: node
linkType: hard

"@babel/types@npm:7.24.7":
version: 7.24.7
resolution: "@babel/types@npm:7.24.7"
dependencies:
"@babel/helper-string-parser": "npm:^7.24.7"
"@babel/helper-validator-identifier": "npm:^7.24.7"
to-fast-properties: "npm:^2.0.0"
checksum: 10/ad3c8c0d6fb4acb0bb74bb5b4bb849b181bf6185677ef9c59c18856c81e43628d0858253cf232f0eca806f02e08eff85a1d3e636a3e94daea737597796b0b430
languageName: node
linkType: hard

"@babel/types@npm:^7.0.0, @babel/types@npm:^7.16.7, @babel/types@npm:^7.17.0, @babel/types@npm:^7.3.0, @babel/types@npm:^7.3.3, @babel/types@npm:^7.4.4":
version: 7.17.0
resolution: "@babel/types@npm:7.17.0"
Expand Down Expand Up @@ -9417,6 +9451,8 @@ __metadata:
version: 0.0.0-use.local
resolution: "@hyperledger/cactus@workspace:."
dependencies:
"@babel/parser": "npm:7.24.7"
"@babel/types": "npm:7.24.7"
"@bufbuild/buf": "npm:1.30.0"
"@bufbuild/protobuf": "npm:1.8.0"
"@bufbuild/protoc-gen-es": "npm:1.8.0"
Expand Down

0 comments on commit 5abbbff

Please sign in to comment.