From 74a247c17854dda5bc1d2c733700dece84be2d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Tue, 26 Nov 2024 11:45:29 +0100 Subject: [PATCH] Added --exclude-regex and --exclude-file --- package.json | 2 +- src/FileUtils.ts | 39 +++++++++++++--- src/Options.ts | 4 +- src/astgen.ts | 16 +++++++ src/index.ts | 4 +- test/astgen.test.ts | 110 +++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 163 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index 1dee537..48e99b1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@joernio/astgen", - "version": "3.20.0", + "version": "3.21.0", "description": "Generate JS/TS AST in json format with Babel", "exports": "./index.js", "keywords": [ diff --git a/src/FileUtils.ts b/src/FileUtils.ts index 313744e..8b75651 100644 --- a/src/FileUtils.ts +++ b/src/FileUtils.ts @@ -1,8 +1,10 @@ +import Options from "./Options"; import * as Defaults from "./Defaults"; import {readdirpPromise} from 'readdirp'; import * as fs from "node:fs"; import nReadlines from "n-readlines"; +import * as path from "node:path" function countFileLines(filePath: string): number { const broadbandLines = new nReadlines(filePath); @@ -13,9 +15,29 @@ function countFileLines(filePath: string): number { return lineNumber } -function ignoreDirectory(dirName: string): boolean { +function dirIsInIgnorePath(options: Options, fullPath: string, ignorePath: string): boolean { + if (path.isAbsolute(ignorePath)) { + return fullPath.startsWith(ignorePath) + } else { + const absIgnorePath = path.join(options.src, ignorePath) + return fullPath.startsWith(absIgnorePath) + } +} + +function fileIsInIgnorePath(options: Options, fullPath: string, ignorePath: string): boolean { + if (path.isAbsolute(ignorePath)) { + return fullPath == ignorePath + } else { + const absIgnorePath = path.join(options.src, ignorePath) + return fullPath == absIgnorePath + } +} + +function ignoreDirectory(options: Options, dirName: string, fullPath: string): boolean { return dirName.startsWith(".") || dirName.startsWith("__") || + options["exclude-file"].some((e: string) => dirIsInIgnorePath(options, fullPath, e)) || + options["exclude-regex"]?.test(fullPath) || Defaults.IGNORE_DIRS.includes(dirName.toLowerCase()) } @@ -35,19 +57,22 @@ function isTooLarge(fileWithDir: string): boolean { return false; } -function ignoreFile(fileName: string, fileWithDir: string, extensions: string[]): boolean { +function ignoreFile(options: Options, fileName: string, fullPath: string, extensions: string[]): boolean { return !extensions.some((e: string) => fileName.endsWith(e)) || fileName.startsWith(".") || fileName.startsWith("__") || Defaults.IGNORE_FILE_PATTERN.test(fileName) || - isEmscripten(fileWithDir) || - isTooLarge(fileWithDir) + options["exclude-file"].some((e: string) => fileIsInIgnorePath(options, fullPath, e)) || + options["exclude-regex"]?.test(fullPath) || + isEmscripten(fullPath) || + isTooLarge(fullPath) } -export async function filesWithExtensions(dir: string, extensions: string[]): Promise { +export async function filesWithExtensions(options: Options, extensions: string[]): Promise { + const dir = options.src const files = await readdirpPromise(dir, { - fileFilter: (f) => !ignoreFile(f.basename, f.fullPath, extensions), - directoryFilter: (d) => !ignoreDirectory(d.basename), + fileFilter: (f) => !ignoreFile(options, f.basename, f.fullPath, extensions), + directoryFilter: (d) => !ignoreDirectory(options, d.basename, d.fullPath), lstat: true }); // @ts-ignore diff --git a/src/Options.ts b/src/Options.ts index 1af99a8..3bbe0dd 100644 --- a/src/Options.ts +++ b/src/Options.ts @@ -3,5 +3,7 @@ export default interface Options { output: string, type?: string, recurse: boolean, - tsTypes: boolean + tsTypes: boolean, + "exclude-file": string[], + "exclude-regex"?: RegExp } diff --git a/src/astgen.ts b/src/astgen.ts index 84f723b..12e3f98 100644 --- a/src/astgen.ts +++ b/src/astgen.ts @@ -35,6 +35,22 @@ async function main(argv: string[]) { type: "boolean", description: "Generate type mappings using the Typescript Compiler API", }) + .option("exclude-file", { + default: [], + type: "string", + array: true, + description: "Exclude this file. Can be specified multiple times. Default is empty." + }) + .option("exclude-regex", { + coerce: (arg: any): RegExp | undefined => { + try { + return new RegExp(arg.toString(), "i") + } catch (err) { + return undefined; + } + }, + description: "Exclude files matching this regex (matches the absolute path)." + }) .version() .help("h").parseSync(); diff --git a/src/index.ts b/src/index.ts index 5fb048f..bc31dc0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -58,7 +58,7 @@ function createTsc(srcFiles: string[]): TscUtils.TscResult | undefined { */ async function createJSAst(options: Options) { try { - const srcFiles: string[] = await FileUtils.filesWithExtensions(options.src, Defaults.JS_EXTENSIONS); + const srcFiles: string[] = await FileUtils.filesWithExtensions(options, Defaults.JS_EXTENSIONS); let ts: TscUtils.TscResult | undefined; if (options.tsTypes) { ts = createTsc(srcFiles); @@ -95,7 +95,7 @@ async function createJSAst(options: Options) { * Generate AST for .vue files */ async function createVueAst(options: Options) { - const srcFiles: string[] = await FileUtils.filesWithExtensions(options.src, [".vue"]); + const srcFiles: string[] = await FileUtils.filesWithExtensions(options, [".vue"]); for (const file of srcFiles) { try { const ast = toVueAst(file); diff --git a/test/astgen.test.ts b/test/astgen.test.ts index e0ecb5d..2bed8d8 100644 --- a/test/astgen.test.ts +++ b/test/astgen.test.ts @@ -14,7 +14,8 @@ describe('astgen basic functionality', () => { type: "js", output: path.join(tmpDir, "ast_out"), recurse: true, - tsTypes: true + tsTypes: true, + "exclude-file": [] }); const resultAst = fs.readFileSync(path.join(tmpDir, "ast_out", "main.js.json")).toString(); expect(resultAst).toContain("\"fullName\":\"" + testFile.replaceAll("\\", "\\\\") + "\""); @@ -24,4 +25,111 @@ describe('astgen basic functionality', () => { fs.rmSync(tmpDir, {recursive: true}); }); + + it('should exclude files by relative file path correctly', async () => { + const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests")); + const testFile = path.join(tmpDir, "main.js"); + fs.writeFileSync(testFile, "console.log(\"Hello, world!\");"); + await start({ + src: tmpDir, + type: "js", + output: path.join(tmpDir, "ast_out"), + recurse: true, + tsTypes: false, + "exclude-file": ["main.js"] + }); + expect(fs.existsSync(path.join(tmpDir, "ast_out", "main.js.json"))).toBeFalsy() + + fs.rmSync(tmpDir, {recursive: true}); + }); + + it('should exclude files by absolute file path correctly', async () => { + const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests")); + const testFile = path.join(tmpDir, "main.js"); + fs.writeFileSync(testFile, "console.log(\"Hello, world!\");"); + await start({ + src: tmpDir, + type: "js", + output: path.join(tmpDir, "ast_out"), + recurse: true, + tsTypes: false, + "exclude-file": [testFile] + }); + expect(fs.existsSync(path.join(tmpDir, "ast_out", "main.js.json"))).toBeFalsy() + + fs.rmSync(tmpDir, {recursive: true}); + }); + + it('should exclude files by relative file path with dir correctly', async () => { + const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests")); + const testFile = path.join(tmpDir, "src", "main.js"); + fs.mkdirSync(path.join(tmpDir, "src")) + fs.writeFileSync(testFile, "console.log(\"Hello, world!\");"); + await start({ + src: tmpDir, + type: "js", + output: path.join(tmpDir, "ast_out"), + recurse: true, + tsTypes: false, + "exclude-file": [path.join("src", "main.js")] + }); + expect(fs.existsSync(path.join(tmpDir, "ast_out", "src", "main.js.json"))).toBeFalsy() + + fs.rmSync(tmpDir, {recursive: true}); + }); + + it('should exclude files by relative dir path correctly', async () => { + const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests")); + const testFile = path.join(tmpDir, "src", "main.js"); + fs.mkdirSync(path.join(tmpDir, "src")) + fs.writeFileSync(testFile, "console.log(\"Hello, world!\");"); + await start({ + src: tmpDir, + type: "js", + output: path.join(tmpDir, "ast_out"), + recurse: true, + tsTypes: false, + "exclude-file": ["src"] + }); + expect(fs.existsSync(path.join(tmpDir, "ast_out", "src", "main.js.json"))).toBeFalsy() + + fs.rmSync(tmpDir, {recursive: true}); + }); + + it('should exclude files by absolute dir path correctly', async () => { + const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests")); + const testFile = path.join(tmpDir, "src", "main.js"); + fs.mkdirSync(path.join(tmpDir, "src")) + fs.writeFileSync(testFile, "console.log(\"Hello, world!\");"); + await start({ + src: tmpDir, + type: "js", + output: path.join(tmpDir, "ast_out"), + recurse: true, + tsTypes: false, + "exclude-file": [path.join(tmpDir, "src")] + }); + expect(fs.existsSync(path.join(tmpDir, "ast_out", "main.js.json"))).toBeFalsy() + + fs.rmSync(tmpDir, {recursive: true}); + }); + + it('should exclude files by regex correctly', async () => { + const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests")); + const testFile = path.join(tmpDir, "main.js"); + fs.writeFileSync(testFile, "console.log(\"Hello, world!\");"); + await start({ + src: tmpDir, + type: "js", + output: path.join(tmpDir, "ast_out"), + recurse: true, + tsTypes: false, + "exclude-file": [], + "exclude-regex": new RegExp(".*main.*", "i") + }); + expect(fs.existsSync(path.join(tmpDir, "ast_out", "main.js.json"))).toBeFalsy() + + fs.rmSync(tmpDir, {recursive: true}); + }); + });