From b98fa013282901ee7eb2054c71dbc763762be964 Mon Sep 17 00:00:00 2001 From: Peter Somogyvari Date: Fri, 31 May 2024 19:12:58 -0700 Subject: [PATCH] build: fix codegen flakiness with sequential execution, handrolled nwget 1. The package.json file of the cmd-api-server package now runs the codegen related scripts sequentially (e.g. using `run-s` instead of `run-p` of `npm-run-all`). This lowers the probability that the download of the openapi-generator .jar file is too late to finish and a crash occurs due to the .jar file not being present on the file-system when it is called upon. 2. Also adding a hand-built `nwget` alternative because it was hanging the process after finishing the download (I've only seen this reproduced locally, but neveretheless it was frustrating) Signed-off-by: Peter Somogyvari --- package.json | 3 +- packages/cactus-cmd-api-server/package.json | 4 +- tools/download-file-to-disk.ts | 105 ++++++++++++++++++++ 3 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 tools/download-file-to-disk.ts diff --git a/package.json b/package.json index 8d6a37e40c..f05f4480d2 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "tools:validate-bundle-names": "TS_NODE_PROJECT=./tools/tsconfig.json node --trace-deprecation --experimental-modules --abort-on-uncaught-exception --loader ts-node/esm --experimental-specifier-resolution=node ./tools/validate-bundle-names.js", "tools:bump-openapi-spec-dep-versions": "TS_NODE_PROJECT=./tools/tsconfig.json node --trace-deprecation --experimental-modules --abort-on-uncaught-exception --loader ts-node/esm --experimental-specifier-resolution=node ./tools/bump-openapi-spec-dep-versions.ts", "tools:create-production-only-archive": "TS_NODE_PROJECT=./tools/tsconfig.json node --trace-deprecation --experimental-modules --abort-on-uncaught-exception --loader ts-node/esm --experimental-specifier-resolution=node ./tools/create-production-only-archive.ts", + "tools:download-file-to-disk": "TS_NODE_PROJECT=./tools/tsconfig.json node --trace-deprecation --experimental-modules --abort-on-uncaught-exception --loader ts-node/esm --experimental-specifier-resolution=node ./tools/download-file-to-disk.ts", "tools:get-latest-sem-ver-git-tag": "TS_NODE_PROJECT=./tools/tsconfig.json node --abort-on-uncaught-exception --loader ts-node/esm --experimental-specifier-resolution=node --no-warnings ./tools/get-latest-sem-ver-git-tag.ts", "tools:generate-sbom": "TS_NODE_PROJECT=tools/tsconfig.json node --experimental-json-modules --trace-deprecation --experimental-modules --abort-on-uncaught-exception --loader ts-node/esm --experimental-specifier-resolution=node ./tools/generate-sbom.ts", "tools:fix-pkg-npm-scope": "TS_NODE_PROJECT=tools/tsconfig.json node --experimental-json-modules --trace-deprecation --experimental-modules --abort-on-uncaught-exception --loader ts-node/esm --experimental-specifier-resolution=node ./tools/custom-checks/check-pkg-npm-scope.ts", @@ -58,7 +59,7 @@ "codegen:lerna": "lerna run codegen", "codegen:warmup-cleancodegendir": "node tools/clear-openapi-codegen-folders.js", "codegen:warmup-mkdir": "make-dir ./node_modules/@openapitools/openapi-generator-cli/versions/", - "codegen:warmup-v6.6.0": "nwget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/6.6.0/openapi-generator-cli-6.6.0.jar -O ./node_modules/@openapitools/openapi-generator-cli/versions/6.6.0.jar", + "codegen:warmup-v6.6.0": "yarn tools:download-file-to-disk --url=https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/6.6.0/openapi-generator-cli-6.6.0.jar --output-file-path=./node_modules/@openapitools/openapi-generator-cli/versions/6.6.0.jar", "watch-other": "lerna run --parallel watch", "watch-tsc": "tsc --build --watch", "watch": "run-p -r watch-*", diff --git a/packages/cactus-cmd-api-server/package.json b/packages/cactus-cmd-api-server/package.json index 7b7c9b6305..64d74549f8 100644 --- a/packages/cactus-cmd-api-server/package.json +++ b/packages/cactus-cmd-api-server/package.json @@ -44,10 +44,10 @@ ], "scripts": { "benchmark": "tsx ./src/test/typescript/benchmark/run-cmd-api-server-benchmark.ts .tmp/benchmark-results/cmd-api-server/run-cmd-api-server-benchmark.ts.log", - "codegen": "run-p 'codegen:*'", + "codegen": "run-s 'codegen:*'", "codegen:openapi": "npm run generate-sdk", "codegen:proto": "run-s proto:openapi proto:protoc-gen-ts", - "generate-sdk": "run-p 'generate-sdk:*'", + "generate-sdk": "run-s 'generate-sdk:*'", "generate-sdk:kotlin": "openapi-generator-cli generate -i ./src/main/json/openapi.json -g kotlin -o ./src/main/kotlin/generated/openapi/kotlin-client/ --reserved-words-mappings protected=protected --ignore-file-override ../../openapi-generator-ignore", "generate-sdk:typescript-axios": "openapi-generator-cli generate -i ./src/main/json/openapi.json -g typescript-axios -o ./src/main/typescript/generated/openapi/typescript-axios --reserved-words-mappings protected=protected --ignore-file-override ../../openapi-generator-ignore", "proto:openapi": "openapi-generator-cli generate -i ./src/main/json/openapi.json -g protobuf-schema --model-name-suffix=PB --additional-properties=packageName=org.hyperledger.cactus.cmd_api_server -o ./src/main/proto/generated/openapi/ -t=./src/main/openapi-generator/templates/protobuf-schema/", diff --git a/tools/download-file-to-disk.ts b/tools/download-file-to-disk.ts new file mode 100644 index 0000000000..35b17079cd --- /dev/null +++ b/tools/download-file-to-disk.ts @@ -0,0 +1,105 @@ +import fs from "node:fs"; +import { Readable } from "stream"; +import { finished } from "stream/promises"; +import { ReadableStream } from "stream/web"; +import { fileURLToPath, parse } from "url"; +import path from "path"; + +import yargs from "yargs"; +import { hideBin } from "yargs/helpers"; +import { RuntimeError } from "run-time-error"; + +const TAG = "[tools/download-file-to-disk.ts]"; + +export interface IDownloadFileToDiskReq { + readonly url: string; + readonly outputFilePath: string; +} + +export interface IDownloadFileToDiskRes { + readonly url: string; + readonly outputFilePath: string; +} + +const nodePath = path.resolve(process.argv[1]); +const modulePath = path.resolve(fileURLToPath(import.meta.url)); +const isRunningDirectlyViaCLI = nodePath === modulePath; + +const main = async (argv: string[], env: NodeJS.ProcessEnv) => { + const req = await createRequest(argv, env); + await downloadFileToDisk(req); +}; + +if (isRunningDirectlyViaCLI) { + main(process.argv, process.env); +} + +async function createRequest( + argv: string[], + env: NodeJS.ProcessEnv, +): Promise { + if (!argv) { + throw new RuntimeError(`Process argv cannot be falsy.`); + } + if (!env) { + throw new RuntimeError(`Process env cannot be falsy.`); + } + + const optOutputFilePath = + "The absolute path on disk where the downloaded file will be streamed."; + + const optUrl = "The URL to download from."; + + const parsedCfg = await yargs(hideBin(argv)) + .env("CACTI_") + .option("url", { + alias: "u", + type: "string", + demandOption: false, + description: optUrl, + default: hideBin(argv)[0], + }) + .option("output-file-path", { + alias: "o", + type: "string", + description: optOutputFilePath, + defaultDescription: "Defaults to the current working directory.", + default: "./", + }).argv; + + const url = parsedCfg.url; + + console.log("%s parsing URL '%s'", TAG, url); + const { pathname } = parse(url); + const pathnameOrDefault = pathname || "new_download_file"; + const filename = path.basename(pathnameOrDefault); + + const endsWithDirSeparator = parsedCfg.outputFilePath.endsWith(path.sep); + + const outputFilePath = endsWithDirSeparator + ? path.join(parsedCfg.outputFilePath, filename) + : parsedCfg.outputFilePath; + + const req: IDownloadFileToDiskReq = { + url, + outputFilePath, + }; + + return req; +} + +export async function downloadFileToDisk( + req: IDownloadFileToDiskReq, +): Promise { + const { url, outputFilePath } = req; + console.log("%s downloading %s into %s ...", TAG, url, outputFilePath); + const stream = fs.createWriteStream(req.outputFilePath); + const { body } = await fetch(req.url); + if (!body) { + throw new RuntimeError("fetching %s did not yield a response body.", url); + } + const bodyNodeJs = body as unknown as ReadableStream; + await finished(Readable.fromWeb(bodyNodeJs).pipe(stream)); + console.log("%s downloaded %s into %s OK", TAG, url, outputFilePath); + return { outputFilePath, url }; +}