Make a tsc plugin for esbuild (#252)

This includes:
- using Brotli to compress final builds
- embed pem files from nsm attestation in code
- make a tsc plugin for esbuild

.github/workflows/manual-test-release.yml

@@ -35,7 +35,7 @@ jobs:
             - run: yarn workspaces focus --all --production
             # package final binaries
             - run: |
-                  yarn dlx @yao-pkg/pkg@5.11.1 ./dist -t node18-${{ matrix.settings.target }} -o bundle/dcli-${{ matrix.settings.target }}${{ matrix.settings.extension }} -C GZip "--public" "--public-packages" "tslib,thirty-two,node-hkdf-sync,vows" "--no-bytecode"
+                  yarn dlx @yao-pkg/pkg@5.11.1 ./dist -t node18-${{ matrix.settings.target }} -o bundle/dcli-${{ matrix.settings.target }}${{ matrix.settings.extension }} -C Brotli "--public" "--public-packages" "tslib,thirty-two,node-hkdf-sync,vows" "--no-bytecode"
 
             - name: Archive binary artifact
               uses: actions/upload-artifact@v4

.github/workflows/release.yml

@@ -38,7 +38,7 @@ jobs:
             - run: yarn workspaces focus --all --production
             # package final binaries
             - run: |
-                  yarn dlx @yao-pkg/pkg@5.11.1 ./dist -t node18-${{ matrix.settings.target }} -o bundle/dcli-${{ matrix.settings.target }}${{ matrix.settings.extension }} -C GZip "--public" "--public-packages" "tslib,thirty-two,node-hkdf-sync,vows" "--no-bytecode"
+                  yarn dlx @yao-pkg/pkg@5.11.1 ./dist -t node18-${{ matrix.settings.target }} -o bundle/dcli-${{ matrix.settings.target }}${{ matrix.settings.extension }} -C Brotli "--public" "--public-packages" "tslib,thirty-two,node-hkdf-sync,vows" "--no-bytecode"
 
             - name: Archive binary artifact
               uses: actions/upload-artifact@v4

package.json

@@ -41,7 +41,6 @@
     "contributors": [],
     "license": "Apache-2.0",
     "nativeDependencies": {
-        "@dashlane/nsm-attestation": "*",
         "better-sqlite3": "*",
         "@json2csv/plainjs": "*",
         "@json2csv/transforms": "*",
@@ -52,6 +51,7 @@
         "node-mac-auth": "*"
     },
     "devDependencies": {
+        "@aivenio/tsc-output-parser": "^2.1.1",
         "@types/async": "^3.2.24",
         "@types/better-sqlite3": "^7.6.10",
         "@types/chai": "^4.3.16",
@@ -67,14 +67,15 @@
         "eslint": "^8.57.0",
         "eslint-config-prettier": "^9.1.0",
         "eslint-plugin-import": "^2.29.1",
+        "execa": "^9.1.0",
         "husky": "^9.0.11",
         "mocha": "^10.4.0",
         "prettier": "^3.2.5",
         "ts-node": "^10.9.2",
         "typescript": "^5.4.5"
     },
     "dependencies": {
-        "@dashlane/nsm-attestation": "^1.0.1",
+        "@dashlane/nsm-attestation": "^1.0.2",
         "@inquirer/prompts": "^5.0.5",
         "@json2csv/plainjs": "^7.0.6",
         "@json2csv/transforms": "^7.0.6",

scripts/build.js

@@ -4,33 +4,103 @@ import os from 'os';
 import fs from 'fs';
 import path from 'path';
 import process from 'process';
-import childProcess from 'child_process';
 import esbuild from 'esbuild';
 import packageJSON from '../package.json' assert { type: 'json' };
 import { fileURLToPath } from 'url';
+import { $ } from "execa";
+import tscOutputParser from '@aivenio/tsc-output-parser';
 
 const platform = os.platform();
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
+const pemReadFilePlugin = {
+    name: 'base64-plugin',
+    setup(build) {
+        build.onLoad({ filter: /\.js$/ }, async (args) => {
+            let contents = fs.readFileSync(args.path, 'utf8');
+
+            const regex = /await\s+fs\.promises\.readFile\(\s*path\.resolve\(__dirname,\s*['"`](.*\.pem)['"`]\s*\)\)/g;
+            let match;
+
+            while ((match = regex.exec(contents)) !== null) {
+                const pemFilePath = path.resolve(path.dirname(args.path), match[1]);
+                const pemContents = fs.readFileSync(pemFilePath, 'utf8');
+                const base64Contents = Buffer.from(pemContents).toString('base64');
+                contents = contents.replace(match[0], `await Promise.resolve(Buffer.from("${base64Contents}", 'base64').toString())`);
+            }
+
+            return {
+                contents,
+                loader: 'js',
+            };
+        });
+    },
+};
+
+const tscDiagnosticToEsbuild = async (
+    diagnostic,
+) => {
+    const lineText =
+        await $`sed -n ${diagnostic.value.cursor.value.line}p ${diagnostic.value.path.value}`;
+
+    const [firstLine, rest] = diagnostic.value.message.value.split("\n", 2);
+
+    return {
+        location: {
+            column: diagnostic.value.cursor.value.col - 1,
+            line: diagnostic.value.cursor.value.line,
+            file: diagnostic.value.path.value,
+            lineText: lineText.stdout,
+        },
+        notes: rest && rest.trim().length > 0 ? [{ text: rest }] : [],
+        text: `${firstLine} [${diagnostic.value.tsError.value.errorString}]`,
+    };
+};
+
+const checkTypesPlugin = () => {
+    return {
+        name: 'check-types',
+        setup(build) {
+            build.onEnd(async (result) => {
+                if (result.errors.length > 0) {
+                    return;
+                }
+
+                const buildArgs = ['--noEmit', '-p', './tsconfig.build.json', '--pretty', 'false'];
+                try {
+                    await $('tsc', buildArgs);
+                } catch (err) {
+                    const tscOutput = tscOutputParser.parse(err.stdout);
+                    const messages = await Promise.all(tscOutput.map(output => tscDiagnosticToEsbuild(output)));
+                    const formatted = await esbuild.formatMessages(
+                        messages,
+                        {
+                            kind: 'error',
+                            color: true,
+                            terminalWidth: 100,
+                        }
+                    );
+                    console.log(formatted.join('\n'));
+                    process.exit(1);
+                }
+            });
+        },
+    };
+};
+
 async function main(argv = process.argv) {
     argv = argv.slice(2);
     const projectRoot = path.join(__dirname, '..');
-    const buildPath = path.join(projectRoot, 'build');
+    const srcPath = path.join(projectRoot, 'src');
     const distPath = path.join(projectRoot, 'dist');
     const gitPath = process.env.GIT_DIR ?? path.join(projectRoot, '.git');
     await fs.promises.rm(distPath, {
         recursive: true,
         force: true,
     });
-    const buildArgs = ['-p', './tsconfig.build.json', ...argv];
-    console.error('Running tsc:');
-    console.error(['tsc', ...buildArgs].join(' '));
-    childProcess.execFileSync('tsc', buildArgs, {
-        stdio: ['inherit', 'inherit', 'inherit'],
-        windowsHide: true,
-        encoding: 'utf-8',
-        shell: platform === 'win32' ? true : false,
+    await fs.promises.mkdir(distPath, {
+        recursive: true,
     });
     // This collects the build metadata and adds it to the build folder so that dynamic imports to it will resolve correctly.
     let gitHead = process.env.COMMIT_HASH;
@@ -52,17 +122,17 @@ async function main(argv = process.argv) {
     console.error('Writing build metadata (build.json):');
     console.error(buildJSON);
     await fs.promises.writeFile(
-        path.join(buildPath, 'build.json'),
+        path.join(distPath, 'build.json'),
         JSON.stringify(buildJSON, null, 2),
     );
     // This specifies import paths that is left as an external require
     // This is kept to packages that have a native binding
     const externalDependencies = Object.keys(packageJSON.nativeDependencies ?? {});
     const esbuildOptions = {
         entryPoints: [
-            path.join(buildPath, 'index.js'),
+            'src/index.ts'
         ],
-        sourceRoot: buildPath,
+        sourceRoot: srcPath,
         bundle: true,
         platform: 'node',
         format: 'cjs',
@@ -74,7 +144,8 @@ async function main(argv = process.argv) {
         minify: true,
         keepNames: true,
         outfile: path.join(distPath, 'index.cjs'),
-        metafile: true
+        metafile: true,
+        plugins: [checkTypesPlugin(), pemReadFilePlugin],
     };
     console.error('Running esbuild:');
     console.error(esbuildOptions);

tsconfig.build.json

@@ -2,7 +2,7 @@
     "extends": "./tsconfig.json",
     "compilerOptions": {
         "rootDir": "./src",
-        "noEmit": false,
+        "noEmit": true,
         "stripInternal": true
     },
     "exclude": ["./documentation/**/*", "./scripts/**/*"]