feat(bundle-source): Support TypeScript type erasure (#2627)

Closes: #2415 

## Description

This change introduces support for TypeScript through type-erasure,
using ts-blank-space, which converts type annotations to equivalent
blank space. As is consistent with `node --experimental-strip-types`,
this only applies to modules with the `.ts`, `.mts`, or `.cts`
extensions in packages that are not under `node_modules`, to discourage
publishing TypeScript as a source language to npm.

### Security Considerations

The choice of `ts-blank-space` is intended to minimize runtime behavior
difference between TypeScript and JavaScript, such that a reviewer or a
debugger of the generated JavaScript aligns with the expected behavior
and original text, to the extent that is possible. This should compose
well with #2444.

### Scaling Considerations

None.

### Documentation Considerations

Contains README and NEWS.

### Testing Considerations

Contains spot check tests for TypeScript in the endoScript and
endoZipBase64 formats. We stand on much more rigorous testing of the
underlying workspace-language-for-extension feature in Compartment
Mapper #2625.

### Compatibility Considerations

This does not break any prior usage.

### Upgrade Considerations

None.

packages/bundle-source/NEWS.md

@@ -1,5 +1,15 @@
 User-visible changes to `@endo/bundle-source`:
 
+# Next release
+
+- Adds support for TypeScript type erasure using
+  [`ts-blank-space`](https://bloomberg.github.io/ts-blank-space/) applied to
+  TypeScript modules with `.ts`, `.mts`, and `.cts` extensions, for any package
+  that is not under a `node_modules` directory, immitating `node
+  --experimental-strip-types`.
+  As with `.js` extensions, the behavior of `.ts` is either consistent with
+  `.mts` or `.cts` depending on the `type` in `package.json`.
+
 # v3.4.0 (2024-08-27)
 
 - Adds support for `--elide-comments` (`-e`) that blanks out the interior of

packages/bundle-source/README.md

@@ -57,6 +57,25 @@ with `@preserve`, `@copyright`, `@license` pragmas or the Internet Explorer
 Comment elision does not strip comments entirely.
 The syntax to begin or end comments remains.
 
+## TypeScript type erasure
+
+TypeScript modules with the `.ts`, `.mts`, and `.cts` extensions in
+packages that are not under a `node_modules` directory are automatically
+converted to JavaScript through type erasure using
+[`ts-blank-space`](https://bloomberg.github.io/ts-blank-space/).
+
+This will not function for packages that are published as their original
+TypeScript sources, as is consistent with `node
+--experimental-strip-types`.
+This will also not function properly for TypeScript modules that have
+[runtime impacting syntax](https://github.com/bloomberg/ts-blank-space/blob/main/docs/unsupported_syntax.md),
+such as `enum`.
+
+This also does not support importing a `.ts` file using the corresponding
+imaginary, generated module with a `.js` extension.
+Use this feature in conjunction with
+[`--allowImportingTsExtensions`](https://www.typescriptlang.org/tsconfig/#allowImportingTsExtensions).
+
 ## Source maps
 
 With the `moduleFormat` of `endoZipBase64`, the bundler can generate source

packages/bundle-source/demo/fortune.ts

@@ -0,0 +1 @@
+export const fortune: string = 'outlook uncertain';

packages/bundle-source/demo/import-ts-as-js.ts

@@ -0,0 +1,2 @@
+// fortune.js does not exist, but fortune.ts does.
+export { fortune } from './fortune.js';

packages/bundle-source/demo/reexport-fortune-ts.js

@@ -0,0 +1,7 @@
+/* eslint-disable @endo/restrict-comparison-operands */
+
+export { fortune } from './fortune.ts';
+
+if (((0).toFixed.apply < Number, String > 1) === true) {
+  throw new Error('JavaScript interpreted as TypeScript');
+}

packages/bundle-source/demo/reexport-fortune-ts.ts

@@ -0,0 +1,5 @@
+export { fortune } from './fortune.ts';
+
+if ((0).toFixed.apply<Number, String>(1) === false) {
+  throw new Error('TypeScript interpreted as JavaScript');
+}

packages/bundle-source/demo/reexport-meaning-js.js

@@ -0,0 +1,7 @@
+/* eslint-disable @endo/restrict-comparison-operands */
+
+export { meaning } from './meaning.js';
+
+if (((0).toFixed.apply < Number, String > 1) === true) {
+  throw new Error('JavaScript interpreted as TypeScript');
+}

packages/bundle-source/demo/reexport-meaning-js.ts

@@ -0,0 +1,5 @@
+export { meaning } from './meaning.js';
+
+if ((0).toFixed.apply<Number, String>(1) === false) {
+  throw new Error('TypeScript interpreted as JavaScript');
+}

packages/bundle-source/package.json

@@ -35,7 +35,8 @@
     "@rollup/plugin-json": "^6.1.0",
     "@rollup/plugin-node-resolve": "^13.0.0",
     "acorn": "^8.2.4",
-    "rollup": "^2.79.1"
+    "rollup": "^2.79.1",
+    "ts-blank-space": "^0.4.1"
   },
   "devDependencies": {
     "@endo/lockdown": "workspace:^",

packages/bundle-source/src/endo.js

@@ -125,10 +125,13 @@ export const makeBundlingKit = (
   };
 
   let parserForLanguage = transparentParserForLanguage;
+
   let moduleTransforms = {};
+
   if (!noTransforms) {
     parserForLanguage = transformingParserForLanguage;
     moduleTransforms = {
+      ...moduleTransforms,
       async mjs(
         sourceBytes,
         specifier,
@@ -162,9 +165,47 @@ export const makeBundlingKit = (
     };
   }
 
+  const mtsParser = {
+    async parse(sourceBytes, ...rest) {
+      const { default: tsBlankSpace } = await import('ts-blank-space');
+      const sourceText = textDecoder.decode(sourceBytes);
+      const objectText = tsBlankSpace(sourceText);
+      const objectBytes = textEncoder.encode(objectText);
+      return parserForLanguage.mjs.parse(objectBytes, ...rest);
+    },
+    heuristicImports: false,
+    synchronous: false,
+  };
+
+  const ctsParser = {
+    async parse(sourceBytes, ...rest) {
+      const { default: tsBlankSpace } = await import('ts-blank-space');
+      const sourceText = textDecoder.decode(sourceBytes);
+      const objectText = tsBlankSpace(sourceText);
+      const objectBytes = textEncoder.encode(objectText);
+      return parserForLanguage.cjs.parse(objectBytes, ...rest);
+    },
+    heuristicImports: true,
+    synchronous: false,
+  };
+
+  parserForLanguage = { ...parserForLanguage, mts: mtsParser, cts: ctsParser };
+
   const sourceMapHook = (sourceMap, sourceDescriptor) => {
     sourceMapJobs.add(writeSourceMap(sourceMap, sourceDescriptor));
   };
 
-  return { sourceMapHook, sourceMapJobs, moduleTransforms, parserForLanguage };
+  const workspaceLanguageForExtension = { mts: 'mts', cts: 'cts' };
+  const workspaceModuleLanguageForExtension = { ts: 'mts' };
+  const workspaceCommonjsLanguageForExtension = { ts: 'cts' };
+
+  return {
+    sourceMapHook,
+    sourceMapJobs,
+    moduleTransforms,
+    parserForLanguage,
+    workspaceLanguageForExtension,
+    workspaceModuleLanguageForExtension,
+    workspaceCommonjsLanguageForExtension,
+  };
 };

packages/bundle-source/src/script.js

@@ -54,29 +54,39 @@ export async function bundleScript(
 
   const entry = url.pathToFileURL(pathResolve(startFilename));
 
-  const { sourceMapHook, sourceMapJobs, moduleTransforms, parserForLanguage } =
-    makeBundlingKit(
-      {
-        pathResolve,
-        userInfo,
-        platform,
-        env,
-        computeSha512,
-      },
-      {
-        cacheSourceMaps,
-        noTransforms,
-        elideComments,
-        commonDependencies,
-        dev,
-      },
-    );
+  const {
+    sourceMapHook,
+    sourceMapJobs,
+    moduleTransforms,
+    parserForLanguage,
+    workspaceLanguageForExtension,
+    workspaceCommonjsLanguageForExtension,
+    workspaceModuleLanguageForExtension,
+  } = makeBundlingKit(
+    {
+      pathResolve,
+      userInfo,
+      platform,
+      env,
+      computeSha512,
+    },
+    {
+      cacheSourceMaps,
+      noTransforms,
+      elideComments,
+      commonDependencies,
+      dev,
+    },
+  );
 
   const source = await makeBundle(powers, entry, {
     dev,
     conditions,
     commonDependencies,
     parserForLanguage,
+    workspaceLanguageForExtension,
+    workspaceCommonjsLanguageForExtension,
+    workspaceModuleLanguageForExtension,
     moduleTransforms,
     sourceMapHook,
   });

packages/bundle-source/src/zip-base64.js

@@ -56,27 +56,37 @@ export async function bundleZipBase64(
 
   const entry = url.pathToFileURL(pathResolve(startFilename));
 
-  const { sourceMapHook, sourceMapJobs, moduleTransforms, parserForLanguage } =
-    makeBundlingKit(
-      {
-        pathResolve,
-        userInfo,
-        platform,
-        env,
-        computeSha512,
-      },
-      {
-        cacheSourceMaps,
-        noTransforms,
-        elideComments,
-        commonDependencies,
-      },
-    );
+  const {
+    sourceMapHook,
+    sourceMapJobs,
+    moduleTransforms,
+    parserForLanguage,
+    workspaceLanguageForExtension,
+    workspaceCommonjsLanguageForExtension,
+    workspaceModuleLanguageForExtension,
+  } = makeBundlingKit(
+    {
+      pathResolve,
+      userInfo,
+      platform,
+      env,
+      computeSha512,
+    },
+    {
+      cacheSourceMaps,
+      noTransforms,
+      elideComments,
+      commonDependencies,
+    },
+  );
 
   const compartmentMap = await mapNodeModules(powers, entry, {
     dev,
     conditions,
     commonDependencies,
+    workspaceLanguageForExtension,
+    workspaceCommonjsLanguageForExtension,
+    workspaceModuleLanguageForExtension,
   });
 
   const { bytes, sha512 } = await makeAndHashArchiveFromMap(

packages/bundle-source/test/endo-script-format.test.js

@@ -4,28 +4,95 @@ import test from '@endo/ses-ava/prepare-endo.js';
 import * as url from 'url';
 import bundleSource from '../src/index.js';
 
-const generate = async (options = {}) => {
-  const entryPath = url.fileURLToPath(
-    new URL(`../demo/meaning.js`, import.meta.url),
-  );
+/**
+ * @template {Partial<object>} Options
+ * @param {string} entry
+ * @param {Options} options
+ */
+const generate = async (entry, options = {}) => {
+  const entryPath = url.fileURLToPath(new URL(entry, import.meta.url));
   return bundleSource(entryPath, {
     format: 'endoScript',
     ...options,
   });
 };
 
 test('endo script format', async t => {
-  const bundle = await generate();
+  const bundle = await generate('../demo/meaning.js');
   t.is(bundle.moduleFormat, 'endoScript');
   const { source } = bundle;
   const compartment = new Compartment();
   const ns = compartment.evaluate(source);
   t.is(ns.meaning, 42);
 });
 
+test('endo script format supports typescript type erasure', async t => {
+  const bundle = await generate('../demo/fortune.ts');
+  t.is(bundle.moduleFormat, 'endoScript');
+  const { source } = bundle;
+  t.notRegex(source, /string/);
+  const compartment = new Compartment();
+  const ns = compartment.evaluate(source);
+  t.is(ns.fortune, 'outlook uncertain');
+});
+
+test('endo script supports reexporting typescript in typescript', async t => {
+  const bundle = await generate('../demo/reexport-fortune-ts.ts');
+  t.is(bundle.moduleFormat, 'endoScript');
+  const { source } = bundle;
+  const compartment = new Compartment();
+  const ns = compartment.evaluate(source);
+  t.is(ns.fortune, 'outlook uncertain');
+});
+
+test('endo script supports reexporting typescript in javascript', async t => {
+  const bundle = await generate('../demo/reexport-fortune-ts.js');
+  t.is(bundle.moduleFormat, 'endoScript');
+  const { source } = bundle;
+  const compartment = new Compartment();
+  const ns = compartment.evaluate(source);
+  t.is(ns.fortune, 'outlook uncertain');
+});
+
+test('endo script supports reexporting javascript in typescript', async t => {
+  const bundle = await generate('../demo/reexport-meaning-js.ts');
+  t.is(bundle.moduleFormat, 'endoScript');
+  const { source } = bundle;
+  const compartment = new Compartment();
+  const ns = compartment.evaluate(source);
+  t.is(ns.meaning, 42);
+});
+
+test('endo script supports reexporting javascript in javascript', async t => {
+  const bundle = await generate('../demo/reexport-meaning-js.js');
+  t.is(bundle.moduleFormat, 'endoScript');
+  const { source } = bundle;
+  const compartment = new Compartment();
+  const ns = compartment.evaluate(source);
+  t.is(ns.meaning, 42);
+});
+
+test.failing(
+  'endo supports importing ts from ts with a js extension',
+  async t => {
+    t.log(`\
+TypeScript with tsc encourages importing with the .js extension, even if
+presumptively generated .js file does not exist but is presumed to be generated
+from the corresponding .ts module. We do not yet implement this.`);
+    const bundle = await generate('../demo/import-ts-as-js.ts');
+    t.is(bundle.moduleFormat, 'endoScript');
+    const { source } = bundle;
+    const compartment = new Compartment();
+    const ns = compartment.evaluate(source);
+    t.is(ns.fortune, 'outlook uncertain');
+  },
+);
+
 test('endo script format is smaller with blank comments', async t => {
-  const bigBundle = await generate();
-  const smallBundle = await generate({ elideComments: true });
+  const bigBundle = await generate('../demo/meaning.js');
+  const smallBundle = await generate('../demo/meaning.js', {
+    elideComments: true,
+  });
   const compartment = new Compartment();
   const ns = compartment.evaluate(smallBundle.source);
   t.is(ns.meaning, 42);

packages/bundle-source/test/typescript.test.js

@@ -0,0 +1,37 @@
+// @ts-check
+import test from '@endo/ses-ava/prepare-endo.js';
+
+import url from 'url';
+import { decodeBase64 } from '@endo/base64';
+import { ZipReader } from '@endo/zip';
+import bundleSource from '../src/index.js';
+
+test('no-transforms applies no transforms', async t => {
+  const entryPath = url.fileURLToPath(
+    new URL(`../demo/fortune.ts`, import.meta.url),
+  );
+  const { endoZipBase64 } = await bundleSource(entryPath, {
+    format: 'endoZipBase64',
+    noTransforms: true,
+  });
+  const endoZipBytes = decodeBase64(endoZipBase64);
+  const zipReader = new ZipReader(endoZipBytes);
+  const compartmentMapBytes = zipReader.read('compartment-map.json');
+  const compartmentMapText = new TextDecoder().decode(compartmentMapBytes);
+  const compartmentMap = JSON.parse(compartmentMapText);
+  const { entry, compartments } = compartmentMap;
+  const compartment = compartments[entry.compartment];
+  const module = compartment.modules[entry.module];
+  // Transformed from TypeScript:
+  t.is(module.parser, 'mjs');
+
+  const moduleBytes = zipReader.read(
+    `${compartment.location}/${module.location}`,
+  );
+  const moduleText = new TextDecoder().decode(moduleBytes);
+  t.is(
+    moduleText.trim(),
+    `export const fortune         = 'outlook uncertain';`,
+    // Erased:           : string
+  );
+});