feat(dev-tools): add ts-transform-remove-glsl-comments (#455)

.eslintignore

@@ -1 +1,2 @@
 dist/
+test-case-*.ts

.prettierignore

@@ -1 +1,2 @@
 **/dist*/**/*.js
+test-case-*.ts

modules/dev-tools/package.json

@@ -36,6 +36,10 @@
     "./ts-transform-append-extension": {
       "require": "./dist/ts-plugins/ts-transform-append-extension/index.cjs",
       "import": "./dist/ts-plugins/ts-transform-append-extension/index.js"
+    },
+    "./ts-transform-remove-glsl-comments": {
+      "require": "./dist/ts-plugins/ts-transform-remove-glsl-comments/index.cjs",
+      "import": "./dist/ts-plugins/ts-transform-remove-glsl-comments/index.js"
     }
   },
   "types": "./dist/index.d.ts",
@@ -86,6 +90,7 @@
     "eslint-plugin-react-hooks": "^4.0.0",
     "glob": "^7.1.4",
     "lerna": "^3.14.1",
+    "minimatch": "^3.0.0",
     "prettier": "3.0.3",
     "prettier-check": "2.0.0",
     "tape": "^4.11.0",
@@ -94,7 +99,6 @@
     "ts-node": "~10.9.0",
     "ts-patch": "^3.1.2",
     "tsconfig-paths": "^4.1.1",
-    "url": "^0.11.0",
     "vite": "^4.0.1",
     "vite-plugin-html": "^3.2.0"
   },

modules/dev-tools/src/ts-plugins/ts-transform-remove-glsl-comments/index.ts

@@ -0,0 +1,109 @@
+/**
+ * TypeScript transform to remove comments and unnecessary white space from GLSL source.
+ * A template string is considered GLSL source if:
+   a) the file matches the pattern specified in the plugin config; or
+   b) it is tagged as glsl`...`
+ * Usage with ts-patch:
+   {
+      "plugins": [
+        {
+          "transform": "ocular-dev-tools/ts-transform-remove-glsl-comments",
+          "pattern": ["*.glsl.ts"]
+        }
+      ]
+   }
+ */
+import * as path from 'path';
+import type {Program, TransformationContext, SourceFile, Node} from 'typescript';
+import type {TransformerExtras, PluginConfig} from 'ts-patch';
+import minimatch from 'minimatch';
+
+// inline comment is only safe to remove if it's followed by a return (i.e. end of comment)
+const INLINE_COMMENT_REGEX = /\s*\/\/.*[\n\r]/g;
+const BLOCK_COMMENT_REGEX = /\s*\/\*(\*(?!\/)|[^*])*\*\//g;
+const WHITESPACE_REGEX = /\s*[\n\r]\s*/gm;
+const DEFAULT_PATTERNS = [];
+
+type RemoveGLSLCommentsPluginConfig = PluginConfig & {
+  /** Glob patterns of shader files to include. */
+  pattern?: string[];
+};
+
+export default function (
+  program: Program,
+  pluginConfig: RemoveGLSLCommentsPluginConfig,
+  {ts}: TransformerExtras
+) {
+  const {pattern = DEFAULT_PATTERNS} = pluginConfig;
+
+  return (ctx: TransformationContext) => {
+    const {factory} = ctx;
+
+    return (sourceFile: SourceFile) => {
+      const isShaderFile = matchFilePath(sourceFile.fileName, pattern);
+
+      function replaceShaderString(node: Node): Node {
+        if (ts.isNoSubstitutionTemplateLiteral(node)) {
+          const text = node.rawText ?? '';
+          // Convert source text to string content
+          const newText = filterShaderSource(text);
+          if (newText === text) {
+            return node;
+          }
+          return factory.createNoSubstitutionTemplateLiteral(newText, newText);
+        }
+        if (ts.isTemplateLiteralToken(node)) {
+          const text = node.rawText ?? '';
+          const newText = filterShaderSource(text);
+          if (newText === text) {
+            return node;
+          }
+          if (ts.isTemplateHead(node)) {
+            return factory.createTemplateHead(newText, newText);
+          }
+          if (ts.isTemplateMiddle(node)) {
+            return factory.createTemplateMiddle(newText, newText);
+          }
+          if (ts.isTemplateTail(node)) {
+            return factory.createTemplateTail(newText, newText);
+          }
+          return node;
+        }
+        return ts.visitEachChild(node, replaceShaderString, ctx);
+      }
+
+      function visit(node: Node): Node {
+        if (
+          ts.isTaggedTemplateExpression(node) &&
+          // First child is the tag identifier
+          node.getChildAt(0).getText() === 'glsl'
+        ) {
+          // Strip the template tag
+          return replaceShaderString(node.getChildAt(1));
+        }
+        if (isShaderFile && ts.isTemplateLiteral(node)) {
+          return replaceShaderString(node);
+        }
+        return ts.visitEachChild(node, visit, ctx);
+      }
+      return ts.visitNode(sourceFile, visit);
+    };
+  };
+}
+
+function matchFilePath(filePath: string, includePatterns: string[]): boolean {
+  const relPath = path.relative(process.env.PWD ?? '', filePath);
+  for (const pattern of includePatterns) {
+    if (minimatch(relPath, pattern)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function filterShaderSource(source: string): string {
+  return source
+    .replace(INLINE_COMMENT_REGEX, '\n')
+    .replace(BLOCK_COMMENT_REGEX, '')
+    .replace(WHITESPACE_REGEX, '\n');
+}

modules/dev-tools/test/index.ts

@@ -3,3 +3,4 @@ import './lib/configuration.spec';
 
 import './ts-plugins/ts-transform-version-inline.spec';
 import './ts-plugins/ts-transform-append-extension.spec';
+import './ts-plugins/ts-transform-remove-glsl-comments/index.spec';

modules/dev-tools/test/ts-plugins/test-transformer.ts

@@ -5,11 +5,13 @@ import type {PluginConfig} from 'ts-patch';
  * Transpile ts code with TypeScript compiler API
  */
 export function transpile({
+  sourceFileName = 'test.ts',
   source,
   transformer,
   config = {},
   outputType = 'js'
 }: {
+  sourceFileName?: string;
   source: string;
   transformer: Function;
   config?: PluginConfig;
@@ -25,7 +27,7 @@ export function transpile({
     }
   });
 
-  project.createSourceFile('test.ts', source);
+  project.createSourceFile(sourceFileName, source);
 
   const customTransformers: ts.CustomTransformers = {};
   const transform: ts.TransformerFactory<ts.SourceFile> = transformer(
@@ -49,3 +51,50 @@ export function transpile({
 
   return result.getFiles()[0].text;
 }
+
+/**
+ * Compare two pieces of source code. Returns a description of the difference, or null if identical.
+ */
+export function assertSourceEqual(
+  actual: string,
+  expected: string,
+  options: {
+    /** If true, ignore difference in indent
+     * @default true
+     */
+    ignoreIndent?: boolean;
+    /** If true, ignore empty lines
+     * @default true
+     */
+    ignoreEmptyLines?: boolean;
+  } = {}
+): true | Error {
+  const {ignoreIndent = true, ignoreEmptyLines = true} = options;
+  const actualLines = actual.split('\n');
+  const expectedLines = expected.split('\n');
+  let i1 = 0;
+  let i2 = 0;
+
+  while (i1 < actualLines.length || i2 < expectedLines.length) {
+    let t1 = actualLines[i1] ?? '';
+    let t2 = expectedLines[i2] ?? '';
+    if (ignoreIndent) {
+      t1 = t1.trimStart();
+      t2 = t2.trimStart();
+    }
+    if (t1 === t2) {
+      i1++;
+      i2++;
+    } else if (ignoreEmptyLines && !t1) {
+      i1++;
+    } else if (ignoreEmptyLines && !t2) {
+      i2++;
+    } else {
+      return new Error(`Mismatch at line ${i1}
+    Actual: ${t1}
+  Expected: ${t2}
+      `);
+    }
+  }
+  return true;
+}

modules/dev-tools/test/ts-plugins/ts-transform-append-extension.spec.ts

@@ -1,5 +1,5 @@
 import test from 'tape-promise/tape';
-import {transpile} from './test-transformer.js';
+import {transpile, assertSourceEqual} from './test-transformer.js';
 // @ts-expect-error Aliased import, remapped to valid path in esm-loader
 import appendExtension from 'ocular-dev-tools/ts-plugins/ts-transform-append-extension';
 
@@ -50,7 +50,7 @@ test('ts-transform-append-extension', (t) => {
       outputType: testCase.config.afterDeclarations ? 'd.ts' : 'js'
     });
 
-    t.is(result.trim(), testCase.output, testCase.title);
+    t.is(assertSourceEqual(result, testCase.output), true, testCase.title);
   }
 
   t.end();

modules/dev-tools/test/ts-plugins/ts-transform-remove-glsl-comments/index.spec.ts

@@ -0,0 +1,61 @@
+import fs from 'fs';
+import path from 'path';
+import {fileURLToPath} from 'node:url';
+
+import test from 'tape-promise/tape';
+import {transpile, assertSourceEqual} from '../test-transformer.js';
+// @ts-expect-error Aliased import, remapped to valid path in esm-loader
+import removeGLSLComments from 'ocular-dev-tools/ts-plugins/ts-transform-remove-glsl-comments';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+function loadSourceFromFile(fileName: string): string {
+  return fs.readFileSync(path.join(__dirname, fileName), 'utf8');
+}
+
+const testCases = [
+  {
+    title: 'no comments',
+    fileName: 'test.glsl.ts',
+    config: {pattern: ['**/*.glsl.ts']},
+    input: 'test-case-0.ts',
+    output: 'test-case-0-expected.ts'
+  },
+  {
+    title: 'remove comments from template literal',
+    fileName: 'test.glsl.ts',
+    config: {pattern: ['**/*.glsl.ts']},
+    input: 'test-case-1.ts',
+    output: 'test-case-1-expected.ts'
+  },
+  {
+    title: 'excluded by file name',
+    fileName: 'test.ts',
+    config: {},
+    input: 'test-case-1.ts',
+    output: 'test-case-1.ts'
+  },
+  {
+    title: 'included by template tag',
+    fileName: 'test.ts',
+    config: {},
+    input: 'test-case-2.ts',
+    output: 'test-case-2-expected.ts'
+  }
+];
+
+test('ts-transform-remove-glsl-comments', (t) => {
+  for (const testCase of testCases) {
+    const result = transpile({
+      sourceFileName: testCase.fileName,
+      source: loadSourceFromFile(testCase.input),
+      transformer: removeGLSLComments,
+      config: testCase.config
+    });
+    const expected = loadSourceFromFile(testCase.output);
+
+    t.is(assertSourceEqual(result, expected, {ignoreEmptyLines: false}), true, testCase.title);
+  }
+
+  t.end();
+});

modules/dev-tools/test/ts-plugins/ts-transform-remove-glsl-comments/test-case-0-expected.ts

@@ -0,0 +1,4 @@
+export function getTime() {
+  // Template literal that is not shader
+  return `The time is ${Date.now()}`;
+}

modules/dev-tools/test/ts-plugins/ts-transform-remove-glsl-comments/test-case-0.ts

@@ -0,0 +1,4 @@
+export function getTime() {
+  // Template literal that is not shader
+  return `The time is ${Date.now()}`;
+}

modules/dev-tools/test/ts-plugins/ts-transform-remove-glsl-comments/test-case-1-expected.ts

@@ -0,0 +1,34 @@
+// Constants
+const COORDINATE_SYSTEM = {
+  CARTESIAN: 0,
+  LNGLAT: 1
+};
+const constantDefinitions = Object.keys(COORDINATE_SYSTEM)
+  .map((key) => `const int COORDINATE_SYSTEM_${key} = ${COORDINATE_SYSTEM[key]};`)
+  .join('\n');
+// Vertex shader
+export const vs = `\
+#version 300 es
+${constantDefinitions}
+in vec4 position;
+in vec4 color;
+uniform mat4 pMatrix;
+uniform mat4 mMatrix;
+uniform float opacity;
+out vec4 vColor;
+main() {
+  gl_Position = pMatrix * mMatrix * position;
+  vColor = vec4(color, color.a * opacity);
+}
+`;
+// Fragment shader
+export const fs = `\
+#version 300 es
+in vec4 vColor;
+main() {
+  if (vColor.a == 0.0) {
+    discard;
+  }
+  gl_FragColor = vColor;
+}
+`;

modules/dev-tools/test/ts-plugins/ts-transform-remove-glsl-comments/test-case-1.ts

@@ -0,0 +1,43 @@
+// Constants
+const COORDINATE_SYSTEM = {
+  CARTESIAN: 0,
+  LNGLAT: 1
+};
+const constantDefinitions = Object.keys(COORDINATE_SYSTEM)
+  .map((key) => `const int COORDINATE_SYSTEM_${key} = ${COORDINATE_SYSTEM[key]};`)
+  .join('\n');
+// Vertex shader
+export const vs = `\
+#version 300 es
+${constantDefinitions}
+
+in vec4 position;
+in vec4 color;
+
+uniform mat4 pMatrix; // Projection matrix
+uniform mat4 mMatrix; // Model matrix
+uniform float opacity;
+
+out vec4 vColor;
+
+main() {
+  gl_Position = pMatrix * mMatrix * position;
+  vColor = vec4(color, /* inline comment */ color.a * opacity);
+}
+`;
+// Fragment shader
+export const fs = `\
+#version 300 es
+
+in vec4 vColor;
+
+main() {
+  if (vColor.a == 0.0) {
+    /*
+      Remove transparent fragment
+    */
+    discard;
+  }
+  gl_FragColor = vColor;
+}
+`;