From 6b4983c2c9ebb42ba637a6d9280f59f46de14d38 Mon Sep 17 00:00:00 2001 From: Cody Bennett Date: Tue, 21 Mar 2023 00:41:03 -0500 Subject: [PATCH 01/64] feat: AST types --- src/ast.ts | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 src/ast.ts diff --git a/src/ast.ts b/src/ast.ts new file mode 100644 index 0000000..d56893e --- /dev/null +++ b/src/ast.ts @@ -0,0 +1,108 @@ +export interface Node {} + +export interface Literal extends Node { + value: string | number | boolean +} + +export interface Identifier extends Node { + value: string +} + +export interface VariableDeclaration extends Node { + name: string + type: Node + value: Node | null + qualifiers: Node[] +} + +export interface BlockStatement extends Node { + body: Node[] +} + +export interface FunctionDeclaration extends Node { + name: string + type: Node | null + args: VariableDeclaration[] + body: BlockStatement +} + +export interface CallExpression extends Node { + callee: Node + args: Node[] +} + +export interface MemberExpression extends Node { + object: Node + property: Node +} + +export interface ArrayExpression extends Node { + members: Node[] +} + +export interface IfStatement extends Node { + test: Node + consequent: Node + alternate: Node | null +} + +export interface WhileStatement extends Node { + test: Node + body: Node +} + +export interface ForStatement extends Node { + init: Node | null + test: Node | null + update: Node | null + body: Node +} + +export interface DoWhileStatement extends Node { + test: Node + body: Node +} + +export interface SwitchCase extends Node { + test: Node | null + consequent: Node[] +} + +export interface SwitchStatement extends Node { + discriminant: Node + cases: SwitchCase[] +} + +export interface StructDeclaration extends Node { + name: string + members: VariableDeclaration[] +} + +export interface ReturnStatement extends Node { + argument: Node | null +} + +export interface UnaryExpression extends Node { + operator: string + argument: Node +} + +export interface BinaryExpression extends Node { + operator: string + left: Node + right: Node +} + +export interface TernaryExpression extends Node { + test: Node + consequent: Node + alternate: Node +} + +export interface ContinueStatement extends Node {} + +export interface BreakStatement extends Node {} + +export interface ReturnStatement extends Node {} + +export interface DiscardStatement extends Node {} From efaffe1a6b05aacd0c5c92fba389f3cda6fb7c68 Mon Sep 17 00:00:00 2001 From: Cody Bennett <23324155+CodyJasonBennett@users.noreply.github.com> Date: Tue, 20 Jun 2023 04:26:54 -0500 Subject: [PATCH 02/64] refactor: narrow ReturnStatement --- src/ast.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ast.ts b/src/ast.ts index d56893e..37c70d0 100644 --- a/src/ast.ts +++ b/src/ast.ts @@ -79,7 +79,7 @@ export interface StructDeclaration extends Node { } export interface ReturnStatement extends Node { - argument: Node | null + argument: Literal | Identifier | UnaryExpression | null } export interface UnaryExpression extends Node { @@ -103,6 +103,4 @@ export interface ContinueStatement extends Node {} export interface BreakStatement extends Node {} -export interface ReturnStatement extends Node {} - export interface DiscardStatement extends Node {} From e8590f349bdaec4773b63d0f2c97588d422030dd Mon Sep 17 00:00:00 2001 From: Cody Bennett <23324155+CodyJasonBennett@users.noreply.github.com> Date: Tue, 20 Jun 2023 20:46:01 -0500 Subject: [PATCH 03/64] feat: parser, type branding --- src/ast.ts | 39 ++++++++++++++++++++++++++++++++------- src/parser.ts | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 src/parser.ts diff --git a/src/ast.ts b/src/ast.ts index 37c70d0..9305020 100644 --- a/src/ast.ts +++ b/src/ast.ts @@ -1,14 +1,17 @@ export interface Node {} export interface Literal extends Node { + __brand: 'Literal' value: string | number | boolean } export interface Identifier extends Node { + __brand: 'Identifier' value: string } export interface VariableDeclaration extends Node { + __brand: 'VariableDeclaration' name: string type: Node value: Node | null @@ -16,10 +19,12 @@ export interface VariableDeclaration extends Node { } export interface BlockStatement extends Node { + __brand: 'BlockStatement' body: Node[] } export interface FunctionDeclaration extends Node { + __brand: 'FunctionDeclaration' name: string type: Node | null args: VariableDeclaration[] @@ -27,31 +32,37 @@ export interface FunctionDeclaration extends Node { } export interface CallExpression extends Node { + __brand: 'CallExpression' callee: Node args: Node[] } export interface MemberExpression extends Node { + __brand: 'MemberExpression' object: Node property: Node } export interface ArrayExpression extends Node { + __brand: 'ArrayExpression' members: Node[] } export interface IfStatement extends Node { + __brand: 'IfStatement' test: Node consequent: Node alternate: Node | null } export interface WhileStatement extends Node { + __brand: 'WhileStatement' test: Node body: Node } export interface ForStatement extends Node { + __brand: 'ForStatement' init: Node | null test: Node | null update: Node | null @@ -59,48 +70,62 @@ export interface ForStatement extends Node { } export interface DoWhileStatement extends Node { + __brand: 'DoWhileStatement' test: Node body: Node } export interface SwitchCase extends Node { + __brand: 'SwitchCase' test: Node | null consequent: Node[] } export interface SwitchStatement extends Node { + __brand: 'SwitchStatement' discriminant: Node cases: SwitchCase[] } export interface StructDeclaration extends Node { + __brand: 'StructDeclaration' name: string members: VariableDeclaration[] } -export interface ReturnStatement extends Node { - argument: Literal | Identifier | UnaryExpression | null -} - export interface UnaryExpression extends Node { + __brand: 'UnaryExpression' operator: string argument: Node } export interface BinaryExpression extends Node { + __brand: 'BinaryExpression' operator: string left: Node right: Node } export interface TernaryExpression extends Node { + __brand: 'TernaryExpression' test: Node consequent: Node alternate: Node } -export interface ContinueStatement extends Node {} +export interface ReturnStatement extends Node { + __brand: 'ReturnStatement' + argument: Literal | Identifier | UnaryExpression | null +} -export interface BreakStatement extends Node {} +export interface ContinueStatement extends Node { + __brand: 'ContinueStatement' +} -export interface DiscardStatement extends Node {} +export interface BreakStatement extends Node { + __brand: 'BreakStatement' +} + +export interface DiscardStatement extends Node { + __brand: 'DiscardStatement' +} diff --git a/src/parser.ts b/src/parser.ts new file mode 100644 index 0000000..45ae382 --- /dev/null +++ b/src/parser.ts @@ -0,0 +1,50 @@ +import { + type Node, + type Literal, + type Identifier, + type VariableDeclaration, + type BlockStatement, + type FunctionDeclaration, + type CallExpression, + type MemberExpression, + type ArrayExpression, + type IfStatement, + type WhileStatement, + type ForStatement, + type DoWhileStatement, + type SwitchCase, + type SwitchStatement, + type StructDeclaration, + type UnaryExpression, + type BinaryExpression, + type TernaryExpression, + type ReturnStatement, + type ContinueStatement, + type BreakStatement, + type DiscardStatement, +} from './ast' + +const isLiteral = (node: Node): node is Literal => (node as any).__brand === 'Literal' +const isIdentifier = (node: Node): node is Identifier => (node as any).__brand === 'Identifier' +const isVariableDeclaration = (node: Node): node is VariableDeclaration => + (node as any).__brand === 'VariableDeclaration' +const isBlockStatement = (node: Node): node is BlockStatement => (node as any).__brand === 'BlockStatement' +const isFunctionDeclaration = (node: Node): node is FunctionDeclaration => + (node as any).__brand === 'FunctionDeclaration' +const isCallExpression = (node: Node): node is CallExpression => (node as any).__brand === 'CallExpression' +const isMemberExpression = (node: Node): node is MemberExpression => (node as any).__brand === 'MemberExpression' +const isArrayExpression = (node: Node): node is ArrayExpression => (node as any).__brand === 'ArrayExpression' +const isIfStatement = (node: Node): node is IfStatement => (node as any).__brand === 'IfStatement' +const isWhileStatement = (node: Node): node is WhileStatement => (node as any).__brand === 'WhileStatement' +const isForStatement = (node: Node): node is ForStatement => (node as any).__brand === 'ForStatement' +const isDoWhileStatement = (node: Node): node is DoWhileStatement => (node as any).__brand === 'DoWhileStatement' +const isSwitchCase = (node: Node): node is SwitchCase => (node as any).__brand === 'SwitchCase' +const isSwitchStatement = (node: Node): node is SwitchStatement => (node as any).__brand === 'SwitchStatement' +const isStructDeclaration = (node: Node): node is StructDeclaration => (node as any).__brand === 'StructDeclaration' +const isUnaryExpression = (node: Node): node is UnaryExpression => (node as any).__brand === 'UnaryExpression' +const isBinaryExpression = (node: Node): node is BinaryExpression => (node as any).__brand === 'BinaryExpression' +const isTernaryExpression = (node: Node): node is TernaryExpression => (node as any).__brand === 'TernaryExpression' +const isReturnStatement = (node: Node): node is ReturnStatement => (node as any).__brand === 'ReturnStatement' +const isContinueStatement = (node: Node): node is ContinueStatement => (node as any).__brand === 'ContinueStatement' +const isBreakStatement = (node: Node): node is BreakStatement => (node as any).__brand === 'BreakStatement' +const isDiscardStatement = (node: Node): node is DiscardStatement => (node as any).__brand === 'DiscardStatement' From 6583245033aca2ed851caffd8bfee2e35b654a38 Mon Sep 17 00:00:00 2001 From: Cody Bennett Date: Wed, 8 Nov 2023 00:54:37 -0600 Subject: [PATCH 04/64] feat(parser): PoC reflection pass --- src/index.ts | 2 + src/parser.ts | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++ tsconfig.json | 2 +- 3 files changed, 109 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 5483538..0550f9e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,5 @@ +export * from './ast' export * from './constants' export * from './minifier' +export * from './parser' export * from './tokenizer' diff --git a/src/parser.ts b/src/parser.ts index 45ae382..a6e35f0 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -23,6 +23,7 @@ import { type BreakStatement, type DiscardStatement, } from './ast' +import { type Token, tokenize } from './tokenizer' const isLiteral = (node: Node): node is Literal => (node as any).__brand === 'Literal' const isIdentifier = (node: Node): node is Identifier => (node as any).__brand === 'Identifier' @@ -48,3 +49,108 @@ const isReturnStatement = (node: Node): node is ReturnStatement => (node as any) const isContinueStatement = (node: Node): node is ContinueStatement => (node as any).__brand === 'ContinueStatement' const isBreakStatement = (node: Node): node is BreakStatement => (node as any).__brand === 'BreakStatement' const isDiscardStatement = (node: Node): node is DiscardStatement => (node as any).__brand === 'DiscardStatement' + +const isOpen = RegExp.prototype.test.bind(/^[\(\[\{]$/) +const isClose = RegExp.prototype.test.bind(/^[\)\]\}]$/) + +function getScopeIndex(token: Token): number { + if (isOpen(token.value)) return 1 + if (isClose(token.value)) return -1 + return 0 +} + +/** + * Parses a string of GLSL or WGSL code into an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree). + */ +export function parse(code: string): Node[] { + // TODO: preserve + const tokens = tokenize(code).filter((token) => token.type !== 'whitespace' && token.type !== 'comment') + let i = 0 + + // TODO: this is GLSL-only, separate language constants + const isType = RegExp.prototype.test.bind(/^(void|bool|float|u?int|[ui]?vec\d|mat\d(x\d)?)$/) + const isStorage = RegExp.prototype.test.bind(/^(uniform|in|out|attribute|varying)$/) + const isQualifier = RegExp.prototype.test.bind(/^(centroid|flat|smooth)$/) + + function getTokensUntil(value: string): Token[] { + const output: Token[] = [] + let scopeIndex = 0 + + while (i < tokens.length) { + const token = tokens[i++] + output.push(token) + + scopeIndex += getScopeIndex(token) + if (scopeIndex === 0 && token.value === value) break + } + + return output + } + + function parseBlock(node: T): T { + if (tokens[i].value === '{') i++ + + let scopeIndex = 0 + + while (i < tokens.length) { + const token = tokens[i++] + + scopeIndex += getScopeIndex(token) + if (scopeIndex < 0) break // skip } + + let statement: Node | null = null + + if (token.type === 'keyword') { + if (isQualifier(token.value) || isStorage(token.value) || isType(token.value) || token.value === 'const') { + if (tokens[i + 2]?.value === '(') { + const body = getTokensUntil('}') + statement = { __brand: 'FunctionDeclaration', body } + } else { + const qualifiers: string[] = [] + while (tokens[i].type !== 'identifier') qualifiers.push(tokens[i++].value) + const type = qualifiers.pop()! + + const body = getTokensUntil(';') + const name = body.shift()!.value + body.pop() // skip ; + + let value = null + if (body.length) { + // TODO: parse expression + } + + statement = { __brand: 'VariableDeclaration', name, type, value, qualifiers } satisfies VariableDeclaration + } + } else if (token.value === 'return') { + const body = getTokensUntil(';') + statement = { __brand: 'ReturnStatement', body } + } else if (token.value === 'if') { + const body = getTokensUntil('}') + statement = { __brand: 'IfStatement', body } + } else if (token.value === 'for') { + const body = getTokensUntil('}') + statement = { __brand: 'ForStatement', body } + } + } + + if (statement) node.body.push(statement) + } + + return node + } + + const program: BlockStatement = { __brand: 'BlockStatement', body: [] } + parseBlock(program) + + return program.body +} + +const glsl = /* glsl */ ` + flat in mat4 test; + + if (true) { + gl_FragColor = vec4(1, 0, 0, 1); // red + } +` + +console.log(parse(glsl)) diff --git a/tsconfig.json b/tsconfig.json index d98d302..9da44d9 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "outDir": "dist", "target": "esnext", "module": "esnext", - "lib": ["esnext", "dom"], + "lib": ["esnext"], "moduleResolution": "node", "strict": true, "pretty": true, From aa29f5d98b8a0c8b107608e76cf854ec1a9c4ffb Mon Sep 17 00:00:00 2001 From: Cody Bennett Date: Wed, 8 Nov 2023 00:56:11 -0600 Subject: [PATCH 05/64] chore: lint --- tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index 9da44d9..d98d302 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "outDir": "dist", "target": "esnext", "module": "esnext", - "lib": ["esnext"], + "lib": ["esnext", "dom"], "moduleResolution": "node", "strict": true, "pretty": true, From 4b5e049e4ec5e61916297a4fb1b8476abd3aefd4 Mon Sep 17 00:00:00 2001 From: Cody Bennett Date: Wed, 8 Nov 2023 00:56:49 -0600 Subject: [PATCH 06/64] refactor: Node => AST for TS --- src/ast.ts | 106 +++++++++++++++++++++++++------------------------- src/parser.ts | 50 ++++++++++++------------ 2 files changed, 78 insertions(+), 78 deletions(-) diff --git a/src/ast.ts b/src/ast.ts index 9305020..66b8f17 100644 --- a/src/ast.ts +++ b/src/ast.ts @@ -1,131 +1,131 @@ -export interface Node {} +export interface AST {} -export interface Literal extends Node { +export interface Literal extends AST { __brand: 'Literal' value: string | number | boolean } -export interface Identifier extends Node { +export interface Identifier extends AST { __brand: 'Identifier' value: string } -export interface VariableDeclaration extends Node { +export interface VariableDeclaration extends AST { __brand: 'VariableDeclaration' name: string - type: Node - value: Node | null - qualifiers: Node[] + type: AST + value: AST | null + qualifiers: AST[] } -export interface BlockStatement extends Node { +export interface BlockStatement extends AST { __brand: 'BlockStatement' - body: Node[] + body: AST[] } -export interface FunctionDeclaration extends Node { +export interface FunctionDeclaration extends AST { __brand: 'FunctionDeclaration' name: string - type: Node | null + type: AST | null args: VariableDeclaration[] body: BlockStatement } -export interface CallExpression extends Node { +export interface CallExpression extends AST { __brand: 'CallExpression' - callee: Node - args: Node[] + callee: AST + args: AST[] } -export interface MemberExpression extends Node { +export interface MemberExpression extends AST { __brand: 'MemberExpression' - object: Node - property: Node + object: AST + property: AST } -export interface ArrayExpression extends Node { +export interface ArrayExpression extends AST { __brand: 'ArrayExpression' - members: Node[] + members: AST[] } -export interface IfStatement extends Node { +export interface IfStatement extends AST { __brand: 'IfStatement' - test: Node - consequent: Node - alternate: Node | null + test: AST + consequent: AST + alternate: AST | null } -export interface WhileStatement extends Node { +export interface WhileStatement extends AST { __brand: 'WhileStatement' - test: Node - body: Node + test: AST + body: AST } -export interface ForStatement extends Node { +export interface ForStatement extends AST { __brand: 'ForStatement' - init: Node | null - test: Node | null - update: Node | null - body: Node + init: AST | null + test: AST | null + update: AST | null + body: AST } -export interface DoWhileStatement extends Node { +export interface DoWhileStatement extends AST { __brand: 'DoWhileStatement' - test: Node - body: Node + test: AST + body: AST } -export interface SwitchCase extends Node { +export interface SwitchCase extends AST { __brand: 'SwitchCase' - test: Node | null - consequent: Node[] + test: AST | null + consequent: AST[] } -export interface SwitchStatement extends Node { +export interface SwitchStatement extends AST { __brand: 'SwitchStatement' - discriminant: Node + discriminant: AST cases: SwitchCase[] } -export interface StructDeclaration extends Node { +export interface StructDeclaration extends AST { __brand: 'StructDeclaration' name: string members: VariableDeclaration[] } -export interface UnaryExpression extends Node { +export interface UnaryExpression extends AST { __brand: 'UnaryExpression' operator: string - argument: Node + argument: AST } -export interface BinaryExpression extends Node { +export interface BinaryExpression extends AST { __brand: 'BinaryExpression' operator: string - left: Node - right: Node + left: AST + right: AST } -export interface TernaryExpression extends Node { +export interface TernaryExpression extends AST { __brand: 'TernaryExpression' - test: Node - consequent: Node - alternate: Node + test: AST + consequent: AST + alternate: AST } -export interface ReturnStatement extends Node { +export interface ReturnStatement extends AST { __brand: 'ReturnStatement' argument: Literal | Identifier | UnaryExpression | null } -export interface ContinueStatement extends Node { +export interface ContinueStatement extends AST { __brand: 'ContinueStatement' } -export interface BreakStatement extends Node { +export interface BreakStatement extends AST { __brand: 'BreakStatement' } -export interface DiscardStatement extends Node { +export interface DiscardStatement extends AST { __brand: 'DiscardStatement' } diff --git a/src/parser.ts b/src/parser.ts index a6e35f0..691782c 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1,5 +1,5 @@ import { - type Node, + type AST, type Literal, type Identifier, type VariableDeclaration, @@ -25,30 +25,30 @@ import { } from './ast' import { type Token, tokenize } from './tokenizer' -const isLiteral = (node: Node): node is Literal => (node as any).__brand === 'Literal' -const isIdentifier = (node: Node): node is Identifier => (node as any).__brand === 'Identifier' -const isVariableDeclaration = (node: Node): node is VariableDeclaration => +const isLiteral = (node: AST): node is Literal => (node as any).__brand === 'Literal' +const isIdentifier = (node: AST): node is Identifier => (node as any).__brand === 'Identifier' +const isVariableDeclaration = (node: AST): node is VariableDeclaration => (node as any).__brand === 'VariableDeclaration' -const isBlockStatement = (node: Node): node is BlockStatement => (node as any).__brand === 'BlockStatement' -const isFunctionDeclaration = (node: Node): node is FunctionDeclaration => +const isBlockStatement = (node: AST): node is BlockStatement => (node as any).__brand === 'BlockStatement' +const isFunctionDeclaration = (node: AST): node is FunctionDeclaration => (node as any).__brand === 'FunctionDeclaration' -const isCallExpression = (node: Node): node is CallExpression => (node as any).__brand === 'CallExpression' -const isMemberExpression = (node: Node): node is MemberExpression => (node as any).__brand === 'MemberExpression' -const isArrayExpression = (node: Node): node is ArrayExpression => (node as any).__brand === 'ArrayExpression' -const isIfStatement = (node: Node): node is IfStatement => (node as any).__brand === 'IfStatement' -const isWhileStatement = (node: Node): node is WhileStatement => (node as any).__brand === 'WhileStatement' -const isForStatement = (node: Node): node is ForStatement => (node as any).__brand === 'ForStatement' -const isDoWhileStatement = (node: Node): node is DoWhileStatement => (node as any).__brand === 'DoWhileStatement' -const isSwitchCase = (node: Node): node is SwitchCase => (node as any).__brand === 'SwitchCase' -const isSwitchStatement = (node: Node): node is SwitchStatement => (node as any).__brand === 'SwitchStatement' -const isStructDeclaration = (node: Node): node is StructDeclaration => (node as any).__brand === 'StructDeclaration' -const isUnaryExpression = (node: Node): node is UnaryExpression => (node as any).__brand === 'UnaryExpression' -const isBinaryExpression = (node: Node): node is BinaryExpression => (node as any).__brand === 'BinaryExpression' -const isTernaryExpression = (node: Node): node is TernaryExpression => (node as any).__brand === 'TernaryExpression' -const isReturnStatement = (node: Node): node is ReturnStatement => (node as any).__brand === 'ReturnStatement' -const isContinueStatement = (node: Node): node is ContinueStatement => (node as any).__brand === 'ContinueStatement' -const isBreakStatement = (node: Node): node is BreakStatement => (node as any).__brand === 'BreakStatement' -const isDiscardStatement = (node: Node): node is DiscardStatement => (node as any).__brand === 'DiscardStatement' +const isCallExpression = (node: AST): node is CallExpression => (node as any).__brand === 'CallExpression' +const isMemberExpression = (node: AST): node is MemberExpression => (node as any).__brand === 'MemberExpression' +const isArrayExpression = (node: AST): node is ArrayExpression => (node as any).__brand === 'ArrayExpression' +const isIfStatement = (node: AST): node is IfStatement => (node as any).__brand === 'IfStatement' +const isWhileStatement = (node: AST): node is WhileStatement => (node as any).__brand === 'WhileStatement' +const isForStatement = (node: AST): node is ForStatement => (node as any).__brand === 'ForStatement' +const isDoWhileStatement = (node: AST): node is DoWhileStatement => (node as any).__brand === 'DoWhileStatement' +const isSwitchCase = (node: AST): node is SwitchCase => (node as any).__brand === 'SwitchCase' +const isSwitchStatement = (node: AST): node is SwitchStatement => (node as any).__brand === 'SwitchStatement' +const isStructDeclaration = (node: AST): node is StructDeclaration => (node as any).__brand === 'StructDeclaration' +const isUnaryExpression = (node: AST): node is UnaryExpression => (node as any).__brand === 'UnaryExpression' +const isBinaryExpression = (node: AST): node is BinaryExpression => (node as any).__brand === 'BinaryExpression' +const isTernaryExpression = (node: AST): node is TernaryExpression => (node as any).__brand === 'TernaryExpression' +const isReturnStatement = (node: AST): node is ReturnStatement => (node as any).__brand === 'ReturnStatement' +const isContinueStatement = (node: AST): node is ContinueStatement => (node as any).__brand === 'ContinueStatement' +const isBreakStatement = (node: AST): node is BreakStatement => (node as any).__brand === 'BreakStatement' +const isDiscardStatement = (node: AST): node is DiscardStatement => (node as any).__brand === 'DiscardStatement' const isOpen = RegExp.prototype.test.bind(/^[\(\[\{]$/) const isClose = RegExp.prototype.test.bind(/^[\)\]\}]$/) @@ -62,7 +62,7 @@ function getScopeIndex(token: Token): number { /** * Parses a string of GLSL or WGSL code into an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree). */ -export function parse(code: string): Node[] { +export function parse(code: string): AST[] { // TODO: preserve const tokens = tokenize(code).filter((token) => token.type !== 'whitespace' && token.type !== 'comment') let i = 0 @@ -98,7 +98,7 @@ export function parse(code: string): Node[] { scopeIndex += getScopeIndex(token) if (scopeIndex < 0) break // skip } - let statement: Node | null = null + let statement: AST | null = null if (token.type === 'keyword') { if (isQualifier(token.value) || isStorage(token.value) || isType(token.value) || token.value === 'const') { From 790523c4c22e47ed4efec8cf0d21f6f50a7735e3 Mon Sep 17 00:00:00 2001 From: Cody Bennett Date: Wed, 8 Nov 2023 00:59:06 -0600 Subject: [PATCH 07/64] chore: inline bitmap sweep --- src/parser.ts | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 691782c..120b882 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -53,12 +53,6 @@ const isDiscardStatement = (node: AST): node is DiscardStatement => (node as any const isOpen = RegExp.prototype.test.bind(/^[\(\[\{]$/) const isClose = RegExp.prototype.test.bind(/^[\)\]\}]$/) -function getScopeIndex(token: Token): number { - if (isOpen(token.value)) return 1 - if (isClose(token.value)) return -1 - return 0 -} - /** * Parses a string of GLSL or WGSL code into an [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree). */ @@ -80,7 +74,9 @@ export function parse(code: string): AST[] { const token = tokens[i++] output.push(token) - scopeIndex += getScopeIndex(token) + if (isOpen(token.value)) scopeIndex++ + if (isClose(token.value)) scopeIndex-- + if (scopeIndex === 0 && token.value === value) break } @@ -88,16 +84,9 @@ export function parse(code: string): AST[] { } function parseBlock(node: T): T { - if (tokens[i].value === '{') i++ - - let scopeIndex = 0 - while (i < tokens.length) { const token = tokens[i++] - scopeIndex += getScopeIndex(token) - if (scopeIndex < 0) break // skip } - let statement: AST | null = null if (token.type === 'keyword') { From 80a15b777037abd4d0540ddbdde5f9d59f8f5395 Mon Sep 17 00:00:00 2001 From: Cody Bennett Date: Wed, 8 Nov 2023 01:41:26 -0600 Subject: [PATCH 08/64] refactor: use classes for AST --- src/ast.ts | 175 ++++++++++++++++++++++++-------------------------- src/parser.ts | 62 ++++-------------- 2 files changed, 98 insertions(+), 139 deletions(-) diff --git a/src/ast.ts b/src/ast.ts index 66b8f17..26a8d56 100644 --- a/src/ast.ts +++ b/src/ast.ts @@ -1,131 +1,126 @@ -export interface AST {} +export abstract class AST {} -export interface Literal extends AST { - __brand: 'Literal' - value: string | number | boolean +export class Literal extends AST { + constructor(public value: string | number | boolean) { + super() + } } -export interface Identifier extends AST { - __brand: 'Identifier' - value: string +export class Identifier extends AST { + constructor(public value: string) { + super() + } } -export interface VariableDeclaration extends AST { - __brand: 'VariableDeclaration' - name: string - type: AST - value: AST | null - qualifiers: AST[] +export class VariableDeclaration extends AST { + constructor(public name: string, public type: AST, public value: AST | null, public qualifiers: AST[]) { + super() + } } -export interface BlockStatement extends AST { - __brand: 'BlockStatement' - body: AST[] +export class BlockStatement extends AST { + constructor(public body: AST[]) { + super() + } } -export interface FunctionDeclaration extends AST { - __brand: 'FunctionDeclaration' - name: string - type: AST | null - args: VariableDeclaration[] - body: BlockStatement +export class FunctionDeclaration extends AST { + constructor( + public name: string, + public type: AST | null, + public args: VariableDeclaration[], + public body: BlockStatement, + ) { + super() + } } -export interface CallExpression extends AST { - __brand: 'CallExpression' - callee: AST - args: AST[] +export class CallExpression extends AST { + constructor(public callee: AST, public args: AST[]) { + super() + } } -export interface MemberExpression extends AST { - __brand: 'MemberExpression' - object: AST - property: AST +export class MemberExpression extends AST { + constructor(public object: AST, public property: AST) { + super() + } } -export interface ArrayExpression extends AST { - __brand: 'ArrayExpression' - members: AST[] +export class ArrayExpression extends AST { + constructor(public members: AST[]) { + super() + } } -export interface IfStatement extends AST { - __brand: 'IfStatement' - test: AST - consequent: AST - alternate: AST | null +export class IfStatement extends AST { + constructor(public test: AST, public consequent: AST, public alternate: AST | null) { + super() + } } -export interface WhileStatement extends AST { - __brand: 'WhileStatement' - test: AST - body: AST +export class WhileStatement extends AST { + constructor(public test: AST, public body: AST) { + super() + } } -export interface ForStatement extends AST { - __brand: 'ForStatement' - init: AST | null - test: AST | null - update: AST | null - body: AST +export class ForStatement extends AST { + constructor(public init: AST | null, public test: AST | null, public update: AST | null, public body: AST) { + super() + } } -export interface DoWhileStatement extends AST { - __brand: 'DoWhileStatement' - test: AST - body: AST +export class DoWhileStatement extends AST { + constructor(public test: AST, public body: AST) { + super() + } } -export interface SwitchCase extends AST { - __brand: 'SwitchCase' - test: AST | null - consequent: AST[] +export class SwitchCase extends AST { + constructor(public test: AST | null, public consequent: AST[]) { + super() + } } -export interface SwitchStatement extends AST { - __brand: 'SwitchStatement' - discriminant: AST - cases: SwitchCase[] +export class SwitchStatement extends AST { + constructor(public discriminant: AST, public cases: SwitchCase[]) { + super() + } } -export interface StructDeclaration extends AST { - __brand: 'StructDeclaration' - name: string - members: VariableDeclaration[] +export class StructDeclaration extends AST { + constructor(public name: string, public members: VariableDeclaration[]) { + super() + } } -export interface UnaryExpression extends AST { - __brand: 'UnaryExpression' - operator: string - argument: AST +export class UnaryExpression extends AST { + constructor(public operator: string, public argument: AST) { + super() + } } -export interface BinaryExpression extends AST { - __brand: 'BinaryExpression' - operator: string - left: AST - right: AST +export class BinaryExpression extends AST { + constructor(public operator: string, public left: AST, public right: AST) { + super() + } } -export interface TernaryExpression extends AST { - __brand: 'TernaryExpression' - test: AST - consequent: AST - alternate: AST +export class TernaryExpression extends AST { + constructor(public test: AST, public consequent: AST, public alternate: AST) { + super() + } } -export interface ReturnStatement extends AST { - __brand: 'ReturnStatement' - argument: Literal | Identifier | UnaryExpression | null +export class ReturnStatement extends AST { + constructor(public argument: Literal | Identifier | UnaryExpression | null) { + super() + } } -export interface ContinueStatement extends AST { - __brand: 'ContinueStatement' -} +export class ContinueStatement extends AST {} -export interface BreakStatement extends AST { - __brand: 'BreakStatement' -} +export class BreakStatement extends AST {} -export interface DiscardStatement extends AST { - __brand: 'DiscardStatement' -} +export class DiscardStatement extends AST {} diff --git a/src/parser.ts b/src/parser.ts index 120b882..8e91357 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1,55 +1,13 @@ import { type AST, - type Literal, - type Identifier, - type VariableDeclaration, - type BlockStatement, - type FunctionDeclaration, - type CallExpression, - type MemberExpression, - type ArrayExpression, - type IfStatement, - type WhileStatement, - type ForStatement, - type DoWhileStatement, - type SwitchCase, - type SwitchStatement, - type StructDeclaration, - type UnaryExpression, - type BinaryExpression, - type TernaryExpression, - type ReturnStatement, - type ContinueStatement, - type BreakStatement, - type DiscardStatement, + BlockStatement, + VariableDeclaration, + ContinueStatement, + BreakStatement, + DiscardStatement, } from './ast' import { type Token, tokenize } from './tokenizer' -const isLiteral = (node: AST): node is Literal => (node as any).__brand === 'Literal' -const isIdentifier = (node: AST): node is Identifier => (node as any).__brand === 'Identifier' -const isVariableDeclaration = (node: AST): node is VariableDeclaration => - (node as any).__brand === 'VariableDeclaration' -const isBlockStatement = (node: AST): node is BlockStatement => (node as any).__brand === 'BlockStatement' -const isFunctionDeclaration = (node: AST): node is FunctionDeclaration => - (node as any).__brand === 'FunctionDeclaration' -const isCallExpression = (node: AST): node is CallExpression => (node as any).__brand === 'CallExpression' -const isMemberExpression = (node: AST): node is MemberExpression => (node as any).__brand === 'MemberExpression' -const isArrayExpression = (node: AST): node is ArrayExpression => (node as any).__brand === 'ArrayExpression' -const isIfStatement = (node: AST): node is IfStatement => (node as any).__brand === 'IfStatement' -const isWhileStatement = (node: AST): node is WhileStatement => (node as any).__brand === 'WhileStatement' -const isForStatement = (node: AST): node is ForStatement => (node as any).__brand === 'ForStatement' -const isDoWhileStatement = (node: AST): node is DoWhileStatement => (node as any).__brand === 'DoWhileStatement' -const isSwitchCase = (node: AST): node is SwitchCase => (node as any).__brand === 'SwitchCase' -const isSwitchStatement = (node: AST): node is SwitchStatement => (node as any).__brand === 'SwitchStatement' -const isStructDeclaration = (node: AST): node is StructDeclaration => (node as any).__brand === 'StructDeclaration' -const isUnaryExpression = (node: AST): node is UnaryExpression => (node as any).__brand === 'UnaryExpression' -const isBinaryExpression = (node: AST): node is BinaryExpression => (node as any).__brand === 'BinaryExpression' -const isTernaryExpression = (node: AST): node is TernaryExpression => (node as any).__brand === 'TernaryExpression' -const isReturnStatement = (node: AST): node is ReturnStatement => (node as any).__brand === 'ReturnStatement' -const isContinueStatement = (node: AST): node is ContinueStatement => (node as any).__brand === 'ContinueStatement' -const isBreakStatement = (node: AST): node is BreakStatement => (node as any).__brand === 'BreakStatement' -const isDiscardStatement = (node: AST): node is DiscardStatement => (node as any).__brand === 'DiscardStatement' - const isOpen = RegExp.prototype.test.bind(/^[\(\[\{]$/) const isClose = RegExp.prototype.test.bind(/^[\)\]\}]$/) @@ -108,8 +66,14 @@ export function parse(code: string): AST[] { // TODO: parse expression } - statement = { __brand: 'VariableDeclaration', name, type, value, qualifiers } satisfies VariableDeclaration + statement = new VariableDeclaration(name, type, value, qualifiers) } + } else if (token.value === 'continue') { + statement = new ContinueStatement() + } else if (token.value === 'break') { + statement = new BreakStatement() + } else if (token.value === 'discard') { + statement = new DiscardStatement() } else if (token.value === 'return') { const body = getTokensUntil(';') statement = { __brand: 'ReturnStatement', body } @@ -128,7 +92,7 @@ export function parse(code: string): AST[] { return node } - const program: BlockStatement = { __brand: 'BlockStatement', body: [] } + const program = new BlockStatement([]) parseBlock(program) return program.body From ba9bb970524428fc11af32e8d63c12e7d9d165ca Mon Sep 17 00:00:00 2001 From: Cody Bennett Date: Wed, 8 Nov 2023 02:01:22 -0600 Subject: [PATCH 09/64] fix: harden qualifiers --- src/parser.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 8e91357..18cc944 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -20,9 +20,8 @@ export function parse(code: string): AST[] { let i = 0 // TODO: this is GLSL-only, separate language constants - const isType = RegExp.prototype.test.bind(/^(void|bool|float|u?int|[ui]?vec\d|mat\d(x\d)?)$/) - const isStorage = RegExp.prototype.test.bind(/^(uniform|in|out|attribute|varying)$/) - const isQualifier = RegExp.prototype.test.bind(/^(centroid|flat|smooth)$/) + const isType = RegExp.prototype.test.bind(/^(void|bool|float|u?int|[uib]?vec\d|mat\d(x\d)?)$/) + const isQualifier = RegExp.prototype.test.bind(/^(in|out|inout|centroid|flat|smooth|invariant|lowp|mediump|highp)$/) function getTokensUntil(value: string): Token[] { const output: Token[] = [] @@ -48,7 +47,7 @@ export function parse(code: string): AST[] { let statement: AST | null = null if (token.type === 'keyword') { - if (isQualifier(token.value) || isStorage(token.value) || isType(token.value) || token.value === 'const') { + if (isQualifier(token.value) || isType(token.value) || token.value === 'const' || token.value === 'uniform') { if (tokens[i + 2]?.value === '(') { const body = getTokensUntil('}') statement = { __brand: 'FunctionDeclaration', body } @@ -106,4 +105,4 @@ const glsl = /* glsl */ ` } ` -console.log(parse(glsl)) +console.log(...parse(glsl)) From 68d0908e24b9678338f7addaa6d6b9374b2fc307 Mon Sep 17 00:00:00 2001 From: Cody Bennett Date: Wed, 8 Nov 2023 02:52:46 -0600 Subject: [PATCH 10/64] refactor: more basic parsing of methods, block statements --- src/parser.ts | 65 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 18cc944..8494519 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1,10 +1,14 @@ import { type AST, BlockStatement, + FunctionDeclaration, VariableDeclaration, ContinueStatement, BreakStatement, DiscardStatement, + ReturnStatement, + IfStatement, + ForStatement, } from './ast' import { type Token, tokenize } from './tokenizer' @@ -48,9 +52,13 @@ export function parse(code: string): AST[] { if (token.type === 'keyword') { if (isQualifier(token.value) || isType(token.value) || token.value === 'const' || token.value === 'uniform') { - if (tokens[i + 2]?.value === '(') { - const body = getTokensUntil('}') - statement = { __brand: 'FunctionDeclaration', body } + if (tokens[i + 1]?.value === '(') { + const type = token.value + const name = tokens[i++].value + // TODO: parse + const args = getTokensUntil(')') as any + const body = getTokensUntil('}') as any + statement = new FunctionDeclaration(name, type, args, body) } else { const qualifiers: string[] = [] while (tokens[i].type !== 'identifier') qualifiers.push(tokens[i++].value) @@ -75,13 +83,48 @@ export function parse(code: string): AST[] { statement = new DiscardStatement() } else if (token.value === 'return') { const body = getTokensUntil(';') - statement = { __brand: 'ReturnStatement', body } + body.pop() // skip; + + let argument = null + if (body.length) { + // TODO: parse expression + } + + statement = new ReturnStatement(argument) } else if (token.value === 'if') { - const body = getTokensUntil('}') - statement = { __brand: 'IfStatement', body } + // TODO: parse expression + const test = getTokensUntil(')') + const consequent = getTokensUntil('}') + const alternate = null + + statement = new IfStatement(test, consequent, alternate) } else if (token.value === 'for') { + const tests: (AST | null)[] = [null, null, null] + let j = 0 + + const loop = getTokensUntil(')') + loop.shift() // skip ( + loop.pop() // skip ) + + let next = loop.shift() + while (next) { + if (next.value === ';') { + j++ + } else { + // TODO: parse expressions + // @ts-ignore + tests[j] ??= [] + // @ts-ignore + tests[j].push(next) + } + + next = loop.shift() + } + + const [init, test, update] = tests const body = getTokensUntil('}') - statement = { __brand: 'ForStatement', body } + + statement = new ForStatement(init, test, update, body) } } @@ -101,6 +144,14 @@ const glsl = /* glsl */ ` flat in mat4 test; if (true) { + discard; + } + + for (int i = 0; i < 10; i++) { + // + } + + void main() { gl_FragColor = vec4(1, 0, 0, 1); // red } ` From a990ca4cf3e2e57a3664229c327aefe6f5f68552 Mon Sep 17 00:00:00 2001 From: Cody Bennett Date: Wed, 8 Nov 2023 02:56:14 -0600 Subject: [PATCH 11/64] chore: update demo --- demo/index.html | 3 +++ src/parser.ts | 1 + 2 files changed, 4 insertions(+) diff --git a/demo/index.html b/demo/index.html index 5956760..d1b2715 100644 --- a/demo/index.html +++ b/demo/index.html @@ -5,6 +5,9 @@ height: 100%; } +