Represent documentation in AST, load into Node (#9205)

Part of #9162.

- Add support for representing and interpreting the full text-literal/documentation syntax (escape codes, platform-independent newlines, string interpolations); build on generalized operations for structured-fields that will simplify future representation of other types like `Vector`.
- Load parsed and interpreted node documentation into `Node`s.

app/gui2/shared/ast/index.ts

@@ -25,7 +25,8 @@ export function asOwned<T>(t: T): Owned<T> {
   return t as Owned<T>
 }
 
-export type NodeChild<T = AstId | SyncTokenId> = { whitespace?: string | undefined; node: T }
+export type NodeChild<T> = { whitespace?: string | undefined; node: T }
+export type RawNodeChild = NodeChild<AstId> | NodeChild<SyncTokenId>
 
 export function newExternalId(): ExternalId {
   return random.uuidv4() as ExternalId

app/gui2/shared/ast/mutableModule.ts

@@ -1,15 +1,7 @@
 import * as random from 'lib0/random'
 import * as Y from 'yjs'
-import {
-  Token,
-  asOwned,
-  isTokenId,
-  newExternalId,
-  subtreeRoots,
-  type AstId,
-  type Owned,
-  type SyncTokenId,
-} from '.'
+import type { AstId, NodeChild, Owned, RawNodeChild, SyncTokenId } from '.'
+import { Token, asOwned, isTokenId, newExternalId, subtreeRoots } from '.'
 import { assert, assertDefined } from '../util/assert'
 import type { SourceRangeEdit } from '../util/data/text'
 import { defaultLocalOrigin, tryAsOrigin, type ExternalId, type Origin } from '../yjsModel'
@@ -38,6 +30,7 @@ export interface Module {
   getToken(token: SyncTokenId): Token
   getToken(token: SyncTokenId | undefined): Token | undefined
   getAny(node: AstId | SyncTokenId): Ast | Token
+  getConcrete(child: RawNodeChild): NodeChild<Ast> | NodeChild<Token>
   has(id: AstId): boolean
 }
 
@@ -322,6 +315,12 @@ export class MutableModule implements Module {
     return isTokenId(node) ? this.getToken(node) : this.get(node)
   }
 
+  getConcrete(child: RawNodeChild): NodeChild<Ast> | NodeChild<Token> {
+    if (isTokenId(child.node))
+      return { whitespace: child.whitespace, node: this.getToken(child.node) }
+    else return { whitespace: child.whitespace, node: this.get(child.node) }
+  }
+
   /** @internal Copy a node into the module, if it is bound to a different module. */
   copyIfForeign<T extends MutableAst>(ast: Owned<T>): Owned<T>
   copyIfForeign<T extends MutableAst>(ast: Owned<T> | undefined): Owned<T> | undefined {

app/gui2/shared/ast/parse.ts

@@ -1,5 +1,5 @@
 import * as map from 'lib0/map'
-import type { AstId, Module, NodeChild, Owned } from '.'
+import type { AstId, Module, NodeChild, Owned, OwnedRefs, TextElement, TextToken } from '.'
 import {
   Token,
   asOwned,
@@ -255,20 +255,16 @@ class Abstractor {
       case RawAst.Tree.Type.TextLiteral: {
         const open = tree.open ? this.abstractToken(tree.open) : undefined
         const newline = tree.newline ? this.abstractToken(tree.newline) : undefined
-        const elements = []
-        for (const e of tree.elements) {
-          elements.push(...this.abstractChildren(e))
-        }
+        const elements = Array.from(tree.elements, (raw) => this.abstractTextElement(raw))
         const close = tree.close ? this.abstractToken(tree.close) : undefined
         node = TextLiteral.concrete(this.module, open, newline, elements, close)
         break
       }
       case RawAst.Tree.Type.Documented: {
         const open = this.abstractToken(tree.documentation.open)
-        const elements = []
-        for (const e of tree.documentation.elements) {
-          elements.push(...this.abstractChildren(e))
-        }
+        const elements = Array.from(tree.documentation.elements, (raw) =>
+          this.abstractTextToken(raw),
+        )
         const newlines = Array.from(tree.documentation.newlines, this.abstractToken.bind(this))
         const expression = tree.expression ? this.abstractTree(tree.expression) : undefined
         node = Documented.concrete(this.module, open, elements, newlines, expression)
@@ -315,8 +311,8 @@ class Abstractor {
     return { whitespace, node }
   }
 
-  private abstractChildren(tree: LazyObject): NodeChild<Owned | Token>[] {
-    const children: NodeChild<Owned | Token>[] = []
+  private abstractChildren(tree: LazyObject): (NodeChild<Owned> | NodeChild<Token>)[] {
+    const children: (NodeChild<Owned> | NodeChild<Token>)[] = []
     const visitor = (child: LazyObject) => {
       if (RawAst.Tree.isInstance(child)) {
         children.push(this.abstractTree(child))
@@ -329,6 +325,42 @@ class Abstractor {
     tree.visitChildren(visitor)
     return children
   }
+
+  private abstractTextElement(raw: RawAst.TextElement): TextElement<OwnedRefs> {
+    switch (raw.type) {
+      case RawAst.TextElement.Type.Newline:
+      case RawAst.TextElement.Type.Escape:
+      case RawAst.TextElement.Type.Section:
+        return this.abstractTextToken(raw)
+      case RawAst.TextElement.Type.Splice:
+        return {
+          type: 'splice',
+          open: this.abstractToken(raw.open),
+          expression: raw.expression && this.abstractTree(raw.expression),
+          close: this.abstractToken(raw.close),
+        }
+    }
+  }
+
+  private abstractTextToken(raw: RawAst.TextElement): TextToken<OwnedRefs> {
+    switch (raw.type) {
+      case RawAst.TextElement.Type.Newline:
+        return { type: 'token', token: this.abstractToken(raw.newline) }
+      case RawAst.TextElement.Type.Escape: {
+        const negativeOneU32 = 4294967295
+        return {
+          type: 'token',
+          token: this.abstractToken(raw.token),
+          interpreted:
+            raw.token.value !== negativeOneU32 ? String.fromCodePoint(raw.token.value) : undefined,
+        }
+      }
+      case RawAst.TextElement.Type.Section:
+        return { type: 'token', token: this.abstractToken(raw.text) }
+      case RawAst.TextElement.Type.Splice:
+        throw new Error('Unreachable: Splice in non-interpolated text field')
+    }
+  }
 }
 
 declare const nodeKeyBrand: unique symbol