From 6c7186264154ee2ea187eef3d4cefd1313a8fe6c Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Mon, 7 Oct 2024 09:08:36 +0200 Subject: [PATCH] Lua: multi-line string literals --- common/space_lua/eval.ts | 1 - common/space_lua/lua.grammar | 14 ++++-- common/space_lua/parse-lua.js | 2 +- common/space_lua/parse.test.ts | 6 ++- common/space_lua/parse.ts | 86 +++++++++++++++++++++++++++++---- plug-api/lib/tree.ts | 4 +- plugs/index/lint.ts | 14 +++++- web/cm_plugins/lua_directive.ts | 37 ++++++++++++-- 8 files changed, 140 insertions(+), 24 deletions(-) diff --git a/common/space_lua/eval.ts b/common/space_lua/eval.ts index aeb77645..0632b10d 100644 --- a/common/space_lua/eval.ts +++ b/common/space_lua/eval.ts @@ -31,7 +31,6 @@ export function evalExpression( try { switch (e.type) { case "String": - // TODO: Deal with escape sequences return e.value; case "Number": return e.value; diff --git a/common/space_lua/lua.grammar b/common/space_lua/lua.grammar index 812393ff..0ac2809d 100644 --- a/common/space_lua/lua.grammar +++ b/common/space_lua/lua.grammar @@ -79,7 +79,7 @@ exp { UnaryExpression | TableConstructor | FunctionDef { kw<"function"> FuncBody } - /*| Query*/ + // | Query } Query { @@ -154,10 +154,8 @@ TableConstructor { "{" (field (fieldsep field)* fieldsep?)? "}" } @tokens { CompareOp { "<" | ">" | $[<>=~/!] "=" } - TagIdentifier { @asciiLetter (@asciiLetter | @digit | "-" | "_" | "/" )* } - word { (std.asciiLetter | "_") (std.digit | std.asciiLetter | "_")* } identifier { word } @@ -165,11 +163,17 @@ TableConstructor { "{" (field (fieldsep field)* fieldsep?)? "}" } stringEscape { "\\" ($[abfnz"'\\] | digit digit? digit?) | "\\x" hex hex | - // NOTE: this should really be /[0-7]hex{5}/ at max, but that's annoying to write "\\u{" hex+ "}" } - simpleString { "'" (stringEscape | ![\r\n\\'])* "'" | '"' (stringEscape | ![\r\n\\"])* '"'} + // Any sequence of characters except two consecutive ]] + longStringContent { (![\]] | $[\]] ![\]])* } + + simpleString { + "'" (stringEscape | ![\r\n\\'])* "'" | + '"' (stringEscape | ![\r\n\\"])* '"' | + '[[' longStringContent ']]' + } hex { $[0-9a-fA-F] } digit { std.digit } diff --git a/common/space_lua/parse-lua.js b/common/space_lua/parse-lua.js index aa918480..872c7baa 100644 --- a/common/space_lua/parse-lua.js +++ b/common/space_lua/parse-lua.js @@ -13,7 +13,7 @@ export const parser = LRParser.deserialize({ ], skippedNodes: [0,1], repeatNodeCount: 9, - tokenData: "6V~RuXY#fYZ$Q[]#f]^$_pq#fqr$grs$rst)Yuv)_vw)dwx)ixy-zyz.Pz{.U{|.Z|}.`}!O.g!O!P/Z!P!Q/p!Q!R0Q!R![1f![!]3j!]!^3w!^!_4O!_!`4b!`!a4j!c!}4|!}#O5_#O#P#w#P#Q5d#Q#R5i#R#S4|#T#o4|#o#p5n#p#q5s#q#r5x#r#s5}~#kS#V~XY#f[]#fpq#f#O#P#w~#zQYZ#f]^#f~$VP#U~]^$Y~$_O#U~~$dP#U~YZ$YT$jP!_!`$mT$rOzT~$uXOY$rZ]$r^r$rrs%bs#O$r#O#P%g#P;'S$r;'S;=`'w<%lO$r~%gO#Z~~%jZrs$rwx$r!Q![&]#O#P$r#T#U$r#U#V$r#Y#Z$r#b#c$r#i#j'}#l#m(p#n#o$r~&`ZOY$rZ]$r^r$rrs%bs!Q$r!Q!['R![#O$r#O#P%g#P;'S$r;'S;=`'w<%lO$r~'UZOY$rZ]$r^r$rrs%bs!Q$r!Q![$r![#O$r#O#P%g#P;'S$r;'S;=`'w<%lO$r~'zP;=`<%l$r~(QP#o#p(T~(WR!Q![(a!c!i(a#T#Z(a~(dS!Q![(a!c!i(a#T#Z(a#q#r$r~(sR!Q![(|!c!i(|#T#Z(|~)PR!Q![$r!c!i$r#T#Z$r~)_O#p~~)dO#m~~)iO#e~~)lXOY)iZ])i^w)iwx%bx#O)i#O#P*X#P;'S)i;'S;=`,i<%lO)i~*[Zrs)iwx)i!Q![*}#O#P)i#T#U)i#U#V)i#Y#Z)i#b#c)i#i#j,o#l#m-b#n#o)i~+QZOY)iZ])i^w)iwx%bx!Q)i!Q![+s![#O)i#O#P*X#P;'S)i;'S;=`,i<%lO)i~+vZOY)iZ])i^w)iwx%bx!Q)i!Q![)i![#O)i#O#P*X#P;'S)i;'S;=`,i<%lO)i~,lP;=`<%l)i~,rP#o#p,u~,xR!Q![-R!c!i-R#T#Z-R~-US!Q![-R!c!i-R#T#Z-R#q#r)i~-eR!Q![-n!c!i-n#T#Z-n~-qR!Q![)i!c!i)i#T#Z)i~.POl~~.UOm~~.ZO#k~~.`O#i~V.gOvR#aS~.lP#j~}!O.o~.tTP~OY.oZ].o^;'S.o;'S;=`/T<%lO.o~/WP;=`<%l.oV/`PgT!O!P/cV/hP!PT!O!P/kQ/pOcQ~/uQ#l~!P!Q/{!_!`$m~0QO#n~~0VUd~!O!P0i!Q![1f!g!h0}!z!{1w#X#Y0}#l#m1w~0lP!Q![0o~0tRd~!Q![0o!g!h0}#X#Y0}~1QQ{|1W}!O1W~1ZP!Q![1^~1cPd~!Q![1^~1kSd~!O!P0i!Q![1f!g!h0}#X#Y0}~1zR!Q![2T!c!i2T#T#Z2T~2YUd~!O!P2l!Q![2T!c!i2T!r!s3^#T#Z2T#d#e3^~2oR!Q![2x!c!i2x#T#Z2x~2}Td~!Q![2x!c!i2x!r!s3^#T#Z2x#d#e3^~3aR{|1W}!O1W!P!Q1W~3oPo~![!]3r~3wOU~V4OOSR#aSV4VQ#uQzT!^!_4]!_!`$mT4bO#gT~4gP#`~!_!`$mV4qQ#vQzT!_!`$m!`!a4wT4|O#hT~5RS#X~!Q![4|!c!}4|#R#S4|#T#o4|~5dOi~~5iOj~~5nO#o~~5sOq~~5xO#d~~5}Ou~~6SP#f~!_!`$m", + tokenData: "7U~RuXY#fYZ$Q[]#f]^$_pq#fqr$grs$rst)Yuv)_vw)dwx)ixy-zyz.Pz{.U{|.Z|}.`}!O.g!O!P/Z!P!Q/p!Q!R0Q!R![1f![!]3j!]!^3w!^!_4O!_!`4b!`!a4j!c!}4|!}#O5_#O#P#w#P#Q6c#Q#R6h#R#S4|#T#o4|#o#p6m#p#q6r#q#r6w#r#s6|~#kS#V~XY#f[]#fpq#f#O#P#w~#zQYZ#f]^#f~$VP#U~]^$Y~$_O#U~~$dP#U~YZ$YT$jP!_!`$mT$rOzT~$uXOY$rZ]$r^r$rrs%bs#O$r#O#P%g#P;'S$r;'S;=`'w<%lO$r~%gO#Z~~%jZrs$rwx$r!Q![&]#O#P$r#T#U$r#U#V$r#Y#Z$r#b#c$r#i#j'}#l#m(p#n#o$r~&`ZOY$rZ]$r^r$rrs%bs!Q$r!Q!['R![#O$r#O#P%g#P;'S$r;'S;=`'w<%lO$r~'UZOY$rZ]$r^r$rrs%bs!Q$r!Q![$r![#O$r#O#P%g#P;'S$r;'S;=`'w<%lO$r~'zP;=`<%l$r~(QP#o#p(T~(WR!Q![(a!c!i(a#T#Z(a~(dS!Q![(a!c!i(a#T#Z(a#q#r$r~(sR!Q![(|!c!i(|#T#Z(|~)PR!Q![$r!c!i$r#T#Z$r~)_O#p~~)dO#m~~)iO#e~~)lXOY)iZ])i^w)iwx%bx#O)i#O#P*X#P;'S)i;'S;=`,i<%lO)i~*[Zrs)iwx)i!Q![*}#O#P)i#T#U)i#U#V)i#Y#Z)i#b#c)i#i#j,o#l#m-b#n#o)i~+QZOY)iZ])i^w)iwx%bx!Q)i!Q![+s![#O)i#O#P*X#P;'S)i;'S;=`,i<%lO)i~+vZOY)iZ])i^w)iwx%bx!Q)i!Q![)i![#O)i#O#P*X#P;'S)i;'S;=`,i<%lO)i~,lP;=`<%l)i~,rP#o#p,u~,xR!Q![-R!c!i-R#T#Z-R~-US!Q![-R!c!i-R#T#Z-R#q#r)i~-eR!Q![-n!c!i-n#T#Z-n~-qR!Q![)i!c!i)i#T#Z)i~.POl~~.UOm~~.ZO#k~~.`O#i~V.gOvR#aS~.lP#j~}!O.o~.tTP~OY.oZ].o^;'S.o;'S;=`/T<%lO.o~/WP;=`<%l.oV/`PgT!O!P/cV/hP!PT!O!P/kQ/pOcQ~/uQ#l~!P!Q/{!_!`$m~0QO#n~~0VUd~!O!P0i!Q![1f!g!h0}!z!{1w#X#Y0}#l#m1w~0lP!Q![0o~0tRd~!Q![0o!g!h0}#X#Y0}~1QQ{|1W}!O1W~1ZP!Q![1^~1cPd~!Q![1^~1kSd~!O!P0i!Q![1f!g!h0}#X#Y0}~1zR!Q![2T!c!i2T#T#Z2T~2YUd~!O!P2l!Q![2T!c!i2T!r!s3^#T#Z2T#d#e3^~2oR!Q![2x!c!i2x#T#Z2x~2}Td~!Q![2x!c!i2x!r!s3^#T#Z2x#d#e3^~3aR{|1W}!O1W!P!Q1W~3oPo~![!]3r~3wOU~V4OOSR#aSV4VQ#uQzT!^!_4]!_!`$mT4bO#gT~4gP#`~!_!`$mV4qQ#vQzT!_!`$m!`!a4wT4|O#hT~5RS#X~!Q![4|!c!}4|#R#S4|#T#o4|~5dPi~!}#O5g~5jTO#P5g#P#Q5y#Q;'S5g;'S;=`6]<%lO5g~5|TO#P5g#P#Q%b#Q;'S5g;'S;=`6]<%lO5g~6`P;=`<%l5g~6hOj~~6mO#o~~6rOq~~6wO#d~~6|Ou~~7RP#f~!_!`$m", tokenizers: [0, 1, 2], topRules: {"Chunk":[0,2]}, dynamicPrecedences: {"110":1}, diff --git a/common/space_lua/parse.test.ts b/common/space_lua/parse.test.ts index 93c848ef..8c6346dc 100644 --- a/common/space_lua/parse.test.ts +++ b/common/space_lua/parse.test.ts @@ -11,6 +11,7 @@ Deno.test("Test Lua parser", () => { parse( `e(1, 1.2, -3.8, +4, #lst, true, false, nil, "string", "", "Hello there \x00", ...)`, ); + parse(`e([[hel]lo]], "Grinny face\\u{1F600}")`); parse(`e(10 << 10, 10 >> 10, 10 & 10, 10 | 10, 10 ~ 10)`); @@ -92,5 +93,8 @@ Deno.test("Test comment handling", () => { --[[ Multi line comment ]] - f()`); + f([[ + hello + -- yo + ]])`); }); diff --git a/common/space_lua/parse.ts b/common/space_lua/parse.ts index 1a55760e..970d51f1 100644 --- a/common/space_lua/parse.ts +++ b/common/space_lua/parse.ts @@ -331,12 +331,56 @@ function parseExpList(t: ParseTree, ctx: ASTCtx): LuaExpression[] { ); } +// In case of quoted strings, remove the quotes and unescape the string +// In case of a [[ type ]] literal string, remove the brackets +function parseString(s: string): string { + if (s.startsWith("[[") && s.endsWith("]]")) { + return s.slice(2, -2); + } + return s.slice(1, -1).replace( + /\\(x[0-9a-fA-F]{2}|u\{[0-9a-fA-F]+\}|[abfnrtv\\'"n])/g, + (match, capture) => { + switch (capture) { + case "a": + return "\x07"; // Bell + case "b": + return "\b"; // Backspace + case "f": + return "\f"; // Form feed + case "n": + return "\n"; // Newline + case "r": + return "\r"; // Carriage return + case "t": + return "\t"; // Horizontal tab + case "v": + return "\v"; // Vertical tab + case "\\": + return "\\"; // Backslash + case '"': + return '"'; // Double quote + case "'": + return "'"; // Single quote + default: + // Handle hexadecimal \x00 + if (capture.startsWith("x")) { + return String.fromCharCode(parseInt(capture.slice(1), 16)); + } + // Handle unicode \u{XXXX} + if (capture.startsWith("u{")) { + const codePoint = parseInt(capture.slice(2, -1), 16); + return String.fromCodePoint(codePoint); + } + return match; // return the original match if nothing fits + } + }, + ); +} + function parseExpression(t: ParseTree, ctx: ASTCtx): LuaExpression { switch (t.type) { case "LiteralString": { - let cleanString = t.children![0].text!; - // Remove quotes etc - cleanString = cleanString.slice(1, -1); + const cleanString = parseString(t.children![0].text!); return { type: "String", value: cleanString, @@ -506,22 +550,46 @@ function parseTableField(t: ParseTree, ctx: ASTCtx): LuaTableField { throw new Error(`Unknown table field type: ${t.type}`); } } - function stripLuaComments(s: string): string { // Strips Lua comments (single-line and multi-line) and replaces them with equivalent length whitespace let result = ""; let inString = false; + let inMultilineString = false; let inComment = false; let inMultilineComment = false; for (let i = 0; i < s.length; i++) { - // Handle string detection (to avoid stripping comments inside strings) - if (s[i] === '"' && !inComment && !inMultilineComment) { + // Handle string detection for single-line strings (to avoid stripping comments inside strings) + if ( + s[i] === '"' && !inComment && !inMultilineComment && !inMultilineString + ) { inString = !inString; } + // Handle multi-line string literals (starting with "[[") + if ( + !inString && !inComment && !inMultilineComment && s[i] === "[" && + s[i + 1] === "[" + ) { + inMultilineString = true; + result += "[["; // Copy "[[" into result + i += 1; // Skip over "[[" + continue; + } + + // Handle end of multi-line string literals (ending with "]]") + if (inMultilineString && s[i] === "]" && s[i + 1] === "]") { + inMultilineString = false; + result += "]]"; // Copy "]]" into result + i += 1; // Skip over "]]" + continue; + } + // Handle single-line comments (starting with "--") - if (!inString && !inMultilineComment && s[i] === "-" && s[i + 1] === "-") { + if ( + !inString && !inMultilineString && !inMultilineComment && s[i] === "-" && + s[i + 1] === "-" + ) { if (s[i + 2] === "[" && s[i + 3] === "[") { // Detect multi-line comment start "--[[" inMultilineComment = true; @@ -546,9 +614,9 @@ function stripLuaComments(s: string): string { continue; } - // Replace comment content with spaces, or copy original content if not in comment + // Replace comment content with spaces, or copy original content if not in comment or multi-line string if (inComment || inMultilineComment) { - result += " "; // Replace comment characters with a space + result += " "; // Replace comment characters with spaces } else { result += s[i]; } diff --git a/plug-api/lib/tree.ts b/plug-api/lib/tree.ts index 50385a1f..9292f538 100644 --- a/plug-api/lib/tree.ts +++ b/plug-api/lib/tree.ts @@ -240,7 +240,9 @@ export function cleanTree(tree: ParseTree, omitTrimmable = true): ParseTree { const parseErrorNodes = collectNodesOfType(tree, "⚠"); if (parseErrorNodes.length > 0) { throw new Error( - `Parse error in: ${renderToText(tree)}`, + `Parse error (${parseErrorNodes[0].from}:${parseErrorNodes[0].to}): ${ + renderToText(tree) + }`, ); } if (tree.text !== undefined) { diff --git a/plugs/index/lint.ts b/plugs/index/lint.ts index e95396f6..e1c4eeba 100644 --- a/plugs/index/lint.ts +++ b/plugs/index/lint.ts @@ -233,9 +233,19 @@ export async function lintLua({ tree }: LintEvent): Promise { try { await lua.parse(luaCode); } catch (e: any) { + const offset = codeText.from!; + let from = codeText.from!; + let to = codeText.to!; + if (e.message.includes("Parse error (")) { + const errorMatch = errorRegex.exec(e.message); + if (errorMatch) { + from = offset + parseInt(errorMatch[1], 10); + to = offset + parseInt(errorMatch[2], 10); + } + } diagnostics.push({ - from: codeText.from!, - to: codeText.to!, + from, + to, severity: "error", message: e.message, }); diff --git a/web/cm_plugins/lua_directive.ts b/web/cm_plugins/lua_directive.ts index 7cf7615b..1048cf6f 100644 --- a/web/cm_plugins/lua_directive.ts +++ b/web/cm_plugins/lua_directive.ts @@ -8,17 +8,25 @@ import { shouldRenderWidgets, } from "./util.ts"; import type { Client } from "../client.ts"; -import { parse } from "$common/space_lua/parse.ts"; +import { parse as parseLua } from "$common/space_lua/parse.ts"; import type { LuaBlock, LuaFunctionCallStatement, } from "$common/space_lua/ast.ts"; import { evalExpression } from "$common/space_lua/eval.ts"; import { luaToString } from "$common/space_lua/runtime.ts"; +import { parse as parseMarkdown } from "$common/markdown_parser/parse_tree.ts"; +import { extendedMarkdownLanguage } from "$common/markdown_parser/parser.ts"; +import { renderMarkdownToHtml } from "../../plugs/markdown/markdown_render.ts"; +import { + isLocalPath, + resolvePath, +} from "@silverbulletmd/silverbullet/lib/resolve"; class LuaDirectiveWidget extends WidgetType { constructor( readonly code: string, + private client: Client, ) { super(); } @@ -38,13 +46,34 @@ class LuaDirectiveWidget extends WidgetType { const span = document.createElement("span"); span.className = "sb-lua-directive"; try { - const parsedLua = parse(`_(${this.code})`) as LuaBlock; + const parsedLua = parseLua(`_(${this.code})`) as LuaBlock; const expr = (parsedLua.statements[0] as LuaFunctionCallStatement).call.args[0]; Promise.resolve(evalExpression(expr, client.clientSystem.spaceLuaEnv.env)) .then((result) => { - span.innerText = luaToString(result); + const mdTree = parseMarkdown( + extendedMarkdownLanguage, + luaToString(result), + ); + + const html = renderMarkdownToHtml(mdTree, { + // Annotate every element with its position so we can use it to put + // the cursor there when the user clicks on the table. + annotationPositions: true, + translateUrls: (url) => { + if (isLocalPath(url)) { + url = resolvePath( + this.client.currentPage, + decodeURI(url), + ); + } + + return url; + }, + preserveAttributes: true, + }, this.client.ui.viewState.allPages); + span.innerHTML = html; }).catch((e) => { console.error("Lua eval error", e); span.innerText = `Lua error: ${e.message}`; @@ -81,7 +110,7 @@ export function luaDirectivePlugin(client: Client) { widgets.push( Decoration.widget({ - widget: new LuaDirectiveWidget(text), + widget: new LuaDirectiveWidget(text, client), }).range(node.to), ); widgets.push(invisibleDecoration.range(node.from, node.to));