diff --git a/packages/pkl.lua/PklProject b/packages/pkl.lua/PklProject index 106e1a3..4e05d78 100644 --- a/packages/pkl.lua/PklProject +++ b/packages/pkl.lua/PklProject @@ -13,9 +13,9 @@ // See the License for the specific language governing permissions and // limitations under the License. //===----------------------------------------------------------------------===// -/// A [Lua](https://www.lua.org) renderer. +/// A parser and renderer for a subset of [Lua](https://www.lua.org). amends "../basePklProject.pkl" package { - version = "1.0.1" + version = "1.1.0" } diff --git a/packages/pkl.lua/lua.pkl b/packages/pkl.lua/lua.pkl index c321761..cd6d587 100644 --- a/packages/pkl.lua/lua.pkl +++ b/packages/pkl.lua/lua.pkl @@ -13,11 +13,13 @@ // See the License for the specific language governing permissions and // limitations under the License. //===----------------------------------------------------------------------===// -/// A renderer for [Lua](https://www.lua.org) configuration files. +/// A [Parser] and [Renderer] for a subset of [Lua](https://www.lua.org). @ModuleInfo { minPklVersion = "0.25.0" } module pkl.lua.lua import "pkl:reflect" +import "pkl:base" +import "pkl:math" import "lua.pkl" local const pathSpecRegex: Regex = let (prop = #"(?:[^\[\]^*.]+)"#) Regex(#""" @@ -58,16 +60,51 @@ local typealias PathEntry = Prop|Key|"^" local const splatKey: Key = new Key { key = "*" } +local const function splitPathConverters(converterMap: Map unknown>): List, (unknown) -> unknown>> = + converterMap + .filter((key, _) -> key is String) + .mapKeys((key, _) -> + if (key.matches(pathSpecRegex)) + if (key == "") List("^") else // PcfRenderer treats empty path as "^" + key + .split(pathSpecSplitRegex) + .map((it) -> + if (it == "^") "^" + else if (it.startsWith("[")) new Key { key = it.substring(1, it.length-1) } + else new Prop { name = it }) + .reverse() + else throw("Converter path `\(key)` has invalid syntax.")) + .entries + /// A string that is a Lua reserved keyword. /// /// These strings are not allowed to be used as identifiers. -typealias LuaKeyword = "and"|"break"|"do"|"else"|"elseif"|"end"|"false"|"for"|"function"|"goto"|"if"|"in"|"local"|"nil"|"not"|"or"|"repeat"|"return"|"then"|"true"|"until"|"while" +@AlsoKnownAs { names { "LuaKeyword" } } +typealias Keyword = "and"|"break"|"do"|"else"|"elseif"|"end"|"false"|"for"|"function"|"goto"|"if"|"in"|"local"|"nil"|"not"|"or"|"repeat"|"return"|"then"|"true"|"until"|"while" + +/// Obsolete alias for [Keyword]. +@Deprecated { message = "Use [Keyword] instead."; replaceWith = "lua.Keyword"; since = "1.1.0" } +typealias LuaKeyword = Keyword /// A string that is a valid Lua identifier. -typealias LuaIdentifier = String(matches(Regex("[a-zA-Z_][a-zA-Z0-9_]*")) && !(this is lua.LuaKeyword)) +@AlsoKnownAs { names { "LuaIdentifier" } } +typealias Identifier = String(matches(Regex("[a-zA-Z_][a-zA-Z0-9_]*")) && !(this is lua.Keyword)) + +/// Obsolete alias for [Identifier]. +@Deprecated { message = "Use [Identifier] instead."; replaceWith = "lua.Identifier"; since = "1.1.0" } +typealias LuaIdentifier = Identifier + +/// Pkl representation of a Lua value. +typealias Value = Null|Boolean|Number|String|Listing|Dynamic|Mapping + +/// Pkl representation of a valid Lua table key. +typealias TableKey = Boolean|Number(!isNaN)|String|Listing|Dynamic|Mapping + +// region Renderer /// Directs [Renderer] to output additional text [before] and/or [after] rendering a [value]. -class LuaRenderDirective { +@AlsoKnownAs { names { "LuaRenderDirective" } } +class RenderDirective { /// The text to output before rendering [value]. before: String? @@ -78,6 +115,10 @@ class LuaRenderDirective { after: String? } +/// Obsolete alias for [RenderDirective]. +@Deprecated { message = "Use [RenderDirective] instead."; replaceWith = "lua.RenderDirective"; since = "1.1.0" } +typealias LuaRenderDirective = RenderDirective + /// Renders values as Lua. class Renderer extends ValueRenderer { /// The characters to use for indenting output. Defaults to two spaces. @@ -100,7 +141,9 @@ class Renderer extends ValueRenderer { /// Value converters to apply before values are rendered. /// /// For more information see [ValueRenderer.converters]. Note that due to language limitations, - /// any entries in a [Dynamic] will have converters applied as though the entry was a property. + /// when rendering a [Dynamic], any entries with a [String] key will have converters applied as + /// though the entry was a property. This means paths like `x[foo]` only apply when rendering + /// a [Mapping], or when a class converter converts an entry key into a [String]. converters: Mapping<(Class|String), (unknown) -> Any> extension = "lua" @@ -112,13 +155,15 @@ class Renderer extends ValueRenderer { function renderDocument(value: Any): String = let (path = List("^")) let (value = convert(value, path)) - if (value is RenderDirective) "\(value.text)\n" - else if (value is LuaRenderDirective) "\(value.before ?? "")\(render(value.value, path, 0))\(value.after ?? "")\n" + if (value is base.RenderDirective) "\(value.text)\n" + else if (value is RenderDirective) "\(value.before ?? "")\(render(value.value, path, 0))\(value.after ?? "")\n" else if (value is Dynamic|Typed) value.toMap().fold("", (acc, k, v) -> - let (path = path.add(Prop(k))) + let (isProp = k is String) + let (k = if (isProp) k else convert(k, null)) + let (path = path.add(if (isProp) Prop(k) else Key(k))) let (v = convert(v, path)) - if (omitNullProperties && v == null && k is String) acc + if (omitNullProperties && v == null && isProp) acc else acc + "\(renderKey(k, "_ENV")) = \(render(v, path, 0))\n") + (if (value is Dynamic) value.toList() else List()).foldIndexed("", (idx, acc, v) -> let (path = path.add(splatKey)) @@ -127,6 +172,7 @@ class Renderer extends ValueRenderer { acc + "_ENV[\(idx+1)] = \(render(v, path, 0))\n") else if (value is Mapping|Map) value.fold("", (acc, k, v) -> + let (k = convert(k, null)) let (path = path.add(Key(k))) let (v = convert(v, path)) acc + "\(renderKey(k, "_ENV")) = \(render(v, path, 0))\n") @@ -138,20 +184,7 @@ class Renderer extends ValueRenderer { // path specs are already in reversed order local pathConverters: List,(unknown) -> Any>> = - converterMap - .filter((key, _) -> key is String) - .mapKeys((key, _) -> - if (key.matches(pathSpecRegex)) - if (key == "") List("^") else // PcfRenderer treats empty path as "^" - key - .split(pathSpecSplitRegex) - .map((it) -> - if (it == "^") "^" - else if (it.startsWith("[")) new Key { key = it.drop(1).dropLast(1) } - else new Prop { name = it }) - .reverse() - else throw("Converter path `\(key)` has invalid syntax.")) - .entries + splitPathConverters(converterMap) // [true] if the converters define any class converters. // For the time being this is limited to any subclasses of [Typed], as this matches current @@ -162,17 +195,19 @@ class Renderer extends ValueRenderer { converterMap.any((k,_) -> k is Class && reflect.Class(k).isSubclassOf(typedClass)) local function convert(value: Any, path: List?): Any = - if (path == null) value else - let (path = path.reverse()) - let (f = pathConverters.findOrNull((p) -> comparePaths(path, p.key))?.value) - let (klass = value.getClass()) - let (f = f ?? converterMap.getOrNull(klass)) - let (f = f ?? if (hasTypedConverters && value is Typed) findTypedConverter(reflect.Class(klass).superclass) else null) // find superclass converters - if (f != null) f.apply(value) else value + let (f = + if (path != null && !pathConverters.isEmpty) + let (path = path.reverse()) + pathConverters.findOrNull((p) -> comparePaths(path, p.key))?.value + else null) + let (klass = value.getClass()) + let (f = f ?? converterMap.getOrNull(klass)) + let (f = f ?? if (hasTypedConverters && value is Typed) findTypedConverter(reflect.Class(klass).superclass) else null) // find superclass converters + if (f != null) f.apply(value) else value // the path and spec must already be reversed local function comparePaths(path: List, pathSpec: List): Boolean = - path.zip(pathSpec).every((p) -> + path.length >= pathSpec.length && path.zip(pathSpec).every((p) -> if (p.second is Prop && p.second.name == "*") p.first is Prop else if (p.second is Key && p.second.key == "*") p.first is Key else p.first == p.second @@ -185,7 +220,7 @@ class Renderer extends ValueRenderer { // endregion // region Rendering functions - local function render(value: Any, path: List?, level: UInt?): String = + local function render(value: Any, path: List, level: UInt?): String = if (value is String) renderString(value, level != null) else if (value is Boolean) value.toString() else if (value is Number) renderNumber(value) @@ -193,17 +228,17 @@ class Renderer extends ValueRenderer { else if (value is Mapping|Map) renderMap(value, path, level) else if (value is Listing|List) renderList(value, path, level) else if (value is Dynamic) renderDynamic(value, path, level) - else if (value is RenderDirective) value.text - else if (value is LuaRenderDirective) "\(value.before ?? "")\(render(value.value, path, level))\(value.after ?? "")" + else if (value is base.RenderDirective) value.text + else if (value is RenderDirective) "\(value.before ?? "")\(render(value.value, path, level))\(value.after ?? "")" else if (value is Typed) renderDynamic(value, path, level) else throw("Cannot render value of type `\(value.getClass())` as Lua.\nValue: \(value)") local function renderKey(key: Any, prefix: String): String = if (key is Null) throw("Lua table keys cannot be null") else if (key is Number && key.isNaN) throw("Lua table keys cannot be NaN") - else if (key is LuaIdentifier) key - else if (key is RenderDirective) key.text - else "\(prefix)[\(render(key, null, null))]" + else if (key is Identifier) key + else if (key is base.RenderDirective) key.text + else "\(prefix)[\(render(key, List(), null))]" local function renderString(value: String, multiline: Boolean): String = let (delim = if (value.contains("\"") && !value.contains("'")) "'" else "\"") @@ -238,7 +273,7 @@ class Renderer extends ValueRenderer { else if (value.isInfinite) "(\(value.sign.toInt())/0)" else value.toString() - local function renderMap(value: Mapping|Map, path: List?, level: UInt?): String = + local function renderMap(value: Mapping|Map, path: List, level: UInt?): String = if (value.isEmpty) "{}" else let (map = value.toMap()) let (multiline = map.length >= multilineThreshold && level != null && value is Mapping) @@ -246,15 +281,16 @@ class Renderer extends ValueRenderer { (if (multiline) "{\n\(indent.repeat(level_!!))" else "{ ") + new Listing { for (k,v in map) { - let (path = path?.add(Key(k))) + let (k = convert(k, null)) + let (path = path.add(Key(k))) "\(renderKey(k, "")) = \(render(convert(v, path), path, level_));" } }.join(if (multiline) "\n\(indent.repeat(level_!!))" else " ") + if (multiline) "\n\(indent.repeat(level!!))}" else " }" - local function renderList(value: Listing|List, path: List?, level: UInt?): String = + local function renderList(value: Listing|List, path: List, level: UInt?): String = if (value.isEmpty) "{}" else - let (path = path?.add(splatKey)) + let (path = path.add(splatKey)) let (multiline = value.length >= multilineThreshold && level != null && !(value is List)) let (level_ = if (multiline) level!! + 1 else level) (if (multiline) "{\n\(indent.repeat(level_!!))" else "{ ") @@ -263,20 +299,22 @@ class Renderer extends ValueRenderer { .join(if (multiline) ",\n\(indent.repeat(level_!!))" else ", ") + if (multiline) "\n\(indent.repeat(level!!))}" else " }" - local function renderDynamic(value: Dynamic|Typed, path: List?, level: UInt?): String = + local function renderDynamic(value: Dynamic|Typed, path: List, level: UInt?): String = let (list = if (value is Dynamic) value.toList() else List()) let (map = value.toMap()) // note: Map.keys.toList() is O(1), other ways of converting are O(n) (as of Pkl 0.25.3) - let (entries = map.keys.toList().mapNonNull((k) -> - let (path = path?.add(Prop(k))) - let (v = convert(map[k], path)) - if (omitNullProperties && v == null && k is String) null + let (entries = map.keys.toList().mapNonNull((k_) -> + let (isProp = k_ is String) + let (k = if (isProp) k_ else convert(k_, null)) + let (path = path.add(if (isProp) Prop(k) else Key(k))) + let (v = convert(map[k_], path)) + if (omitNullProperties && v == null && isProp) null else Pair(k, Pair(path, v)) )) if (entries.isEmpty && list.isEmpty) "{}" else let (multiline = entries.length + list.length >= multilineThreshold && level != null) let (level_ = if (multiline) level!! + 1 else level) - let (listPath = if (list.isEmpty) null else path?.add(splatKey)) + let (listPath = if (list.isEmpty) path else path.add(splatKey)) (if (multiline) "{\n\(indent.repeat(level_!!))" else "{ ") + new Listing { for (kpv in entries) { // kpv = Pair(key, Pair(path, value)) @@ -294,3 +332,560 @@ class Renderer extends ValueRenderer { // endregion } + +// endregion +// region Parser + +/// A parser for a strict subset of Lua. +/// +/// This parser can handle Lua files that consist of comments and `key=value` lines, where the key is a Lua identifier +/// and the value is a literal string, number, boolean, `nil`, or table. Expressions are not supported. At the top level +/// the key cannot be the identifier `_ENV` unless it is followed by a subscript expression, as in `_ENV[key]=value`. +/// An `_ENV` subscript like this allows the top-level to contain keys that are not Lua identifiers. +/// +/// When parsing nested tables, tables using key/value syntax (`{ [key] = value; … }`) will be parsed as list elements +/// if the key is integral and equal to the next unused index, otherwise they will be treated as map entries. Be aware +/// that the order of keys is important here; `{ [0] = "a"; [1] = "b"; [2] = "c" }` will be parsed as a list whereas +/// `{ [2] = "c"; [1] = "b"; [0] = "a" }` will be parsed as a map despite being equivalent Lua tables. See [useDynamic] +/// for details on the type used to represent nested tables. +/// +/// When parsing `_ENV[key]=value` statements at the top-level, if the subscript key is an integral value and +/// [useDynamic] is [true] then it will be parsed as a list element in the same fashion as nested tables. However if +/// [useDynamic] is [false] then integral keys will not be treated any differently than other keys. +/// +/// Lua values are mapped to Pkl values as follows: +/// +/// **Lua type** | **Pkl type** +/// -------------|------------- +/// nil | [Null] +/// boolean | [Boolean] +/// number | [Number] +/// string | [String] +/// table | [Dynamic] or [Mapping]/[Listing] depending on [Parser.useDynamic] +/// +/// # Example +/// +/// This is a sample Lua file that can be parsed with this Parser. +/// ```lua +/// --[[ +/// This file has a header comment. +/// ]] +/// foo="bar" +/// count=2 +/// -- line comment here +/// enable=true +/// frob=nil +/// ports={80, 443} +/// ips={ +/// localhost = "127.0.0.1"; +/// ["example.com"] = "93.184.215.14"; +/// } +/// _ENV[" "]="space" +/// ``` +class Parser { + /// Determines what the parser produces when parsing Lua. + /// + /// If [true] (the default), the parse result is a [Dynamic], otherwise it's a [Mapping]. + /// + /// For nested tables, if [true] every nested table is a [Dynamic], otherwise a nested table will be a [Mapping] if it + /// contains key/value pairs, a [Listing] if it contains elements, or it will throw an error if it contains both. If + /// [false] then empty tables will be represented as empty [Listing]s. + /// + /// If [useDynamic] is [true], Lua keys named "default" will be shadowed by the built-in [Dynamic.default] property. + useDynamic: Boolean = true + + /// Value converters to apply to parsed values. + /// + /// For further information see [ValueRenderer.converters]. Table entries with string keys are treated as properties. + /// This means paths like `x[foo]` will never match anything. + converters: Mapping<(Class|String), (unknown) -> Any> + + // region Parsing functions + + /// Parses [source] as a strict subset of Lua. + /// + /// Throws if an error occurs during parsing. + /// + /// If [source] is a [Resource], the resource URI is included in parse error messages. + /// + /// In the absence of converters this will return a [Dynamic] or a [Mapping][Mapping]. + function parse(source: Resource|String): unknown = + let (uri = if (source is Resource) source.uri else null) + let (source = if (source is Resource) source.text else source) + let (source = source.replaceAll(Regex(#"\r\n?|\n\r"#), "\n")) // normalize line endings + let (tokens = tokenRegex.findMatchesIn(source)) + let (state: ParseState = tokens.fold(new ParseState { path = List("^") }, (state: ParseState, token) -> + // the way parsing works is by folding over a list of tokens and maintaining state in the ParseState object. + // since we don't have enums with associated values, we instead rely on knowing which combinations of state fields + // are reachable and which combinations will never occur. This is a complete list of all valid combinations, where + // the `type` column is "root" for ParseState and "child" for ChildParseState and the other columns are fields: + // + // # | type | op | key | negate | valid next token + // ---|-------|---------|--------|--------|----------------- + // 1a | root | null | null | null | identifier or ";" or EOF + // 1b | child | null | null | null | identifier or value or "[" or "}" or "-" + // 2 | child | null | null | bool | number or "-" + // 3 | root | null | "_ENV" | null | "[" + // 4 | any | null | !null¹ | null | "=" + // 5 | any | "=" | !null | null | value or "-" or "{" + // 6 | any | "=" | !null | bool | number or "-" + // 7 | any | "[" | null | null | value or "-" or "{" + // 8 | any | "[" | null | bool | number or "-" + // 9 | any | "key" | !null | null | "]" + // 10 | any | "]" | !null | null | "=" + // 11 | child | "value" | null | null | "," or ";" or "}" + // + // ¹In state #4, if the type is root then the key cannot be "_ENV" (see state #2). + if (validateToken(token, source, uri).value.startsWith("--")) state // skip comment + else if (state.op == null) + if (state.key == null) + if (state is ChildParseState) + // state #1b/#2, state is ChildParseState + if (state.negate == null && token.value.matches(identifierRegex)) + if (token.value is Keyword) throwError("Unexpected keyword `\(token.value)`", source, uri, token) + else (state) { key = token.value; mapStart = super.mapStart ?? token } // -> #4 + else if (state.negate == null && token.value == "[") + (state) { op = "[" } // -> #7 + else if (state.negate == null && token.value == "}") + let (value = state.toValue(useDynamic, source, uri)) + let (parent = state.parent) + if (parent.op == "[") // parent is in state #7 + parent.setKey(convertKey(value, state), state.brace) // -> #9 + else + parent.put(convert(value, parent.path.add(Prop(parent.key!!))), state.brace, useDynamic) // -> #11 or #1a + else if (token.value == "-") + state.negate() // -> #2 + else + let (value = parseValue(source, uri, state, token, "identifier or value or [ or }")) + state.add(convert(value, state.path.add(splatKey)), token) // -> #11 + else + // state #1a, state is ParseState + if (token.value.matches(identifierRegex)) + if (token.value is Keyword) throwError("Unexpected keyword `\(token.value)`", source, uri, token) + else (state) { key = token.value } // -> #4 + else if (token.value == ";") + state // stay in state #1a + else throwExpected("identifier or ;", source, uri, token) + else if (state.key == "_ENV" && !(state is ChildParseState)) + // state #2, state is ParseState + if (token.value == "[") + (state) { key = null; op = "[" } // -> #7 + else if (token.value == "=") throwError("_ENV cannot be assigned to directly", source, uri, token) + else throwExpected("[", source, uri, token) + else if (token.value == "=") // key is !null + // state #4 + (state) { op = "=" } // -> #5 + else throwExpected("=", source, uri, token) + else if (state.op == "=") + // state #5/#6 + if (token.value == "-") + state.negate() // -> #6 + else if (token.value == "{" && state.negate == null) + new ChildParseState { parent = state; brace = token; path = state.path.add(Prop(state.key!!)) } // -> #1b + else + let (value = parseValue(source, uri, state, token, "value or {")) + state.put(convert(value, state.path.add(Prop(state.key!!))), token, useDynamic) // -> #11 or #1a + else if (state.op == "[") + // state #7/#8 + if (token.value == "-") state.negate() // -> #8 + else if (token.value == "{" && state.negate == null) + new ChildParseState { parent = state; brace = token; path = List() } // -> #1b + else + let (value = parseValue(source, uri, state, token, "value or {")) + if (value == null) throwError("Table key cannot be nil", source, uri, token) + else if (value is Number && value.isNaN) throwError("Table key cannot be NaN", source, uri, token) + else state.setKey(convertKey(value, state), token) // -> #9 + else if (state.op == "key") + // state #9 + if (token.value == "]") (state) { op = "]" } // -> #10 + else throwExpected("]", source, uri, token) + else if (state.op == "]") + // state #10 + if (token.value == "=") (state) { op = "=" } // -> #5 + else throwExpected("=", source, uri, token) + else if (state.op == "value" && state is ChildParseState) + // state #11, state is ChildParseState + if (token.value is ","|";") (state) { op = null } // -> #1b + else if (token.value == "}") + let (value = state.toValue(useDynamic, source, uri)) + let (parent = state.parent) + if (parent.op == "[") // parent is in state #7 + parent.setKey(convertKey(value, parent), state.brace) // -> #9 + else + parent.put(convert(value, parent.path.add(Prop(parent.key!!))), state.brace, useDynamic) // -> #11 or #1a + else throwExpected(", or ; or }", source, uri, token) + else + // invalid state, we can't ever get here + throwError("Internal error; invalid state", source, uri, token) + )) + // We're at EOF, check if we allow EOF here + if (state.negate != null) throwExpected("number", source, uri, null) // state #2/#6/#8 + else if (state.op == null) + if (state.key == null) + if (state is ChildParseState) throwExpected("identifier or value or [ or }", source, uri, null) // state #1b + else convert(state.toValue(useDynamic, source, uri), state.path) // state #1a + else if (state.key == "_ENV" && !(state is ChildParseState)) throwExpected("[", source, uri, null) // state #3 + else throwExpected("=", source, uri, null) // state #4 + else if (state.op is "="|"[") throwExpected("value or {", source, uri, null) // state #5/#7 + else if (state.op == "key") throwExpected("]", source, uri, null) // state #9 + else if (state.op == "]") throwExpected("=", source, uri, null) // state #10 + else /* op is "value" */ throwExpected(", or ; or }", source, uri, null) // state #11 + + local function parseValue(source: String, uri: Uri?, state: ParseState, token: RegexMatch, expected: String): (Boolean|Number|String)? = + let (value = if (token.value == "nil") null + else if (token.value == "true") true + else if (token.value == "false") false + else if (token.value.startsWith(Regex(#"\.?0[xX]"#))) + parseHexLiteral(token, state.negate ?? false, source, uri) + else if (token.value.startsWith(Regex(#"\.?[0-9]"#))) + parseDecLiteral(token, state.negate ?? false, source, uri) + else if (token.value.startsWith(Regex(#"["']"#))) + parseShortString(token, source, uri) + else if (token.value.startsWith(Regex(#"\[=*+\["#))) + parseLongString(token) + else throwExpected(if (state.negate == null) expected else "number", source, uri, token)) + if (state.negate != null && !(value is Number)) + throwError("Attempted to negate non-numeric value", source, uri, token) + else value + + local function parseHexLiteral(token: RegexMatch, negate: Boolean, source: String, uri: Uri?): Number = + let (match = hexLiteralRegex.matchEntire(token.value) ?? throwError("Invalid numeric literal: \(token.value)", source, uri, token)) + let (intPart = match.groups[1]!!.value) + let (fracPart = match.groups[2]?.value ?? "") + let (exp = match.groups[3]?.value?.toInt()) + let (base = if (exp != null || !fracPart.isEmpty) 0.0 else 0) + let (intValue: Number = + if (negate) intPart.chars.fold(base, (acc: Number, it) -> acc * 16 - parseHex(it)) + else intPart.chars.fold(base, (acc: Number, it) -> acc * 16 + parseHex(it))) + let (fracValue = fracPart.chars.foldBack(0.0, (it, acc: Float) -> acc / 16 + parseHex(it) / 16) as Float) + let (value = if (fracPart.isEmpty) intValue else if (negate) intValue - fracValue else intValue + fracValue) + if (exp != null) value * (2.0 ** exp) + else value + + local function parseDecLiteral(token: RegexMatch, negate: Boolean, source: String, uri: Uri?): Number = + let (value = if (negate) "-\(token.value)" else token.value) + (if (value.contains(Regex("[.eE]"))) value.toFloatOrNull() else value.toIntOrNull()) + ?? throwError("Invalid numeric literal: \(token.value)", source, uri, token) + + local function parseShortString(token: RegexMatch, source: String, uri: Uri?): String = + let (value = token.value.substring(1, token.value.length-1)) // drop quotes + value.replaceAllMapped(Regex(#"(?s-U)\\(z[ \f\n\t\x0b]*|x\p{XDigit}{1,2}|\d{1,3}+|u\{\p{XDigit}*}?|.)"#), (it) -> + let (value = it.groups[1]!!.value) + if (value.startsWith("z")) "" + else if (value == "\n") "\n" + else if (value == "a") "\u{7}" + else if (value == "b") "\u{8}" + else if (value == "f") "\u{c}" + else if (value == "n") "\n" + else if (value == "r") "\r" + else if (value == "t") "\t" + else if (value == "v") "\u{b}" + else if (value is "\\"|"\""|"'") value + else if (value.startsWith("x")) + if (!value.matches(Regex(#"(?-U)x\p{XDigit}{2}"#))) throwError("Invalid hex escape in string: \(it.value)", source, uri, it) + else + let (c = parseHexOctet(value.drop(1))) + if (c > 0x7f) throwError("Non-ascii hex escape in string: \(it.value)", source, uri, it) + else c.toChar() + else if (value.startsWith("u")) + if (!value.matches(Regex(#"(?-U)u\{\p{XDigit}{1,8}}"#))) throwError("Invalid unicode escape in string: \(it.value)", source, uri, it) + else + let (c = parseHex32(value.substring(2, value.length-1).padStart(8, "0"))) + if (c > 0x10FFFF) throwError("Out-of-range unicode escape in string: \(it.value)", source, uri, it) + else c.toChar() + else if (value.matches(Regex(#"[0-9]{1,3}"#))) + let (c = value.toInt()) + if (c > 0x7f) throwError("Non-ascii decimal escape in string: \(it.value)", source, uri, it) + else c.toChar() + else throwError("Invalid backslash in string: \(it.value)", source, uri, it)) + + local function parseLongString(token: RegexMatch): String = + // we know we start with [=…[ and end with ]=…] (end was validated in validateToken) + // group 5 is starting equals, group 6 is the whole end + let (value = token.value.substring(token.groups[5]!!.end-token.start+1, token.groups[6]!!.start-token.start)) + if (value.startsWith("\n")) value.drop(1) + else value + + // endregion + // region Converters + + // path specs are already in reversed order + local pathConverters: List, (unknown) -> Any>> = + splitPathConverters(converters.toMap()) + + local function convert(value: Value, path: List): unknown = + let (path = path.reverse()) + let (f = pathConverters.findOrNull((p) -> comparePaths(path, p.key))?.value) + let (f = f ?? converters.getOrNull(value.getClass())) + if (f != null) f.apply(value) else value + + local function convertKey(value: Value, state: ParseState): unknown = + if (state.isListIndex(value) && (useDynamic || state is ChildParseState)) value // don't convert indices + else if (value is String) value // String keys are treated as properties + else + let (f = converters.getOrNull(value.getClass())) + if (f != null) f.apply(value) else value + + // the path and spec must already be reversed + local function comparePaths(path: List, pathSpec: List): Boolean = + path.length >= pathSpec.length && path.zip(pathSpec).every((p) -> + if (p.second is Prop && p.second.name == "*") p.first is Prop + else if (p.second is Key && p.second.key == "*") p.first is Key + else p.first == p.second + ) + + // endregion +} + +// region State + +local open class ParseState { + map: Map // note: can't provide key/value types, converters can return non-Lua types + list: List + key: Any = null + op: ("["|"key"|"]"|"=")? // op is "["? iff key is null + negate: Boolean? // negative numbers are unary negation + path: List + + function toValue(useDynamic: Boolean, _, _): Dynamic|Mapping = + if (useDynamic) (map.toDynamic()) { + ...list + } else map.toMapping() // list should be empty + + function put(value, token, useDynamic: Boolean): ParseState = + if (useDynamic && isListIndex(key)) add(value, token) + else (this) { + map = super.map.put(super.key!!, value) + key = null + op = null + negate = null + } + + // used for _ENV[0]=value expressions when useDynamic is true + function add(value, _): ParseState = + (this) { + list = super.list.add(value) + key = null + op = null + negate = null + } + + // only use this for [key]= keys + function setKey(value, _): ParseState = + (this) { + key = value + op = "key" + negate = null + } + + function negate(): ParseState = + (this) { negate = !(super.negate ?? false) } + + function isListIndex(key): Boolean = + // note: Lua indexes are 1-based + key is Int && key == list.length + 1 +} + +local class ChildParseState extends ParseState { + parent: ParseState + brace: RegexMatch // token for opening {, used for error reporting + op: ("["|"key"|"]"|"="|"value")? // op is "["|"value"? iff key is null + mapStart: RegexMatch? // non-null if !map.isEmpty + listStart: RegexMatch? // non-null if !list.isEmpty + + function toValue(useDynamic: Boolean, source: String, uri: Uri?): Dynamic|Mapping|Listing = + if (useDynamic) (map.toDynamic()) { + ...list + } else if (map.isEmpty) list.toListing() + else if (list.isEmpty) map.toMapping() + else throwError2("Table has both list elements and map entries", source, uri, mapStart!!, "first map entry", listStart!!, "first list entry") + + function put(value, token: RegexMatch, _): ChildParseState = + if (isListIndex(key)) add(value, token) + else (this) { + map = super.map.put(super.key!!, value) + key = null + op = "value" + negate = null + } + + function add(value, token: RegexMatch): ChildParseState = + (this) { + list = super.list.add(value) + key = null // key could be [int] + op = "value" + negate = null + listStart = super.listStart ?? token + } + + function setKey(value, token: RegexMatch): ChildParseState = + (this) { + key = value + op = "key" + negate = null + mapStart = + // don't update mapStart if this will become a listing element + if (isListIndex(value)) super.mapStart + else super.mapStart ?? token + } +} + +// endregion +// region Tokens + +// Regex that matches a single Lua token, or an invalid character. +// This regex assumes line endings have already been normalized, so no carriage returns exist. +// Error states: +// - Group 2 is "" +// - Group 3 is not "\""? +// - Group 4 is not "'"? +// - Group 6 is "" +// - Last group is non-null +local const tokenRegex: Regex = Regex(##""" + (?x-uU) + --(?: # comment + \[(?=*)\[ # long comment (equals are GROUP 1) + (?>(?> + [^]] + | ](?!\k]) + )*) + (]\k]|\z) # match end or EOF (GROUP 2) + | .* # short comment + ) + | [\w&&\D]\w* # identifier + | "(?>(?> # short literal string (") + [^\\\n"] + | \\(?>z[\ \f\n\t\x0b]*|x\p{XDigit}{2}|\d{1,3}|u\{\p{XDigit}+}|(?s:.)|\z) + )*)("|(?s:.)|\z) # match string truncated by newline or EOF (GROUP 3) + | '(?>(?> # short literal string (') + [^\\\n'] + | \\(?>z[\ \f\n\t\x0b]*|x\p{XDigit}{2}|\d{1,3}|u\{\p{XDigit}+}|(?s:.)|\z) + )*)('|(?s:.)|\z) # match string truncated by newline or EOF (GROUP 4) + | \[(?=*)\[ # long literal string (equals are GROUP 5) + (?>(?> + [^]] + | ](?!\k]) + )*) + (]\k]|\z) # match end or EOF (GROUP 6) + # for the numeric literals, they consume extra periods and hex digits and throw a parse error + # this regex aims to match everything Lua 5.3 itself will tokenize as numeric + | \.?0[xX](?:(?:[pP][-+]?)?[.\p{XDigit}])* # hex numeric literal + | \.?\d(?:(?:[eE][-+]?)?[.\p{XDigit}])* # dec numeric literal + | [-+*%^\#&|(){}\[\];,] # single-char operators + | <[<=]? | >[>=]? | //? | ~=? | ==? | ::? | \.{1,3} # multi-char operators + | ([^\ \f\n\t\x0b]) # invalid token (last group) + """##) + +// checks the error states documented on tokenRegex +// returns the same token, or throws an error +local const function validateToken(token: RegexMatch, source: String, uri: Uri?): RegexMatch = + if (token.groups[2]?.value == "") throwError("Expected ]\(token.groups[1]!!.value)], found EOF", source, uri, token.groups[2]!!) + else let (g = token.groups[3]) if (g != null && g.value != "\"") throwExpected("\"", source, uri, g) + else let (g = token.groups[4]) if (g != null && g.value != "'") throwExpected("'", source, uri, g) + else if (token.groups[6]?.value == "") throwError("Expected ]\(token.groups[5]!!.value)], found EOF", source, uri, token.groups[6]!!) + else let (g = token.groups.last) if (g != null) throwError("Illegal token \(g.value)", source, uri, g) + else token + +// groups: +// 1 - integral part (String) +// 2 - fractional part (String?) +// 3 - exponent (String(!isEmpty)?) +local const hexLiteralRegex: Regex = Regex(#"(?-U)0[xX](?=\.?\p{XDigit})(\p{XDigit}*)(?:\.(\p{XDigit}*))?(?:[pP]([-+]?\d+))?"#) + +local const identifierRegex = Regex("[a-zA-Z_][a-zA-Z0-9_]*") + +// endregion +// region Errors + +local class ErrorLocation { + row1: Int // 1-based row + col1: Int // 1-based column + line: String // with prefix + marker: String +} + +local const function errorLocation(source: String, token: RegexMatch): ErrorLocation = + let (lineOffset = (source.take(token.start).lastIndexOfOrNull("\n") ?? -1) + 1) + // locate the entire line the token starts on + let (lineEndOffset = source.drop(token.start).indexOfOrNull("\n")) + let (source = if (lineEndOffset != null) source.take(token.start + lineEndOffset) else source) + // zero-width split so we don't lose any blank lines from the end + let (lines = source.split(Regex(#"(?<=\n)"#))) + let (_line = lines.last) + let (_row1 = lines.length) + let (col0 = math.min(token.start - lineOffset, _line.length) as Int) // min() is just in case + let (rowPrefix = "\(_row1) | ") + let (markerPrefix = " ".repeat(rowPrefix.length - 3) + " | ") + new ErrorLocation { + row1 = _row1 + col1 = col0 + 1 + line = "\(rowPrefix)\(_line)" + marker = markerPrefix + " ".repeat(col0) + "^".repeat(math.max(1, math.min(_line.length - col0, token.end - token.start)) as UInt) + } + +local const function throwError(msg: String, source: String, uri: Uri?, token: RegexMatch): nothing = + let (loc = errorLocation(source, token)) + let (errMsg = "\(msg)\n\n\(loc.line)\n\(loc.marker)\nat \(uri ?? ""):\(loc.row1):\(loc.col1)") + throw(errMsg) + +local const function throwError2(msg: String, source: String, uri: Uri?, token1: RegexMatch, note1: String, token2: RegexMatch, note2: String): nothing = + let (loc1 = errorLocation(source, token1)) + let (loc2 = errorLocation(source, token2)) + let (errMsg = + if (loc1.row1 == loc2.row1) // same line + if (loc1.col1 <= loc2.col1) // loc1 comes first + "\(msg)\n\n\(loc1.line)\n\(loc1.marker) \(note1)\n\(loc2.marker) \(note2)\nat \(uri ?? ""):\(loc1.row1):\(loc1.col1)" + else // loc2 comes first + "\(msg)\n\n\(loc1.line)\n\(loc2.marker) \(note2)\n\(loc1.marker) \(note1)\nat \(uri ?? ""):\(loc1.row1):\(loc2.col1)" + else if (loc1.row1 < loc2.row1) // loc1 comes first + "\(msg)\n\n\(loc1.line)\n\(loc1.marker) \(note1)\n\(loc2.line)\n\(loc2.marker) \(note2)\nat \(uri ?? ""):\(loc1.row1):\(loc1.col1)" + else // loc2 comes first + "\(msg)\n\n\(loc2.line)\n\(loc2.marker) \(note2)\n\(loc1.line)\n\(loc1.marker) \(note1)\nat \(uri ?? ""):\(loc2.row1):\(loc2.col1)") + throw(errMsg) + +local const function throwExpected(expected: String, source: String, uri: Uri?, token: RegexMatch?): nothing = + let (found = + if (token == null) "EOF" + else if (token.value == "\n") "newline" + else if (token.value.isEmpty) "EOF" + else "token `\(token.value.replaceAllMapped(Regex(#"[\n\\]"#), (it) -> if (it.value == "\n") "\\n" else #"\\"#))`") + throwError("Expected \(expected), found \(found)", source, uri, token ?? new RegexMatch { value = ""; start = source.length; end = source.length }) + +// endregion +// endregion +// region Hex + +/// parseHex tranforms a single hexadecimal character into its unsigned integer representation. +local const function parseHex(digit: Char): UInt8 = nybbleLut.getOrNull(digit) ?? throw("Unrecognized hex digit: \(digit)") + +/// parseHexOctet tranforms a two hexadecimal characters into its unsigned integer representation. +local const function parseHexOctet(octet: String(length == 2)): UInt8 = byteLut.getOrNull(octet) ?? throw("Unrecognized hex octet: \(octet)") + +/// parseHex32 transforms an 8 character hexidecimal string into its UInt32 representation. +local const function parseHex32(s: String(length == 8)): UInt32 = + IntSeq(0, 7) + .step(2) + .map((it) -> s.substring(it, it + 2)) + .fold(0, (acc, it) -> acc.shl(8) + parseHexOctet(it)) + +local const nybbleLut = new Mapping { + for (n in IntSeq(0, 9)) { + [n.toString()] = n + } + for (n in IntSeq(0xa, 0xf)) { + [n.toRadixString(16)] = n + [n.toRadixString(16).toUpperCase()] = n + } +} + +local const byteLut = new Mapping { + for (k,v in nybbleLut) { + for (k2,v2 in nybbleLut) { + ["\(k)\(k2)"] = v.shl(4) + v2 + } + } +} + +// endregion + +output {} // makes the above endregion comment work diff --git a/packages/pkl.lua/tests/fixtures/comment.lua b/packages/pkl.lua/tests/fixtures/comment.lua new file mode 100644 index 0000000..a581b7a --- /dev/null +++ b/packages/pkl.lua/tests/fixtures/comment.lua @@ -0,0 +1,15 @@ +-- This is a copy of the sample file from the Parser doc comment +--[[ +This file has a header comment. +]] +foo="bar" +count=2 +-- line comment here +enable=true +frob=nil +ports={80, 443} +ips={ + localhost = "127.0.0.1"; + ["example.com"] = "93.184.215.14"; +} +_ENV[" "]="space" \ No newline at end of file diff --git a/packages/pkl.lua/tests/fixtures/sample.lua b/packages/pkl.lua/tests/fixtures/sample.lua new file mode 100644 index 0000000..98f5478 --- /dev/null +++ b/packages/pkl.lua/tests/fixtures/sample.lua @@ -0,0 +1,24 @@ +--[[ +This is an example config file written in Lua. +It consists of key/value pairs and only literals, no expressions. +]] +greeting="Hello, world!" +snippet=[=[ +Long Lua strings can be written inside brackets like this: [[ +This is a multiline string. +It ends at the close double-bracket.]] +]=] +hex_floats={ + 0x10.0, + 0x0.fffff, + -0x32p3, +} +tableKeys={ + identifier = true; + ["string"] = "yes"; + [42] = "the meaning of life"; + [3.14159] = "pi"; + [true] = "very true"; + [{1, 2, 3}] = "even tables can be keys"; + -- but nil and NaN cannot +} diff --git a/packages/pkl.lua/tests/parser.pkl b/packages/pkl.lua/tests/parser.pkl new file mode 100644 index 0000000..4bfe4b7 --- /dev/null +++ b/packages/pkl.lua/tests/parser.pkl @@ -0,0 +1,326 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// +module pkl.lua.tests.parser + +amends "pkl:test" + +import "../lua.pkl" +import "pkl:math" + +local parser: lua.Parser = new {} + +local function parseValue(s: String): lua.Value = + let (map = (parser) { useDynamic = false }.parse("value=\(s)")) + if (map.length == 1) map["value"] + else throw("source parsed as more than one value: \(s)") + +local function parseDynamic(s: String): lua.Value = + let (map = parser.parse("value=\(s)").toMap()) + if (map.length == 1) map["value"] + else throw("source parsed as more than one value: \(s)") + +facts { + ["hex floats"] { + // with the way we parse hex floats, check for precision issues + parseValue("0xaBcD") is Int(this == 0xabcd) + parseValue("0X80.0") is Float(this == 0x80) + parseValue("0x80.8") is Float(this == 128.5) + parseValue("0x7f.f0") is Float(this == 127.9375) + parseValue("0x7f.84") is Float(this == 127.515625) + parseValue("0x7f.abc") is Float(this == 127.6708984375) + parseValue("-0x7f.abc") is Float(this == -127.6708984375) + parseValue("0x1p1") is Float(this == 2.0) + parseValue("0x1p+1") is Float(this == 2.0) + parseValue("0x1p2") is Float(this == 4.0) + parseValue("0x1p-1") is Float(this == 0.5) + parseValue("0x80.8p1") is Float(this == 257.0) + parseValue("0x7fffffffffffffff") is Int(this == math.maxInt) + parseValue("-0x8000000000000000") is Int(this == math.minInt) + parseValue("0x1.fffffffffffffp+1023") is Float(this == math.maxFiniteFloat) + parseValue("-0x1.fffffffffffffp+1023") is Float(this == math.minFiniteFloat) + } + ["negative numbers"] { + // negative numbers are a unary negation on a positive number + parseValue("-1") == -1 + // we support double-negation because the alternative is giving the user an error like + // error: expected number, found token '-' + // and that's just confusing. + parseValue("- -1") == 1 + parseValue("- - - 1") == -1 + parseValue("-\n1") == -1 + parseValue("\(math.maxUInt)") == math.maxUInt + parseValue("\(math.maxInt)") == math.maxInt + parseValue("\(math.minInt)") == math.minInt + parseValue("-0.1") == -0.1 + parseValue("-0xABC") == -0xABC + parseValue("-2e1") == -2e1 + } + ["tables with useDynamic=true"] { + // note: new Dynamic { 1; 2 } == new Dynamic { [0] = 1; [1] = 2 }, so we have to take care with comparisons + parseDynamic("{}") == new Dynamic {} + parseDynamic("{1,2}") is Dynamic(this.toList() == List(1,2) && this.toMap().isEmpty) + parseDynamic("{a=1;b=2}") == new Dynamic { a = 1; b = 2 } + parseDynamic(#"{["a"]=1;["b"]=2}"#) == new Dynamic { a = 1; b = 2 } + parseDynamic("{[1]=1;[2]=2}") is Dynamic(this.toList() == List(1,2) && this.toMap().isEmpty) + parseDynamic("{[2]=2;[1]=1}") is Dynamic(this.toList() == List(1) && this.toMap() == Map(2,2)) + parseDynamic("{[0]=1;[2]=2}") is Dynamic(this.toList().isEmpty && this.toMap() == Map(0,1,2,2)) + parseDynamic("{[{}]=1}") == new Dynamic { [new Dynamic {}] = 1 } + parseDynamic("{[{1,2}]=1}") == new Dynamic { [new Dynamic { 1; 2 }] = 1 } + parseDynamic("{[{a=1}]=1}") == new Dynamic { [new Dynamic { a = 1 }] = 1 } + } + ["tables with useDynamic=false"] { + parseValue("{}") == new Listing {} + parseValue("{1,2}") == new Listing { 1; 2 } + parseValue("{a=1;b=2}") == new Mapping { ["a"] = 1; ["b"] = 2 } + parseValue(#"{["a"]=1;["b"]=2}"#) == new Mapping { ["a"] = 1; ["b"] = 2 } + parseValue("{[1]=1;[2]=2}") == new Listing { 1; 2 } + module.catch(() -> parseValue("{[2]=2;[1]=1}")).startsWith("Table has both list elements and map entries") + parseValue("{[2]=1;[3]=2}") == new Mapping { [2] = 1; [3] = 2 } + parseValue("{[0]=1;[2]=2}") == new Mapping { [0] = 1; [2] = 2 } + parseValue("{[{}]=1}") == new Mapping { [new Listing {}] = 1 } + parseValue("{[{1,2}]=1}") == new Mapping { [new Listing { 1; 2 }] = 1 } + parseValue("{[{a=1}]=1}") == new Mapping { [new Mapping { ["a"] = 1 }] = 1 } + } + ["_ENV[key] with useDynamic=true"] { + // note: new Dynamic { 1; 2 } == new Dynamic { [0] = 1; [1] = 2 }, so we have to take care with comparisons + parser.parse("_ENV[true]=1") == new Dynamic { [true] = 1 } + parser.parse(#"_ENV["foo"]=1"#) == new Dynamic { foo = 1 } + parser.parse("_ENV[1]=2") is Dynamic(this.toList() == List(2) && this.toMap().isEmpty) + parser.parse("_ENV[0]=2") is Dynamic(this.toList().isEmpty && this.toMap() == Map(0, 2)) + parser.parse("_ENV[-1]=2") is Dynamic(this.toList().isEmpty && this.toMap() == Map(-1, 2)) + parser.parse("_ENV[2]=2;_ENV[1]=1") is Dynamic(this.toList() == List(1) && this.toMap() == Map(2,2)) + parser.parse("_ENV[{a=1}]=2") == new Dynamic { [new Dynamic { a = 1 }] = 2 } + } + ["_ENV[key] with useDynamic=false"] { + local mapParser = (parser) { useDynamic = false } + mapParser.parse("_ENV[true]=1") == new Mapping { [true] = 1 } + mapParser.parse(#"_ENV["foo"]=1"#) == new Mapping { ["foo"] = 1 } + mapParser.parse("_ENV[1]=2") == new Mapping { [1] = 2 } + mapParser.parse("_ENV[0]=2") == new Mapping { [0] = 2 } + mapParser.parse("_ENV[-1]=2") == new Mapping { [-1] = 2 } + mapParser.parse("_ENV[2]=2;_ENV[1]=1") == new Mapping { [2] = 2; [1] = 1 } + mapParser.parse("_ENV[{a=1}]=2") == new Mapping { [new Mapping { ["a"] = 1 }] = 2 } + } + ["errors"] { + // these are facts instead of examples so we can preserve formatting in the error strings, since module.catch + // replaces newlines with spaces. + module.catch(() -> parser.parse("foo=")) == """ + Expected value or {, found EOF + + 1 | foo= + | ^ + at :1:5 + """.replaceAll("\n", " ") + module.catch(() -> parser.parse("foo=3\nbar=")) == """ + Expected value or {, found EOF + + 2 | bar= + | ^ + at :2:5 + """.replaceAll("\n", " ") + module.catch(() -> parser.parse(new Resource { text = "foo=1.2.3"; uri = "uri:path/to/input.lua" })) == """ + Invalid numeric literal: 1.2.3 + + 1 | foo=1.2.3 + | ^^^^^ + at uri:path/to/input.lua:1:5 + """.replaceAll("\n", " ") + module.catch(() -> (parser) { useDynamic = false }.parse("foo={1, [true]=2}")) == """ + Table has both list elements and map entries + + 1 | foo={1, [true]=2} + | ^ first list entry + | ^^^^ first map entry + at :1:6 + """.replaceAll("\n", " ") + module.catch(() -> (parser) { useDynamic = false }.parse("foo={[true]=1, 2}")) == """ + Table has both list elements and map entries + + 1 | foo={[true]=1, 2} + | ^^^^ first map entry + | ^ first list entry + at :1:7 + """.replaceAll("\n", " ") + module.catch(() -> (parser) { useDynamic = false }.parse("foo={\n [true]=1;\n 2\n}")) == """ + Table has both list elements and map entries + + 2 | [true]=1; + | ^^^^ first map entry + 3 | 2 + | ^ first list entry + at :2:4 + """.replaceAll("\n", " ") + module.catch(() -> (parser) { useDynamic = false }.parse("foo={\n 1;\n [true]=2\n}")) == """ + Table has both list elements and map entries + + 2 | 1; + | ^ first list entry + 3 | [true]=2 + | ^^^^ first map entry + at :2:3 + """.replaceAll("\n", " ") + module.catch(() -> parser.parse("[1]=1")) == """ + Expected identifier or ;, found token `[` + + 1 | [1]=1 + | ^ + at :1:1 + """.replaceAll("\n", " ") + module.catch(() -> parser.parse("_ENV=1")) == """ + _ENV cannot be assigned to directly + + 1 | _ENV=1 + | ^ + at :1:5 + """.replaceAll("\n", " ") + } +} + +examples { + ["empty"] { + parser.parse("") + } + ["null"] { + parser.parse("foo=nil") + } + ["boolean"] { + parser.parse("foo=true\nbar=false") + } + ["number"] { + parser.parse(""" + zero=0 + one=1 + negative=-1 + maxInt32=2147483647 + minInt32=-2147483648 + zerof=0.0 + float=5.32 + negfloat=-10.26 + hex=0xaBcD + hexf=0X80.0 + hexf2=0x80.8 -- 128.5 + hexf3=0x7f.f0 -- 127.9375 + hexf4=0x7f.84 -- 127.515625 + hexf5=0x7f.abc -- 127.6708984375 + hexp=0x1p1 -- 2.0 + hexp=0x1p+1 -- 2.0 + hexp2=0x1p2 -- 4.0 + hexp3=0x1p-1 -- 0.5 + hexp4=0x80.8p1 -- 257.0 + """) + } + ["string"] { + parser.parse(#""" + s="hello world" + single = 'one\'two' + -- line comment + double = "one\"two" + --[[ + long comment + ]] + escapes = "\a\b\f\n\r\t\v" + hex = --[=[ comment]] ]=] "\x00\x3a\x3A\x7f" + dec = "\0\58\058\0580\127" + u = "\u{0}\u{300a}\u{300B}\u{10FFFF}" + newline = "foo\ + bar" + z = "foo\z + bar" + long=[[foo]] + long2=[[ + foo]] + long3=[[ + foo]] + long4=[=[]]]=] + long5=[===[]==]]====]]===] + """#) + } + ["class converters"] { + (parser) { converters { [Int] = (it) -> it + 1 } }.parse("foo=1;bar=2;baz=1.0") + (parser) { converters { [Int] = (it) -> it + 1 } }.parse("foo={1, 2}") + (parser) { converters { [Int] = (it) -> it + 1 } }.parse("foo={[1]=1;[2]=2}") + (parser) { converters { [Int] = (it) -> it + 1 } }.parse("foo={[2]=2;[3]=3}") // not listing elements! + (parser) { converters { [Int] = (it) -> it + 1 } }.parse("_ENV[1]=1;_ENV[2]=2") + (parser) { converters { [Int] = (it) -> it + 1 } }.parse("_ENV[2]=2;_ENV[3]=3") // not listing elements! + (parser) { converters { [String] = (it) -> "\(it)!" } }.parse(#"foo="bar""#) + (parser) { converters { [String] = (it) -> "\(it)!" } }.parse(#"_ENV["foo"]="bar""#) + fixupTableKeys((parser) { converters { [String] = (it) -> "\(it)!" } }.parse(#"table={["foo"]="bar"}"#)) + fixupTableKeys((parser) { converters { [Int] = (it) -> it + 1 } }.parse("_ENV[{a=1}]=true")) + (parser) { converters { [Dynamic] = (it) -> (it) { done = true } } }.parse("foo=1") + } + ["path converters"] { + (parser) { converters { ["^"] = (it) -> (it) { done = true } } }.parse("foo=1") + (parser) { converters { [""] = (it) -> (it) { done = true } } }.parse("foo=1") + (parser) { converters { ["foo"] = (it) -> it + 1 } }.parse("foo=1") + (parser) { converters { ["^foo"] = (it) -> it + 1 } }.parse("foo=1") + (parser) { converters { ["a.foo"] = (it) -> it + 1 } }.parse("foo=1") + (parser) { converters { ["foo.a"] = (it) -> it + 1 } }.parse("foo={a=1;b=2}") + (parser) { converters { ["^a"] = (it) -> it + 1 } }.parse("a=1;foo={a=1}") + (parser) { converters { ["*"] = (it) -> it + 1 } }.parse("a=1;_ENV[true]=1") + (parser) { converters { ["[*]"] = (it) -> it + 1 } }.parse(#"a=1;_ENV["b"]=1;_ENV[true]=1"#) + (parser) { useDynamic = false; converters { ["[*]"] = (it) -> it + 1 } }.parse(#"foo={a=1;["b"]=1;[true]=1}"#) + } + ["path converters apply after converting keys"] { + // first validate that paths "x[42]" and "x.42" won't match an integral key + (parser) { converters { ["x[42]"] = (it) -> "matched: \(it)"; ["x.42"] = (it) -> "matched: \(it)" } }.parse("x={[42]=true}") + // and validate that the path "x.42" will match a string key + (parser) { converters { ["x.42"] = (it) -> "matched: \(it)" } }.parse(#"x={["42"]=true}"#) + // now if we convert the integral key to a string, it should match "x.42" + (parser) { converters { [Int] = (it) -> it.toString(); ["x.42"] = (it) -> "matched: \(it)" } }.parse("x={[42]=true}") + } + ["path converters in tables as table keys"] { + fixupTableKeys((parser) { converters { ["^"] = (it) -> (it) { done = true } } }.parse("_ENV[{a=1}]=1")) // ^ doesn't match the key table + fixupTableKeys((parser) { converters { ["a"] = (it) -> it + 1 } }.parse("_ENV[{a=1;b=1}]=1;a=1")) + fixupTableKeys((parser) { converters { ["^a"] = (it) -> it + 1 } }.parse("_ENV[{a=1;b=1}]=1;a=1")) // ^ doesn't match the key table + fixupTableKeys((parser) { converters { ["a.b"] = (it) -> it + 1 } }.parse("foo={[{a={b=1};b=1}]=1}")) + } + ["converters can return non-Lua types"] { + (parser) { converters { ["^"] = (it) -> Pair("foo", it.foo) } }.parse("foo=1") + (parser) { converters { [Int] = (it) -> Pair("int", it) } }.parse("foo=1") + (parser) { converters { [Int] = (it) -> Pair("int", it) } }.parse("_ENV[5]=true") + (parser) { converters { [Int] = (it) -> Pair("int", it) } }.parse("foo={[5]=true}") + (parser) { converters { [Int] = (it) -> Pair("int", it) }; useDynamic = false }.parse("_ENV[5]=true") + (parser) { converters { [Int] = (it) -> Pair("int", it) }; useDynamic = false }.parse("foo={[5]=true}") + } + ["fixtures"] { + fixupTableKeys(parser.parse(read("fixtures/sample.lua"))) + parser.parse(read("fixtures/comment.lua")) + } +} + +// When rendering parser.pkl-actual.pcf, any table keys that are objects just render as `new { … }`, and this produces +// an error "Cannot tell which parent to amend". This function replaces any such keys with a rendered string. This does +// mean that parser.pkl-expected.pcf needs to use rendered strings here instead. +local function fixupTableKeys(value: Dynamic|Mapping|Listing): Dynamic|Mapping|Listing = + if (value is Listing) + value.toList().map((v) -> + if (v is Dynamic|Mapping|Listing) fixupTableKeys(v) + else v + ).toListing() + else + let (mapf = (k, v) -> Pair( + if (k is Object) "new \(k.getClass().simpleName) \(new PcfRenderer { indent = "" }.renderValue(k).replaceAll("\n", " "))" else k, + if (v is Dynamic|Mapping|Listing) fixupTableKeys(v) else v + )) + let (valueMap = value.toMap()) + if (value is Dynamic) + let (map1 = valueMap.filter((k,_) -> !(k is Object)).map(mapf)) + let (map2 = valueMap.filter((k,_) -> k is Object).map(mapf)) + (map1.toDynamic()) { + ...map2 + ...value.toList() + } + else valueMap.map(mapf).toMapping() diff --git a/packages/pkl.lua/tests/parser.pkl-expected.pcf b/packages/pkl.lua/tests/parser.pkl-expected.pcf new file mode 100644 index 0000000..94a0d77 Binary files /dev/null and b/packages/pkl.lua/tests/parser.pkl-expected.pcf differ diff --git a/packages/pkl.lua/tests/lua.pkl b/packages/pkl.lua/tests/renderer.pkl similarity index 80% rename from packages/pkl.lua/tests/lua.pkl rename to packages/pkl.lua/tests/renderer.pkl index 102d7ed..36b7ec0 100644 --- a/packages/pkl.lua/tests/lua.pkl +++ b/packages/pkl.lua/tests/renderer.pkl @@ -13,17 +13,17 @@ // See the License for the specific language governing permissions and // limitations under the License. //===----------------------------------------------------------------------===// -module pkl.lua.tests.lua +module pkl.lua.tests.renderer amends "pkl:test" import "../lua.pkl" local function RenderDirective(text_: String): RenderDirective = new { text = text_ } -local function LuaRenderDirective(before_: String?, value_: Any, after_: String?): lua.LuaRenderDirective = new { before = before_; value = value_; after = after_ } +local function LuaRenderDirective(before_: String?, value_: Any, after_: String?): lua.RenderDirective = new { before = before_; value = value_; after = after_ } facts { - ["RenderDirective"] { + ["base.RenderDirective"] { new lua.Renderer {}.renderDocument(RenderDirective("foo")) == "foo\n" new lua.Renderer {}.renderDocument(new Dynamic { [RenderDirective("foo bar")] = 1}) == "foo bar = 1\n" new lua.Renderer {}.renderValue(Map(RenderDirective("a.b"), 1)) == "{ a.b = 1; }" @@ -33,7 +33,7 @@ facts { new lua.Renderer {}.renderValue(Map("a", RenderDirective("1 + 2"))) == "{ a = 1 + 2; }" new lua.Renderer { multilineThreshold = 1 }.renderValue(new Dynamic { a { [RenderDirective("a\nb")] = RenderDirective("c\nd") }}) == "{\n a = {\n a\nb = c\nd;\n };\n}" } - ["LuaRenderDirective"] { + ["lua.RenderDirective"] { new lua.Renderer {}.renderDocument(LuaRenderDirective(null, 1, null)) == "1\n" new lua.Renderer {}.renderDocument(LuaRenderDirective(null, new Dynamic { a = 1 }, null)) == "{ a = 1; }\n" new lua.Renderer {}.renderDocument(LuaRenderDirective("x", new Dynamic { a = 1 }, "y")) == "x{ a = 1; }y\n" @@ -149,12 +149,12 @@ facts { ["path converters"] { new lua.Renderer { converters { - ["^"] = (it) -> new Dynamic { a = 1 } + ["^"] = (_) -> new Dynamic { a = 1 } } }.renderDocument(new Dynamic { b = 2 }) == "a = 1\n" new lua.Renderer { converters { - [""] = (it) -> new Dynamic { a = 1 } + [""] = (_) -> new Dynamic { a = 1 } } }.renderDocument(new Dynamic { b = 2 }) == "a = 1\n" new lua.Renderer { @@ -210,9 +210,61 @@ facts { } }.renderDocument(source) == new lua.Renderer {}.renderDocument(expected) } + ["path converters apply after converting keys"] { + // first validate that "x.42" matches a string key but not an integral key, and "x[42]" doesn't match + new lua.Renderer { + converters { + ["x[42]"] = (it) -> "unexpected match: \(it)" + ["x.42"] = (it) -> "matched: \(it)" + } + }.renderDocument(new Dynamic { x { [42] = true; ["42"] = true } }) == new lua.Renderer {}.renderDocument(new Dynamic { x { [42] = true; ["42"] = "matched: true" } }) + // now validate that converting the key makes it match "x[42]" instead of "x.42" (as we know it's a key, not a property) + new lua.Renderer { + converters { + [Int] = (it) -> it.toString() + ["x.42"] = (it) -> "unexpected match: \(it)" + ["x[42]"] = (it) -> "matched: \(it)" + } + }.renderDocument(new Dynamic { x { [42] = true } }) == new lua.Renderer {}.renderDocument(new Dynamic { x { ["42"] = "matched: true" } }) + // same thing with Mapping, validate that "x[42]" matches a string key but not an integral key, and "x.42" doesn't match + new lua.Renderer { + converters { + ["x.42"] = (it) -> "unexpected match: \(it)" + ["x[42]"] = (it) -> "matched: \(it)" + } + }.renderDocument(new Dynamic { x = new Mapping { [42] = true; ["42"] = true } }) == new lua.Renderer {}.renderDocument(new Dynamic { x { [42] = true; ["42"] = "matched: true" } }) + // now validate that converting the key makes it match "x[42]" (but not "x.42") + new lua.Renderer { + converters { + [Int] = (it) -> it.toString() + ["x.42"] = (it) -> "unexpected match: \(it)" + ["x[42]"] = (it) -> "matched: \(it)" + } + }.renderDocument(new Dynamic { x = new Mapping { [42] = true } }) == new lua.Renderer {}.renderDocument(new Dynamic { x { ["42"] = "matched: true" } }) + } + ["path converters in objects used as mapping keys"] { + let (source = new Dynamic { a = 1; [new Dynamic { a = 1; b { a = 1; c = 1 }; c = 1 }] = 1 }) + let (expected = new Dynamic { a = 11; [new Dynamic { a = 2; b { a = 2; c = 3 }; c = 1 }] = 1 }) + new lua.Renderer { + converters { + ["^a"] = (it) -> it + 10 // ^ won't match in table key + ["a"] = (it) -> it + 1 + ["b.c"] = (it) -> it + 2 + } + }.renderDocument(source) == new lua.Renderer {}.renderDocument(expected) + let (source = new Dynamic { foo { a = 1; [new Dynamic { a = 1; b { a = 1; c = 1 }; c = 1 }] = 1 } }) + let (expected = new Dynamic { foo { a = 11; [new Dynamic { a = 2; b { a = 2; c = 3 }; c = 1 }] = 1 } }) + new lua.Renderer { + converters { + ["^*.a"] = (it) -> it + 10 // ^ won't match in table key + ["a"] = (it) -> it + 1 + ["b.c"] = (it) -> it + 2 + } + }.renderDocument(source) == new lua.Renderer {}.renderDocument(expected) + } ["class converters"] { let (source = new Dynamic { a = 1; b = "two"; c = true; nest { 1; "one"; a = 1; b = "two"; c = true }; map = new Mapping { ["a"] = 1; ["b"] = "two"; ["c"] = true }; list = new Listing { 1; "two"; true } }) - let (expected = new Dynamic { a = 2; b = "two!"; c = true; nest { 2; "one!"; a = 2; b = "two!"; c = true }; map { a = 2; b = "two!"; c = true }; list { 2; "two!"; true } }) + let (expected = new Dynamic { a = 2; b = "two!"; c = true; nest { 2; "one!"; a = 2; b = "two!"; c = true }; map { ["a!"] = 2; ["b!"] = "two!"; ["c!"] = true }; list { 2; "two!"; true } }) new lua.Renderer { converters { [Int] = (it) -> it + 1 @@ -222,17 +274,28 @@ facts { // class converters prefer the most specific class new lua.Renderer { converters { - [Base] = (it) -> "base" - [Child] = (it) -> "child" + [Base] = (_) -> "base" + [Child] = (_) -> "child" } }.renderDocument(new Dynamic { base = new Base {}; child = new Child {} }) == new lua.Renderer {}.renderDocument(new Dynamic { base = "base"; child = "child" }) // class converters match subclasses new lua.Renderer { converters { - [Base] = (it) -> "base" + [Base] = (_) -> "base" } }.renderDocument(new Dynamic { child = new Child {} }) == new lua.Renderer {}.renderDocument(new Dynamic { child = "base" }) } + ["class converters in mapping keys"] { + // remember, string mapping keys in Dynamic are treated as properties + let (source = new Dynamic { [5] = 1; ["a"] = true; foo { [5] = 1; ["a"] = true }; map = new Mapping { [5] = 1; ["a"] = true } }) + let (expected = new Dynamic { [6] = 2; ["a"] = true; foo { [6] = 2; ["a"] = true }; map { [6] = 2; ["a!"] = true } }) + new lua.Renderer { + converters { + [Int] = (it) -> it + 1 + [String] = (it) -> "\(it)!" + } + }.renderDocument(source) == new lua.Renderer {}.renderDocument(expected) + } ["NaN and Infinity"] { new lua.Renderer {}.renderValue(Infinity) == "(1/0)" new lua.Renderer {}.renderValue(-Infinity) == "(-1/0)" diff --git a/packages/pkl.lua/tests/lua.pkl-expected.pcf b/packages/pkl.lua/tests/renderer.pkl-expected.pcf similarity index 100% rename from packages/pkl.lua/tests/lua.pkl-expected.pcf rename to packages/pkl.lua/tests/renderer.pkl-expected.pcf