Merge pull request #28 from fink-lang/improve-str-escaping

improve str escape char handling and template nesting

Author: kollhof

package-lock.json

@@ -30,16 +30,16 @@
     "release": "semantic-release"
   },
   "dependencies": {
-    "@fink/prattler": "^1.2.0"
+    "@fink/prattler": "^2.0.0"
   },
   "devDependencies": {
     "@fink/cli": "^2.1.0",
     "@fink/jest": "^1.1.0",
-    "@fink/larix": "^4.5.1",
-    "@fink/loxia": "^4.5.0",
+    "@fink/larix": "^4.5.2",
+    "@fink/loxia": "^4.5.1",
     "commitizen": "^4.0.3",
     "cz-conventional-changelog": "^3.1.0",
-    "jest-cli": "^25.1.0",
+    "jest-cli": "^25.2.3",
     "npx-run": "^2.1.2",
     "semantic-release": "^17.0.4"
   },

package.json

@@ -2,7 +2,7 @@
 
 {parse_expr} = import '../..'
 
-# TODO: should not need to escape `
-describe:: 'import \`foobar\`', fn:
+
+describe:: 'import `foobar`', fn:
   it:: 'parses import', fn:
-    parse_expr('import \`./foobar\`') eq snapshot
+    parse_expr('import `./foobar`') eq snapshot

src/lang/import/index.test.fnk

@@ -5,24 +5,24 @@ jsx-elem (1:0-3:9) Foobar
   jsx-attr (1:8-1:10) ni
     null
   :
-  jsx-text (1:10-1:11)
+  jsx-text (1:11-2:2)
     "\\n  "
   jsx-elem (2:2-2:10) Spam
-  jsx-text (2:8-2:10)
+  jsx-text (2:10-3:0)
     "\\n"
 `;
 
 exports[`JSX <Foobar>...</Foobar> parses elem with expr in body: <Foobar> {...} </Foobar> 1`] = `
 jsx-elem (1:0-3:9) Foobar
   :
-  jsx-text (1:7-1:8)
+  jsx-text (1:8-2:2)
     "\\n  "
   jsx-expr-container (2:2-2:8)
     block (2:2-2:8)
       arithm + (2:3-2:8)
         number (2:3-2:4) 1
         number (2:7-2:8) 2
-  jsx-text (2:8-2:9)
+  jsx-text (2:9-3:0)
     "\\n"
 `;
 
@@ -45,14 +45,14 @@ jsx-elem (1:0-1:15) Foobar
 exports[`JSX <Foobar>...</Foobar> parses self closing elem with str attr: <Foobar spam="ni" /> 1`] = `
 jsx-elem (1:0-1:20) Foobar
   jsx-attr (1:8-1:17) spam
-        jsx-string (1:13-1:17)
+        jsx-string (1:14-1:16)
       'ni'
 `;
 
 exports[`JSX <Foobar>...</Foobar> parses self closing elem with str attr: <Foobar spam='ni' /> 1`] = `
 jsx-elem (1:0-1:20) Foobar
   jsx-attr (1:8-1:17) spam
-        jsx-string (1:13-1:17)
+        jsx-string (1:14-1:16)
       'ni'
 `;
 

src/lang/jsx/index.test.fnk.snap

@@ -15,7 +15,7 @@ jsx_expr_container = fn ctx:
 
 
 jsx_txt = fn ctx:
-  [{text, loc}, next_ctx] = collect_text(ctx, '</', '<', '{')
+  [{value: text, loc}, next_ctx] = collect_text(ctx, rx/'<\/|<|{/)
 
   [{type: 'jsx-text', value: text, loc}, next_ctx]
 
@@ -48,9 +48,9 @@ jsx_body = fn name, ctx:
 
 
 jsx_string_val = fn ctx:
-  [value, next_ctx] = collect_text(ctx, curr_value(ctx))
+  [{value, loc}, next_ctx] = collect_text(ctx, curr_value(ctx))
 
-  [{type: 'jsx-string', value: value.text, loc: value.loc}, next_ctx]
+  [{type: 'jsx-string', value, loc}, next_ctx]
 
 
 jsx_attr_value = fn ctx:

src/lang/jsx/jsx.fnk

@@ -24,8 +24,7 @@ add_object_literal = fn ctx:
 
 add_string_literal = fn ctx:
   pipe ctx:
-    # TODO: bug in lorix prevents the use of '`'
-    add_operator_like(string(`\``))
+    add_operator_like(string('`'))
     add_operator_like(string(`'`))
     add_operator_like(string(`"`))
 

src/lang/literals/index.fnk

@@ -8,10 +8,9 @@
 
 
 get_text = fn ctx, op:
-  [{text, loc: {start, end}}, next_ctx] = collect_text(ctx, op)
+  [{value: text, loc: {start, end}}, next_ctx] = collect_text(ctx, op)
 
-  # TODO: previous larix version has a bug to terminate regex at $
-  [escaped] = text.match(rx/\\*\\$/) || ['']
+  [escaped] = text.match(rx/\\*$/)
 
   match true:
     escaped.length % 2 == 0:

src/lang/literals/regex.fnk

@@ -27,9 +27,9 @@ describe:: 'regex - parsing failures', fn:
     (
       fn: parse_expr(`rx/foo.bar`, 'test.fnk')
     ) to_throw `
-      test.fnk:1:7
+      test.fnk:1:10
       1| rx/foo.bar
-                ^
+                   ^
 
       Unexpected end of code.
     `
@@ -39,9 +39,9 @@ describe:: 'regex - parsing failures', fn:
     (
       fn: parse_expr(`rx/foo.bar/n`, 'test.fnk')
     ) to_throw `
-      test.fnk:1:11
+      test.fnk:1:12
       1| rx/foo.bar/n
-                    ^
+                     ^
 
       Unexpected end of code.
     `

src/lang/literals/regex.test.fnk

@@ -17,17 +17,19 @@ get_expr_part = fn ctx:
 
 
 get_str_part = fn ctx, op:
-  # TODO: should not need to escape `$` unless it is `${`
-  [{text, loc: {start, end}}, next_ctx] = collect_text(ctx, op, '\$')
+  [{value: text, loc: {start, end}}, next_ctx] = match op:
+    '"': collect_text(ctx, rx/"|\${/)
+    '`': collect_text(ctx, rx/`|\${/)
+    else: collect_text(ctx, rx/'|\${/)
 
-  # TODO: previous larix version has a bug to terminate regex at $
-  [escaped] = text.match(rx/\\*\\$/) || ['']
+
+  # TODO: can we do the escapeing check in the regex above?
+  [escaped] = text.match(rx/\\*$/)
 
   match true:
     escaped.length % 2 == 0:
       [{type: 'string:text', value: text, loc: {start, end}}, next_ctx]
 
-    # TODO: should check for ${ and continue too
     else:
       [str_part, final_ctx] = get_str_part(next_ctx, op)
 
@@ -63,18 +65,19 @@ get_unindented_text = fn ctx, op:
     else:
       first
 
-  [...parts] = pipe rest:
+  [...rest_parts] = pipe rest:
     map part:
-      match true:
-        (part.type == 'string:text'):
+      match part:
+        {type: 'string:text'}:
           {...part, value: unindent_text(part.value, ind)}
 
         else:
           part
 
+  parts = [first_part, ...rest_parts]
   {end} = curr_loc(next_ctx)
 
-  [{type: 'string', op, parts: [first_part, ...parts], loc: {start, end}}, next_ctx]
+  [{type: 'string', op, parts, loc: {start, end}}, next_ctx]
 
 
 string = fn op:

src/lang/literals/string.fnk

@@ -1,20 +1,18 @@
-{describe, it, eq, to_throw, snapshot} = import '../../testing/jest'
+{describe, it, it_only, eq, to_throw, snapshot} = import '../../testing/jest'
 
 {parse_expr, parse} = import '../../'
 
 
 describe:: 'strings', fn:
 
   it:: 'parses empty strings', fn:
-    # TODO: should not need to escape `
-    parse_expr('\`\`') eq snapshot
+    parse_expr('``') eq snapshot
     parse_expr('""') eq snapshot
     parse_expr(`''`) eq snapshot
 
 
   it:: 'parses single line strings', fn:
-    # TODO: should not need to escape `
-    parse_expr('\`foo bar\`') eq snapshot
+    parse_expr('`foo bar`') eq snapshot
     parse_expr('"foo bar"') eq snapshot
     parse_expr(`'foo bar'`) eq snapshot
 
@@ -28,22 +26,21 @@ describe:: 'strings', fn:
 
 
   it:: 'parses escape characters', fn:
-    # TODO: should not need to escape `
-    parse_expr('\` \\n \\t \\\\ \\\` \`') eq snapshot
-    parse_expr(`' \\n \\t \\\\ \\\` '`) eq snapshot
-    parse_expr(`" \\n \\t \\\\ \\\` \\\\"`) eq snapshot
+    # TODO: should not need to escape \$
+    # TODO: add test for \${foo}
+    parse_expr('` \$ \\n \\t \\` \\\\`') eq snapshot
+    parse_expr(`' \$ \\n \\t \\' \\\\'`) eq snapshot
+    parse_expr(`" \$ \\n \\t \\" \\\\"`) eq snapshot
 
 
   it:: 'parses tagged strings', fn:
-    # TODO: should not need to escape `
-    parse_expr('foo\`bar spam\`') eq snapshot
+    parse_expr('foo`bar spam`') eq snapshot
     parse_expr(`foo'bar spam'`) eq snapshot
     parse_expr(`foo"bar spam"`) eq snapshot
 
 
   it:: 'parses template strings with expressions', fn:
-    # TODO: should not need to escape `
-    parse_expr('\`foo \${bar + spam} spam\`') eq snapshot
+    parse_expr('`foo \${bar + spam} spam`') eq snapshot
     parse_expr(`'foo \${bar + spam} spam'`) eq snapshot
     parse_expr(`"foo \${bar + spam} spam"`) eq snapshot
 
@@ -58,6 +55,12 @@ describe:: 'strings', fn:
     `) eq snapshot
 
 
+  it:: 'parses nested template strings', fn:
+    parse_expr('
+      `\\n\${foo(bar, `  \${spam}`)}\\n`
+    ') eq snapshot
+
+
   it:: `handles default indentation for lbp`, fn:
     parse(`
       'foobar'
@@ -71,9 +74,9 @@ describe:: 'string - parsing failures', fn:
     (
       fn: parse_expr('"foo bar,', 'test.fnk')
     ) to_throw `
-      test.fnk:1:8
+      test.fnk:1:9
       1| "foo bar,
-                 ^
+                  ^
 
       Unexpected end of code.
     `
@@ -89,5 +92,3 @@ describe:: 'string - parsing failures', fn:
 
       Expected identifier before tagged string.
     `
-
-

src/lang/literals/string.test.fnk

@@ -24,18 +24,18 @@ string ' (1:0-1:2)
 `;
 
 exports[`strings parses escape characters 1`] = `
-string \` (1:0-1:15)
-  \` \\n \\t \\\\ \\\` \`
+string \` (1:0-1:16)
+  \` $ \\n \\t \\\` \\\\\`
 `;
 
 exports[`strings parses escape characters 2`] = `
-string ' (1:0-1:15)
-  \` \\n \\t \\\\ \\\` \`
+string ' (1:0-1:16)
+  \` $ \\n \\t \\' \\\\\`
 `;
 
 exports[`strings parses escape characters 3`] = `
-string " (1:0-1:17)
-  \` \\n \\t \\\\ \\\` \\\\\`
+string " (1:0-1:16)
+  \` $ \\n \\t \\" \\\\\`
 `;
 
 exports[`strings parses multi line strings 1`] = `
@@ -61,6 +61,20 @@ string ' (1:0-5:1)
   \`
 `;
 
+exports[`strings parses nested template strings 1`] = `
+string \` (1:0-1:30)
+  \`\\n\`
+    call (1:5-1:26)
+      ident (1:5-1:8) foo
+      ident (1:9-1:12) bar
+      string \` (1:14-1:25)
+        \`  \`
+              ident (1:19-1:23) spam
+        \`\`
+  
+  \`\\n\`
+`;
+
 exports[`strings parses single line strings 1`] = `
 string \` (1:0-1:9)
   \`foo bar\`

src/lang/literals/string.test.fnk.snap

@@ -70,7 +70,7 @@ serialize_str = fn node, serialize, indent:
   [lne, ...lns] = node.parts
     .map(fn part:
       match part.type:
-        'string:text': `\`${part.value}\``
+        'string:text': '`${part.value}`'
         else:
           value = serialize(part, '  ${indent}')
           `\n${value}\n`