Fix text parsing issues (#86)

travisbrown · May 19, 2020 · 361ba37 · 361ba37
1 parent 25f94f3
commit 361ba37
Show file tree

Hide file tree

Showing 2 changed files with 57 additions and 33 deletions.
diff --git a/modules/parser/src/main/java/org/dhallj/parser/support/ParsingHelpers.java b/modules/parser/src/main/java/org/dhallj/parser/support/ParsingHelpers.java
@@ -59,13 +59,25 @@ private static String unescapeText(String in) {
       if (in.charAt(i) == '\\') {
         i += 1;
         char next = in.charAt(i);
-        if (next == '"' || next == '$') {
+        if (next == '"' || next == '$' || next == '/') {
           builder.append(next);
         } else if (next == 'u') {
-          // TODO: handle braced escapes.
-          long code = Long.parseLong(in.substring(i + 1, i + 5), 16);
-          builder.append((char) code);
-          i += 4;
+          char escapeFirst = in.charAt(i + 1);
+
+          if (escapeFirst == '{') {
+            int len = 0;
+            while (in.charAt(i + 2 + len) != '}') {
+              len += 1;
+            }
+
+            int code = Integer.parseInt(in.substring(i + 2, i + 2 + len), 16);
+            builder.appendCodePoint(code);
+            i += len + 2;
+          } else {
+            int code = Integer.parseInt(in.substring(i + 1, i + 5), 16);
+            builder.append((char) code);
+            i += 4;
+          }
         } else {
           builder.append('\\');
           builder.append(next);
@@ -110,33 +122,31 @@ static final Expr.Parsed makeTextLiteral(
 
   static final void dedent(String[] input) {
     List<Character> candidate = null;
+    String[][] partLines = new String[input.length][];
 
-    input[0] = "\n" + input[0];
-
-    for (int i = 0; i < input.length; i++) {
+    for (int i = 0; i < input.length; i += 1) {
       String part = input[i].replace("\r\n", "\n");
-      input[i] = part;
+      String[] lines = part.split("\n", -1);
+      partLines[i] = lines;
+
+      for (int j = (i == 0) ? 0 : 1; j < lines.length; j += 1) {
+        String line = lines[j];
 
-      for (int j = 0; j < part.length(); j++) {
-        // Check if this character is a newline (but not before a blank line).
-        if ((part.charAt(j) == '\n') && (j == part.length() - 1 || part.charAt(j + 1) != '\n')) {
+        if (line.length() > 0 || j == lines.length - 1) {
           if (candidate == null) {
-            candidate = new ArrayList<Character>();
-            for (int k = j + 1; k < part.length(); k++) {
-              char c = part.charAt(k);
+            candidate = new ArrayList<>();
+            for (int k = 0; k < line.length(); k += 1) {
+              char c = line.charAt(k);
               if (c == ' ' || c == '\t') {
                 candidate.add(c);
               } else {
                 break;
               }
             }
           } else {
-            for (int k = j + 1; k < part.length(); k++) {
-              if (candidate.size() >= k - j
-                  && part.charAt(k) != candidate.get(k - j - 1).charValue()) {
-                for (int r = candidate.size() - 1; r >= k - j - 1; r--) {
-                  candidate.remove(r);
-                }
+            for (int k = 0; k < candidate.size(); k += 1) {
+              if (k == line.length() || line.charAt(k) != candidate.get(k).charValue()) {
+                candidate = candidate.subList(0, k);
                 break;
               }
             }
@@ -145,22 +155,35 @@ static final void dedent(String[] input) {
       }
     }
 
-    if (!candidate.isEmpty()) {
-      StringBuilder builder = new StringBuilder();
-      for (Character c : candidate) {
-        builder.append(c);
-      }
-      String target = builder.toString();
+    int stripCount = candidate == null ? 0 : candidate.size();
 
-      for (int i = 0; i < input.length; i++) {
-        input[i] = input[i].replace("\n" + target, "\n");
+    if (stripCount == 0) {
+      for (int i = 0; i < input.length; i += 1) {
+        input[i] = reEscape(input[i]);
       }
-    }
+    } else {
+      StringBuilder builder = new StringBuilder();
 
-    input[0] = input[0].substring(1);
+      for (int i = 0; i < input.length; i += 1) {
+        builder.setLength(0);
 
-    for (int i = 0; i < input.length; i += 1) {
-      input[i] = reEscape(input[i]);
+        String[] lines = partLines[i];
+
+        for (int j = 0; j < lines.length; j += 1) {
+          if (lines[j].length() != 0) {
+            if (i > 0 && j == 0) {
+              builder.append(lines[j]);
+            } else {
+              builder.append(lines[j].substring(stripCount));
+            }
+          }
+          if (j < lines.length - 1) {
+            builder.append("\n");
+          }
+        }
+
+        input[i] = reEscape(builder.toString());
+      }
     }
   }
 

diff --git a/tests/src/test/scala/org/dhallj/tests/acceptance/AcceptanceSuites.scala b/tests/src/test/scala/org/dhallj/tests/acceptance/AcceptanceSuites.scala
@@ -28,6 +28,7 @@ class TypeCheckingFailureUnitSuite extends TypeCheckingFailureSuite("type-infere
 class TypeCheckingPreludeSuite extends TypeCheckingSuite("type-inference/success/prelude", true)
 
 class ParsingUnitSuite extends ParsingSuite("parser/success/unit")
+class ParsingTextSuite extends ParsingSuite("parser/success/text")
 class ParsingOtherSuite extends ParsingSuite("parser/success")
 
 class ParsingFailureUnitSuite extends ParsingFailureSuite("parser/failure/unit")