Fix #128

yhirose · Nov 9, 2020 · b92da07 · b92da07
1 parent 11ed83e
commit b92da07
Show file tree

Hide file tree

Showing 4 changed files with 103 additions and 10 deletions.
diff --git a/docs/native.js b/docs/native.js
diff --git a/docs/native.wasm b/docs/native.wasm
diff --git a/peglib.h b/peglib.h
@@ -827,7 +827,7 @@ class Context {
   std::vector<Definition *> rule_stack;
   std::vector<std::vector<std::shared_ptr<Ope>>> args_stack;
 
-  bool in_token = false;
+  size_t in_token_boundary_count = 0;
 
   std::shared_ptr<Ope> whitespaceOpe;
   bool in_whitespace = false;
@@ -2392,7 +2392,7 @@ inline size_t parse_literal(const char *s, size_t n, SemanticValues &sv,
   }
 
   // Skip whiltespace
-  if (!c.in_token) {
+  if (!c.in_token_boundary_count) {
     if (c.whitespaceOpe) {
       auto len = c.whitespaceOpe->parse(s + i, n - i, sv, c, dt);
       if (fail(len)) { return static_cast<size_t>(-1); }
@@ -2457,16 +2457,22 @@ inline size_t LiteralString::parse_core(const char *s, size_t n,
 inline size_t TokenBoundary::parse_core(const char *s, size_t n,
                                         SemanticValues &sv, Context &c,
                                         any &dt) const {
-  c.in_token = true;
-  auto se = make_scope_exit([&]() { c.in_token = false; });
-  auto len = ope_->parse(s, n, sv, c, dt);
+  size_t len;
+  {
+    c.in_token_boundary_count++;
+    auto se = make_scope_exit([&]() { c.in_token_boundary_count--; });
+    len = ope_->parse(s, n, sv, c, dt);
+  }
+
   if (success(len)) {
     sv.tokens.emplace_back(std::make_pair(s, len));
 
-    if (c.whitespaceOpe) {
-      auto l = c.whitespaceOpe->parse(s + len, n - len, sv, c, dt);
-      if (fail(l)) { return static_cast<size_t>(-1); }
-      len += l;
+    if (!c.in_token_boundary_count) {
+      if (c.whitespaceOpe) {
+        auto l = c.whitespaceOpe->parse(s + len, n - len, sv, c, dt);
+        if (fail(l)) { return static_cast<size_t>(-1); }
+        len += l;
+      }
     }
   }
   return len;

diff --git a/test/test2.cc b/test/test2.cc
@@ -3,6 +3,93 @@
 
 using namespace peg;
 
+TEST_CASE("Token boundary 1", "[token boundary]")
+{
+    parser pg(R"(
+        ROOT        <- TOP
+        TOP         <- 'a' 'b' 'c'
+        %whitespace <- [ \t\r\n]*
+    )");
+
+    REQUIRE(pg.parse(" a  b  c "));
+}
+
+TEST_CASE("Token boundary 2", "[token boundary]")
+{
+    parser pg(R"(
+        ROOT        <- TOP
+        TOP         <- < 'a' 'b' 'c' >
+        %whitespace <- [ \t\r\n]*
+    )");
+
+    REQUIRE(!pg.parse(" a  b  c "));
+}
+
+TEST_CASE("Token boundary 3", "[token boundary]")
+{
+    parser pg(R"(
+        ROOT        <- TOP
+        TOP         <- < 'a' B 'c' >
+        B           <- 'b'
+        %whitespace <- [ \t\r\n]*
+    )");
+
+    REQUIRE(!pg.parse(" a  b  c "));
+}
+
+TEST_CASE("Token boundary 4", "[token boundary]")
+{
+    parser pg(R"(
+        ROOT        <- TOP
+        TOP         <- < A 'b' 'c' >
+        A           <- 'a'
+        %whitespace <- [ \t\r\n]*
+    )");
+
+    REQUIRE(!pg.parse(" a  b  c "));
+}
+
+TEST_CASE("Token boundary 5", "[token boundary]")
+{
+    parser pg(R"(
+        ROOT        <- TOP
+        TOP         <- A < 'b' C >
+        A           <- 'a'
+        C           <- 'c'
+        %whitespace <- [ \t\r\n]*
+    )");
+
+    REQUIRE(!pg.parse(" a  b  c "));
+}
+
+TEST_CASE("Token boundary 6", "[token boundary]")
+{
+    parser pg(R"(
+        ROOT        <- TOP
+        TOP         <- < A > B C
+        A           <- 'a'
+        B           <- 'b'
+        C           <- 'c'
+        %whitespace <- [ \t\r\n]*
+    )");
+
+    REQUIRE(pg.parse(" a  b  c "));
+}
+
+TEST_CASE("Token boundary 7", "[token boundary]")
+{
+    parser pg(R"(
+        ROOT        <- TOP
+        TOP         <- < A B C >
+        A           <- 'a'
+        B           <- 'b'
+        C           <- 'c'
+        %whitespace <- [ \t\r\n]*
+    )");
+
+    REQUIRE(!pg.parse(" a  b  c "));
+}
+
 TEST_CASE("Infinite loop 1", "[infinite loop]")
 {
     parser pg(R"(