fix: Allow braces in non-repetition context

Author: tkarabela

src/regex_automata/parser/tokenizer.py

@@ -3,14 +3,15 @@
 from .tokens import Token, LPar, RPar, Repetition, Pipe, CharacterSet, BoundaryAssertion, BoundaryAssertionSemantic
 from ..automata.rangeset import RangeSet, WORD_RANGESET, NONWORD_RANGESET, DIGIT_RANGESET, NONDIGIT_RANGESET, \
     WHITESPACE_RANGESET, NONWHITESPACE_RANGESET
-from ..errors import TokenizerError, UnsupportedSyntaxError
+from ..errors import TokenizerError, UnsupportedSyntaxError, RegexAutomataError
 from ..regex.flags import PatternFlag
 
 
 class Tokenizer:
     class Reader:
         def __init__(self, tokenizer: "Tokenizer") -> None:
             self.tokenizer = tokenizer
+            self.tokenizer_initial_pos = tokenizer.pos
             self.start = tokenizer.pos + 1
             self.end = self.start
 
@@ -42,6 +43,14 @@ def span(self) -> tuple[int, int]:
         def text(self) -> str:
             return self.tokenizer.text[self.start:self.end]
 
+        def reset(self) -> None:
+            self.tokenizer.pos = self.tokenizer_initial_pos
+            self.start = self.tokenizer.pos + 1
+            self.end = self.start
+
+    class FailedToReadBraceRepetition(Exception):
+        pass
+
     def __init__(self, text: str, flags: PatternFlag = PatternFlag.NOFLAG) -> None:
         self.text = text
         self.flags = flags
@@ -88,7 +97,11 @@ def get_tokens(self) -> Iterator[Token]:
                 case ")":
                     yield self.read_RPar(reader)
                 case "*" | "?" | "+" | "{":
-                    yield self.read_Repetition(reader)
+                    try:
+                        yield self.read_Repetition(reader)
+                    except self.FailedToReadBraceRepetition:
+                        reader.reset()
+                        yield self.read_CharacterSet(reader)
                 case "|":
                     yield self.read_Pipe(reader)
                 case ".":
@@ -132,45 +145,47 @@ def read_Repetition(self, reader: Reader) -> Repetition:
                     self.error(f"{c}{c2} quantifier is not supported", unsupported=True)
                 return Repetition(reader.span, reader.text, 1, None)
             case "{":
-                read_lower_limit = False
-                c = self.peek()
-                if c is None:
-                    self.error("bad repetition definition")
-                elif c == ",":
-                    rmin = 0
-                elif c.isdigit():
-                    rmin = reader.read_number()
-                    read_lower_limit = True
-                else:
-                    self.error("bad repetition definition")
-
-                c = self.peek()
-                if c is None:
-                    self.error("bad repetition definition")
-                elif c == ",":
-                    reader.read(",")
-
+                try:
+                    read_lower_limit = False
                     c = self.peek()
-                    if c == "}":
-                        rmax = None
-                    elif c is not None and c.isdigit():
-                        rmax = reader.read_number()
+                    if c is None:
+                        self.error("bad repetition definition")
+                    elif c == ",":
+                        rmin = 0
+                    elif c.isdigit():
+                        rmin = reader.read_number()
+                        read_lower_limit = True
                     else:
                         self.error("bad repetition definition")
-                elif c == "}":
-                    if not read_lower_limit:
-                        self.error("bad repetition definition (braced definition missing both limits)")
-                    rmax = rmin
-                else:
-                    self.error("bad repetition definition")
 
-                reader.read("}")
+                    c = self.peek()
+                    if c is None:
+                        self.error("bad repetition definition")
+                    elif c == ",":
+                        reader.read(",")
+
+                        c = self.peek()
+                        if c == "}":
+                            rmax = None
+                        elif c is not None and c.isdigit():
+                            rmax = reader.read_number()
+                        else:
+                            self.error("bad repetition definition")
+                    elif c == "}":
+                        if not read_lower_limit:
+                            self.error("bad repetition definition (braced definition missing both limits)")
+                        rmax = rmin
+                    else:
+                        self.error("bad repetition definition")
 
-                if (c2 := self.peek()) in ("?", "+"):
-                    self.error(f"{{...}}{c2} quantifier is not supported", unsupported=True)
+                    reader.read("}")
 
-                return Repetition(reader.span, reader.text, rmin, rmax)
+                    if (c2 := self.peek()) in ("?", "+"):
+                        self.error(f"{{...}}{c2} quantifier is not supported", unsupported=True)
 
+                    return Repetition(reader.span, reader.text, rmin, rmax)
+                except RegexAutomataError as e:
+                    raise self.FailedToReadBraceRepetition() from e
             case _:
                 self.error("bad repetition definition")
 

tests/test_parser.py

@@ -43,8 +43,12 @@ def test_parse_tree_union_parens():
     )
 
 
-@pytest.mark.parametrize("pattern", ["\\", "[a", "[a-bc-", "[]",
-                                     "{", "{123", "{123,", "{,123", "{,123,}", "{123,456,}"])
+@pytest.mark.parametrize("pattern", ["\\", "[a", "[a-bc-", "[]"])
 def test_tokenizer_errors_in_pattern_malformed(pattern):
     with pytest.raises(TokenizerError):
         list(Tokenizer(pattern).get_tokens())
+
+
+@pytest.mark.parametrize("pattern", ["{", "{123", "{123,", "{,123", "{,123,}", "{123,456,}"])
+def test_tokenizer_errors_in_pattern_nonmalformed(pattern):
+    list(Tokenizer(pattern).get_tokens())

tests/test_regex.py

@@ -46,6 +46,8 @@
                           # ("a||c|", "", True),
                           # ("a||c|", "a", True),
                           # ("a||c|", "c", True),
+                          ("{[^}]*}", "{foo bar}", True),
+                          ("x{3}{[^}]*}{3}", "xxx{foo bar}}}", True),
                           ])
 def test_fullmatch_regex(pattern: str, s: str, result: bool):
     assert (regex_automata.fullmatch(pattern, s) is not None) is result