I don't like that zfill(3)

but realistically, a sentence with more than 1000 protected patterns?
hplt-project · Oct 30, 2023 · 97747e9 · 97747e9
1 parent 23571f7
commit 97747e9
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/sacremoses/tokenize.py b/sacremoses/tokenize.py
@@ -458,6 +458,7 @@ def tokenize(
                 for protected_pattern in protected_patterns
                 for match in protected_pattern.finditer(text)
             ]
+            assert len(protected_tokens) <= 1000 # so we don't run out of the zfill(3) space.
 
             # Apply the protected_patterns, longest match first.
             for i, token in sorted(enumerate(protected_tokens), key=lambda pair:len(pair[1]), reverse=True):