We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 4e303e6 commit 810279bCopy full SHA for 810279b
src/tokeneer.rs
@@ -26,10 +26,12 @@ impl<M: Method> Tokeneer<M> {
26
pub fn encode(&self, text: &str) -> Vec<utok> {
27
let mut ans = Vec::new();
28
let mut start = 0;
29
- for m in self.special_regex.find_iter(text) {
30
- ans.extend(self.method.encode(&text[start..m.start()]));
31
- ans.extend_from_slice(&self.special[m.as_str()]);
32
- start = m.end();
+ if !self.special_regex.as_str().is_empty() {
+ for m in self.special_regex.find_iter(text) {
+ ans.extend(self.method.encode(&text[start..m.start()]));
+ ans.extend_from_slice(&self.special[m.as_str()]);
33
+ start = m.end();
34
+ }
35
}
36
ans.extend(self.method.encode(&text[start..]));
37
ans
0 commit comments