Skip to content

Commit 810279b

Browse files
committed
fix(tokeneer): 如果没有提供任何特殊词,则不进行匹配
Signed-off-by: YdrMaster <ydrml@hotmail.com>
1 parent 4e303e6 commit 810279b

File tree

1 file changed

+6
-4
lines changed

1 file changed

+6
-4
lines changed

src/tokeneer.rs

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,12 @@ impl<M: Method> Tokeneer<M> {
2626
pub fn encode(&self, text: &str) -> Vec<utok> {
2727
let mut ans = Vec::new();
2828
let mut start = 0;
29-
for m in self.special_regex.find_iter(text) {
30-
ans.extend(self.method.encode(&text[start..m.start()]));
31-
ans.extend_from_slice(&self.special[m.as_str()]);
32-
start = m.end();
29+
if !self.special_regex.as_str().is_empty() {
30+
for m in self.special_regex.find_iter(text) {
31+
ans.extend(self.method.encode(&text[start..m.start()]));
32+
ans.extend_from_slice(&self.special[m.as_str()]);
33+
start = m.end();
34+
}
3335
}
3436
ans.extend(self.method.encode(&text[start..]));
3537
ans

0 commit comments

Comments
 (0)