Merge #52

52: Introduce a new default normalizer that removes zeroes from tokens r=Kerollmops a=Kerollmops



Co-authored-by: Kerollmops <clement@meilisearch.com>

src/analyzer.rs

@@ -4,7 +4,7 @@ use fst::Set;
 use once_cell::sync::Lazy;
 
 use crate::detection::is_latin;
-use crate::normalizer::{Normalizer, DeunicodeNormalizer, LowercaseNormalizer};
+use crate::normalizer::{DeunicodeNormalizer, LowercaseNormalizer, Normalizer, ZeroesRemover};
 use crate::processors::{PreProcessor, IdentityPreProcessor, ProcessedText, ChineseTranslationPreProcessor};
 use crate::token_classifier::TokenClassifier;
 use crate::Token;
@@ -22,7 +22,11 @@ impl Default for Pipeline {
     fn default() -> Self {
         // Hotfix: make a common default normalizer for every pipeline
         let deunicoder = DeunicodeNormalizer::new(&|text: &str| !text.chars().next().map_or(true, is_latin));
-        let normalizer: Vec<Box<dyn Normalizer>> = vec![Box::new(deunicoder), Box::new(LowercaseNormalizer)];
+        let normalizer: Vec<Box<dyn Normalizer>> = vec![
+            Box::new(deunicoder),
+            Box::new(LowercaseNormalizer),
+            Box::new(ZeroesRemover),
+        ];
 
         Self {
             pre_processor: Box::new(IdentityPreProcessor),

src/normalizer/mod.rs

@@ -1,12 +1,14 @@
 mod identity;
 mod lowercase;
 mod deunicoder;
+mod zeroes_remover;
 
 use crate::Token;
 
 pub use identity::IdentityNormalizer;
 pub use lowercase::LowercaseNormalizer;
 pub use deunicoder::DeunicodeNormalizer;
+pub use zeroes_remover::ZeroesRemover;
 
 pub trait Normalizer: Sync + Send {
     fn normalize<'a>(&self, token: Token<'a>) -> Token<'a>;

src/normalizer/zeroes_remover.rs

@@ -0,0 +1,31 @@
+use std::borrow::Cow;
+
+use super::Normalizer;
+use crate::Token;
+
+pub struct ZeroesRemover;
+
+impl Normalizer for ZeroesRemover {
+    fn normalize<'a>(&self, mut token: Token<'a>) -> Token<'a> {
+        if token.word.chars().any(|c| c == '\0') {
+            token.word = Cow::Owned(token.word.chars().filter(|c| *c != '\0').collect());
+        }
+        token
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn remove_zeroes() {
+        let s = "\0loloo\0";
+        let token = Token {
+            word: Cow::Borrowed(s),
+            ..Token::default()
+        };
+        let token = ZeroesRemover.normalize(token);
+        assert!(!token.word.chars().any(|c| c == '\0'));
+    }
+}