From 9a1cba6a83fcd048ffd0c91ee6320cc9a9136d04 Mon Sep 17 00:00:00 2001
From: Lukas Kalbertodt <lukas.kalbertodt@gmail.com>
Date: Wed, 26 Jun 2024 11:46:33 +0200
Subject: [PATCH 01/25] Add null byte as hard context separator

This allows one to use \0 as artificial separator, for example when concatting lots of small strings into a large string. See this discussion for context: https://github.com/orgs/meilisearch/discussions/744
---
 charabia/src/separators.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/charabia/src/separators.rs b/charabia/src/separators.rs
index 60a00483..1ca63860 100644
--- a/charabia/src/separators.rs
+++ b/charabia/src/separators.rs
@@ -11,11 +11,11 @@
 /// - Zl Line Separator
 /// - Zp Paragraph Separator
 /// - Zs Space Separator
-/// plus ". ", ", " and ។ល។" (៘ decomposition) to categorize them as hard separators
+/// plus "\0", ". ", ", " and ។ល។" (៘ decomposition) to categorize them as hard separators
 /// and "`" to understand markdown formatted text
 #[rustfmt::skip]
 pub const DEFAULT_SEPARATORS: &[&str] = &[
-    ". ", ", ", "_", "‿", "⁀", "⁔", "︳", "︴", "﹍", "﹎", "﹏", "＿", "-", "֊", "־", "᐀", "᠆", "‐", "‒", "–",
+    "\0", ". ", ", ", "_", "‿", "⁀", "⁔", "︳", "︴", "﹍", "﹎", "﹏", "＿", "-", "֊", "־", "᐀", "᠆", "‐", "‒", "–",
     "—", "―", "⸗", "⸚", "⸺", "⸻", "⹀", "〜", "〰", "゠", "︱", "︲", "﹘", "﹣", "－", "𐺭", ")",
     "]", "}", "༻", "༽", "᚜", "⁆", "⁾", "₎", "⌉", "⌋", "〉", "❩", "❫", "❭", "❯", "❱", "❳", "❵", "⟆",
     "⟧", "⟩", "⟫", "⟭", "⟯", "⦄", "⦆", "⦈", "⦊", "⦌", "⦎", "⦐", "⦒", "⦔", "⦖", "⦘", "⧙", "⧛", "⧽",
@@ -64,6 +64,7 @@ pub const DEFAULT_SEPARATORS: &[&str] = &[
 
 #[rustfmt::skip]
 pub const CONTEXT_SEPARATORS: &[&str] = &[
+    "\0", // Null byte, can be used as artificial separator
     "᠆", // Mongolian Todo Soft Hyphen, mark the end of a paragraph.
     "᚛", "᚜", // Oghams, mark start and end of text
     "!", ". ", ", ", ";", "?", "¡", "§", "¶", "¿", ";", // Latin

From b28ac5c3810acbc63b7e089212023863aeb1793d Mon Sep 17 00:00:00 2001
From: Tamo <tamo@meilisearch.com>
Date: Tue, 2 Jul 2024 14:42:07 +0200
Subject: [PATCH 02/25] update dependencies

---
 charabia/Cargo.toml      | 12 ++++++------
 irg-kvariants/Cargo.toml |  4 ++--
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/charabia/Cargo.toml b/charabia/Cargo.toml
index 15e01d5a..05b437b1 100644
--- a/charabia/Cargo.toml
+++ b/charabia/Cargo.toml
@@ -12,11 +12,11 @@ categories = ["text-processing"]
 exclude = ["dictionaries/txt/thai/words.txt"]
 
 [dependencies]
-aho-corasick = "1.1.2"
+aho-corasick = "1.1.3"
 cow-utils = "0.1"
 csv = "1.3.0"
-deunicode = "1.4.2"
-either = "1.9.0"
+deunicode = "1.6.0"
+either = "1.13.0"
 finl_unicode = { version= "1.2.0", optional = true }
 fst = "0.4"
 jieba-rs = { version = "0.6", optional = true }
@@ -29,10 +29,10 @@ pinyin = { version = "0.10", default-features = false, features = [
   "with_tone",
 ], optional = true }
 wana_kana = { version = "3.0.0", optional = true }
-unicode-normalization = "0.1.22"
+unicode-normalization = "0.1.23"
 irg-kvariants = "0.1.0"
-litemap = "0.7.2"
-zerovec = "0.10.1"
+litemap = "0.7.3"
+zerovec = "0.10.4"
 
 [features]
 default = ["chinese", "hebrew", "japanese", "thai", "korean", "greek", "latin-camelcase", "latin-snakecase", "khmer", "vietnamese"]
diff --git a/irg-kvariants/Cargo.toml b/irg-kvariants/Cargo.toml
index 08ae6121..cb1bc69a 100644
--- a/irg-kvariants/Cargo.toml
+++ b/irg-kvariants/Cargo.toml
@@ -11,8 +11,8 @@ repository = "https://github.com/meilisearch/charabia"
 [dependencies]
 csv = "1.3.0"
 once_cell = "1.19.0"
-serde = { version = "1.0.196", features = ["derive"] }
+serde = { version = "1.0.203", features = ["derive"] }
 
 [build-dependencies]
 csv = "1.3.0"
-serde = { version = "1.0.196", features = ["derive"] }
+serde = { version = "1.0.203", features = ["derive"] }

From c5ca9206dbff984ec71d4f3f3597e4e6f90e2150 Mon Sep 17 00:00:00 2001
From: Tamo <tamo@meilisearch.com>
Date: Tue, 2 Jul 2024 14:47:17 +0200
Subject: [PATCH 03/25] update lindera

---
 charabia/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charabia/Cargo.toml b/charabia/Cargo.toml
index 05b437b1..3573efa2 100644
--- a/charabia/Cargo.toml
+++ b/charabia/Cargo.toml
@@ -24,7 +24,7 @@ once_cell = "1.19.0"
 serde = "1.0"
 slice-group-by = "0.3.1"
 whatlang = "0.16.4"
-lindera = { version = "=0.31.0", default-features = false, optional = true }
+lindera = { version = "=0.32.2", default-features = false, optional = true }
 pinyin = { version = "0.10", default-features = false, features = [
   "with_tone",
 ], optional = true }

From 3e188098d364fd6596c7c121657a0915c116a057 Mon Sep 17 00:00:00 2001
From: Tamo <tamo@meilisearch.com>
Date: Tue, 2 Jul 2024 14:51:32 +0200
Subject: [PATCH 04/25] update jieba

---
 charabia/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charabia/Cargo.toml b/charabia/Cargo.toml
index 3573efa2..63d7c679 100644
--- a/charabia/Cargo.toml
+++ b/charabia/Cargo.toml
@@ -19,7 +19,7 @@ deunicode = "1.6.0"
 either = "1.13.0"
 finl_unicode = { version= "1.2.0", optional = true }
 fst = "0.4"
-jieba-rs = { version = "0.6", optional = true }
+jieba-rs = { version = "0.7", optional = true }
 once_cell = "1.19.0"
 serde = "1.0"
 slice-group-by = "0.3.1"

From 99e8b38840201a813b631f415a86be9d9ed3f739 Mon Sep 17 00:00:00 2001
From: Tamo <tamo@meilisearch.com>
Date: Tue, 2 Jul 2024 14:53:53 +0200
Subject: [PATCH 05/25] prepare for release

---
 charabia/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charabia/Cargo.toml b/charabia/Cargo.toml
index 63d7c679..2eace07f 100644
--- a/charabia/Cargo.toml
+++ b/charabia/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "charabia"
-version = "0.8.11"
+version = "0.8.12"
 license = "MIT"
 authors = ["Many <many@meilisearch.com>"]
 edition = "2021"

From b6a3943b291386bbc4326937610e2c1d44c84aa8 Mon Sep 17 00:00:00 2001
From: Tamo <tamo@meilisearch.com>
Date: Tue, 2 Jul 2024 15:40:07 +0200
Subject: [PATCH 06/25] update internal dependencies for release

---
 charabia/Cargo.toml      | 2 +-
 irg-kvariants/Cargo.toml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/charabia/Cargo.toml b/charabia/Cargo.toml
index 2eace07f..b7a31676 100644
--- a/charabia/Cargo.toml
+++ b/charabia/Cargo.toml
@@ -30,7 +30,7 @@ pinyin = { version = "0.10", default-features = false, features = [
 ], optional = true }
 wana_kana = { version = "3.0.0", optional = true }
 unicode-normalization = "0.1.23"
-irg-kvariants = "0.1.0"
+irg-kvariants = "0.1.1"
 litemap = "0.7.3"
 zerovec = "0.10.4"
 
diff --git a/irg-kvariants/Cargo.toml b/irg-kvariants/Cargo.toml
index cb1bc69a..f32d005f 100644
--- a/irg-kvariants/Cargo.toml
+++ b/irg-kvariants/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "irg-kvariants"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 license = "MIT"
 description = "A simple wrapper around kvariant from hfhchan/irg"

From cf3916a40c4d86de5695bf7d56589420270dcd92 Mon Sep 17 00:00:00 2001
From: Tamo <tamo@meilisearch.com>
Date: Tue, 2 Jul 2024 15:45:35 +0200
Subject: [PATCH 07/25] use local dependencies when doing local testing

---
 charabia/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charabia/Cargo.toml b/charabia/Cargo.toml
index b7a31676..6086c9ce 100644
--- a/charabia/Cargo.toml
+++ b/charabia/Cargo.toml
@@ -30,7 +30,7 @@ pinyin = { version = "0.10", default-features = false, features = [
 ], optional = true }
 wana_kana = { version = "3.0.0", optional = true }
 unicode-normalization = "0.1.23"
-irg-kvariants = "0.1.1"
+irg-kvariants = { path = "../irg-kvariants", version = "=0.1.1" }
 litemap = "0.7.3"
 zerovec = "0.10.4"
 

From 28d1305da7f3b5e0831514ea8ef3a3b9ea680209 Mon Sep 17 00:00:00 2001
From: ManyTheFish <many@meilisearch.com>
Date: Wed, 10 Jul 2024 08:36:32 +0200
Subject: [PATCH 08/25] Simplify lang detection

---
 charabia/Cargo.toml           |  6 +-----
 charabia/src/detection/mod.rs | 17 ++++++-----------
 charabia/src/segmenter/mod.rs |  4 +++-
 charabia/src/tokenizer.rs     |  5 ++---
 4 files changed, 12 insertions(+), 20 deletions(-)

diff --git a/charabia/Cargo.toml b/charabia/Cargo.toml
index 6086c9ce..69562d5b 100644
--- a/charabia/Cargo.toml
+++ b/charabia/Cargo.toml
@@ -13,15 +13,13 @@ exclude = ["dictionaries/txt/thai/words.txt"]
 
 [dependencies]
 aho-corasick = "1.1.3"
-cow-utils = "0.1"
 csv = "1.3.0"
-deunicode = "1.6.0"
 either = "1.13.0"
 finl_unicode = { version= "1.2.0", optional = true }
 fst = "0.4"
 jieba-rs = { version = "0.7", optional = true }
 once_cell = "1.19.0"
-serde = "1.0"
+serde = "1.0.192"
 slice-group-by = "0.3.1"
 whatlang = "0.16.4"
 lindera = { version = "=0.32.2", default-features = false, optional = true }
@@ -31,8 +29,6 @@ pinyin = { version = "0.10", default-features = false, features = [
 wana_kana = { version = "3.0.0", optional = true }
 unicode-normalization = "0.1.23"
 irg-kvariants = { path = "../irg-kvariants", version = "=0.1.1" }
-litemap = "0.7.3"
-zerovec = "0.10.4"
 
 [features]
 default = ["chinese", "hebrew", "japanese", "thai", "korean", "greek", "latin-camelcase", "latin-snakecase", "khmer", "vietnamese"]
diff --git a/charabia/src/detection/mod.rs b/charabia/src/detection/mod.rs
index 0c5781b0..f965be83 100644
--- a/charabia/src/detection/mod.rs
+++ b/charabia/src/detection/mod.rs
@@ -1,5 +1,3 @@
-use std::collections::HashMap;
-
 pub use script_language::{Language, Script};
 use whatlang::Detector;
 
@@ -12,11 +10,11 @@ pub struct StrDetection<'o, 'al> {
     inner: &'o str,
     pub script: Option<Script>,
     pub language: Option<Language>,
-    allow_list: Option<&'al HashMap<Script, Vec<Language>>>,
+    allow_list: Option<&'al [Language]>,
 }
 
 impl<'o, 'al> StrDetection<'o, 'al> {
-    pub fn new(inner: &'o str, allow_list: Option<&'al HashMap<Script, Vec<Language>>>) -> Self {
+    pub fn new(inner: &'o str, allow_list: Option<&'al [Language]>) -> Self {
         Self { inner, script: None, language: None, allow_list }
     }
 
@@ -27,8 +25,7 @@ impl<'o, 'al> StrDetection<'o, 'al> {
 
     pub fn language(&mut self) -> Language {
         let inner = self.inner;
-        let script = self.script();
-        *self.language.get_or_insert_with(|| Self::detect_lang(inner, script, self.allow_list))
+        *self.language.get_or_insert_with(|| Self::detect_lang(inner, self.allow_list))
     }
 
     /// detect script with whatlang,
@@ -41,11 +38,9 @@ impl<'o, 'al> StrDetection<'o, 'al> {
     /// if no language is detected, return Language::Other
     fn detect_lang(
         text: &str,
-        script: Script,
-        allow_list: Option<&HashMap<Script, Vec<Language>>>,
+        allow_list: Option<&[Language]>,
     ) -> Language {
         let detector = allow_list
-            .and_then(|allow_list| allow_list.get(&script))
             .map(|allow_list| allow_list.iter().map(|lang| (*lang).into()).collect())
             .map(Detector::with_allowlist)
             .unwrap_or_default();
@@ -57,14 +52,14 @@ impl<'o, 'al> StrDetection<'o, 'al> {
 pub trait Detect<'o, 'al> {
     fn detect(
         &'o self,
-        allow_list: Option<&'al HashMap<Script, Vec<Language>>>,
+        allow_list: Option<&'al [Language]>,
     ) -> StrDetection<'o, 'al>;
 }
 
 impl<'o, 'al> Detect<'o, 'al> for &str {
     fn detect(
         &'o self,
-        allow_list: Option<&'al HashMap<Script, Vec<Language>>>,
+        allow_list: Option<&'al [Language]>,
     ) -> StrDetection<'o, 'al> {
         StrDetection::new(self, allow_list)
     }
diff --git a/charabia/src/segmenter/mod.rs b/charabia/src/segmenter/mod.rs
index 579d3389..8aa5eadb 100644
--- a/charabia/src/segmenter/mod.rs
+++ b/charabia/src/segmenter/mod.rs
@@ -52,6 +52,8 @@ pub static SEGMENTERS: Lazy<HashMap<(Script, Language), Box<dyn Segmenter>>> = L
     vec![
         // latin segmenter
         ((Script::Latin, Language::Other), Box::new(LatinSegmenter) as Box<dyn Segmenter>),
+        #[cfg(feature = "swedish-recomposition")]
+        ((Script::Latin, Language::Swe), Box::new(LatinSegmenter) as Box<dyn Segmenter>),
         // chinese segmenter
         #[cfg(feature = "chinese-segmentation")]
         ((Script::Cj, Language::Cmn), Box::new(ChineseSegmenter) as Box<dyn Segmenter>),
@@ -271,7 +273,7 @@ fn segmenter<'b>(detector: &mut StrDetection) -> &'b dyn Segmenter {
 #[derive(Debug, Clone, Default)]
 pub struct SegmenterOption<'tb> {
     pub aho: Option<AhoCorasick>,
-    pub allow_list: Option<&'tb HashMap<Script, Vec<Language>>>,
+    pub allow_list: Option<&'tb [Language]>,
 }
 
 /// Trait defining a segmenter.
diff --git a/charabia/src/tokenizer.rs b/charabia/src/tokenizer.rs
index e9b5f4dd..7851f50b 100644
--- a/charabia/src/tokenizer.rs
+++ b/charabia/src/tokenizer.rs
@@ -1,10 +1,9 @@
 use std::borrow::Cow;
-use std::collections::HashMap;
 
 use aho_corasick::{AhoCorasick, MatchKind};
 use fst::Set;
 
-use crate::detection::{Language, Script};
+use crate::detection::Language;
 use crate::normalizer::{NormalizedTokenIter, NormalizerOption};
 use crate::segmenter::{Segment, SegmentedStrIter, SegmentedTokenIter, SegmenterOption};
 use crate::separators::DEFAULT_SEPARATORS;
@@ -301,7 +300,7 @@ impl<'tb, A: AsRef<[u8]>> TokenizerBuilder<'tb, A> {
     /// # Arguments
     ///
     /// * `allow_list` - a `HashMap` of the selection of languages associated with a script to limit during autodetection.
-    pub fn allow_list(&mut self, allow_list: &'tb HashMap<Script, Vec<Language>>) -> &mut Self {
+    pub fn allow_list(&mut self, allow_list: &'tb [Language]) -> &mut Self {
         self.segmenter_option.allow_list = Some(allow_list);
         self
     }

From a7d34a148c5bb300f159b59961fd5583ec97653f Mon Sep 17 00:00:00 2001
From: ManyTheFish <many@meilisearch.com>
Date: Thu, 11 Jul 2024 17:23:28 +0200
Subject: [PATCH 09/25] Remove Charabia Language layer

---
 charabia/src/detection/mod.rs                 |  26 ++--
 charabia/src/detection/script_language.rs     | 117 +-----------------
 .../src/normalizer/swedish_recomposition.rs   |   2 +-
 charabia/src/segmenter/mod.rs                 |  61 +++++----
 4 files changed, 47 insertions(+), 159 deletions(-)

diff --git a/charabia/src/detection/mod.rs b/charabia/src/detection/mod.rs
index f965be83..3bfa5d53 100644
--- a/charabia/src/detection/mod.rs
+++ b/charabia/src/detection/mod.rs
@@ -23,9 +23,14 @@ impl<'o, 'al> StrDetection<'o, 'al> {
         *self.script.get_or_insert_with(|| Self::detect_script(inner))
     }
 
-    pub fn language(&mut self) -> Language {
+    pub fn language(&mut self) -> Option<Language> {
         let inner = self.inner;
-        *self.language.get_or_insert_with(|| Self::detect_lang(inner, self.allow_list))
+        self.language = match self.language.take() {
+            Some(lang) => Some(lang),
+            None => Self::detect_lang(inner, self.allow_list),
+        };
+
+        self.language
     }
 
     /// detect script with whatlang,
@@ -36,31 +41,22 @@ impl<'o, 'al> StrDetection<'o, 'al> {
 
     /// detect lang with whatlang
     /// if no language is detected, return Language::Other
-    fn detect_lang(
-        text: &str,
-        allow_list: Option<&[Language]>,
-    ) -> Language {
+    fn detect_lang(text: &str, allow_list: Option<&[Language]>) -> Option<Language> {
         let detector = allow_list
             .map(|allow_list| allow_list.iter().map(|lang| (*lang).into()).collect())
             .map(Detector::with_allowlist)
             .unwrap_or_default();
 
-        detector.detect_lang(text).map(Language::from).unwrap_or_default()
+        detector.detect_lang(text).map(Language::from)
     }
 }
 
 pub trait Detect<'o, 'al> {
-    fn detect(
-        &'o self,
-        allow_list: Option<&'al [Language]>,
-    ) -> StrDetection<'o, 'al>;
+    fn detect(&'o self, allow_list: Option<&'al [Language]>) -> StrDetection<'o, 'al>;
 }
 
 impl<'o, 'al> Detect<'o, 'al> for &str {
-    fn detect(
-        &'o self,
-        allow_list: Option<&'al [Language]>,
-    ) -> StrDetection<'o, 'al> {
+    fn detect(&'o self, allow_list: Option<&'al [Language]>) -> StrDetection<'o, 'al> {
         StrDetection::new(self, allow_list)
     }
 }
diff --git a/charabia/src/detection/script_language.rs b/charabia/src/detection/script_language.rs
index 96bea342..554aec65 100644
--- a/charabia/src/detection/script_language.rs
+++ b/charabia/src/detection/script_language.rs
@@ -5,122 +5,7 @@ use quickcheck::{Arbitrary, Gen};
 
 use super::chars;
 
-macro_rules! make_language {
-    ($($language:tt), +) => {
-        #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
-        pub enum Language {
-            $($language),+,
-            Other,
-        }
-        impl From<whatlang::Lang> for Language {
-            fn from(other: whatlang::Lang) -> Language {
-                match other {
-                    $(whatlang::Lang::$language => Language::$language), +
-                }
-            }
-        }
-
-        impl From<Language> for whatlang::Lang {
-            fn from(other: Language) -> whatlang::Lang {
-                match other {
-                    $(Language::$language => whatlang::Lang::$language), +,
-                    _other => whatlang::Lang::Eng,
-                }
-            }
-        }
-
-        impl Default for Language {
-            fn default() -> Self {
-                Self::Other
-            }
-        }
-
-        impl Language {
-            pub fn name(&self) -> &'static str {
-                match self {
-                    $(Language::$language => whatlang::Lang::$language.code()), +,
-                    _other => "other",
-                }
-            }
-
-            pub fn from_name<S: AsRef<str>>(code: S) -> Language {
-                whatlang::Lang::from_code(code.as_ref()).map(Language::from).unwrap_or_default()
-            }
-        }
-    };
-}
-
-make_language! {
-    Epo,
-    Eng,
-    Rus,
-    Cmn,
-    Spa,
-    Por,
-    Ita,
-    Ben,
-    Fra,
-    Deu,
-    Ukr,
-    Kat,
-    Ara,
-    Hin,
-    Jpn,
-    Heb,
-    Yid,
-    Pol,
-    Amh,
-    Jav,
-    Kor,
-    Nob,
-    Dan,
-    Swe,
-    Fin,
-    Tur,
-    Nld,
-    Hun,
-    Ces,
-    Ell,
-    Bul,
-    Bel,
-    Mar,
-    Kan,
-    Ron,
-    Slv,
-    Hrv,
-    Srp,
-    Mkd,
-    Lit,
-    Lav,
-    Est,
-    Tam,
-    Vie,
-    Urd,
-    Tha,
-    Guj,
-    Uzb,
-    Pan,
-    Aze,
-    Ind,
-    Tel,
-    Pes,
-    Mal,
-    Ori,
-    Mya,
-    Nep,
-    Sin,
-    Khm,
-    Tuk,
-    Aka,
-    Zul,
-    Sna,
-    Afr,
-    Lat,
-    Slk,
-    Cat,
-    Tgl,
-    Hye
-}
+pub use whatlang::Lang as Language;
 
 macro_rules! make_script {
     ($($script:tt), +) => {
diff --git a/charabia/src/normalizer/swedish_recomposition.rs b/charabia/src/normalizer/swedish_recomposition.rs
index 298ab60a..de84cdbb 100644
--- a/charabia/src/normalizer/swedish_recomposition.rs
+++ b/charabia/src/normalizer/swedish_recomposition.rs
@@ -77,7 +77,7 @@ impl Normalizer for SwedishRecompositionNormalizer {
 
     // Returns `true` if the Normalizer should be used.
     fn should_normalize(&self, token: &Token) -> bool {
-        token.script == Script::Latin && MATCHING_STR.is_match(token.lemma())
+        token.language == Some(Language::Swedish) && MATCHING_STR.is_match(token.lemma())
     }
 }
 
diff --git a/charabia/src/segmenter/mod.rs b/charabia/src/segmenter/mod.rs
index 8aa5eadb..9fe489df 100644
--- a/charabia/src/segmenter/mod.rs
+++ b/charabia/src/segmenter/mod.rs
@@ -48,32 +48,39 @@ mod utils;
 /// A segmenter assigned to `Language::Other` is considered as the default `Segmenter` for any `Language` that uses the assigned `Script`.
 /// For example, [`LatinSegmenter`] is assigned to `(Script::Latin, Language::Other)`,
 /// meaning that `LatinSegmenter` is the default `Segmenter` for any `Language` that uses `Latin` `Script`.
-pub static SEGMENTERS: Lazy<HashMap<(Script, Language), Box<dyn Segmenter>>> = Lazy::new(|| {
-    vec![
-        // latin segmenter
-        ((Script::Latin, Language::Other), Box::new(LatinSegmenter) as Box<dyn Segmenter>),
-        #[cfg(feature = "swedish-recomposition")]
-        ((Script::Latin, Language::Swe), Box::new(LatinSegmenter) as Box<dyn Segmenter>),
-        // chinese segmenter
-        #[cfg(feature = "chinese-segmentation")]
-        ((Script::Cj, Language::Cmn), Box::new(ChineseSegmenter) as Box<dyn Segmenter>),
-        // japanese segmenter
-        #[cfg(feature = "japanese")]
-        ((Script::Cj, Language::Jpn), Box::new(JapaneseSegmenter) as Box<dyn Segmenter>),
-        // korean segmenter
-        #[cfg(feature = "korean")]
-        ((Script::Hangul, Language::Kor), Box::new(KoreanSegmenter) as Box<dyn Segmenter>),
-        // thai segmenter
-        #[cfg(feature = "thai")]
-        ((Script::Thai, Language::Tha), Box::new(ThaiSegmenter) as Box<dyn Segmenter>),
-        #[cfg(feature = "khmer")]
-        ((Script::Khmer, Language::Khm), Box::new(KhmerSegmenter) as Box<dyn Segmenter>),
-        // arabic segmenter
-        ((Script::Arabic, Language::Ara), Box::new(ArabicSegmenter) as Box<dyn Segmenter>),
-    ]
-    .into_iter()
-    .collect()
-});
+pub static SEGMENTERS: Lazy<HashMap<(Script, Option<Language>), Box<dyn Segmenter>>> =
+    Lazy::new(|| {
+        vec![
+            // latin segmenter
+            ((Script::Latin, None), Box::new(LatinSegmenter) as Box<dyn Segmenter>),
+            #[cfg(feature = "swedish-recomposition")]
+            ((Script::Latin, Some(Language::Swe)), Box::new(LatinSegmenter) as Box<dyn Segmenter>),
+            // chinese segmenter
+            #[cfg(feature = "chinese-segmentation")]
+            ((Script::Cj, Some(Language::Cmn)), Box::new(ChineseSegmenter) as Box<dyn Segmenter>),
+            // japanese segmenter
+            #[cfg(feature = "japanese")]
+            ((Script::Cj, Some(Language::Jpn)), Box::new(JapaneseSegmenter) as Box<dyn Segmenter>),
+            // korean segmenter
+            #[cfg(feature = "korean")]
+            (
+                (Script::Hangul, Some(Language::Kor)),
+                Box::new(KoreanSegmenter) as Box<dyn Segmenter>,
+            ),
+            // thai segmenter
+            #[cfg(feature = "thai")]
+            ((Script::Thai, Some(Language::Tha)), Box::new(ThaiSegmenter) as Box<dyn Segmenter>),
+            #[cfg(feature = "khmer")]
+            ((Script::Khmer, Some(Language::Khm)), Box::new(KhmerSegmenter) as Box<dyn Segmenter>),
+            // arabic segmenter
+            (
+                (Script::Arabic, Some(Language::Ara)),
+                Box::new(ArabicSegmenter) as Box<dyn Segmenter>,
+            ),
+        ]
+        .into_iter()
+        .collect()
+    });
 
 /// Picked [`Segmenter`] when no segmenter is specialized to the detected [`Script`].
 pub static DEFAULT_SEGMENTER: Lazy<Box<dyn Segmenter>> = Lazy::new(|| Box::new(LatinSegmenter));
@@ -263,7 +270,7 @@ fn segmenter<'b>(detector: &mut StrDetection) -> &'b dyn Segmenter {
             let detected_language = detector.language();
             SEGMENTERS
                 .get(&(detected_script, detected_language))
-                .or_else(|| SEGMENTERS.get(&(detected_script, Language::Other)))
+                .or_else(|| SEGMENTERS.get(&(detected_script, None)))
                 .unwrap_or(&DEFAULT_SEGMENTER)
         }
     }

From f9732b2ce1526c121a8b6f8635476d0b7c2de66c Mon Sep 17 00:00:00 2001
From: ManyTheFish <many@meilisearch.com>
Date: Mon, 22 Jul 2024 15:57:22 +0200
Subject: [PATCH 10/25] Allow to dynamicaly change the language allow list when
 tokenizing a text

---
 charabia/src/detection/script_language.rs | 123 ++++++++++++++++++++--
 charabia/src/normalizer/mod.rs            |  13 ++-
 charabia/src/segmenter/mod.rs             |  67 +++++++-----
 charabia/src/tokenizer.rs                 |  65 +++++++++---
 4 files changed, 215 insertions(+), 53 deletions(-)

diff --git a/charabia/src/detection/script_language.rs b/charabia/src/detection/script_language.rs
index 554aec65..5c1bc2f3 100644
--- a/charabia/src/detection/script_language.rs
+++ b/charabia/src/detection/script_language.rs
@@ -2,14 +2,121 @@ use core::str::FromStr;
 
 #[cfg(test)]
 use quickcheck::{Arbitrary, Gen};
+use serde::{Deserialize, Serialize};
 
 use super::chars;
 
-pub use whatlang::Lang as Language;
+macro_rules! make_language {
+    ($($language:tt), +) => {
+        #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize, PartialOrd, Ord)]
+        pub enum Language {
+            $($language),+,
+        }
+        impl From<whatlang::Lang> for Language {
+            fn from(other: whatlang::Lang) -> Language {
+                match other {
+                    $(whatlang::Lang::$language => Language::$language), +
+                }
+            }
+        }
+
+        impl From<Language> for whatlang::Lang {
+            fn from(other: Language) -> whatlang::Lang {
+                match other {
+                    $(Language::$language => whatlang::Lang::$language), +,
+                }
+            }
+        }
+
+        impl Language {
+            pub fn code(&self) -> &'static str {
+                match self {
+                    $(Language::$language => whatlang::Lang::$language.code()), +,
+                }
+            }
+
+            pub fn from_code<S: AsRef<str>>(code: S) -> Option<Language> {
+                whatlang::Lang::from_code(code.as_ref()).map(Language::from)
+            }
+        }
+    };
+}
+
+make_language! {
+    Epo,
+    Eng,
+    Rus,
+    Cmn,
+    Spa,
+    Por,
+    Ita,
+    Ben,
+    Fra,
+    Deu,
+    Ukr,
+    Kat,
+    Ara,
+    Hin,
+    Jpn,
+    Heb,
+    Yid,
+    Pol,
+    Amh,
+    Jav,
+    Kor,
+    Nob,
+    Dan,
+    Swe,
+    Fin,
+    Tur,
+    Nld,
+    Hun,
+    Ces,
+    Ell,
+    Bul,
+    Bel,
+    Mar,
+    Kan,
+    Ron,
+    Slv,
+    Hrv,
+    Srp,
+    Mkd,
+    Lit,
+    Lav,
+    Est,
+    Tam,
+    Vie,
+    Urd,
+    Tha,
+    Guj,
+    Uzb,
+    Pan,
+    Aze,
+    Ind,
+    Tel,
+    Pes,
+    Mal,
+    Ori,
+    Mya,
+    Nep,
+    Sin,
+    Khm,
+    Tuk,
+    Aka,
+    Zul,
+    Sna,
+    Afr,
+    Lat,
+    Slk,
+    Cat,
+    Tgl,
+    Hye
+}
 
 macro_rules! make_script {
     ($($script:tt), +) => {
-        #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
+        #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize, PartialOrd, Ord)]
         pub enum Script {
             $($script),+,
             Cj,
@@ -246,12 +353,12 @@ mod test {
 
     #[test]
     fn from_into_language() {
-        assert_eq!(Language::Eng.name(), "eng");
-        assert_eq!(Language::from_name("eng"), Language::Eng);
-        assert_eq!(Language::Jpn.name(), "jpn");
-        assert_eq!(Language::from_name("jpn"), Language::Jpn);
-        assert_eq!(Language::Cmn.name(), "cmn");
-        assert_eq!(Language::from_name("cmn"), Language::Cmn);
+        assert_eq!(Language::Eng.code(), "eng");
+        assert_eq!(Language::from_code("eng"), Language::Eng);
+        assert_eq!(Language::Jpn.code(), "jpn");
+        assert_eq!(Language::from_code("jpn"), Language::Jpn);
+        assert_eq!(Language::Cmn.code(), "cmn");
+        assert_eq!(Language::from_code("cmn"), Language::Cmn);
     }
 
     #[test]
diff --git a/charabia/src/normalizer/mod.rs b/charabia/src/normalizer/mod.rs
index e4551961..fa28e83c 100644
--- a/charabia/src/normalizer/mod.rs
+++ b/charabia/src/normalizer/mod.rs
@@ -81,12 +81,12 @@ pub(crate) const DEFAULT_NORMALIZER_OPTION: NormalizerOption = NormalizerOption
 };
 
 /// Iterator over Normalized [`Token`]s.
-pub struct NormalizedTokenIter<'o, 'tb> {
-    token_iter: SegmentedTokenIter<'o, 'tb>,
+pub struct NormalizedTokenIter<'o, 'aho, 'lang, 'tb> {
+    token_iter: SegmentedTokenIter<'o, 'aho, 'lang>,
     options: &'tb NormalizerOption<'tb>,
 }
 
-impl<'o> Iterator for NormalizedTokenIter<'o, '_> {
+impl<'o> Iterator for NormalizedTokenIter<'o, '_, '_, '_> {
     type Item = Token<'o>;
 
     fn next(&mut self) -> Option<Self::Item> {
@@ -232,11 +232,14 @@ impl From<String> for CharOrStr {
     }
 }
 
-impl<'o, 'tb> SegmentedTokenIter<'o, 'tb> {
+impl<'o, 'aho, 'lang> SegmentedTokenIter<'o, 'aho, 'lang> {
     /// Normalize [`Token`]s using all the compatible Normalizers.
     ///
     /// A Latin `Token` would not be normalized the same as a Chinese `Token`.
-    pub fn normalize(self, options: &'tb NormalizerOption<'tb>) -> NormalizedTokenIter<'o, 'tb> {
+    pub fn normalize<'tb>(
+        self,
+        options: &'tb NormalizerOption<'tb>,
+    ) -> NormalizedTokenIter<'o, 'aho, 'lang, 'tb> {
         NormalizedTokenIter { token_iter: self, options }
     }
 }
diff --git a/charabia/src/segmenter/mod.rs b/charabia/src/segmenter/mod.rs
index 9fe489df..0841f6b1 100644
--- a/charabia/src/segmenter/mod.rs
+++ b/charabia/src/segmenter/mod.rs
@@ -90,13 +90,13 @@ pub static DEFAULT_SEPARATOR_AHO: Lazy<AhoCorasick> = Lazy::new(|| {
 });
 
 /// Iterator over segmented [`Token`]s.
-pub struct SegmentedTokenIter<'o, 'tb> {
-    inner: SegmentedStrIter<'o, 'tb>,
+pub struct SegmentedTokenIter<'o, 'aho, 'lang> {
+    inner: SegmentedStrIter<'o, 'aho, 'lang>,
     char_index: usize,
     byte_index: usize,
 }
 
-impl<'o> Iterator for SegmentedTokenIter<'o, '_> {
+impl<'o> Iterator for SegmentedTokenIter<'o, '_, '_> {
     type Item = Token<'o>;
 
     fn next(&mut self) -> Option<Self::Item> {
@@ -120,24 +120,31 @@ impl<'o> Iterator for SegmentedTokenIter<'o, '_> {
     }
 }
 
-impl<'o, 'tb> From<SegmentedStrIter<'o, 'tb>> for SegmentedTokenIter<'o, 'tb> {
-    fn from(segmented_str_iter: SegmentedStrIter<'o, 'tb>) -> Self {
+impl<'o, 'aho, 'lang> From<SegmentedStrIter<'o, 'aho, 'lang>>
+    for SegmentedTokenIter<'o, 'aho, 'lang>
+{
+    fn from(segmented_str_iter: SegmentedStrIter<'o, 'aho, 'lang>) -> Self {
         Self { inner: segmented_str_iter, char_index: 0, byte_index: 0 }
     }
 }
 
-pub struct SegmentedStrIter<'o, 'tb> {
+pub struct SegmentedStrIter<'o, 'aho, 'lang> {
     inner: Box<dyn Iterator<Item = &'o str> + 'o>,
     current: Box<dyn Iterator<Item = &'o str> + 'o>,
-    aho_iter: Option<AhoSegmentedStrIter<'o, 'tb>>,
+    aho_iter: Option<AhoSegmentedStrIter<'o, 'aho>>,
     segmenter: &'static dyn Segmenter,
-    options: &'tb SegmenterOption<'tb>,
+    aho: Option<&'aho AhoCorasick>,
+    allow_list: Option<&'lang [Language]>,
     script: Script,
     language: Option<Language>,
 }
 
-impl<'o, 'tb> SegmentedStrIter<'o, 'tb> {
-    pub fn new(original: &'o str, options: &'tb SegmenterOption<'tb>) -> Self {
+impl<'o, 'aho, 'lang> SegmentedStrIter<'o, 'aho, 'lang> {
+    pub fn new(
+        original: &'o str,
+        aho: Option<&'aho AhoCorasick>,
+        allow_list: Option<&'lang [Language]>,
+    ) -> Self {
         let mut current_script = Script::Other;
         let mut group_id = 0;
         let inner = original.linear_group_by_key(move |c| {
@@ -158,14 +165,15 @@ impl<'o, 'tb> SegmentedStrIter<'o, 'tb> {
             current: Box::new(None.into_iter()),
             aho_iter: None,
             segmenter: &*DEFAULT_SEGMENTER,
-            options,
+            aho,
+            allow_list,
             script: Script::Other,
             language: None,
         }
     }
 }
 
-impl<'o, 'tb> Iterator for SegmentedStrIter<'o, 'tb> {
+impl<'o, 'aho, 'lang> Iterator for SegmentedStrIter<'o, 'aho, 'lang> {
     type Item = &'o str;
 
     fn next(&mut self) -> Option<Self::Item> {
@@ -180,13 +188,13 @@ impl<'o, 'tb> Iterator for SegmentedStrIter<'o, 'tb> {
                 }
                 None => {
                     let text = self.inner.next()?;
-                    let mut detector = text.detect(self.options.allow_list);
+                    let mut detector = text.detect(self.allow_list);
                     self.segmenter = segmenter(&mut detector);
                     self.script = detector.script();
                     self.language = detector.language;
                     self.aho_iter = Some(AhoSegmentedStrIter::new(
                         text,
-                        self.options.aho.as_ref().unwrap_or(&DEFAULT_SEPARATOR_AHO),
+                        self.aho.unwrap_or(&DEFAULT_SEPARATOR_AHO),
                     ));
 
                     self.next()
@@ -328,16 +336,17 @@ pub trait Segment<'o> {
     /// assert_eq!(lemma, "quick");
     /// assert_eq!(kind, TokenKind::Unknown);
     /// ```
-    fn segment(&self) -> SegmentedTokenIter<'o, 'o> {
+    fn segment(&self) -> SegmentedTokenIter<'o, 'o, 'o> {
         self.segment_str().into()
     }
 
     /// Segments the provided text creating an Iterator over Tokens where you can specify an allowed list of languages to be used with a script.
-    fn segment_with_option<'tb>(
+    fn segment_with_option<'aho, 'lang>(
         &self,
-        options: &'tb SegmenterOption<'tb>,
-    ) -> SegmentedTokenIter<'o, 'tb> {
-        self.segment_str_with_option(options).into()
+        aho: Option<&'aho AhoCorasick>,
+        allow_list: Option<&'lang [Language]>,
+    ) -> SegmentedTokenIter<'o, 'aho, 'lang> {
+        self.segment_str_with_option(aho, allow_list).into()
     }
 
     /// Segments the provided text creating an Iterator over `&str`.
@@ -355,24 +364,26 @@ pub trait Segment<'o> {
     /// assert_eq!(segments.next(), Some(" "));
     /// assert_eq!(segments.next(), Some("quick"));
     /// ```
-    fn segment_str(&self) -> SegmentedStrIter<'o, 'o> {
-        self.segment_str_with_option(&SegmenterOption { aho: None, allow_list: None })
+    fn segment_str(&self) -> SegmentedStrIter<'o, 'o, 'o> {
+        self.segment_str_with_option(None, None)
     }
 
     /// Segments the provided text creating an Iterator over `&str` where you can specify an allowed list of languages to be used with a script.
     ///
-    fn segment_str_with_option<'tb>(
+    fn segment_str_with_option<'aho, 'lang>(
         &self,
-        options: &'tb SegmenterOption<'tb>,
-    ) -> SegmentedStrIter<'o, 'tb>;
+        aho: Option<&'aho AhoCorasick>,
+        allow_list: Option<&'lang [Language]>,
+    ) -> SegmentedStrIter<'o, 'aho, 'lang>;
 }
 
 impl<'o> Segment<'o> for &'o str {
-    fn segment_str_with_option<'tb>(
+    fn segment_str_with_option<'aho, 'lang>(
         &self,
-        options: &'tb SegmenterOption<'tb>,
-    ) -> SegmentedStrIter<'o, 'tb> {
-        SegmentedStrIter::new(self, options)
+        aho: Option<&'aho AhoCorasick>,
+        allow_list: Option<&'lang [Language]>,
+    ) -> SegmentedStrIter<'o, 'aho, 'lang> {
+        SegmentedStrIter::new(self, aho, allow_list)
     }
 }
 
diff --git a/charabia/src/tokenizer.rs b/charabia/src/tokenizer.rs
index 7851f50b..c8617715 100644
--- a/charabia/src/tokenizer.rs
+++ b/charabia/src/tokenizer.rs
@@ -10,12 +10,12 @@ use crate::separators::DEFAULT_SEPARATORS;
 use crate::Token;
 
 /// Iterator over tuples of [`&str`] (part of the original text) and [`Token`].
-pub struct ReconstructedTokenIter<'o, 'tb> {
-    token_iter: NormalizedTokenIter<'o, 'tb>,
+pub struct ReconstructedTokenIter<'o, 'aho, 'lang, 'tb> {
+    token_iter: NormalizedTokenIter<'o, 'aho, 'lang, 'tb>,
     original: &'o str,
 }
 
-impl<'o> Iterator for ReconstructedTokenIter<'o, '_> {
+impl<'o> Iterator for ReconstructedTokenIter<'o, '_, '_, '_> {
     type Item = (&'o str, Token<'o>);
 
     fn next(&mut self) -> Option<Self::Item> {
@@ -99,8 +99,8 @@ impl Tokenize<'_> for &str {
 /// See [`TokenizerBuilder`] to know how to build a [`Tokenizer`].
 #[derive(Debug)]
 pub struct Tokenizer<'tb> {
-    segmenter_option: Cow<'tb, SegmenterOption<'tb>>,
-    normalizer_option: Cow<'tb, NormalizerOption<'tb>>,
+    pub segmenter_option: Cow<'tb, SegmenterOption<'tb>>,
+    pub normalizer_option: Cow<'tb, NormalizerOption<'tb>>,
 }
 
 impl<'tb> Tokenizer<'tb> {
@@ -108,23 +108,55 @@ impl<'tb> Tokenizer<'tb> {
     ///
     /// The provided text is segmented creating tokens,
     /// then tokens are normalized and classified depending on the list of normalizers and classifiers in [`normalizer::NORMALIZERS`].
-    pub fn tokenize<'t, 'o>(&'t self, original: &'o str) -> NormalizedTokenIter<'o, 't> {
-        original.segment_with_option(&self.segmenter_option).normalize(&self.normalizer_option)
+    pub fn tokenize<'t, 'o>(&'t self, original: &'o str) -> NormalizedTokenIter<'o, 't, 't, 't> {
+        original
+            .segment_with_option(
+                self.segmenter_option.aho.as_ref(),
+                self.segmenter_option.allow_list,
+            )
+            .normalize(&self.normalizer_option)
+    }
+
+    /// Creates an Iterator over [`Token`]s.
+    ///
+    /// The provided text is segmented creating tokens,
+    /// then tokens are normalized and classified depending on the list of normalizers and classifiers in [`normalizer::NORMALIZERS`].
+    ///
+    /// # Arguments
+    ///
+    /// * `allow_list` - a slice of [`Language`] to allow during autodetection.
+    pub fn tokenize_with_allow_list<'t, 'o, 'lang>(
+        &'t self,
+        original: &'o str,
+        allow_list: Option<&'lang [Language]>,
+    ) -> NormalizedTokenIter<'o, 't, 'lang, 't> {
+        original
+            .segment_with_option(self.segmenter_option.aho.as_ref(), allow_list)
+            .normalize(&self.normalizer_option)
     }
 
     /// Same as [`tokenize`] but attaches each [`Token`] to its corresponding portion of the original text.
-    pub fn reconstruct<'t, 'o>(&'t self, original: &'o str) -> ReconstructedTokenIter<'o, 't> {
+    pub fn reconstruct<'t, 'o>(
+        &'t self,
+        original: &'o str,
+    ) -> ReconstructedTokenIter<'o, 't, 't, 't> {
         ReconstructedTokenIter { original, token_iter: self.tokenize(original) }
     }
 
     /// Segments the provided text creating an Iterator over [`Token`].
-    pub fn segment<'t, 'o>(&'t self, original: &'o str) -> SegmentedTokenIter<'o, 't> {
-        original.segment_with_option(&self.segmenter_option)
+    pub fn segment<'t, 'o>(&'t self, original: &'o str) -> SegmentedTokenIter<'o, 't, 't> {
+        original.segment_with_option(
+            self.segmenter_option.aho.as_ref(),
+            self.segmenter_option.allow_list,
+        )
     }
 
     /// Segments the provided text creating an Iterator over `&str`.
-    pub fn segment_str<'t, 'o>(&'t self, original: &'o str) -> SegmentedStrIter<'o, 't> {
-        original.segment_str_with_option(&self.segmenter_option)
+    pub fn segment_str<'t, 'o>(&'t self, original: &'o str) -> SegmentedStrIter<'o, 't, 't> {
+        original.segment_str_with_option(
+            self.segmenter_option.aho.as_ref(),
+            self.segmenter_option.allow_list,
+        )
     }
 }
 
@@ -353,6 +385,15 @@ impl<'tb, A: AsRef<[u8]>> TokenizerBuilder<'tb, A> {
             segmenter_option: Cow::Owned(self.segmenter_option),
         }
     }
+
+    /// Build the tokenizer options consumming self.
+    ///
+    /// This method allows to drop the tokenizer builder without having to drop the options themselves.
+    pub fn into_options(mut self) -> (NormalizerOption<'tb>, SegmenterOption<'tb>) {
+        drop(self.build());
+
+        (self.normalizer_option, self.segmenter_option)
+    }
 }
 
 impl Default for TokenizerBuilder<'_, Vec<u8>> {

From 599bb3b41b5d0d30c4d9695cfc2453cc30508b98 Mon Sep 17 00:00:00 2001
From: ManyTheFish <many@meilisearch.com>
Date: Mon, 22 Jul 2024 16:04:57 +0200
Subject: [PATCH 11/25] Fix tests

---
 charabia/src/detection/script_language.rs     |  6 +-
 .../src/normalizer/swedish_recomposition.rs   |  8 ++-
 charabia/src/segmenter/latin/mod.rs           |  2 +-
 charabia/src/segmenter/mod.rs                 | 61 +++++++++----------
 4 files changed, 38 insertions(+), 39 deletions(-)

diff --git a/charabia/src/detection/script_language.rs b/charabia/src/detection/script_language.rs
index 5c1bc2f3..1c904d87 100644
--- a/charabia/src/detection/script_language.rs
+++ b/charabia/src/detection/script_language.rs
@@ -354,11 +354,11 @@ mod test {
     #[test]
     fn from_into_language() {
         assert_eq!(Language::Eng.code(), "eng");
-        assert_eq!(Language::from_code("eng"), Language::Eng);
+        assert_eq!(Language::from_code("eng"), Some(Language::Eng));
         assert_eq!(Language::Jpn.code(), "jpn");
-        assert_eq!(Language::from_code("jpn"), Language::Jpn);
+        assert_eq!(Language::from_code("jpn"), Some(Language::Jpn));
         assert_eq!(Language::Cmn.code(), "cmn");
-        assert_eq!(Language::from_code("cmn"), Language::Cmn);
+        assert_eq!(Language::from_code("cmn"), Some(Language::Cmn));
     }
 
     #[test]
diff --git a/charabia/src/normalizer/swedish_recomposition.rs b/charabia/src/normalizer/swedish_recomposition.rs
index de84cdbb..9a38da41 100644
--- a/charabia/src/normalizer/swedish_recomposition.rs
+++ b/charabia/src/normalizer/swedish_recomposition.rs
@@ -5,7 +5,7 @@ use once_cell::sync::Lazy;
 
 use super::Normalizer;
 use crate::normalizer::NormalizerOption;
-use crate::{Script, Token};
+use crate::{Language, Token};
 
 static MATCHING_STR: Lazy<AhoCorasick> = Lazy::new(|| {
     AhoCorasick::new(&["A\u{30a}", "a\u{30a}", "A\u{308}", "a\u{308}", "O\u{308}", "o\u{308}"])
@@ -77,7 +77,7 @@ impl Normalizer for SwedishRecompositionNormalizer {
 
     // Returns `true` if the Normalizer should be used.
     fn should_normalize(&self, token: &Token) -> bool {
-        token.language == Some(Language::Swedish) && MATCHING_STR.is_match(token.lemma())
+        token.language == Some(Language::Swe) && MATCHING_STR.is_match(token.lemma())
     }
 }
 
@@ -101,6 +101,7 @@ mod test {
     use crate::normalizer::test::test_normalizer;
     use crate::normalizer::Normalizer;
     use crate::token::TokenKind;
+    use crate::Script;
 
     // base tokens to normalize.
     fn tokens() -> Vec<Token<'static>> {
@@ -109,6 +110,7 @@ mod test {
             char_end: 13,
             byte_end: 19,
             script: Script::Latin,
+            language: Some(Language::Swe),
             ..Default::default()
         }]
     }
@@ -121,6 +123,7 @@ mod test {
             char_end: 13,
             byte_end: 19,
             script: Script::Latin,
+            language: Some(Language::Swe),
             ..Default::default()
         }]
     }
@@ -148,6 +151,7 @@ mod test {
             ]),
             script: Script::Latin,
             kind: TokenKind::Word,
+            language: Some(Language::Swe),
             ..Default::default()
         }]
     }
diff --git a/charabia/src/segmenter/latin/mod.rs b/charabia/src/segmenter/latin/mod.rs
index 88a9fafa..7f978c18 100644
--- a/charabia/src/segmenter/latin/mod.rs
+++ b/charabia/src/segmenter/latin/mod.rs
@@ -40,5 +40,5 @@ mod test {
         "snake", "_", "case",
     ];
 
-    test_segmenter!(LatinSegmenter, TEXT, SEGMENTED, TOKENIZED, Script::Latin, Language::Other);
+    test_segmenter!(LatinSegmenter, TEXT, SEGMENTED, TOKENIZED, Script::Latin, Language::Eng);
 }
diff --git a/charabia/src/segmenter/mod.rs b/charabia/src/segmenter/mod.rs
index 0841f6b1..f41f18aa 100644
--- a/charabia/src/segmenter/mod.rs
+++ b/charabia/src/segmenter/mod.rs
@@ -37,6 +37,8 @@ mod thai;
 #[cfg(any(feature = "thai", feature = "khmer"))]
 mod utils;
 
+pub type SegmenterMap = HashMap<(Script, Option<Language>), Box<dyn Segmenter>>;
+
 /// List of used [`Segmenter`]s linked to their corresponding [`Script`] and [`Language`].
 ///
 /// This list is used after `Script` and `Language` detection to pick the specialized [`Segmenter`].
@@ -48,39 +50,32 @@ mod utils;
 /// A segmenter assigned to `Language::Other` is considered as the default `Segmenter` for any `Language` that uses the assigned `Script`.
 /// For example, [`LatinSegmenter`] is assigned to `(Script::Latin, Language::Other)`,
 /// meaning that `LatinSegmenter` is the default `Segmenter` for any `Language` that uses `Latin` `Script`.
-pub static SEGMENTERS: Lazy<HashMap<(Script, Option<Language>), Box<dyn Segmenter>>> =
-    Lazy::new(|| {
-        vec![
-            // latin segmenter
-            ((Script::Latin, None), Box::new(LatinSegmenter) as Box<dyn Segmenter>),
-            #[cfg(feature = "swedish-recomposition")]
-            ((Script::Latin, Some(Language::Swe)), Box::new(LatinSegmenter) as Box<dyn Segmenter>),
-            // chinese segmenter
-            #[cfg(feature = "chinese-segmentation")]
-            ((Script::Cj, Some(Language::Cmn)), Box::new(ChineseSegmenter) as Box<dyn Segmenter>),
-            // japanese segmenter
-            #[cfg(feature = "japanese")]
-            ((Script::Cj, Some(Language::Jpn)), Box::new(JapaneseSegmenter) as Box<dyn Segmenter>),
-            // korean segmenter
-            #[cfg(feature = "korean")]
-            (
-                (Script::Hangul, Some(Language::Kor)),
-                Box::new(KoreanSegmenter) as Box<dyn Segmenter>,
-            ),
-            // thai segmenter
-            #[cfg(feature = "thai")]
-            ((Script::Thai, Some(Language::Tha)), Box::new(ThaiSegmenter) as Box<dyn Segmenter>),
-            #[cfg(feature = "khmer")]
-            ((Script::Khmer, Some(Language::Khm)), Box::new(KhmerSegmenter) as Box<dyn Segmenter>),
-            // arabic segmenter
-            (
-                (Script::Arabic, Some(Language::Ara)),
-                Box::new(ArabicSegmenter) as Box<dyn Segmenter>,
-            ),
-        ]
-        .into_iter()
-        .collect()
-    });
+pub static SEGMENTERS: Lazy<SegmenterMap> = Lazy::new(|| {
+    vec![
+        // latin segmenter
+        ((Script::Latin, None), Box::new(LatinSegmenter) as Box<dyn Segmenter>),
+        #[cfg(feature = "swedish-recomposition")]
+        ((Script::Latin, Some(Language::Swe)), Box::new(LatinSegmenter) as Box<dyn Segmenter>),
+        // chinese segmenter
+        #[cfg(feature = "chinese-segmentation")]
+        ((Script::Cj, Some(Language::Cmn)), Box::new(ChineseSegmenter) as Box<dyn Segmenter>),
+        // japanese segmenter
+        #[cfg(feature = "japanese")]
+        ((Script::Cj, Some(Language::Jpn)), Box::new(JapaneseSegmenter) as Box<dyn Segmenter>),
+        // korean segmenter
+        #[cfg(feature = "korean")]
+        ((Script::Hangul, Some(Language::Kor)), Box::new(KoreanSegmenter) as Box<dyn Segmenter>),
+        // thai segmenter
+        #[cfg(feature = "thai")]
+        ((Script::Thai, Some(Language::Tha)), Box::new(ThaiSegmenter) as Box<dyn Segmenter>),
+        #[cfg(feature = "khmer")]
+        ((Script::Khmer, Some(Language::Khm)), Box::new(KhmerSegmenter) as Box<dyn Segmenter>),
+        // arabic segmenter
+        ((Script::Arabic, Some(Language::Ara)), Box::new(ArabicSegmenter) as Box<dyn Segmenter>),
+    ]
+    .into_iter()
+    .collect()
+});
 
 /// Picked [`Segmenter`] when no segmenter is specialized to the detected [`Script`].
 pub static DEFAULT_SEGMENTER: Lazy<Box<dyn Segmenter>> = Lazy::new(|| Box::new(LatinSegmenter));

From a95a9217265cee515708a679a2ed08ced1ac58a3 Mon Sep 17 00:00:00 2001
From: Many the fish <many@meilisearch.com>
Date: Tue, 23 Jul 2024 11:06:32 +0200
Subject: [PATCH 12/25] Update charabia/src/tokenizer.rs

Co-authored-by: Louis Dureuil <louis@meilisearch.com>
---
 charabia/src/tokenizer.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charabia/src/tokenizer.rs b/charabia/src/tokenizer.rs
index c8617715..a20a3f8b 100644
--- a/charabia/src/tokenizer.rs
+++ b/charabia/src/tokenizer.rs
@@ -386,7 +386,7 @@ impl<'tb, A: AsRef<[u8]>> TokenizerBuilder<'tb, A> {
         }
     }
 
-    /// Build the tokenizer options consumming self.
+    /// Build the tokenizer options consuming self.
     ///
     /// This method allows to drop the tokenizer builder without having to drop the options themselves.
     pub fn into_options(mut self) -> (NormalizerOption<'tb>, SegmenterOption<'tb>) {

From 357e53b50775416311faea699fd9a7e32eaa90ef Mon Sep 17 00:00:00 2001
From: ManyTheFish <many@meilisearch.com>
Date: Tue, 23 Jul 2024 14:57:13 +0200
Subject: [PATCH 13/25] fix

---
 charabia/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charabia/src/lib.rs b/charabia/src/lib.rs
index bfabcde1..0d2a5ac0 100644
--- a/charabia/src/lib.rs
+++ b/charabia/src/lib.rs
@@ -56,7 +56,7 @@ mod detection;
 mod token;
 mod tokenizer;
 
-pub use detection::{Language, Script};
+pub use detection::{Language, Script, StrDetection};
 pub use normalizer::Normalize;
 pub use segmenter::Segment;
 pub use token::{SeparatorKind, Token, TokenKind};

From 2992583137458afcebff5d44cae93fa46d9cf664 Mon Sep 17 00:00:00 2001
From: ManyTheFish <many@meilisearch.com>
Date: Tue, 23 Jul 2024 15:04:48 +0200
Subject: [PATCH 14/25] Remove into_option method

---
 charabia/src/tokenizer.rs | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/charabia/src/tokenizer.rs b/charabia/src/tokenizer.rs
index a20a3f8b..ba39fb83 100644
--- a/charabia/src/tokenizer.rs
+++ b/charabia/src/tokenizer.rs
@@ -385,15 +385,6 @@ impl<'tb, A: AsRef<[u8]>> TokenizerBuilder<'tb, A> {
             segmenter_option: Cow::Owned(self.segmenter_option),
         }
     }
-
-    /// Build the tokenizer options consuming self.
-    ///
-    /// This method allows to drop the tokenizer builder without having to drop the options themselves.
-    pub fn into_options(mut self) -> (NormalizerOption<'tb>, SegmenterOption<'tb>) {
-        drop(self.build());
-
-        (self.normalizer_option, self.segmenter_option)
-    }
 }
 
 impl Default for TokenizerBuilder<'_, Vec<u8>> {

From 330da23ab09f59071f01f345a9894f25bdd0ce87 Mon Sep 17 00:00:00 2001
From: Trevor Phillips <phillitr@oregonstate.edu>
Date: Tue, 23 Jul 2024 13:44:56 -0600
Subject: [PATCH 15/25] Add math symbols to default separators list

---
 charabia/src/separators.rs | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/charabia/src/separators.rs b/charabia/src/separators.rs
index 1ca63860..93399395 100644
--- a/charabia/src/separators.rs
+++ b/charabia/src/separators.rs
@@ -59,7 +59,19 @@ pub const DEFAULT_SEPARATORS: &[&str] = &[
     "𑪠", "𑪡", "𑪢", "𑱁", "𑱂", "𑱃", "𑱄", "𑱅", "𑱰", "𑱱", "𑻷", "𑻸", "𑿿", "𒑰", "𒑱", "𒑲", "𒑳", "𒑴", "𖩮",
     "𖩯", "𖫵", "𖬷", "𖬸", "𖬹", "𖬺", "𖬻", "𖭄", "𖺗", "𖺘", "𖺙", "𖺚", "𖿢", "𛲟", "𝪇", "𝪈", "𝪉", "𝪊", "𝪋",
     "𞥞", "𞥟", "\n", "\r", "\u{2029}", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ",
-    " ", "　", "`", "\t"
+    " ", "　", "`", "\t" "+", "-", "±", "×", "÷", "−", "∓", "∔", "∕", "∖", "∗", "∘", "∙", "√", "∛",
+    "∜", "∝", "∞", "∟", "∠", "∡", "∢", "∣", "∤", "∥", "∧", "∨", "∩", "∪", "∫", "∬", "∭", "∮", "∯",
+    "∰", "∱", "∲", "∳", "∴", "∵", "∶", "∷", "∸", "∹", "∺", "∻", "∼", "∽", "∾", "∿", "≀", "≁", "≂",
+    "≃", "≄", "≅", "≆", "≇", "≈", "≉", "≊", "≋", "≌", "≍", "≎", "≏", "≐", "≑", "≒", "≓", "≔", "≕",
+    "≖", "≗", "≘", "≙", "≚", "≛", "≜", "≝", "≞", "≟", "≠", "≡", "≢", "≣", "≤", "≥", "≦", "≧", "≨",
+    "≩", "≪", "≫", "≬", "≭", "≮", "≯", "≰", "≱", "≲", "≳", "≴", "≵", "≶", "≷", "≸", "≹", "≺",
+    "≻", "≼", "≽", "≾", "≿", "⊀", "⊁", "⊂", "⊃", "⊄", "⊅", "⊆", "⊇", "⊈", "⊉", "⊊", "⊋", "⊌", "⊍",
+    "⊏", "⊐", "⊑", "⊒", "⊓", "⊔", "⊕", "⊖", "⊗", "⊘", "⊙", "⊚", "⊛", "⊜", "⊝", "⊞", "⊟", "⊠", "⊡",
+    "⊢", "⊣", "⊤", "⊥", "⊦", "⊧", "⊨", "⊩", "⊪", "⊫", "⊬", "⊭", "⊮", "⊯", "⊰", "⊱", "⊲", "⊳", "⊴",
+    "⊵", "⊶", "⊷", "⊸", "⊹", "⊺", "⊻", "⊼", "⊽", "⊾", "⊿", "⋀", "⋁", "⋂", "⋃", "⋄", "⋅", "⋆", "⋇",
+    "⋈", "⋉", "⋊", "⋋", "⋌", "⋍", "⋎", "⋏", "⋐", "⋑", "⋒", "⋓", "⋔", "⋕", "⋖", "⋗", "⋘", "⋙", "⋚",
+    "⋛", "⋜", "⋝", "⋞", "⋟", "⋠", "⋡", "⋢", "⋣", "⋤", "⋥", "⋦", "⋧", "⋨", "⋩", "⋪", "⋫", "⋬", "⋭",
+    "⋮", "⋯", "⋰", "⋱", "⋲", "⋳", "⋴", "⋵", "⋶", "⋷", "⋸", "⋹", "⋺", "⋻", "⋼", "⋽", "⋾", "⋿"
 ];
 
 #[rustfmt::skip]

From 33f75189761a0df43b24a7cd76c01cbd2c0db16a Mon Sep 17 00:00:00 2001
From: Trevor Phillips <phillitr@oregonstate.edu>
Date: Tue, 23 Jul 2024 13:48:14 -0600
Subject: [PATCH 16/25] Add math symbols to default separators list

---
 charabia/src/separators.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/charabia/src/separators.rs b/charabia/src/separators.rs
index 93399395..31db94aa 100644
--- a/charabia/src/separators.rs
+++ b/charabia/src/separators.rs
@@ -8,6 +8,7 @@
 /// - Pi Initial Punctuation
 /// - Po Other Punctuation
 /// - Ps Open Punctuation
+/// - Sm Math Symbols
 /// - Zl Line Separator
 /// - Zp Paragraph Separator
 /// - Zs Space Separator

From a43e27e623a070142d6cdd6f88eb4ccc18c8150c Mon Sep 17 00:00:00 2001
From: Trevor Phillips <phillitr@oregonstate.edu>
Date: Tue, 23 Jul 2024 13:48:46 -0600
Subject: [PATCH 17/25] Add math symbols to default separators list

---
 charabia/src/separators.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charabia/src/separators.rs b/charabia/src/separators.rs
index 31db94aa..91436d3a 100644
--- a/charabia/src/separators.rs
+++ b/charabia/src/separators.rs
@@ -8,7 +8,7 @@
 /// - Pi Initial Punctuation
 /// - Po Other Punctuation
 /// - Ps Open Punctuation
-/// - Sm Math Symbols
+/// - Sm Math Symbol
 /// - Zl Line Separator
 /// - Zp Paragraph Separator
 /// - Zs Space Separator

From 953eb2edd73cf3981eb9a61e92df427b706dd407 Mon Sep 17 00:00:00 2001
From: Trevor Glenn Phillips <phillitr@oregonstate.edu>
Date: Tue, 23 Jul 2024 14:12:50 -0600
Subject: [PATCH 18/25] Fix comma typo

---
 charabia/src/separators.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charabia/src/separators.rs b/charabia/src/separators.rs
index 91436d3a..c8212046 100644
--- a/charabia/src/separators.rs
+++ b/charabia/src/separators.rs
@@ -60,7 +60,7 @@ pub const DEFAULT_SEPARATORS: &[&str] = &[
     "𑪠", "𑪡", "𑪢", "𑱁", "𑱂", "𑱃", "𑱄", "𑱅", "𑱰", "𑱱", "𑻷", "𑻸", "𑿿", "𒑰", "𒑱", "𒑲", "𒑳", "𒑴", "𖩮",
     "𖩯", "𖫵", "𖬷", "𖬸", "𖬹", "𖬺", "𖬻", "𖭄", "𖺗", "𖺘", "𖺙", "𖺚", "𖿢", "𛲟", "𝪇", "𝪈", "𝪉", "𝪊", "𝪋",
     "𞥞", "𞥟", "\n", "\r", "\u{2029}", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ",
-    " ", "　", "`", "\t" "+", "-", "±", "×", "÷", "−", "∓", "∔", "∕", "∖", "∗", "∘", "∙", "√", "∛",
+    " ", "　", "`", "\t", "+", "-", "±", "×", "÷", "−", "∓", "∔", "∕", "∖", "∗", "∘", "∙", "√", "∛",
     "∜", "∝", "∞", "∟", "∠", "∡", "∢", "∣", "∤", "∥", "∧", "∨", "∩", "∪", "∫", "∬", "∭", "∮", "∯",
     "∰", "∱", "∲", "∳", "∴", "∵", "∶", "∷", "∸", "∹", "∺", "∻", "∼", "∽", "∾", "∿", "≀", "≁", "≂",
     "≃", "≄", "≅", "≆", "≇", "≈", "≉", "≊", "≋", "≌", "≍", "≎", "≏", "≐", "≑", "≒", "≓", "≔", "≕",

From cbeb12a57fa78c02f8cc498a47c2c3ac2ccfbc4f Mon Sep 17 00:00:00 2001
From: Many the fish <many@meilisearch.com>
Date: Wed, 24 Jul 2024 17:04:09 +0200
Subject: [PATCH 19/25] Update charabia/src/tokenizer.rs

---
 charabia/src/tokenizer.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/charabia/src/tokenizer.rs b/charabia/src/tokenizer.rs
index ba39fb83..36b8443b 100644
--- a/charabia/src/tokenizer.rs
+++ b/charabia/src/tokenizer.rs
@@ -99,8 +99,8 @@ impl Tokenize<'_> for &str {
 /// See [`TokenizerBuilder`] to know how to build a [`Tokenizer`].
 #[derive(Debug)]
 pub struct Tokenizer<'tb> {
-    pub segmenter_option: Cow<'tb, SegmenterOption<'tb>>,
-    pub normalizer_option: Cow<'tb, NormalizerOption<'tb>>,
+    segmenter_option: Cow<'tb, SegmenterOption<'tb>>,
+    normalizer_option: Cow<'tb, NormalizerOption<'tb>>,
 }
 
 impl<'tb> Tokenizer<'tb> {

From 927990954a077868214b5962c77167cab28f2b24 Mon Sep 17 00:00:00 2001
From: ManyTheFish <ManyTheFish@users.noreply.github.com>
Date: Thu, 25 Jul 2024 11:47:39 +0000
Subject: [PATCH 20/25] Update version for the next release (v0.9.0) in
 Cargo.toml files

---
 charabia/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charabia/Cargo.toml b/charabia/Cargo.toml
index 69562d5b..3a5efe7f 100644
--- a/charabia/Cargo.toml
+++ b/charabia/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "charabia"
-version = "0.8.12"
+version = "0.9.0"
 license = "MIT"
 authors = ["Many <many@meilisearch.com>"]
 edition = "2021"

From fc236dd419285239922c0b04623a14536323ce86 Mon Sep 17 00:00:00 2001
From: ManyTheFish <many@meilisearch.com>
Date: Thu, 25 Jul 2024 14:22:41 +0200
Subject: [PATCH 21/25] Activate swedish by default

---
 charabia/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charabia/Cargo.toml b/charabia/Cargo.toml
index 3a5efe7f..6bf05688 100644
--- a/charabia/Cargo.toml
+++ b/charabia/Cargo.toml
@@ -31,7 +31,7 @@ unicode-normalization = "0.1.23"
 irg-kvariants = { path = "../irg-kvariants", version = "=0.1.1" }
 
 [features]
-default = ["chinese", "hebrew", "japanese", "thai", "korean", "greek", "latin-camelcase", "latin-snakecase", "khmer", "vietnamese"]
+default = ["chinese", "hebrew", "japanese", "thai", "korean", "greek", "latin-camelcase", "latin-snakecase", "khmer", "vietnamese", "swedish-recomposition"]
 
 # allow chinese specialized tokenization
 chinese = ["chinese-segmentation", "chinese-normalization"]

From 9daa84b0628a4e06534fbfb1725c28bed9fefbca Mon Sep 17 00:00:00 2001
From: ManyTheFish <many@meilisearch.com>
Date: Thu, 25 Jul 2024 15:14:05 +0200
Subject: [PATCH 22/25] fix clippy

---
 charabia/src/normalizer/arabic.rs                | 4 ++--
 charabia/src/normalizer/swedish_recomposition.rs | 2 +-
 charabia/src/separators.rs                       | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/charabia/src/normalizer/arabic.rs b/charabia/src/normalizer/arabic.rs
index b12812f0..306b06e1 100644
--- a/charabia/src/normalizer/arabic.rs
+++ b/charabia/src/normalizer/arabic.rs
@@ -8,8 +8,8 @@ use crate::{Script, Token};
 /// - normalizing the arabic Alef 'أ','إ','آ','ٱ' to 'ا'
 /// - normalizing the arabic Yeh 'ى' to 'ي'
 /// - Normalizing the arabic Taa Marbuta 'ة' to 'ه'
-/// https://en.wikipedia.org/wiki/Arabic_alphabet
-/// https://en.wikipedia.org/wiki/Kashida
+///   https://en.wikipedia.org/wiki/Arabic_alphabet
+///   https://en.wikipedia.org/wiki/Kashida
 
 pub struct ArabicNormalizer;
 
diff --git a/charabia/src/normalizer/swedish_recomposition.rs b/charabia/src/normalizer/swedish_recomposition.rs
index 9a38da41..cd77e28e 100644
--- a/charabia/src/normalizer/swedish_recomposition.rs
+++ b/charabia/src/normalizer/swedish_recomposition.rs
@@ -8,7 +8,7 @@ use crate::normalizer::NormalizerOption;
 use crate::{Language, Token};
 
 static MATCHING_STR: Lazy<AhoCorasick> = Lazy::new(|| {
-    AhoCorasick::new(&["A\u{30a}", "a\u{30a}", "A\u{308}", "a\u{308}", "O\u{308}", "o\u{308}"])
+    AhoCorasick::new(["A\u{30a}", "a\u{30a}", "A\u{308}", "a\u{308}", "O\u{308}", "o\u{308}"])
         .unwrap()
 });
 
diff --git a/charabia/src/separators.rs b/charabia/src/separators.rs
index c8212046..29fc26ac 100644
--- a/charabia/src/separators.rs
+++ b/charabia/src/separators.rs
@@ -12,8 +12,8 @@
 /// - Zl Line Separator
 /// - Zp Paragraph Separator
 /// - Zs Space Separator
-/// plus "\0", ". ", ", " and ។ល។" (៘ decomposition) to categorize them as hard separators
-/// and "`" to understand markdown formatted text
+///   plus "\0", ". ", ", " and ។ល។" (៘ decomposition) to categorize them as hard separators
+///   and "`" to understand markdown formatted text
 #[rustfmt::skip]
 pub const DEFAULT_SEPARATORS: &[&str] = &[
     "\0", ". ", ", ", "_", "‿", "⁀", "⁔", "︳", "︴", "﹍", "﹎", "﹏", "＿", "-", "֊", "־", "᐀", "᠆", "‐", "‒", "–",

From 6debc92637392674eb963389de1023802630be74 Mon Sep 17 00:00:00 2001
From: ToshinoriTakahashi <dr.t.takahashi.0917@gmail.com>
Date: Sun, 18 Aug 2024 13:15:07 +0900
Subject: [PATCH 23/25] Add turkish feature

---
 charabia/Cargo.toml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/charabia/Cargo.toml b/charabia/Cargo.toml
index 6bf05688..f9e2aef3 100644
--- a/charabia/Cargo.toml
+++ b/charabia/Cargo.toml
@@ -31,7 +31,7 @@ unicode-normalization = "0.1.23"
 irg-kvariants = { path = "../irg-kvariants", version = "=0.1.1" }
 
 [features]
-default = ["chinese", "hebrew", "japanese", "thai", "korean", "greek", "latin-camelcase", "latin-snakecase", "khmer", "vietnamese", "swedish-recomposition"]
+default = ["chinese", "hebrew", "japanese", "thai", "korean", "greek", "latin-camelcase", "latin-snakecase", "khmer", "vietnamese", "swedish-recomposition", "turkish"]
 
 # allow chinese specialized tokenization
 chinese = ["chinese-segmentation", "chinese-normalization"]
@@ -71,6 +71,9 @@ latin-snakecase = ["dep:finl_unicode"]
 # force Charabia to recompose Swedish characters
 swedish-recomposition = []
 
+# allow turkish specialized tokenization
+turkish = []
+
 [dev-dependencies]
 criterion = "0.5"
 jemallocator = "0.5.4"

From a8014ed8d0048fb7fa58132172896d6c90ecaf76 Mon Sep 17 00:00:00 2001
From: ToshinoriTakahashi <dr.t.takahashi.0917@gmail.com>
Date: Mon, 19 Aug 2024 01:34:07 +0900
Subject: [PATCH 24/25] Add turkish normalizer definition and tests

---
 charabia/src/normalizer/mod.rs     |   6 +
 charabia/src/normalizer/turkish.rs | 421 +++++++++++++++++++++++++++++
 2 files changed, 427 insertions(+)
 create mode 100644 charabia/src/normalizer/turkish.rs

diff --git a/charabia/src/normalizer/mod.rs b/charabia/src/normalizer/mod.rs
index fa28e83c..2cc31ad2 100644
--- a/charabia/src/normalizer/mod.rs
+++ b/charabia/src/normalizer/mod.rs
@@ -17,6 +17,8 @@ use self::nonspacing_mark::NonspacingMarkNormalizer;
 use self::quote::QuoteNormalizer;
 #[cfg(feature = "swedish-recomposition")]
 use self::swedish_recomposition::SwedishRecompositionNormalizer;
+#[cfg(feature = "turkish")]
+pub use self::turkish::TurkishNormalizer;
 #[cfg(feature = "vietnamese")]
 pub use self::vietnamese::VietnameseNormalizer;
 use crate::segmenter::SegmentedTokenIter;
@@ -39,6 +41,8 @@ mod nonspacing_mark;
 mod quote;
 #[cfg(feature = "swedish-recomposition")]
 mod swedish_recomposition;
+#[cfg(feature = "turkish")]
+mod turkish;
 #[cfg(feature = "vietnamese")]
 mod vietnamese;
 
@@ -71,6 +75,8 @@ pub static LOSSY_NORMALIZERS: Lazy<Vec<Box<dyn Normalizer>>> = Lazy::new(|| {
         Box::new(NonspacingMarkNormalizer),
         #[cfg(feature = "vietnamese")]
         Box::new(VietnameseNormalizer),
+        #[cfg(feature = "turkish")]
+        Box::new(TurkishNormalizer),
     ]
 });
 
diff --git a/charabia/src/normalizer/turkish.rs b/charabia/src/normalizer/turkish.rs
new file mode 100644
index 00000000..03dfb0d7
--- /dev/null
+++ b/charabia/src/normalizer/turkish.rs
@@ -0,0 +1,421 @@
+use super::{CharNormalizer, CharOrStr};
+use crate::{Script, Token};
+
+/// Turkish specialized [`Normalizer`].
+///
+/// Turkish text should be normalized by:
+/// - Normalizing the Turkish alphabet: 'ı' to 'i'
+///
+/// There are other peculiar characters in the Turkish alphabet[1],
+/// but this `Normalizer` only supports 'ı', as normalization
+/// is already achieved by the existing `Normalizer` used
+/// in the case of `Script::Latin`.
+///
+/// [1]: https://en.wikipedia.org/wiki/Turkish_alphabet
+pub struct TurkishNormalizer;
+
+impl CharNormalizer for TurkishNormalizer {
+    fn normalize_char(&self, c: char) -> Option<CharOrStr> {
+        match c {
+            'ı' => Some("i".to_string().into()),
+            _ => Some(c.into()),
+        }
+    }
+
+    fn should_normalize(&self, token: &Token) -> bool {
+        token.script == Script::Latin && token.lemma.chars().any(is_should_normalize)
+    }
+}
+
+fn is_should_normalize(c: char) -> bool {
+    c == 'ı'
+}
+
+#[cfg(test)]
+mod test {
+    use std::borrow::Cow::Owned;
+
+    use crate::normalizer::test::test_normalizer;
+    use crate::normalizer::{Normalizer, NormalizerOption};
+    use crate::token::TokenKind;
+
+    // base tokens to normalize.
+    fn tokens() -> Vec<Token<'static>> {
+        vec![
+            // Turkish alphabet
+            Token {
+                lemma: Owned("ABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZ".to_string()),
+                char_end: 29,
+                byte_end: 35,
+                script: Script::Latin,
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("abcçdefgğhıijklmnoöprsştuüvyz".to_string()),
+                char_end: 29,
+                byte_end: 35,
+                script: Script::Latin,
+                ..Default::default()
+            },
+            // Turkish texts containing 'ı'
+            Token {
+                lemma: Owned("çalışma".to_string()),
+                char_end: 7,
+                byte_end: 10,
+                script: Script::Latin,
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("şarkı".to_string()),
+                char_end: 5,
+                byte_end: 7,
+                script: Script::Latin,
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("ışık".to_string()),
+                char_end: 4,
+                byte_end: 7,
+                script: Script::Latin,
+                ..Default::default()
+            },
+            // Turkish texts without 'ı'
+            // - verify that the complete pipeline normalizes turkish text as expected
+            Token {
+                lemma: Owned("günlük".to_string()),
+                char_end: 6,
+                byte_end: 8,
+                script: Script::Latin,
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("İstanbul".to_string()),
+                char_end: 8,
+                byte_end: 9,
+                script: Script::Latin,
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("İstasyon".to_string()),
+                char_end: 8,
+                byte_end: 9,
+                script: Script::Latin,
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("ömür".to_string()),
+                char_end: 4,
+                byte_end: 6,
+                script: Script::Latin,
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("ütü".to_string()),
+                char_end: 3,
+                byte_end: 5,
+                script: Script::Latin,
+                ..Default::default()
+            },
+        ]
+    }
+
+    // expected result of the current Normalizer.
+    fn normalizer_result() -> Vec<Token<'static>> {
+        vec![
+            // Turkish alphabet
+            Token {
+                lemma: Owned("ABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZ".to_string()),
+                char_end: 29,
+                byte_end: 35,
+                script: Script::Latin,
+                char_map: None,
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("abcçdefgğhiijklmnoöprsştuüvyz".to_string()),
+                char_end: 29,
+                byte_end: 35,
+                script: Script::Latin,
+                char_map: Some(vec![
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 2),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 2),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 2),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 2),
+                    (1, 1),
+                    (1, 1),
+                    (2, 2),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                ]),
+                ..Default::default()
+            },
+            // Turkish texts containing 'ı'
+            Token {
+                lemma: Owned("çalişma".to_string()),
+                char_end: 7,
+                byte_end: 10,
+                script: Script::Latin,
+                char_map: Some(vec![(2, 2), (1, 1), (1, 1), (2, 1), (2, 2), (1, 1), (1, 1)]),
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("şarki".to_string()),
+                char_end: 5,
+                byte_end: 7,
+                script: Script::Latin,
+                char_map: Some(vec![(2, 2), (1, 1), (1, 1), (1, 1), (2, 1)]),
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("işik".to_string()),
+                char_end: 4,
+                byte_end: 7,
+                script: Script::Latin,
+                char_map: Some(vec![(2, 1), (2, 2), (2, 1), (1, 1)]),
+                ..Default::default()
+            },
+            // Turkish texts without 'ı'
+            // - verify that the complete pipeline normalizes turkish text as expected
+            Token {
+                lemma: Owned("günlük".to_string()),
+                char_end: 6,
+                byte_end: 8,
+                script: Script::Latin,
+                char_map: None,
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("İstanbul".to_string()),
+                char_end: 8,
+                byte_end: 9,
+                script: Script::Latin,
+                char_map: None,
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("İstasyon".to_string()),
+                char_end: 8,
+                byte_end: 9,
+                script: Script::Latin,
+                char_map: None,
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("ömür".to_string()),
+                char_end: 4,
+                byte_end: 6,
+                script: Script::Latin,
+                char_map: None,
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("ütü".to_string()),
+                char_end: 3,
+                byte_end: 5,
+                script: Script::Latin,
+                char_map: None,
+                ..Default::default()
+            },
+        ]
+    }
+
+    // expected result of the complete Normalizer pipeline.
+    fn normalized_tokens() -> Vec<Token<'static>> {
+        vec![
+            // Turkish alphabet
+            Token {
+                lemma: Owned("abccdefgghiijklmnooprsstuuvyz".to_string()),
+                char_end: 29,
+                byte_end: 35,
+                script: Script::Latin,
+                kind: TokenKind::Word,
+                char_map: Some(vec![
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                ]),
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("abccdefgghiijklmnooprsstuuvyz".to_string()),
+                char_end: 29,
+                byte_end: 35,
+                script: Script::Latin,
+                kind: TokenKind::Word,
+                char_map: Some(vec![
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                ]),
+                ..Default::default()
+            },
+            // Turkish texts containing 'ı'
+            Token {
+                lemma: Owned("calisma".to_string()),
+                char_end: 7,
+                byte_end: 10,
+                script: Script::Latin,
+                kind: TokenKind::Word,
+                char_map: Some(vec![(2, 1), (1, 1), (1, 1), (2, 1), (2, 1), (1, 1), (1, 1)]),
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("sarki".to_string()),
+                char_end: 5,
+                byte_end: 7,
+                script: Script::Latin,
+                kind: TokenKind::Word,
+                char_map: Some(vec![(2, 1), (1, 1), (1, 1), (1, 1), (2, 1)]),
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("isik".to_string()),
+                char_end: 4,
+                byte_end: 7,
+                script: Script::Latin,
+                kind: TokenKind::Word,
+                char_map: Some(vec![(2, 1), (2, 1), (2, 1), (1, 1)]),
+                ..Default::default()
+            },
+            // Turkish texts without 'ı'
+            // - verify that the complete pipeline normalizes turkish text as expected
+            Token {
+                lemma: Owned("gunluk".to_string()),
+                char_end: 6,
+                byte_end: 8,
+                script: Script::Latin,
+                kind: TokenKind::Word,
+                char_map: Some(vec![(1, 1), (2, 1), (1, 1), (1, 1), (2, 1), (1, 1)]),
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("istanbul".to_string()),
+                char_end: 8,
+                byte_end: 9,
+                script: Script::Latin,
+                kind: TokenKind::Word,
+                char_map: Some(vec![
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                ]),
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("istasyon".to_string()),
+                char_end: 8,
+                byte_end: 9,
+                script: Script::Latin,
+                kind: TokenKind::Word,
+                char_map: Some(vec![
+                    (2, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                    (1, 1),
+                ]),
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("omur".to_string()),
+                char_end: 4,
+                byte_end: 6,
+                script: Script::Latin,
+                kind: TokenKind::Word,
+                char_map: Some(vec![(2, 1), (1, 1), (2, 1), (1, 1)]),
+                ..Default::default()
+            },
+            Token {
+                lemma: Owned("utu".to_string()),
+                char_end: 3,
+                byte_end: 5,
+                script: Script::Latin,
+                kind: TokenKind::Word,
+                char_map: Some(vec![(2, 1), (1, 1), (2, 1)]),
+                ..Default::default()
+            },
+        ]
+    }
+
+    test_normalizer!(TurkishNormalizer, tokens(), normalizer_result(), normalized_tokens());
+}

From d365fcfaed68e2ef90288bd9100c347efae8a172 Mon Sep 17 00:00:00 2001
From: ToshinoriTakahashi <dr.t.takahashi.0917@gmail.com>
Date: Mon, 19 Aug 2024 01:43:41 +0900
Subject: [PATCH 25/25] Update README.md

---
 charabia/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charabia/README.md b/charabia/README.md
index 9bc0d01b..4ee97707 100644
--- a/charabia/README.md
+++ b/charabia/README.md
@@ -16,7 +16,7 @@ Charabia provides a simple API to segment, normalize, or tokenize (segment + nor
 
 |  Script / Language  |                           specialized segmentation                            | specialized normalization | Segmentation Performance level | Tokenization Performance level |
 |---------------------|-------------------------------------------------------------------------------|---------------------------|-------------------|---|
-| **Latin** | ✅ CamelCase segmentation | ✅ [compatibility decomposition](https://unicode.org/reports/tr15/) + lowercase + [nonspacing-marks](https://www.compart.com/en/unicode/category/Mn) removal + `Ð vs Đ` spoofing normalization         | 🟩 ~23MiB/sec    | 🟨 ~9MiB/sec    |
+| **Latin** | ✅ CamelCase segmentation | ✅ [compatibility decomposition](https://unicode.org/reports/tr15/) + lowercase + [nonspacing-marks](https://www.compart.com/en/unicode/category/Mn) removal + `Ð vs Đ` spoofing normalization + `ı` normalization | 🟩 ~23MiB/sec    | 🟨 ~9MiB/sec    |
 | **Greek** | ❌ | ✅ [compatibility decomposition](https://unicode.org/reports/tr15/) + lowercase + final sigma normalization         | 🟩 ~27MiB/sec    | 🟨 ~8MiB/sec    |
 | **Cyrillic** - **Georgian** | ❌ | ✅ [compatibility decomposition](https://unicode.org/reports/tr15/) + lowercase          | 🟩 ~27MiB/sec    | 🟨 ~9MiB/sec    |
 | **Chinese** **CMN** 🇨🇳 | ✅ [jieba](https://github.com/messense/jieba-rs) | ✅ [compatibility decomposition](https://unicode.org/reports/tr15/) + kvariant conversion | 🟨 ~10MiB/sec    | 🟧 ~5MiB/sec    |