style: 整理代码，添加注释

Signed-off-by: YdrMaster <ydrml@hotmail.com>

Author: YdrMaster

src/bpe/algorithm.rs

@@ -7,7 +7,7 @@ use std::{
     ops::Range,
 };
 
-pub struct BpeTokenizer<'v, 't> {
+pub struct MergeState<'v, 't> {
     text: &'t [u8],
     bpe: &'v Bpe,
     marks: Vec<Mark>,
@@ -26,7 +26,7 @@ pub struct Iter<'a> {
 }
 
 impl Bpe {
-    pub fn build_tokenizer<'v, 't>(&'v self, text: &'t str) -> BpeTokenizer<'v, 't> {
+    pub fn begin_merge<'v, 't>(&'v self, text: &'t str) -> MergeState<'v, 't> {
         let mut marks = vec![Mark::unk(self.unk); text.len()];
         let mut merges = BinaryHeap::new();
 
@@ -55,7 +55,7 @@ impl Bpe {
             };
         }
 
-        BpeTokenizer {
+        MergeState {
             text: text.as_bytes(),
             bpe: self,
             marks,
@@ -119,7 +119,7 @@ impl PartialOrd for Merge {
     }
 }
 
-impl BpeTokenizer<'_, '_> {
+impl MergeState<'_, '_> {
     /// 尝试执行一次合并，返回是否成功执行了一次合并。
     pub fn merge(&mut self) -> bool {
         // 一次合并将涉及至多 4 个 token：
@@ -203,7 +203,7 @@ impl BpeTokenizer<'_, '_> {
     }
 }
 
-impl<'v> IntoIterator for BpeTokenizer<'v, '_> {
+impl<'v> IntoIterator for MergeState<'v, '_> {
     type Item = utok;
     type IntoIter = IntoIter<'v>;
     #[inline]
@@ -244,7 +244,7 @@ impl Iterator for Iter<'_> {
     }
 }
 
-impl fmt::Display for BpeTokenizer<'_, '_> {
+impl fmt::Display for MergeState<'_, '_> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         use std::str::{from_utf8, from_utf8_unchecked};
 

src/bpe/mod.rs

@@ -3,8 +3,9 @@
 mod algorithm;
 
 use crate::{
-    functions::{collect_vocabs_with_hint, CompressedVocab},
-    utok, Method,
+    utok,
+    vocab::{CollectedVocab, CompressedVocab},
+    Method,
 };
 use std::{
     collections::{HashMap, HashSet},
@@ -37,6 +38,7 @@ struct TokenMeta {
     rank: u32,
 }
 
+// SAFETY: TokenMeta 中的指针是指向 Bpe 内容的自引用指针，且仅用于不可变引用。
 unsafe impl Send for TokenMeta {}
 unsafe impl Sync for TokenMeta {}
 
@@ -96,8 +98,15 @@ impl Bpe {
         is_byte: impl IntoIterator<Item = bool>,
         unk: utok,
     ) -> Self {
-        let (vocabs, bytes, total_len) =
-            collect_vocabs_with_hint(vocabs.into_iter().map(|s| s.as_bytes()), is_byte, unk);
+        let CollectedVocab {
+            vocabs,
+            total_len,
+            bytes,
+        } = CollectedVocab::collect_with_hint(
+            vocabs.into_iter().map(|s| s.as_bytes()),
+            is_byte,
+            unk,
+        );
         let CompressedVocab { vocabs, slices } = CompressedVocab::new(&vocabs, total_len);
         // 收集合词评分
         let scores = scores.into_iter().collect::<Vec<_>>();
@@ -189,7 +198,7 @@ impl Method for Bpe {
     }
     #[inline]
     fn encode(&self, text: &str) -> impl IntoIterator<Item = utok> + '_ {
-        let mut tokenizer = self.build_tokenizer(text);
+        let mut tokenizer = self.begin_merge(text);
         while tokenizer.merge() {}
         tokenizer.into_iter()
     }

src/functions.rs

@@ -1,9 +1,9 @@
 #![deny(warnings)]
 
 mod bpe;
-mod functions;
 mod lpe;
 mod tokeneer;
+mod vocab;
 
 pub use bpe::Bpe;
 pub use lpe::Lpe;

src/lib.rs

@@ -1,8 +1,9 @@
 //! l-p-e for Longest Prefix Encoding
 
 use crate::{
-    functions::{collect_vocabs, CompressedVocab},
-    utok, Method,
+    utok,
+    vocab::{CollectedVocab, CompressedVocab},
+    Method,
 };
 use patricia_tree::PatriciaMap;
 use std::{collections::HashSet, pin::Pin};
@@ -37,7 +38,11 @@ impl Lpe {
     }
 
     pub fn new<'a>(vocabs: impl IntoIterator<Item = &'a [u8]>, unk: utok) -> Self {
-        let (vocabs, bytes, total_len) = collect_vocabs(vocabs, unk);
+        let CollectedVocab {
+            vocabs,
+            total_len,
+            bytes,
+        } = CollectedVocab::collect(vocabs, unk);
         let CompressedVocab { vocabs, slices } = CompressedVocab::new(&vocabs, total_len);
         let tokens = slices
             .into_iter()