chore: remove normalizer from training pipeline

DVDAGames · Feb 11, 2025 · 75e25fd · 75e25fd
1 parent 8f02592
commit 75e25fd
Show file tree

Hide file tree

Showing 4 changed files with 3 additions and 8 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -75,4 +75,4 @@ ignore = [
     "E402",   # false positives for local imports
     "E501",   # line too long
     "TRY003", # external messages in exceptions are too verbose
-]
+]
diff --git a/scripts/train.py b/scripts/train.py
@@ -4,7 +4,6 @@
 from tokenizers import Regex, Tokenizer
 from tokenizers.decoders import ByteLevel as ByteLevelDecoder
 from tokenizers.models import BPE
-from tokenizers.normalizers import NFC
 from tokenizers.pre_tokenizers import Split
 from tokenizers.processors import ByteLevel as ByteLevelProcessor
 from tokenizers.trainers import BpeTrainer
@@ -68,8 +67,6 @@
     ),
 )
 
-tokenizer.normalizer = NFC()
-
 tokenizer.pre_tokenizer = Split(
     pattern=Regex(TOKENIZER_CHUNK_PATTERN),
     behavior="isolated",

diff --git a/src/pgn_tokenizer/config/pgn-tokenizer.json b/src/pgn_tokenizer/config/pgn-tokenizer.json
@@ -40,9 +40,7 @@
       "special": true
     }
   ],
-  "normalizer": {
-    "type": "NFC"
-  },
+  "normalizer": null,
   "pre_tokenizer": {
     "type": "Split",
     "pattern": {

diff --git a/uv.lock b/uv.lock