1 file changed
+1
-1
lines changedSubmodule openvino_tokenizers updated 15 files
- CMakeLists.txt+1-1
- pyproject.toml+3-3
- python/openvino_tokenizers/__init__.py+17-2
- python/openvino_tokenizers/build_tokenizer.py+4-4
- python/openvino_tokenizers/hf_parser.py+8-4
- python/openvino_tokenizers/str_pack.py-62
- python/openvino_tokenizers/tokenizer_pipeline.py+40-29
- python/openvino_tokenizers/utils.py+42-3
- src/regex_split.cpp+26-9
- src/special_tokens_split.cpp+1-1
- src/string_tensor_pack.cpp-33
- src/string_tensor_pack.hpp+8-10
- src/utils.cpp+2-2
- tests/layer_tests.py+5-6
- tests/stats.json+8.3k-8.3k
0 commit comments