1 file changed
+1
-1
lines changedSubmodule openvino_tokenizers updated 55 files
- .github/dependabot.yml+33
- .github/dependency_review.yml+18
- .github/labeler.yml+25-16
- .github/workflows/labeler.yml+11-3
- .github/workflows/linux.yml+62-11
- .github/workflows/mac.yml+61-10
- .github/workflows/sdl.yml+8-1
- .github/workflows/windows.yml+62-10
- CMakeLists.txt+6-1
- Jenkinsfile+3
- README.md+76-5
- pyproject.toml+6-4
- python/openvino_tokenizers/__init__.py+7-5
- python/openvino_tokenizers/build_tokenizer.py+76
- python/openvino_tokenizers/cli.py+3-1
- python/openvino_tokenizers/hf_parser.py+19-16
- python/openvino_tokenizers/str_pack.py+1-1
- python/openvino_tokenizers/tokenizer_pipeline.py+108-10
- python/openvino_tokenizers/utils.py+4-2
- requirements-build.txt+1-1
- src/CMakeLists.txt+17-5
- src/bpe_tokenizer.cpp+24-28
- src/bpe_tokenizer.hpp+1-1
- src/case_fold.cpp+22-6
- src/case_fold.hpp+9-2
- src/equal_str.cpp+73
- src/equal_str.hpp+40
- src/fuze.cpp+40
- src/fuze.hpp+35
- src/ov_extension.cpp+28-11
- src/ragged_to_ragged.cpp+82
- src/ragged_to_ragged.hpp+41
- src/ragged_to_sparse.cpp+47
- src/ragged_to_sparse.hpp+36
- src/regex_normalization.cpp+31-7
- src/regex_normalization.hpp+13-7
- src/regex_split.cpp+20-5
- src/regex_split.hpp+7-4
- src/sentence_piece.cpp+56-81
- src/sentence_piece.hpp+3-3
- src/tensorflow_translators.cpp+371-104
- src/tensorflow_translators.hpp+9-4
- src/tokenizer.hpp+6
- src/trie_tokenizer.cpp+111
- src/trie_tokenizer.hpp+54
- src/utils.cpp+14-18
- src/utils.hpp+3-1
- src/vocab_decoder.cpp+11-16
- src/vocab_decoder.hpp+1-2
- src/vocab_encoder.cpp+74
- src/vocab_encoder.hpp+45
- src/wordpiece_tokenizer.cpp+22-25
- src/wordpiece_tokenizer.hpp+1-1
- tests/pass_rates.json+1-1
- tests/tokenizers_test.py+2-2
0 commit comments