take sentencepiece chages

pavel-esir · Aug 26, 2024 · a3fa982 · a3fa982
1 parent 207a124
commit a3fa982
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/.github/workflows/causal_lm_cpp.yml b/.github/workflows/causal_lm_cpp.yml
@@ -54,6 +54,7 @@ jobs:
           | diff <(timeout 50s samples/python/greedy_causal_lm/greedy_causal_lm.py ./open_llama_3b_v2/ "return 0") -
 
   cpp-beam_search_causal_lm-ubuntu:
+    runs-on: ubuntu-22.04-16-cores
     strategy:
       matrix:
         executable:
@@ -387,7 +388,7 @@ jobs:
           echo "Alan Turing was a" passed
 
   cpp-prompt_lookup_decoding_lm-ubuntu:
-    runs-on: ubuntu-20.04-16-cores
+    runs-on: ubuntu-22.04-16-cores
     steps:
       - uses: actions/checkout@v4
         with:

diff --git a/thirdparty/openvino_tokenizers b/thirdparty/openvino_tokenizers
+0 −2		.github/workflows/linux.yml
+6 −2		CMakeLists.txt
+46 −22		README.md
+1 −1		python/openvino_tokenizers/convert_tokenizer.py
+58 −28		python/openvino_tokenizers/hf_parser.py
+4 −0		python/openvino_tokenizers/tokenizer_pipeline.py
+1 −12		src/CMakeLists.txt
+14 −19		src/ov_extension.cpp
+17 −1		src/sentence_piece.cpp
+16 −19		src/tensorflow_translators.cpp
+1 −5		src/tensorflow_translators.hpp
+1 −0		tests/conftest.py
+1 −1		tests/pass_rates.json
+1,515 −339		tests/stats.json
+216 −108		tests/tokenizers_test.py