Update number of expected exported tokenizers (#940)

* update number of expected exported tokenizers

* tokenizer version

* warning

* fix last test

Author: IlyasMoutawwakil

optimum/exporters/openvino/convert.py

@@ -41,6 +41,7 @@
     _torch_version,
     _transformers_version,
     compare_versions,
+    is_tokenizers_version,
     is_transformers_version,
 )
 from optimum.utils import DEFAULT_DUMMY_SHAPES, is_diffusers_available
@@ -730,6 +731,12 @@ def export_tokenizer(
     except ModuleNotFoundError:
         return
 
+    if is_tokenizers_version(">", "0.19"):
+        logger.warning(
+            "Exporting tokenizers to OpenVINO is not supported for tokenizers version > 0.19. "
+            "Please downgrade to tokenizers version <= 0.19 to export tokenizers to OpenVINO."
+        )
+
     if not isinstance(output, Path):
         output = Path(output)
 

optimum/intel/utils/import_utils.py

@@ -43,6 +43,13 @@
     except importlib_metadata.PackageNotFoundError:
         _transformers_available = False
 
+_tokenizers_available = importlib.util.find_spec("tokenizers") is not None
+_tokenizers_version = "N/A"
+if _tokenizers_available:
+    try:
+        _tokenizers_version = importlib_metadata.version("tokenizers")
+    except importlib_metadata.PackageNotFoundError:
+        _tokenizers_available = False
 
 _torch_available = importlib.util.find_spec("torch") is not None
 _torch_version = "N/A"
@@ -181,6 +188,10 @@ def is_transformers_available():
     return _transformers_available
 
 
+def is_tokenizers_available():
+    return _tokenizers_available
+
+
 def is_neural_compressor_available():
     return _neural_compressor_available
 
@@ -340,6 +351,15 @@ def is_transformers_version(operation: str, version: str):
     return compare_versions(parse(_transformers_version), operation, version)
 
 
+def is_tokenizers_version(operation: str, version: str):
+    """
+    Compare the current Tokenizers version to a given reference with an operation.
+    """
+    if not _tokenizers_available:
+        return False
+    return compare_versions(parse(_tokenizers_version), operation, version)
+
+
 def is_optimum_version(operation: str, version: str):
     return compare_versions(parse(_optimum_version), operation, version)
 

tests/openvino/test_exporters_cli.py

@@ -48,6 +48,7 @@
 from optimum.intel.utils.import_utils import (
     compare_versions,
     is_openvino_tokenizers_available,
+    is_tokenizers_version,
 )
 
 
@@ -73,17 +74,17 @@ class OVCLIExportTestCase(unittest.TestCase):
         ("image-to-image", "stable-diffusion-xl-refiner"),
     )
     EXPECTED_NUMBER_OF_TOKENIZER_MODELS = {
-        "gpt2": 2,
+        "gpt2": 2 if is_tokenizers_version("<", "0.20") else 0,
         "t5": 0,  # no .model file in the repository
         "albert": 0,  # not supported yet
         "distilbert": 1,  # no detokenizer
-        "roberta": 2,
+        "roberta": 2 if is_tokenizers_version("<", "0.20") else 0,
         "vit": 0,  # no tokenizer for image model
         "wav2vec2": 0,  # no tokenizer
         "bert": 1,  # no detokenizer
-        "blenderbot": 2,
-        "stable-diffusion": 2,
-        "stable-diffusion-xl": 4,
+        "blenderbot": 2 if is_tokenizers_version("<", "0.20") else 0,
+        "stable-diffusion": 2 if is_tokenizers_version("<", "0.20") else 0,
+        "stable-diffusion-xl": 4 if is_tokenizers_version("<", "0.20") else 0,
     }
 
     SUPPORTED_SD_HYBRID_ARCHITECTURES = (