Update OV Tokenizers Availability Check

Author: apaniukov

optimum/commands/export/openvino.py

@@ -123,7 +123,7 @@ def parse_args_openvino(parser: "ArgumentParser"):
     optional_group.add_argument(
         "--convert-tokenizer",
         action="store_true",
-        help="[Deprecated]  Add converted tokenizer and detokenizer with OpenVINO Tokenizers.",
+        help="[Deprecated] Add converted tokenizer and detokenizer with OpenVINO Tokenizers.",
     )
 
 

optimum/exporters/openvino/__main__.py

@@ -192,12 +192,6 @@ def main_export(
                 f"The task could not be automatically inferred as this is available only for models hosted on the Hugging Face Hub. Please provide the argument --task with the relevant task from {', '.join(TasksManager.get_all_tasks())}. Detailed error: {e}"
             )
 
-    if convert_tokenizer and not is_openvino_tokenizers_available():
-        logger.warning(
-            "`convert_tokenizer` requires openvino-tokenizers, please install it with `pip install optimum-intel[openvino-tokenizers]`"
-        )
-        convert_tokenizer = False
-
     do_gptq_patching = False
     custom_architecture = False
     loading_kwargs = {}

optimum/intel/utils/import_utils.py

@@ -94,21 +94,43 @@
         _openvino_tokenizers_available = False
 
 if _openvino_tokenizers_available and _openvino_tokenizers_version != "N/A":
-    _compatible_openvino_version = next(
-        (
-            requirement.split("==")[-1]
-            for requirement in importlib_metadata.requires("openvino-tokenizers")
-            if requirement.startswith("openvino==")
-        ),
-        "",
-    )
-    _openvino_tokenizers_available = _compatible_openvino_version == ov_major_version
+    _is_ovt_dev_version = "dev" in _openvino_tokenizers_version
+    _ov_version = importlib_metadata.version("openvino")
+    _is_ov_dev_version = "dev" in _ov_version
+    if _is_ovt_dev_version:
+        _compatible_openvino_major_version, _, _dev_date = _openvino_tokenizers_version.rsplit(".", 2)
+        _compatible_ov_version = _compatible_openvino_major_version + "." + _dev_date
+        _compatible_ovt_version = _ov_version.replace("dev", "0.dev")
+    else:
+        _compatible_ov_version = _openvino_tokenizers_version.rsplit(".", 1)[0]
+        _compatible_ovt_version = _ov_version + ".0"
+
+    _openvino_tokenizers_available = _ov_version == _compatible_ov_version
+
     if not _openvino_tokenizers_available:
+        _update_ov_command = (
+            f"pip install {'--pre' if _is_ovt_dev_version else ''} -U openvino=={_compatible_ov_version} "
+            + (
+                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
+                if _is_ovt_dev_version
+                else ""
+            )
+        ).strip()
+        _update_ovt_command = (
+            f"pip install {'--pre' if _is_ov_dev_version else ''} -U openvino-tokenizers=={_compatible_ovt_version} "
+            + (
+                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
+                if _is_ov_dev_version
+                else ""
+            )
+        ).strip()
         logger.warning(
             "OpenVINO Tokenizer version is not compatible with OpenVINO version. "
-            f"Installed OpenVINO version: {ov_major_version},"
-            f"OpenVINO Tokenizers requires {_compatible_openvino_version}. "
-            f"OpenVINO Tokenizers models will not be added during export."
+            f"Installed OpenVINO version: {_ov_version}, "
+            f"OpenVINO Tokenizers requires {_compatible_ov_version}. "
+            "OpenVINO Tokenizers models will not be added during export. "
+            f"Update OpenVINO with \n{_update_ov_command}\n"
+            f"Or update OpenVINO Tokenizers with \n{_update_ovt_command}"
         )
 
 _nncf_available = importlib.util.find_spec("nncf") is not None