Apply Review Comments

huggingface · May 7, 2024 · ed5cbb9 · ed5cbb9
1 parent 1f44ce9
commit ed5cbb9
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 11 deletions.
diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py
@@ -226,6 +226,9 @@ def run(self):
             )
             library_name = "transformers"
 
+        if self.args.convert_tokenizer:
+            logger.warning("`--convert-tokenizer` option is deprecated. Tokenizer will be converted by default.")
+
         if (
             library_name == "diffusers"
             and ov_config
@@ -261,6 +264,9 @@ def run(self):
             )
             model.save_pretrained(self.args.output)
 
+            if self.args.disable_convert_tokenizer:
+                return
+
             # not export when using other exporters
             from ...exporters.openvino.convert import export_tokenizer
 
@@ -273,9 +279,6 @@ def run(self):
             if tokenizer_2 is not None:
                 export_tokenizer(tokenizer_2, output / "tokenizer_2")
         else:
-            if self.args.convert_tokenizer:
-                logger.warning("`--convert-tokenizer` option is deprecated. Tokenizer will be converted by default.")
-
             # TODO : add input shapes
             main_export(
                 model_name_or_path=self.args.model,

diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py
@@ -358,7 +358,7 @@ class StoreAttr(object):
     # hide openvino import when using other exporters
     # avoid circular import
     from optimum.exporters.openvino.convert import export_tokenizer
-    from optimum.intel.openvino.utils import OV_TOKENIZER_FLOLDER
+    from optimum.intel.openvino.utils import OV_TOKENIZER_FOLDER
 
     if convert_tokenizer and is_openvino_tokenizers_available():
         if library_name != "diffusers":
@@ -369,7 +369,7 @@ class StoreAttr(object):
 
             if tokenizer is not None:
                 try:
-                    export_tokenizer(tokenizer, output / OV_TOKENIZER_FLOLDER)
+                    export_tokenizer(tokenizer, output / OV_TOKENIZER_FOLDER)
                 except Exception as exception:
                     logger.warning(
                         "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer "

diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py
@@ -18,7 +18,7 @@
 import os
 from glob import glob
 from pathlib import Path
-from typing import List, Union
+from typing import Tuple, Union
 
 import numpy as np
 from huggingface_hub import model_info
@@ -34,7 +34,7 @@
 OV_DECODER_NAME = "openvino_decoder_model.xml"
 OV_DECODER_WITH_PAST_NAME = "openvino_decoder_with_past_model.xml"
 
-OV_TOKENIZER_FLOLDER = "openvino_tokenizer"
+OV_TOKENIZER_FOLDER = "openvino_tokenizer"
 OV_TOKENIZER_NAME = "openvino_tokenizer{}.xml"
 OV_DETOKENIZER_NAME = "openvino_detokenizer{}.xml"
 
@@ -111,9 +111,7 @@
 }
 
 
-NEED_CONVERT_TO_FAST_TOKENIZER: List[type(PreTrainedTokenizer)] = [
-    CLIPTokenizer,
-]
+NEED_CONVERT_TO_FAST_TOKENIZER: Tuple[type(PreTrainedTokenizer)] = (CLIPTokenizer,)
 
 
 def maybe_convert_tokenizer_to_fast(
@@ -122,7 +120,7 @@ def maybe_convert_tokenizer_to_fast(
     if isinstance(hf_tokenizer, PreTrainedTokenizerFast):
         return hf_tokenizer
 
-    if any(isinstance(hf_tokenizer, slow_class) for slow_class in NEED_CONVERT_TO_FAST_TOKENIZER):
+    if isinstance(hf_tokenizer, NEED_CONVERT_TO_FAST_TOKENIZER):
         try:
             return AutoTokenizer.from_pretrained(tokenizer_path)
         except Exception: