From 1878ed9dd7f3164a9fa012defdf4dae7428fec97 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Wed, 28 Feb 2024 14:12:30 +0100 Subject: [PATCH 1/8] Added deepcopying of inputs collected by InferRequestWrapper. Added a test covering the fixed issue. --- optimum/intel/openvino/quantization.py | 3 +- tests/openvino/test_quantization.py | 41 ++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index 5fd743c5b4..b900a09949 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import inspect import logging import os @@ -102,7 +103,7 @@ def start_async( *, shared_memory: Any = None, ): - self.data_cache.append(inputs) + self.data_cache.append(copy.deepcopy(inputs)) self.request.infer(inputs, share_inputs, share_outputs=True) def wait(self): diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 07a9f14774..721a0f7379 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -16,10 +16,13 @@ import tempfile import unittest +from collections import defaultdict from functools import partial +from itertools import islice import evaluate import numpy as np +import torch from datasets import load_dataset from parameterized import parameterized import openvino.runtime as ov @@ -30,6 +33,7 @@ AutoModelForCausalLM, AutoModelForTokenClassification, AutoTokenizer, + AutoProcessor, TrainingArguments, default_data_collator, ) @@ -45,6 +49,7 @@ OVModelForSeq2SeqLM, OVModelForSequenceClassification, OVModelForTokenClassification, + OVModelForSpeechSeq2Seq, OVStableDiffusionPipeline, OVStableDiffusionXLPipeline, OVQuantizer, @@ -54,6 +59,7 @@ from optimum.intel.openvino.configuration import INT8_WEIGHT_COMPRESSION_CONFIG +from optimum.intel.openvino.quantization import InferRequestWrapper from optimum.intel.utils.import_utils import is_openvino_version from utils_tests import MODEL_NAMES, get_num_quantized_nodes, _ARCHITECTURES_TO_EXPECTED_INT8 @@ -589,3 +595,38 @@ def compute_metrics(p): tokens = tokenizer("This is a sample input", return_tensors="pt") outputs = model(**tokens) self.assertTrue("logits" in outputs) + + +class InferRequestWrapperTest(unittest.TestCase): + MODEL_ID = ("openai/whisper-tiny.en",) + DATASET_ID = ("hf-internal-testing/librispeech_asr_dummy",) + + @staticmethod + def _extract_input_features(processor, sample): + input_features = processor( + sample["audio"]["array"], + sampling_rate=sample["audio"]["sampling_rate"], + return_tensors="pt", + ).input_features + return input_features + + @parameterized.expand(zip(MODEL_ID, DATASET_ID)) + def test_calibration_data_uniqueness(self, model_id, dataset_id): + ov_model = OVModelForSpeechSeq2Seq.from_pretrained(model_id, export=True, compile=True) + processor = AutoProcessor.from_pretrained(model_id) + + dataset = load_dataset(dataset_id, "clean", split="validation") + calibration_data = [] + ov_model.decoder_with_past.request = InferRequestWrapper(ov_model.decoder_with_past.request, calibration_data) + for data in islice(dataset, 2): + input_features = self._extract_input_features(processor, data) + ov_model.generate(input_features) + + data_hashes_per_key = defaultdict(list) + for inputs_dict in calibration_data: + for k, v in inputs_dict.items(): + x = (v.numpy() if isinstance(v, torch.Tensor) else v).copy() + data_hashes_per_key[k].append(hash(x.tobytes())) + for k, data_hashes in data_hashes_per_key.items(): + # All hashes must not be equal because calibration dataset contains at least 2 different samples + assert any(data_hashes[0] != it for it in data_hashes), f"Collected samples are all equal for input {k}" From a05a6d1a9b54248e40c8f7669123e92c02c51f39 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Wed, 28 Feb 2024 14:16:30 +0100 Subject: [PATCH 2/8] Phrasing tweaks --- tests/openvino/test_quantization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 721a0f7379..63b91d0bd5 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -628,5 +628,5 @@ def test_calibration_data_uniqueness(self, model_id, dataset_id): x = (v.numpy() if isinstance(v, torch.Tensor) else v).copy() data_hashes_per_key[k].append(hash(x.tobytes())) for k, data_hashes in data_hashes_per_key.items(): - # All hashes must not be equal because calibration dataset contains at least 2 different samples - assert any(data_hashes[0] != it for it in data_hashes), f"Collected samples are all equal for input {k}" + # All hashes can not be equal because calibration dataset contains at least 2 different samples + assert any(data_hashes[0] != it for it in data_hashes), f"Collected tensors are all equal for input {k}" From d0b3978acdbbf3eb55170eebc6ae4377d358df37 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Wed, 28 Feb 2024 15:29:43 +0100 Subject: [PATCH 3/8] Add soundfile to test requirements --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1701af990c..ee3bdf2f0b 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ "timm", "invisible-watermark>=0.2.0", "auto-gptq", + "soundfile" ] QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241"] From fe84cac8cfd5bb07b409a3f0fee776a86b2d83d0 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Wed, 28 Feb 2024 16:25:24 +0100 Subject: [PATCH 4/8] Added librosa to test requirements --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ee3bdf2f0b..563d0e8eed 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,8 @@ "timm", "invisible-watermark>=0.2.0", "auto-gptq", - "soundfile" + "librosa", + "soundfile", ] QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241"] From f3a84a4e7f563b3bdcf6a8fb80d7f98ca0256b27 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Wed, 28 Feb 2024 16:56:36 +0100 Subject: [PATCH 5/8] Added copying to other data cache appends --- optimum/intel/openvino/quantization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index b900a09949..f7fedacf6f 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -88,11 +88,11 @@ def __init__(self, request, data_cache=None): self.data_cache = data_cache def __call__(self, *args, **kwargs): - self.data_cache.append(*args) + self.data_cache.append(copy.deepcopy(args)) return self.request(*args, **kwargs) def infer(self, inputs: Any = None, share_inputs: bool = False): - self.data_cache.append(inputs) + self.data_cache.append(copy.deepcopy(inputs)) return self.request.infer(inputs, share_inputs) def start_async( From 412a10cd5bbe430b5b5cfc9749b9ea88ef535318 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Wed, 28 Feb 2024 21:04:19 +0100 Subject: [PATCH 6/8] Remove the need for real test data --- setup.py | 2 -- tests/openvino/test_quantization.py | 19 +++++++++---------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index 563d0e8eed..1701af990c 100644 --- a/setup.py +++ b/setup.py @@ -34,8 +34,6 @@ "timm", "invisible-watermark>=0.2.0", "auto-gptq", - "librosa", - "soundfile", ] QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241"] diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 63b91d0bd5..7e020a7f93 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -18,7 +18,6 @@ import unittest from collections import defaultdict from functools import partial -from itertools import islice import evaluate import numpy as np @@ -599,27 +598,27 @@ def compute_metrics(p): class InferRequestWrapperTest(unittest.TestCase): MODEL_ID = ("openai/whisper-tiny.en",) - DATASET_ID = ("hf-internal-testing/librispeech_asr_dummy",) @staticmethod - def _extract_input_features(processor, sample): + def _generate_random_audio_data(processor): + t = np.linspace(0, 1.0, int(1000), endpoint=False) + audio_data = 0.5 * np.sin((2 + np.random.random()) * np.pi * t) input_features = processor( - sample["audio"]["array"], - sampling_rate=sample["audio"]["sampling_rate"], + audio_data, + sampling_rate=16000, return_tensors="pt", ).input_features return input_features - @parameterized.expand(zip(MODEL_ID, DATASET_ID)) - def test_calibration_data_uniqueness(self, model_id, dataset_id): + @parameterized.expand(MODEL_ID) + def test_calibration_data_uniqueness(self, model_id): ov_model = OVModelForSpeechSeq2Seq.from_pretrained(model_id, export=True, compile=True) processor = AutoProcessor.from_pretrained(model_id) - dataset = load_dataset(dataset_id, "clean", split="validation") calibration_data = [] ov_model.decoder_with_past.request = InferRequestWrapper(ov_model.decoder_with_past.request, calibration_data) - for data in islice(dataset, 2): - input_features = self._extract_input_features(processor, data) + for _ in range(2): + input_features = self._generate_random_audio_data(processor) ov_model.generate(input_features) data_hashes_per_key = defaultdict(list) From 3b078e1d0dd6baadb16f95999dfa17f99662660a Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Thu, 29 Feb 2024 11:18:05 +0100 Subject: [PATCH 7/8] Process __call__ call properly --- optimum/intel/openvino/quantization.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index f7fedacf6f..3cce386187 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -88,7 +88,10 @@ def __init__(self, request, data_cache=None): self.data_cache = data_cache def __call__(self, *args, **kwargs): - self.data_cache.append(copy.deepcopy(args)) + # If __call__ is invoked then self.request must be an instance of CompiledModel + signature = inspect.signature(self.request) + bound_args = signature.bind(*args, **kwargs).arguments + self.data_cache.append(copy.deepcopy(bound_args["inputs"])) return self.request(*args, **kwargs) def infer(self, inputs: Any = None, share_inputs: bool = False): From bdc6f049d831d97846378a909d98a47220332df4 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Thu, 29 Feb 2024 20:25:33 +0100 Subject: [PATCH 8/8] Addressed suggested changes --- tests/openvino/test_quantization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 7e020a7f93..16f848da9e 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -628,4 +628,4 @@ def test_calibration_data_uniqueness(self, model_id): data_hashes_per_key[k].append(hash(x.tobytes())) for k, data_hashes in data_hashes_per_key.items(): # All hashes can not be equal because calibration dataset contains at least 2 different samples - assert any(data_hashes[0] != it for it in data_hashes), f"Collected tensors are all equal for input {k}" + self.assertTrue(any(data_hashes[0] != it for it in data_hashes))