Add test for OpenVINO Examples (#602)

* added tests for ov examples

* modified tests for ov examples

* modified tests for ov examples

* modified tests for ov examples

* added tests for ov examples

* added a new file for testing examples

* added dependencies for ov examples

* added config path

* removed audio test

* removed SD test

* removed audio test

* changed installation packages

* added stable diffusion

* removed stable diffusion

* changed installation packages

* added nncf package

* only audio classification

* audio image classification

* audio image q-answering

* image text q-answering

* combined notebooks and examples

* including only openvino examples

* including only openvino examples

* added combined test for notebooks and examples

* added combined test for notebooks and examples

* added SD for examples

* added SD for OV examples

* Running only SD

* Running only SD

* added test for ov examples

* test for ov examples

* torch cpu

* test for ov examples

* added a test script for OV examples v1

* added a test script for OV examples v2

* added test for ov examples

* added test for ov examples v3

* added test for ov examples v4

* added test for ov examples v5

* added test for ov examples v6

* added test for ov examples v7

* replaced suggested models

.github/workflows/test_openvino_examples.yml

@@ -0,0 +1,47 @@
+name: OpenVINO - Examples Test
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron:  '14 3 * * 1'  # run weekly: every Monday at 3:14
+  push:
+    paths:
+    - '.github/workflows/test_openvino_examples.yml'
+    - 'examples/openvino/*'
+  pull_request:
+    paths:
+    - '.github/workflows/test_openvino_examples.yml'
+    - 'examples/openvino/*'
+
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.10"]
+
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        pip install optimum[openvino] jstyleson nncf
+        pip install -r examples/openvino/audio-classification/requirements.txt
+        pip install -r examples/openvino/image-classification/requirements.txt
+        pip install -r examples/openvino/question-answering/requirements.txt
+        pip install -r examples/openvino/text-classification/requirements.txt
+
+    - name: Test examples
+      run: |
+        python examples/openvino/test_examples.py

examples/openvino/test_examples.py

@@ -0,0 +1,145 @@
+#  Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import os
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+SRC_DIRS = [
+    os.path.join(os.path.dirname(__file__), dirname)
+    for dirname in [
+        "text-classification",
+        "question-answering",
+        "audio-classification",
+        "image-classification",
+    ]
+]
+sys.path.extend(SRC_DIRS)
+
+if SRC_DIRS is not None:
+    import run_image_classification
+    import run_audio_classification
+    import run_glue
+    import run_qa
+
+
+class TestExamples(unittest.TestCase):
+    def test_audio_classification(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            test_args = f"""
+                run_audio_classification.py  
+                --model_name_or_path hf-internal-testing/tiny-random-Wav2Vec2Model  
+                --nncf_compression_config  examples/openvino/audio-classification/configs/wav2vec2-base-qat.json  
+                --dataset_name superb  
+                --dataset_config_name ks  
+                --max_train_samples 10  
+                --max_eval_samples 2  
+                --remove_unused_columns False  
+                --do_train  
+                --learning_rate 3e-5  
+                --max_length_seconds 1  
+                --attention_mask False  
+                --warmup_ratio 0.1  
+                --num_train_epochs 1  
+                --gradient_accumulation_steps 1  
+                --dataloader_num_workers 1  
+                --logging_strategy steps  
+                --logging_steps 1  
+                --evaluation_strategy epoch  
+                --save_strategy epoch  
+                --load_best_model_at_end False  
+                --seed 42
+                --output_dir {tmp_dir}  
+                --overwrite_output_dir
+                """.split()
+
+            with patch.object(sys, "argv", test_args):
+                run_audio_classification.main()
+
+    def test_image_classification(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            test_args = f"""
+                run_image_classification.py  
+                --model_name_or_path nateraw/vit-base-beans  
+                --dataset_name beans  
+                --max_train_samples 10  
+                --max_eval_samples 2  
+                --remove_unused_columns False  
+                --do_train
+                --do_eval  
+                --learning_rate 2e-5  
+                --num_train_epochs 1  
+                --logging_strategy steps  
+                --logging_steps 1  
+                --evaluation_strategy epoch  
+                --save_strategy epoch  
+                --save_total_limit 1  
+                --seed 1337
+                --output_dir {tmp_dir}
+                """.split()
+
+            with patch.object(sys, "argv", test_args):
+                run_image_classification.main()
+
+    def test_text_classification(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            test_args = f"""  
+                run_glue.py  
+                --model_name_or_path hf-internal-testing/tiny-random-DistilBertForSequenceClassification  
+                --task_name sst2  
+                --max_train_samples 10  
+                --max_eval_samples 2   
+                --overwrite_output_dir  
+                --do_train
+                --do_eval  
+                --max_seq_length 128  
+                --learning_rate 1e-5  
+                --optim adamw_torch  
+                --num_train_epochs 1  
+                --logging_steps 1  
+                --evaluation_strategy steps  
+                --eval_steps 1  
+                --save_strategy epoch  
+                --seed 42
+                --output_dir {tmp_dir}
+                """.split()
+
+            with patch.object(sys, "argv", test_args):
+                run_glue.main()
+
+    def test_question_answering(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            test_args = f"""
+                run_qa.py  
+                --model_name_or_path hf-internal-testing/tiny-random-DistilBertForQuestionAnswering  
+                --dataset_name squad  
+                --do_train  
+                --do_eval  
+                --max_train_samples 10  
+                --max_eval_samples 2  
+                --learning_rate 3e-5  
+                --num_train_epochs 1  
+                --max_seq_length 384  
+                --doc_stride 128  
+                --overwrite_output_dir
+                --output_dir {tmp_dir}
+                """.split()
+
+            with patch.object(sys, "argv", test_args):
+                run_qa.main()
+
+
+if __name__ == "__main__":
+    unittest.main()