[alt] add a --testsfile option for running tests from a txt file. (n…

…od-ai#332)

Example usage to run `vai-vision-int8` models from alt_e2eshark:

```
python run.py --testsfile="onnx_tests/models/external_lists/vai-vision-int8.txt" -v --report --report-file="reports/vai-vision-int8.md"
```

Would generate a report like:

## Summary

|Stage|Count|
|--|--|
| Total | 78 |
| PASS | 52 |
| Numerics | 26 |
| results-summary | 0 |
| postprocessing | 0 |
| compiled_inference | 0 |
| native_inference | 0 |
| construct_inputs | 0 |
| compilation | 0 |
| preprocessing | 0 |
| import_model | 0 |
| setup | 0 |

## Test Run Detail 
 Test was run with the following arguments:
Namespace(device='local-task', backend='llvm-cpu',
iree_compile_args=None, mode='onnx-iree', torchtolinalg=False,
stages=None, skip_stages=None, load_inputs=False, groups='all',
test_filter=None,
testsfile='onnx_tests/models/external_lists/vai-vision-int8.txt',
tolerance=None, verbose=True, rundirectory='test-run',
no_artifacts=False, report=True, report_file='vai_vision_int8.md')

| Test | Exit Status | Notes |
|--|--|--|
| DarkNet53_vaiq | PASS | |
| dla169_vaiq | Numerics | |
| efficientnet_b0.ra_in1k_vaiq | Numerics | |
| EfficientNet_b0_vaiq | Numerics | |
| efficientnet_b1.ft_in1k_vaiq | PASS | |
| EfficientNet_b1_vaiq | PASS | |
| efficientnet_b2.ra_in1k_vaiq | Numerics | |
| EfficientNet_b2_vaiq | Numerics | |
| efficientnet_b3.ra2_in1k_vaiq | Numerics | |
| EfficientNet_b3_vaiq | Numerics | |
| efficientnet_b4.ra2_in1k_vaiq | Numerics | |
| EfficientNet_b4_vaiq | Numerics | |
| efficientnet_b5.sw_in12k_vaiq | Numerics | |
| EfficientNet_b5_vaiq | PASS | |
| EfficientNet_b6_vaiq | PASS | |
| EfficientNet_b7_vaiq | Numerics | |
| efficientnet_el.ra_in1k_vaiq | PASS | |
| efficientnet_el_pruned.in1k_vaiq | PASS | |
| efficientnet_em.ra2_in1k_vaiq | PASS | |
| efficientnet_es.ra_in1k_vaiq | PASS | |
| efficientnet_es_pruned.in1k_vaiq | PASS | |
| efficientnet_lite0.ra_in1k_vaiq | PASS | |
| EfficientNet_v2_l_vaiq | Numerics | |
| EfficientNet_v2_m_vaiq | Numerics | |
| EfficientNet_v2_s_vaiq | Numerics | |
| efficientnetv2_rw_m.agc_in1k_vaiq | Numerics | |
| efficientnetv2_rw_s.ra2_in1k_vaiq | PASS | |
| efficientnetv2_rw_t.ra2_in1k_vaiq | PASS | |
| fbnetc_100.rmsp_in1k_vaiq | PASS | |
| gernet_l.idstcv_in1k_vaiq | PASS | |
| gernet_m.idstcv_in1k_vaiq | PASS | |
| gernet_s.idstcv_in1k_vaiq | PASS | |
| GoogLeNet_vaiq | PASS | |
| inception_v3.tf_in1k_vaiq | PASS | |
| Inception_v3_vaiq | PASS | |
| inception_v4.tf_in1k_vaiq | PASS | |
| lcnet_050.ra2_in1k_vaiq | Numerics | |
| lcnet_075.ra2_in1k_vaiq | Numerics | |
| lcnet_100.ra2_in1k_vaiq | Numerics | |
| mnasnet_100.rmsp_in1k_vaiq | PASS | |
| mnasnet_small.lamb_in1k_vaiq | PASS | |
| mobilenetv2_050.lamb_in1k_vaiq | PASS | |
| mobilenetv2_100.ra_in1k_vaiq | PASS | |
| mobilenetv2_110d.ra_in1k_vaiq | PASS | |
| mobilenetv2_120d.ra_in1k_vaiq | PASS | |
| mobilenetv2_140.ra_in1k_vaiq | PASS | |
| MobileNetV2_vaiq | PASS | |
| mobilenetv3_large_100.ra_in1k_vaiq | PASS | |
| MobileNetV3_large_vaiq | PASS | |
| mobilenetv3_small_050.lamb_in1k_vaiq | Numerics | |
| mobilenetv3_small_075.lamb_in1k_vaiq | Numerics | |
| mobilenetv3_small_100.lamb_in1k_vaiq | Numerics | |
| MobileNetV3_small_vaiq | Numerics | |
| ResNet101_vaiq | PASS | |
| ResNet152_vaiq | PASS | |
| ResNet18_vaiq | PASS | |
| resnet32ts.ra2_in1k_vaiq | PASS | |
| resnet33ts.ra2_in1k_vaiq | PASS | |
| ResNet34_vaiq | PASS | |
| resnet50.a1_in1k_vaiq | PASS | |
| ResNet50_vaiq | PASS | |
| RRDB_ESRGAN_pro_vaiq | Numerics | |
| RRDB_ESRGAN_vaiq | Numerics | |
| SqueezeNet_1_0_vaiq | Numerics | |
| SqueezeNet_1_1_vaiq | Numerics | |
| VGG11_bn_vaiq | PASS | |
| VGG11_vaiq | PASS | |
| VGG13_bn_vaiq | PASS | |
| VGG13_vaiq | PASS | |
| VGG16_bn_vaiq | PASS | |
| VGG16_vaiq | PASS | |
| VGG19_bn_vaiq | PASS | |
| VGG19_vaiq | PASS | |
| WideResNet_101_2_vaiq | PASS | |
| WideResNet_50_2_vaiq | PASS | |
| YoloNetV3_vaiq | PASS | |
| Yolov8m_vaiq | PASS | |
| Yolov8n_vaiq | PASS | |

alt_e2eshark/e2e_testing/storage.py

@@ -3,11 +3,12 @@
 # Licensed under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-import io
+import json
 import numpy
 import struct
 import torch
-from typing import Tuple, Optional
+from typing import Tuple, Optional, Dict, List, Any, Union
+from pathlib import Path
 
 def get_shape_string(torch_tensor):
     input_shape = list(torch_tensor.shape)
@@ -120,6 +121,15 @@ def write_inference_input_bin_file(modelinput, modelinputbinfilename):
         f.write(bytearr)
         f.close()
 
+def load_test_txt_file(filepath : Union[str, Path]) -> List[str]:
+    with open(filepath, "r") as file:
+        contents = file.read().split()
+    return contents
+
+def load_json_dict(filepath: Union[str, Path]) -> Dict[str, Any]:
+    with open(filepath) as contents:
+        loaded_dict = json.load(contents)
+    return loaded_dict
 
 class TestTensors:
     """storage class for tuples of tensor-like objects"""

alt_e2eshark/onnx_tests/models/azure_models.py

@@ -3,17 +3,27 @@
 # Licensed under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+from pathlib import Path
 from ..helper_classes import AzureDownloadableModel
 from e2e_testing.registry import register_test
+from e2e_testing.storage import load_test_txt_file
 
-# If the azure model is simple enough to use the default input generation in e2e_testing/onnx_utils.py, then add the name to the list here.
-# Note: several of these models will likely fail numerics without further post-processing. Consider writing a custom child class if further post-processing is desired. 
-model_names = [
-    "RAFT_vaiq_int8",
-    "pytorch-3dunet_vaiq_int8",
-    "FCN_vaiq_int8",
-    "u-net_brain_mri_vaiq_int8",
+this_file = Path(__file__)
+lists_dir = (this_file.parent).joinpath("external_lists")
+
+model_names = load_test_txt_file(lists_dir.joinpath("shark-test-suite.txt"))
+for i in range(1,4):
+    model_names += load_test_txt_file(lists_dir.joinpath(f"vai-hf-cnn-fp32-shard{i}.txt"))
+    model_names += load_test_txt_file(lists_dir.joinpath(f"vai-int8-p0p1-shard{i}.txt"))
+model_names += load_test_txt_file(lists_dir.joinpath("vai-vision-int8.txt"))
+
+custom_registry = [
+    "opt-125M-awq",
+    "opt-125m-gptq",
+    "DeepLabV3_resnet50_vaiq_int8",
 ]
 
-for t in model_names:
+# for simple models without dim params or additional customization, we should be able to register them directly with AzureDownloadableModel
+# TODO: many of the models in the text files loaded from above will likely need to be registered with an alternative test info class.
+for t in set(model_names).difference(custom_registry):
     register_test(AzureDownloadableModel, t)

alt_e2eshark/onnx_tests/models/external_lists/shark-test-suite-pytorch.txt

@@ -0,0 +1,28 @@
+bart-large
+beit-base-patch16-224-pt22k-ft22k
+bert-large-uncased
+bge-base-en-v1.5
+deit-small-distilled-patch16-224
+dlrm
+gemma-7b
+gpt2-xl
+gpt2
+llama2-7b-GPTQ
+llama2-7b-hf
+miniLM-L12-H384-uncased
+mit-b0
+mobilebert-uncased
+opt-1.3b
+opt-125M
+opt-125m-gptq
+opt-350m
+phi-1_5
+phi-2
+resnet50
+stablelm-3b-4e1t
+t5-base
+t5-large
+vit-base-patch16-224
+whisper-base
+whisper-medium
+whisper-small

alt_e2eshark/onnx_tests/models/external_lists/shark-test-suite.txt

@@ -0,0 +1,32 @@
+AlexNet_vaiq_int8
+CSP-DarkNet_vaiq_int8
+ConvNeXt_vaiq_int8
+DarkNet53_vaiq_int8
+DeepLabV3_resnet50_vaiq_int8
+DenseNet201_vaiq_int8
+EfficientNet_v2_s_vaiq_int8
+FCN_vaiq_int8
+GoogLeNet_vaiq_int8
+Inception_v4_vaiq_int8
+KeypointRCNN_vaiq_int8
+LRASPP_vaiq_int8
+MNASNet_1_3_vaiq_int8
+MobileNetV3_small_vaiq_int8
+RAFT_vaiq_int8
+RDN_pytorch_vaiq_int8
+RRDB_ESRGAN_vaiq_int8
+RegNet_y_8gf_vaiq_int8
+ResNet152_vaiq_int8
+ShuffleNet_v2_x2_0_vaiq_int8
+SqueezeNet_1_1_vaiq_int8
+U-2-Net_vaiq_int8
+VGG11_bn_vaiq_int8
+VGG19_vaiq_int8
+VideoResNet_vaiq_int8
+WideResNet_50_2_vaiq_int8
+YoloNetV3_vaiq_int8
+pytorch-3dunet_vaiq_int8
+resnet50_vaiq_int8
+retinanet_resnet50_fpn_vaiq_int8
+u-net_brain_mri_vaiq_int8
+yolov8n_vaiq_int8

alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard1.txt

@@ -0,0 +1,175 @@
+bat_resnext26ts.ch_in1k
+beit_base_patch16_224.in22k_ft_in22k_in1k
+beit_base_patch16_384.in22k_ft_in22k_in1k
+beit_large_patch16_224.in22k_ft_in22k_in1k
+beit_large_patch16_384.in22k_ft_in22k_in1k
+beit_large_patch16_512.in22k_ft_in22k_in1k
+beitv2_base_patch16_224.in1k_ft_in22k_in1k
+beitv2_large_patch16_224.in1k_ft_in22k_in1k
+botnet26t_256
+cait_m36_384
+cait_m48_448
+cait_s24_224
+cait_s24_384
+cait_s36_384
+cait_xs24_384
+cait_xxs24_224
+cait_xxs24_384
+cait_xxs36_224
+cait_xxs36_384
+coat_lite_mini
+coat_lite_small
+coat_lite_tiny
+coat_mini
+coat_tiny
+coatnet_0_rw_224.sw_in1k
+coatnet_1_rw_224.sw_in1k
+coatnet_2_rw_224.sw_in12k
+coatnet_2_rw_224.sw_in12k_ft_in1k
+coatnet_3_rw_224.sw_in12k
+coatnet_bn_0_rw_224.sw_in1k
+coatnet_nano_rw_224.sw_in1k
+coatnet_rmlp_1_rw2_224.sw_in12k
+coatnet_rmlp_1_rw2_224.sw_in12k_ft_in1k
+coatnet_rmlp_1_rw_224.sw_in1k
+coatnet_rmlp_2_rw_224.sw_in12k
+coatnet_rmlp_2_rw_224.sw_in12k_ft_in1k
+coatnet_rmlp_2_rw_224.sw_in1k
+coatnet_rmlp_2_rw_384.sw_in12k_ft_in1k
+coatnet_rmlp_nano_rw_224.sw_in1k
+coatnext_nano_rw_224.sw_in1k
+convit_base
+convit_small
+convit_tiny
+convnext_atto.d2_in1k
+convnext_atto_ols.a2_in1k
+convnext_base.clip_laion2b
+convnext_base.clip_laion2b_augreg
+convnext_base.clip_laion2b_augreg_ft_in1k
+convnext_base.clip_laiona
+convnext_base.clip_laiona_320
+convnext_base.clip_laiona_augreg_320
+convnext_base.clip_laiona_augreg_ft_in1k_384
+convnext_base.fb_in1k
+convnext_base.fb_in22k_ft_in1k
+convnext_base.fb_in22k_ft_in1k_384
+convnext_femto.d1_in1k
+convnext_femto_ols.d1_in1k
+convnext_large.fb_in1k
+convnext_large.fb_in22k_ft_in1k
+convnext_large.fb_in22k_ft_in1k_384
+convnext_large_mlp.clip_laion2b_augreg
+convnext_large_mlp.clip_laion2b_augreg_ft_in1k
+convnext_large_mlp.clip_laion2b_augreg_ft_in1k_384
+convnext_large_mlp.clip_laion2b_ft_320
+convnext_large_mlp.clip_laion2b_ft_soup_320
+convnext_nano.d1h_in1k
+convnext_nano.in12k
+convnext_nano.in12k_ft_in1k
+convnext_nano_ols.d1h_in1k
+convnext_pico.d1_in1k
+convnext_pico_ols.d1_in1k
+convnext_small.fb_in1k
+convnext_small.fb_in22k_ft_in1k
+convnext_small.fb_in22k_ft_in1k_384
+convnext_small.in12k
+convnext_small.in12k_ft_in1k
+convnext_small.in12k_ft_in1k_384
+convnext_tiny.fb_in1k
+convnext_tiny.fb_in22k_ft_in1k
+convnext_tiny.fb_in22k_ft_in1k_384
+convnext_tiny.in12k
+convnext_tiny.in12k_ft_in1k
+convnext_tiny.in12k_ft_in1k_384
+convnext_tiny_hnf.a2h_in1k
+convnext_xlarge.fb_in22k_ft_in1k
+convnext_xlarge.fb_in22k_ft_in1k_384
+convnext_xxlarge.clip_laion2b_rewind
+convnext_xxlarge.clip_laion2b_soup
+convnextv2_atto.fcmae
+convnextv2_atto.fcmae_ft_in1k
+convnextv2_base.fcmae
+convnextv2_base.fcmae_ft_in1k
+convnextv2_base.fcmae_ft_in22k_in1k
+convnextv2_base.fcmae_ft_in22k_in1k_384
+convnextv2_femto.fcmae
+convnextv2_femto.fcmae_ft_in1k
+convnextv2_huge.fcmae
+convnextv2_huge.fcmae_ft_in1k
+convnextv2_huge.fcmae_ft_in22k_in1k_384
+convnextv2_huge.fcmae_ft_in22k_in1k_512
+convnextv2_large.fcmae
+convnextv2_large.fcmae_ft_in1k
+convnextv2_large.fcmae_ft_in22k_in1k
+convnextv2_large.fcmae_ft_in22k_in1k_384
+convnextv2_nano.fcmae
+convnextv2_nano.fcmae_ft_in1k
+convnextv2_nano.fcmae_ft_in22k_in1k
+convnextv2_nano.fcmae_ft_in22k_in1k_384
+convnextv2_pico.fcmae
+convnextv2_pico.fcmae_ft_in1k
+convnextv2_tiny.fcmae
+convnextv2_tiny.fcmae_ft_in1k
+convnextv2_tiny.fcmae_ft_in22k_in1k
+convnextv2_tiny.fcmae_ft_in22k_in1k_384
+crossvit_15_240
+crossvit_15_dagger_240
+crossvit_15_dagger_408
+crossvit_18_240
+crossvit_18_dagger_240
+crossvit_18_dagger_408
+crossvit_9_240
+crossvit_9_dagger_240
+crossvit_base_240
+crossvit_small_240
+crossvit_tiny_240
+darknetaa53
+davit_base.msft_in1k
+davit_small.msft_in1k
+davit_tiny.msft_in1k
+deit3_base_patch16_224.fb_in1k
+deit3_base_patch16_224.fb_in22k_ft_in1k
+deit3_base_patch16_384.fb_in1k
+deit3_base_patch16_384.fb_in22k_ft_in1k
+deit3_huge_patch14_224.fb_in1k
+deit3_huge_patch14_224.fb_in22k_ft_in1k
+deit3_large_patch16_224.fb_in1k
+deit3_large_patch16_224.fb_in22k_ft_in1k
+deit3_large_patch16_384.fb_in1k
+deit3_large_patch16_384.fb_in22k_ft_in1k
+deit3_medium_patch16_224.fb_in1k
+deit3_medium_patch16_224.fb_in22k_ft_in1k
+deit3_small_patch16_224.fb_in1k
+deit3_small_patch16_224.fb_in22k_ft_in1k
+deit3_small_patch16_384.fb_in1k
+deit3_small_patch16_384.fb_in22k_ft_in1k
+deit_base_distilled_patch16_224.fb_in1k
+deit_base_distilled_patch16_384.fb_in1k
+deit_base_patch16_224.fb_in1k
+deit_base_patch16_384.fb_in1k
+deit_small_distilled_patch16_224.fb_in1k
+deit_small_patch16_224.fb_in1k
+deit_tiny_distilled_patch16_224.fb_in1k
+deit_tiny_patch16_224.fb_in1k
+densenet201
+dm_nfnet_f2.dm_in1k
+dm_nfnet_f3.dm_in1k
+dm_nfnet_f4.dm_in1k
+dm_nfnet_f5.dm_in1k
+dm_nfnet_f6.dm_in1k
+eca_botnext26ts_256
+ecaresnet269d
+edgenext_base
+edgenext_small
+edgenext_small_rw
+edgenext_x_small
+edgenext_xx_small
+efficientformer_l1.snap_dist_in1k
+efficientformer_l3.snap_dist_in1k
+efficientformer_l7.snap_dist_in1k
+efficientformerv2_l.snap_dist_in1k
+efficientformerv2_s0.snap_dist_in1k
+efficientformerv2_s1.snap_dist_in1k
+efficientformerv2_s2.snap_dist_in1k
+efficientnet_b1_pruned.in1k
+efficientnet_b2_pruned.in1k