From d045c434cedc4634062f86ef1ca841139ac48444 Mon Sep 17 00:00:00 2001 From: zjgarvey <47986913+zjgarvey@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:05:13 -0700 Subject: [PATCH] [alt] add a `--testsfile` option for running tests from a txt file. (#332) Example usage to run `vai-vision-int8` models from alt_e2eshark: ``` python run.py --testsfile="onnx_tests/models/external_lists/vai-vision-int8.txt" -v --report --report-file="reports/vai-vision-int8.md" ``` Would generate a report like: ## Summary |Stage|Count| |--|--| | Total | 78 | | PASS | 52 | | Numerics | 26 | | results-summary | 0 | | postprocessing | 0 | | compiled_inference | 0 | | native_inference | 0 | | construct_inputs | 0 | | compilation | 0 | | preprocessing | 0 | | import_model | 0 | | setup | 0 | ## Test Run Detail Test was run with the following arguments: Namespace(device='local-task', backend='llvm-cpu', iree_compile_args=None, mode='onnx-iree', torchtolinalg=False, stages=None, skip_stages=None, load_inputs=False, groups='all', test_filter=None, testsfile='onnx_tests/models/external_lists/vai-vision-int8.txt', tolerance=None, verbose=True, rundirectory='test-run', no_artifacts=False, report=True, report_file='vai_vision_int8.md') | Test | Exit Status | Notes | |--|--|--| | DarkNet53_vaiq | PASS | | | dla169_vaiq | Numerics | | | efficientnet_b0.ra_in1k_vaiq | Numerics | | | EfficientNet_b0_vaiq | Numerics | | | efficientnet_b1.ft_in1k_vaiq | PASS | | | EfficientNet_b1_vaiq | PASS | | | efficientnet_b2.ra_in1k_vaiq | Numerics | | | EfficientNet_b2_vaiq | Numerics | | | efficientnet_b3.ra2_in1k_vaiq | Numerics | | | EfficientNet_b3_vaiq | Numerics | | | efficientnet_b4.ra2_in1k_vaiq | Numerics | | | EfficientNet_b4_vaiq | Numerics | | | efficientnet_b5.sw_in12k_vaiq | Numerics | | | EfficientNet_b5_vaiq | PASS | | | EfficientNet_b6_vaiq | PASS | | | EfficientNet_b7_vaiq | Numerics | | | efficientnet_el.ra_in1k_vaiq | PASS | | | efficientnet_el_pruned.in1k_vaiq | PASS | | | efficientnet_em.ra2_in1k_vaiq | PASS | | | efficientnet_es.ra_in1k_vaiq | PASS | | | efficientnet_es_pruned.in1k_vaiq | PASS | | | efficientnet_lite0.ra_in1k_vaiq | PASS | | | EfficientNet_v2_l_vaiq | Numerics | | | EfficientNet_v2_m_vaiq | Numerics | | | EfficientNet_v2_s_vaiq | Numerics | | | efficientnetv2_rw_m.agc_in1k_vaiq | Numerics | | | efficientnetv2_rw_s.ra2_in1k_vaiq | PASS | | | efficientnetv2_rw_t.ra2_in1k_vaiq | PASS | | | fbnetc_100.rmsp_in1k_vaiq | PASS | | | gernet_l.idstcv_in1k_vaiq | PASS | | | gernet_m.idstcv_in1k_vaiq | PASS | | | gernet_s.idstcv_in1k_vaiq | PASS | | | GoogLeNet_vaiq | PASS | | | inception_v3.tf_in1k_vaiq | PASS | | | Inception_v3_vaiq | PASS | | | inception_v4.tf_in1k_vaiq | PASS | | | lcnet_050.ra2_in1k_vaiq | Numerics | | | lcnet_075.ra2_in1k_vaiq | Numerics | | | lcnet_100.ra2_in1k_vaiq | Numerics | | | mnasnet_100.rmsp_in1k_vaiq | PASS | | | mnasnet_small.lamb_in1k_vaiq | PASS | | | mobilenetv2_050.lamb_in1k_vaiq | PASS | | | mobilenetv2_100.ra_in1k_vaiq | PASS | | | mobilenetv2_110d.ra_in1k_vaiq | PASS | | | mobilenetv2_120d.ra_in1k_vaiq | PASS | | | mobilenetv2_140.ra_in1k_vaiq | PASS | | | MobileNetV2_vaiq | PASS | | | mobilenetv3_large_100.ra_in1k_vaiq | PASS | | | MobileNetV3_large_vaiq | PASS | | | mobilenetv3_small_050.lamb_in1k_vaiq | Numerics | | | mobilenetv3_small_075.lamb_in1k_vaiq | Numerics | | | mobilenetv3_small_100.lamb_in1k_vaiq | Numerics | | | MobileNetV3_small_vaiq | Numerics | | | ResNet101_vaiq | PASS | | | ResNet152_vaiq | PASS | | | ResNet18_vaiq | PASS | | | resnet32ts.ra2_in1k_vaiq | PASS | | | resnet33ts.ra2_in1k_vaiq | PASS | | | ResNet34_vaiq | PASS | | | resnet50.a1_in1k_vaiq | PASS | | | ResNet50_vaiq | PASS | | | RRDB_ESRGAN_pro_vaiq | Numerics | | | RRDB_ESRGAN_vaiq | Numerics | | | SqueezeNet_1_0_vaiq | Numerics | | | SqueezeNet_1_1_vaiq | Numerics | | | VGG11_bn_vaiq | PASS | | | VGG11_vaiq | PASS | | | VGG13_bn_vaiq | PASS | | | VGG13_vaiq | PASS | | | VGG16_bn_vaiq | PASS | | | VGG16_vaiq | PASS | | | VGG19_bn_vaiq | PASS | | | VGG19_vaiq | PASS | | | WideResNet_101_2_vaiq | PASS | | | WideResNet_50_2_vaiq | PASS | | | YoloNetV3_vaiq | PASS | | | Yolov8m_vaiq | PASS | | | Yolov8n_vaiq | PASS | | --- alt_e2eshark/e2e_testing/storage.py | 14 +- .../onnx_tests/models/azure_models.py | 26 ++- .../shark-test-suite-pytorch.txt | 28 +++ .../external_lists/shark-test-suite.txt | 32 ++++ .../external_lists/vai-hf-cnn-fp32-shard1.txt | 175 +++++++++++++++++ .../external_lists/vai-hf-cnn-fp32-shard2.txt | 177 ++++++++++++++++++ .../external_lists/vai-hf-cnn-fp32-shard3.txt | 177 ++++++++++++++++++ .../external_lists/vai-int8-p0p1-shard1.txt | 162 ++++++++++++++++ .../external_lists/vai-int8-p0p1-shard2.txt | 162 ++++++++++++++++ .../external_lists/vai-int8-p0p1-shard3.txt | 162 ++++++++++++++++ .../models/external_lists/vai-vision-int8.txt | 78 ++++++++ .../onnx_tests/models/vision_models.py | 7 +- alt_e2eshark/run.py | 25 ++- 13 files changed, 1204 insertions(+), 21 deletions(-) create mode 100644 alt_e2eshark/onnx_tests/models/external_lists/shark-test-suite-pytorch.txt create mode 100644 alt_e2eshark/onnx_tests/models/external_lists/shark-test-suite.txt create mode 100644 alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard1.txt create mode 100644 alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard2.txt create mode 100644 alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard3.txt create mode 100644 alt_e2eshark/onnx_tests/models/external_lists/vai-int8-p0p1-shard1.txt create mode 100644 alt_e2eshark/onnx_tests/models/external_lists/vai-int8-p0p1-shard2.txt create mode 100644 alt_e2eshark/onnx_tests/models/external_lists/vai-int8-p0p1-shard3.txt create mode 100644 alt_e2eshark/onnx_tests/models/external_lists/vai-vision-int8.txt diff --git a/alt_e2eshark/e2e_testing/storage.py b/alt_e2eshark/e2e_testing/storage.py index 9b0a3484..28a0976e 100644 --- a/alt_e2eshark/e2e_testing/storage.py +++ b/alt_e2eshark/e2e_testing/storage.py @@ -3,11 +3,12 @@ # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -import io +import json import numpy import struct import torch -from typing import Tuple, Optional +from typing import Tuple, Optional, Dict, List, Any, Union +from pathlib import Path def get_shape_string(torch_tensor): input_shape = list(torch_tensor.shape) @@ -120,6 +121,15 @@ def write_inference_input_bin_file(modelinput, modelinputbinfilename): f.write(bytearr) f.close() +def load_test_txt_file(filepath : Union[str, Path]) -> List[str]: + with open(filepath, "r") as file: + contents = file.read().split() + return contents + +def load_json_dict(filepath: Union[str, Path]) -> Dict[str, Any]: + with open(filepath) as contents: + loaded_dict = json.load(contents) + return loaded_dict class TestTensors: """storage class for tuples of tensor-like objects""" diff --git a/alt_e2eshark/onnx_tests/models/azure_models.py b/alt_e2eshark/onnx_tests/models/azure_models.py index 644ff7a7..84fb386f 100644 --- a/alt_e2eshark/onnx_tests/models/azure_models.py +++ b/alt_e2eshark/onnx_tests/models/azure_models.py @@ -3,17 +3,27 @@ # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +from pathlib import Path from ..helper_classes import AzureDownloadableModel from e2e_testing.registry import register_test +from e2e_testing.storage import load_test_txt_file -# If the azure model is simple enough to use the default input generation in e2e_testing/onnx_utils.py, then add the name to the list here. -# Note: several of these models will likely fail numerics without further post-processing. Consider writing a custom child class if further post-processing is desired. -model_names = [ - "RAFT_vaiq_int8", - "pytorch-3dunet_vaiq_int8", - "FCN_vaiq_int8", - "u-net_brain_mri_vaiq_int8", +this_file = Path(__file__) +lists_dir = (this_file.parent).joinpath("external_lists") + +model_names = load_test_txt_file(lists_dir.joinpath("shark-test-suite.txt")) +for i in range(1,4): + model_names += load_test_txt_file(lists_dir.joinpath(f"vai-hf-cnn-fp32-shard{i}.txt")) + model_names += load_test_txt_file(lists_dir.joinpath(f"vai-int8-p0p1-shard{i}.txt")) +model_names += load_test_txt_file(lists_dir.joinpath("vai-vision-int8.txt")) + +custom_registry = [ + "opt-125M-awq", + "opt-125m-gptq", + "DeepLabV3_resnet50_vaiq_int8", ] -for t in model_names: +# for simple models without dim params or additional customization, we should be able to register them directly with AzureDownloadableModel +# TODO: many of the models in the text files loaded from above will likely need to be registered with an alternative test info class. +for t in set(model_names).difference(custom_registry): register_test(AzureDownloadableModel, t) diff --git a/alt_e2eshark/onnx_tests/models/external_lists/shark-test-suite-pytorch.txt b/alt_e2eshark/onnx_tests/models/external_lists/shark-test-suite-pytorch.txt new file mode 100644 index 00000000..7e185cbb --- /dev/null +++ b/alt_e2eshark/onnx_tests/models/external_lists/shark-test-suite-pytorch.txt @@ -0,0 +1,28 @@ +bart-large +beit-base-patch16-224-pt22k-ft22k +bert-large-uncased +bge-base-en-v1.5 +deit-small-distilled-patch16-224 +dlrm +gemma-7b +gpt2-xl +gpt2 +llama2-7b-GPTQ +llama2-7b-hf +miniLM-L12-H384-uncased +mit-b0 +mobilebert-uncased +opt-1.3b +opt-125M +opt-125m-gptq +opt-350m +phi-1_5 +phi-2 +resnet50 +stablelm-3b-4e1t +t5-base +t5-large +vit-base-patch16-224 +whisper-base +whisper-medium +whisper-small \ No newline at end of file diff --git a/alt_e2eshark/onnx_tests/models/external_lists/shark-test-suite.txt b/alt_e2eshark/onnx_tests/models/external_lists/shark-test-suite.txt new file mode 100644 index 00000000..5089dcd6 --- /dev/null +++ b/alt_e2eshark/onnx_tests/models/external_lists/shark-test-suite.txt @@ -0,0 +1,32 @@ +AlexNet_vaiq_int8 +CSP-DarkNet_vaiq_int8 +ConvNeXt_vaiq_int8 +DarkNet53_vaiq_int8 +DeepLabV3_resnet50_vaiq_int8 +DenseNet201_vaiq_int8 +EfficientNet_v2_s_vaiq_int8 +FCN_vaiq_int8 +GoogLeNet_vaiq_int8 +Inception_v4_vaiq_int8 +KeypointRCNN_vaiq_int8 +LRASPP_vaiq_int8 +MNASNet_1_3_vaiq_int8 +MobileNetV3_small_vaiq_int8 +RAFT_vaiq_int8 +RDN_pytorch_vaiq_int8 +RRDB_ESRGAN_vaiq_int8 +RegNet_y_8gf_vaiq_int8 +ResNet152_vaiq_int8 +ShuffleNet_v2_x2_0_vaiq_int8 +SqueezeNet_1_1_vaiq_int8 +U-2-Net_vaiq_int8 +VGG11_bn_vaiq_int8 +VGG19_vaiq_int8 +VideoResNet_vaiq_int8 +WideResNet_50_2_vaiq_int8 +YoloNetV3_vaiq_int8 +pytorch-3dunet_vaiq_int8 +resnet50_vaiq_int8 +retinanet_resnet50_fpn_vaiq_int8 +u-net_brain_mri_vaiq_int8 +yolov8n_vaiq_int8 \ No newline at end of file diff --git a/alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard1.txt b/alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard1.txt new file mode 100644 index 00000000..90449a87 --- /dev/null +++ b/alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard1.txt @@ -0,0 +1,175 @@ +bat_resnext26ts.ch_in1k +beit_base_patch16_224.in22k_ft_in22k_in1k +beit_base_patch16_384.in22k_ft_in22k_in1k +beit_large_patch16_224.in22k_ft_in22k_in1k +beit_large_patch16_384.in22k_ft_in22k_in1k +beit_large_patch16_512.in22k_ft_in22k_in1k +beitv2_base_patch16_224.in1k_ft_in22k_in1k +beitv2_large_patch16_224.in1k_ft_in22k_in1k +botnet26t_256 +cait_m36_384 +cait_m48_448 +cait_s24_224 +cait_s24_384 +cait_s36_384 +cait_xs24_384 +cait_xxs24_224 +cait_xxs24_384 +cait_xxs36_224 +cait_xxs36_384 +coat_lite_mini +coat_lite_small +coat_lite_tiny +coat_mini +coat_tiny +coatnet_0_rw_224.sw_in1k +coatnet_1_rw_224.sw_in1k +coatnet_2_rw_224.sw_in12k +coatnet_2_rw_224.sw_in12k_ft_in1k +coatnet_3_rw_224.sw_in12k +coatnet_bn_0_rw_224.sw_in1k +coatnet_nano_rw_224.sw_in1k +coatnet_rmlp_1_rw2_224.sw_in12k +coatnet_rmlp_1_rw2_224.sw_in12k_ft_in1k +coatnet_rmlp_1_rw_224.sw_in1k +coatnet_rmlp_2_rw_224.sw_in12k +coatnet_rmlp_2_rw_224.sw_in12k_ft_in1k +coatnet_rmlp_2_rw_224.sw_in1k +coatnet_rmlp_2_rw_384.sw_in12k_ft_in1k +coatnet_rmlp_nano_rw_224.sw_in1k +coatnext_nano_rw_224.sw_in1k +convit_base +convit_small +convit_tiny +convnext_atto.d2_in1k +convnext_atto_ols.a2_in1k +convnext_base.clip_laion2b +convnext_base.clip_laion2b_augreg +convnext_base.clip_laion2b_augreg_ft_in1k +convnext_base.clip_laiona +convnext_base.clip_laiona_320 +convnext_base.clip_laiona_augreg_320 +convnext_base.clip_laiona_augreg_ft_in1k_384 +convnext_base.fb_in1k +convnext_base.fb_in22k_ft_in1k +convnext_base.fb_in22k_ft_in1k_384 +convnext_femto.d1_in1k +convnext_femto_ols.d1_in1k +convnext_large.fb_in1k +convnext_large.fb_in22k_ft_in1k +convnext_large.fb_in22k_ft_in1k_384 +convnext_large_mlp.clip_laion2b_augreg +convnext_large_mlp.clip_laion2b_augreg_ft_in1k +convnext_large_mlp.clip_laion2b_augreg_ft_in1k_384 +convnext_large_mlp.clip_laion2b_ft_320 +convnext_large_mlp.clip_laion2b_ft_soup_320 +convnext_nano.d1h_in1k +convnext_nano.in12k +convnext_nano.in12k_ft_in1k +convnext_nano_ols.d1h_in1k +convnext_pico.d1_in1k +convnext_pico_ols.d1_in1k +convnext_small.fb_in1k +convnext_small.fb_in22k_ft_in1k +convnext_small.fb_in22k_ft_in1k_384 +convnext_small.in12k +convnext_small.in12k_ft_in1k +convnext_small.in12k_ft_in1k_384 +convnext_tiny.fb_in1k +convnext_tiny.fb_in22k_ft_in1k +convnext_tiny.fb_in22k_ft_in1k_384 +convnext_tiny.in12k +convnext_tiny.in12k_ft_in1k +convnext_tiny.in12k_ft_in1k_384 +convnext_tiny_hnf.a2h_in1k +convnext_xlarge.fb_in22k_ft_in1k +convnext_xlarge.fb_in22k_ft_in1k_384 +convnext_xxlarge.clip_laion2b_rewind +convnext_xxlarge.clip_laion2b_soup +convnextv2_atto.fcmae +convnextv2_atto.fcmae_ft_in1k +convnextv2_base.fcmae +convnextv2_base.fcmae_ft_in1k +convnextv2_base.fcmae_ft_in22k_in1k +convnextv2_base.fcmae_ft_in22k_in1k_384 +convnextv2_femto.fcmae +convnextv2_femto.fcmae_ft_in1k +convnextv2_huge.fcmae +convnextv2_huge.fcmae_ft_in1k +convnextv2_huge.fcmae_ft_in22k_in1k_384 +convnextv2_huge.fcmae_ft_in22k_in1k_512 +convnextv2_large.fcmae +convnextv2_large.fcmae_ft_in1k +convnextv2_large.fcmae_ft_in22k_in1k +convnextv2_large.fcmae_ft_in22k_in1k_384 +convnextv2_nano.fcmae +convnextv2_nano.fcmae_ft_in1k +convnextv2_nano.fcmae_ft_in22k_in1k +convnextv2_nano.fcmae_ft_in22k_in1k_384 +convnextv2_pico.fcmae +convnextv2_pico.fcmae_ft_in1k +convnextv2_tiny.fcmae +convnextv2_tiny.fcmae_ft_in1k +convnextv2_tiny.fcmae_ft_in22k_in1k +convnextv2_tiny.fcmae_ft_in22k_in1k_384 +crossvit_15_240 +crossvit_15_dagger_240 +crossvit_15_dagger_408 +crossvit_18_240 +crossvit_18_dagger_240 +crossvit_18_dagger_408 +crossvit_9_240 +crossvit_9_dagger_240 +crossvit_base_240 +crossvit_small_240 +crossvit_tiny_240 +darknetaa53 +davit_base.msft_in1k +davit_small.msft_in1k +davit_tiny.msft_in1k +deit3_base_patch16_224.fb_in1k +deit3_base_patch16_224.fb_in22k_ft_in1k +deit3_base_patch16_384.fb_in1k +deit3_base_patch16_384.fb_in22k_ft_in1k +deit3_huge_patch14_224.fb_in1k +deit3_huge_patch14_224.fb_in22k_ft_in1k +deit3_large_patch16_224.fb_in1k +deit3_large_patch16_224.fb_in22k_ft_in1k +deit3_large_patch16_384.fb_in1k +deit3_large_patch16_384.fb_in22k_ft_in1k +deit3_medium_patch16_224.fb_in1k +deit3_medium_patch16_224.fb_in22k_ft_in1k +deit3_small_patch16_224.fb_in1k +deit3_small_patch16_224.fb_in22k_ft_in1k +deit3_small_patch16_384.fb_in1k +deit3_small_patch16_384.fb_in22k_ft_in1k +deit_base_distilled_patch16_224.fb_in1k +deit_base_distilled_patch16_384.fb_in1k +deit_base_patch16_224.fb_in1k +deit_base_patch16_384.fb_in1k +deit_small_distilled_patch16_224.fb_in1k +deit_small_patch16_224.fb_in1k +deit_tiny_distilled_patch16_224.fb_in1k +deit_tiny_patch16_224.fb_in1k +densenet201 +dm_nfnet_f2.dm_in1k +dm_nfnet_f3.dm_in1k +dm_nfnet_f4.dm_in1k +dm_nfnet_f5.dm_in1k +dm_nfnet_f6.dm_in1k +eca_botnext26ts_256 +ecaresnet269d +edgenext_base +edgenext_small +edgenext_small_rw +edgenext_x_small +edgenext_xx_small +efficientformer_l1.snap_dist_in1k +efficientformer_l3.snap_dist_in1k +efficientformer_l7.snap_dist_in1k +efficientformerv2_l.snap_dist_in1k +efficientformerv2_s0.snap_dist_in1k +efficientformerv2_s1.snap_dist_in1k +efficientformerv2_s2.snap_dist_in1k +efficientnet_b1_pruned.in1k +efficientnet_b2_pruned.in1k \ No newline at end of file diff --git a/alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard2.txt b/alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard2.txt new file mode 100644 index 00000000..8f7ba8b3 --- /dev/null +++ b/alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard2.txt @@ -0,0 +1,177 @@ +efficientnet_b3_pruned.in1k +efficientnet_b5.in12k +efficientnet_b5.in12k_ft_in1k +eva_giant_patch14_224.clip_ft_in1k +eva_giant_patch14_336.clip_ft_in1k +eva_giant_patch14_336.m30m_ft_in22k_in1k +eva_giant_patch14_560.m30m_ft_in22k_in1k +eva_large_patch14_196.in22k_ft_in1k +eva_large_patch14_196.in22k_ft_in22k_in1k +eva_large_patch14_336.in22k_ft_in1k +eva_large_patch14_336.in22k_ft_in22k_in1k +flexivit_base.1200ep_in1k +flexivit_base.300ep_in1k +flexivit_base.600ep_in1k +flexivit_large.1200ep_in1k +flexivit_large.300ep_in1k +flexivit_large.600ep_in1k +flexivit_small.1200ep_in1k +flexivit_small.300ep_in1k +flexivit_small.600ep_in1k +focalnet_base_lrf.ms_in1k +focalnet_base_srf.ms_in1k +focalnet_small_lrf.ms_in1k +focalnet_small_srf.ms_in1k +focalnet_tiny_lrf.ms_in1k +focalnet_tiny_srf.ms_in1k +gcvit_base +gcvit_small +gcvit_tiny +gcvit_xtiny +gcvit_xxtiny +gluon_xception65 +gmixer_24_224.ra3_in1k +gmlp_s16_224.ra3_in1k +ig_resnext101_32x48d +jx_nest_base +jx_nest_small +jx_nest_tiny +lambda_resnet26t +lambda_resnet50ts +levit_128.fb_dist_in1k +levit_128s.fb_dist_in1k +levit_192.fb_dist_in1k +levit_256.fb_dist_in1k +levit_384.fb_dist_in1k +levit_conv_128.fb_dist_in1k +levit_conv_128s.fb_dist_in1k +levit_conv_192.fb_dist_in1k +levit_conv_256.fb_dist_in1k +levit_conv_384.fb_dist_in1k +maxvit_base_tf_224.in1k +maxvit_base_tf_384.in1k +maxvit_base_tf_384.in21k_ft_in1k +maxvit_base_tf_512.in1k +maxvit_base_tf_512.in21k_ft_in1k +maxvit_large_tf_224.in1k +maxvit_large_tf_384.in1k +maxvit_large_tf_384.in21k_ft_in1k +maxvit_large_tf_512.in1k +maxvit_large_tf_512.in21k_ft_in1k +maxvit_nano_rw_256.sw_in1k +maxvit_rmlp_base_rw_224.sw_in12k +maxvit_rmlp_base_rw_224.sw_in12k_ft_in1k +maxvit_rmlp_base_rw_384.sw_in12k_ft_in1k +maxvit_rmlp_nano_rw_256.sw_in1k +maxvit_rmlp_pico_rw_256.sw_in1k +maxvit_rmlp_small_rw_224.sw_in1k +maxvit_rmlp_tiny_rw_256.sw_in1k +maxvit_small_tf_224.in1k +maxvit_small_tf_384.in1k +maxvit_small_tf_512.in1k +maxvit_tiny_rw_224.sw_in1k +maxvit_tiny_tf_224.in1k +maxvit_tiny_tf_384.in1k +maxvit_tiny_tf_512.in1k +maxvit_xlarge_tf_384.in21k_ft_in1k +maxvit_xlarge_tf_512.in21k_ft_in1k +maxxvit_rmlp_nano_rw_256.sw_in1k +maxxvit_rmlp_small_rw_256.sw_in1k +maxxvitv2_nano_rw_256.sw_in1k +maxxvitv2_rmlp_base_rw_224.sw_in12k +maxxvitv2_rmlp_base_rw_224.sw_in12k_ft_in1k +maxxvitv2_rmlp_base_rw_384.sw_in12k_ft_in1k +mixer_b16_224.goog_in21k_ft_in1k +mixer_b16_224.miil_in21k_ft_in1k +mixer_l16_224.goog_in21k_ft_in1k +mobilevit_s +mobilevit_xs +mobilevit_xxs +mobilevitv2_050 +mobilevitv2_075 +mobilevitv2_100 +mobilevitv2_125 +mobilevitv2_150 +mobilevitv2_150_384_in22ft1k +mobilevitv2_150_in22ft1k +mobilevitv2_175 +mobilevitv2_175_384_in22ft1k +mobilevitv2_175_in22ft1k +mobilevitv2_200 +mobilevitv2_200_384_in22ft1k +mobilevitv2_200_in22ft1k +mvitv2_base +mvitv2_large +mvitv2_small +mvitv2_tiny +nasnetalarge +pit_b_224 +pit_b_distilled_224 +pit_s_224 +pit_s_distilled_224 +pit_ti_224 +pit_ti_distilled_224 +pit_xs_224 +pit_xs_distilled_224 +pnasnet5large +poolformer_m36 +poolformer_m48 +poolformer_s12 +poolformer_s24 +poolformer_s36 +pvt_v2_b0 +pvt_v2_b1 +pvt_v2_b2 +pvt_v2_b2_li +pvt_v2_b3 +pvt_v2_b4 +pvt_v2_b5 +regnety_120.sw_in12k +regnety_1280.seer +regnety_1280.seer_ft_in1k +regnety_1280.swag_ft_in1k +regnety_1280.swag_lc_in1k +regnety_160.deit_in1k +regnety_160.sw_in12k +regnety_2560.seer_ft_in1k +regnety_320.seer +regnety_640.seer +resmlp_big_24_224.fb_in1k +resmlp_big_24_224.fb_in22k_ft_in1k +resnest200e +resnest269e +resnetrs270 +resnetrs350 +resnetrs420 +resnetv2_101x3_bit.goog_in21k_ft_in1k +resnetv2_152x4_bit.goog_in21k_ft_in1k +rexnetr_200.sw_in12k +rexnetr_300.sw_in12k +sebotnet33ts_256 +sequencer2d_l +swin_base_patch4_window12_384.ms_in1k +swin_base_patch4_window12_384.ms_in22k_ft_in1k +swin_base_patch4_window7_224.ms_in1k +swin_base_patch4_window7_224.ms_in22k_ft_in1k +swin_large_patch4_window12_384.ms_in22k_ft_in1k +swin_large_patch4_window7_224.ms_in22k_ft_in1k +swin_s3_base_224.ms_in1k +swin_s3_small_224.ms_in1k +swin_s3_tiny_224.ms_in1k +swin_small_patch4_window7_224.ms_in1k +swin_small_patch4_window7_224.ms_in22k_ft_in1k +swin_tiny_patch4_window7_224.ms_in1k +swin_tiny_patch4_window7_224.ms_in22k_ft_in1k +swinv2_base_window12to16_192to256.ms_in22k_ft_in1k +swinv2_base_window12to24_192to384.ms_in22k_ft_in1k +swinv2_base_window16_256.ms_in1k +swinv2_base_window8_256.ms_in1k +swinv2_cr_small_224.sw_in1k +swinv2_cr_small_ns_224.sw_in1k +swinv2_cr_tiny_ns_224.sw_in1k +swinv2_large_window12to16_192to256.ms_in22k_ft_in1k +swinv2_large_window12to24_192to384.ms_in22k_ft_in1k +swinv2_small_window16_256.ms_in1k +swinv2_small_window8_256.ms_in1k +swinv2_tiny_window16_256.ms_in1k +swinv2_tiny_window8_256.ms_in1k \ No newline at end of file diff --git a/alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard3.txt b/alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard3.txt new file mode 100644 index 00000000..7eb12e95 --- /dev/null +++ b/alt_e2eshark/onnx_tests/models/external_lists/vai-hf-cnn-fp32-shard3.txt @@ -0,0 +1,177 @@ +tf_efficientnet_b0.aa_in1k +tf_efficientnet_b0.ap_in1k +tf_efficientnet_b0.ns_jft_in1k +tf_efficientnet_b1.aa_in1k +tf_efficientnet_b1.ap_in1k +tf_efficientnet_b1.ns_jft_in1k +tf_efficientnet_b2.aa_in1k +tf_efficientnet_b2.ap_in1k +tf_efficientnet_b2.ns_jft_in1k +tf_efficientnet_b3.aa_in1k +tf_efficientnet_b3.ap_in1k +tf_efficientnet_b3.ns_jft_in1k +tf_efficientnet_b4.aa_in1k +tf_efficientnet_b4.ap_in1k +tf_efficientnet_b4.ns_jft_in1k +tf_efficientnet_b5.ap_in1k +tf_efficientnet_b5.ns_jft_in1k +tf_efficientnet_b5.ra_in1k +tf_efficientnet_b6.aa_in1k +tf_efficientnet_b6.ap_in1k +tf_efficientnet_b6.ns_jft_in1k +tf_efficientnet_b7.ap_in1k +tf_efficientnet_b7.ns_jft_in1k +tf_efficientnet_b7.ra_in1k +tf_efficientnet_b8.ap_in1k +tf_efficientnet_b8.ra_in1k +tf_efficientnet_l2.ns_jft_in1k +tf_efficientnet_l2.ns_jft_in1k_475 +tf_efficientnetv2_l.in1k +tf_efficientnetv2_l.in21k_ft_in1k +tf_efficientnetv2_m.in1k +tf_efficientnetv2_m.in21k_ft_in1k +tf_efficientnetv2_s.in1k +tf_efficientnetv2_s.in21k_ft_in1k +tf_efficientnetv2_xl.in21k_ft_in1k +tf_mixnet_l.in1k +tf_mixnet_m.in1k +tf_mixnet_s.in1k +tf_mobilenetv3_large_075.in1k +tf_mobilenetv3_large_100.in1k +tinynet_b.in1k +tinynet_c.in1k +tinynet_e.in1k +tnt_s_patch16_224 +twins_pcpvt_base +twins_pcpvt_large +twins_pcpvt_small +twins_svt_base +twins_svt_large +twins_svt_small +visformer_small +vit_base_patch16_224.augreg2_in21k_ft_in1k +vit_base_patch16_224.augreg_in1k +vit_base_patch16_224.augreg_in21k_ft_in1k +vit_base_patch16_224.orig_in21k_ft_in1k +vit_base_patch16_224.sam +vit_base_patch16_224_miil.in21k_ft_in1k +vit_base_patch16_384.augreg_in1k +vit_base_patch16_384.augreg_in21k_ft_in1k +vit_base_patch16_384.orig_in21k_ft_in1k +vit_base_patch16_clip_224.laion2b_ft_in12k +vit_base_patch16_clip_224.laion2b_ft_in12k_in1k +vit_base_patch16_clip_224.laion2b_ft_in1k +vit_base_patch16_clip_224.openai +vit_base_patch16_clip_224.openai_ft_in12k +vit_base_patch16_clip_224.openai_ft_in12k_in1k +vit_base_patch16_clip_224.openai_ft_in1k +vit_base_patch16_clip_384.laion2b_ft_in12k_in1k +vit_base_patch16_clip_384.laion2b_ft_in1k +vit_base_patch16_clip_384.openai_ft_in12k_in1k +vit_base_patch16_clip_384.openai_ft_in1k +vit_base_patch16_rpn_224.in1k +vit_base_patch32_224.augreg_in1k +vit_base_patch32_224.augreg_in21k_ft_in1k +vit_base_patch32_224.sam +vit_base_patch32_384.augreg_in1k +vit_base_patch32_384.augreg_in21k_ft_in1k +vit_base_patch32_clip_224.laion2b +vit_base_patch32_clip_224.laion2b_ft_in12k_in1k +vit_base_patch32_clip_224.laion2b_ft_in1k +vit_base_patch32_clip_224.openai +vit_base_patch32_clip_224.openai_ft_in1k +vit_base_patch32_clip_384.laion2b_ft_in12k_in1k +vit_base_patch32_clip_384.openai_ft_in12k_in1k +vit_base_patch32_clip_448.laion2b_ft_in12k_in1k +vit_base_patch8_224.augreg2_in21k_ft_in1k +vit_base_patch8_224.augreg_in21k_ft_in1k +vit_base_r50_s16_384.orig_in21k_ft_in1k +vit_giant_patch14_clip_224.laion2b +vit_gigantic_patch14_clip_224.laion2b +vit_huge_patch14_clip_224.laion2b +vit_huge_patch14_clip_224.laion2b_ft_in12k +vit_huge_patch14_clip_224.laion2b_ft_in12k_in1k +vit_huge_patch14_clip_224.laion2b_ft_in1k +vit_huge_patch14_clip_336.laion2b_ft_in12k_in1k +vit_large_patch14_clip_224.laion2b +vit_large_patch14_clip_224.laion2b_ft_in12k +vit_large_patch14_clip_224.laion2b_ft_in12k_in1k +vit_large_patch14_clip_224.laion2b_ft_in1k +vit_large_patch14_clip_224.openai +vit_large_patch14_clip_224.openai_ft_in12k +vit_large_patch14_clip_224.openai_ft_in12k_in1k +vit_large_patch14_clip_224.openai_ft_in1k +vit_large_patch14_clip_336.laion2b_ft_in12k_in1k +vit_large_patch14_clip_336.laion2b_ft_in1k +vit_large_patch14_clip_336.openai_ft_in12k_in1k +vit_large_patch16_224.augreg_in21k_ft_in1k +vit_large_patch16_384.augreg_in21k_ft_in1k +vit_large_patch32_384.orig_in21k_ft_in1k +vit_large_r50_s32_224.augreg_in21k_ft_in1k +vit_large_r50_s32_384.augreg_in21k_ft_in1k +vit_medium_patch16_gap_240.in12k +vit_medium_patch16_gap_256.in12k_ft_in1k +vit_medium_patch16_gap_384.in12k_ft_in1k +vit_relpos_base_patch16_224.sw_in1k +vit_relpos_base_patch16_clsgap_224.sw_in1k +vit_relpos_base_patch32_plus_rpn_256.sw_in1k +vit_relpos_medium_patch16_224.sw_in1k +vit_relpos_medium_patch16_cls_224.sw_in1k +vit_relpos_medium_patch16_rpn_224.sw_in1k +vit_relpos_small_patch16_224.sw_in1k +vit_small_patch16_224.augreg_in1k +vit_small_patch16_224.augreg_in21k_ft_in1k +vit_small_patch16_384.augreg_in1k +vit_small_patch16_384.augreg_in21k_ft_in1k +vit_small_patch32_224.augreg_in21k_ft_in1k +vit_small_patch32_384.augreg_in21k_ft_in1k +vit_small_r26_s32_224.augreg_in21k_ft_in1k +vit_small_r26_s32_384.augreg_in21k_ft_in1k +vit_srelpos_medium_patch16_224.sw_in1k +vit_srelpos_small_patch16_224.sw_in1k +vit_tiny_patch16_224.augreg_in21k_ft_in1k +vit_tiny_patch16_384.augreg_in21k_ft_in1k +vit_tiny_r_s16_p8_224.augreg_in21k_ft_in1k +vit_tiny_r_s16_p8_384.augreg_in21k_ft_in1k +xcit_large_24_p16_224 +xcit_large_24_p16_224_dist +xcit_large_24_p16_384_dist +xcit_large_24_p8_224 +xcit_large_24_p8_224_dist +xcit_large_24_p8_384_dist +xcit_medium_24_p16_224 +xcit_medium_24_p16_224_dist +xcit_medium_24_p16_384_dist +xcit_medium_24_p8_224 +xcit_medium_24_p8_224_dist +xcit_medium_24_p8_384_dist +xcit_nano_12_p16_224 +xcit_nano_12_p16_224_dist +xcit_nano_12_p16_384_dist +xcit_nano_12_p8_224 +xcit_nano_12_p8_224_dist +xcit_nano_12_p8_384_dist +xcit_small_12_p16_224 +xcit_small_12_p16_224_dist +xcit_small_12_p16_384_dist +xcit_small_12_p8_224 +xcit_small_12_p8_224_dist +xcit_small_12_p8_384_dist +xcit_small_24_p16_224 +xcit_small_24_p16_224_dist +xcit_small_24_p16_384_dist +xcit_small_24_p8_224 +xcit_small_24_p8_224_dist +xcit_small_24_p8_384_dist +xcit_tiny_12_p16_224 +xcit_tiny_12_p16_224_dist +xcit_tiny_12_p16_384_dist +xcit_tiny_12_p8_224 +xcit_tiny_12_p8_224_dist +xcit_tiny_12_p8_384_dist +xcit_tiny_24_p16_224 +xcit_tiny_24_p16_224_dist +xcit_tiny_24_p16_384_dist +xcit_tiny_24_p8_224 +xcit_tiny_24_p8_224_dist +xcit_tiny_24_p8_384_dist \ No newline at end of file diff --git a/alt_e2eshark/onnx_tests/models/external_lists/vai-int8-p0p1-shard1.txt b/alt_e2eshark/onnx_tests/models/external_lists/vai-int8-p0p1-shard1.txt new file mode 100644 index 00000000..03f6a606 --- /dev/null +++ b/alt_e2eshark/onnx_tests/models/external_lists/vai-int8-p0p1-shard1.txt @@ -0,0 +1,162 @@ +adv_inception_v3_vaiq +cs3darknet_focus_l_vaiq +cs3darknet_focus_l_train_vaiq +cs3darknet_focus_m_vaiq +cs3darknet_focus_m_train_vaiq +cs3darknet_l_vaiq +cs3darknet_l_train_vaiq +cs3darknet_m_vaiq +cs3darknet_m_train_vaiq +cs3darknet_x_vaiq +cs3darknet_x_train_vaiq +cs3edgenet_x_vaiq +cs3edgenet_x_train_vaiq +cs3sedarknet_l_vaiq +cs3sedarknet_l_train_vaiq +cs3sedarknet_x_vaiq +cs3sedarknet_x_train_vaiq +cs3se_edgenet_x_vaiq +cs3se_edgenet_x_train_vaiq +cspdarknet53_vaiq +cspresnet50_vaiq +cspresnext50_vaiq +densenet121_vaiq +densenet121_test_vaiq +densenet161_vaiq +densenet169_vaiq +densenetblur121d_vaiq +densenetblur121d_test_vaiq +dla102_vaiq +dla102x2_vaiq +dla102x_vaiq +dla169_vaiq +dla34_vaiq +dla46_c_vaiq +dla46x_c_vaiq +dla60_vaiq +dla60_res2net_vaiq +dla60_res2next_vaiq +dla60x_c_vaiq +dla60x_vaiq +dm_nfnet_f0.dm_in1k_vaiq +dm_nfnet_f0.dm_in1k_train_vaiq +dm_nfnet_f1.dm_in1k_vaiq +dm_nfnet_f1.dm_in1k_train_vaiq +dpn107_vaiq +dpn131_vaiq +dpn68b_vaiq +dpn68b_test_vaiq +dpn68_vaiq +dpn92_vaiq +dpn98_vaiq +eca_nfnet_l0.ra2_in1k_vaiq +eca_nfnet_l0.ra2_in1k_train_vaiq +eca_nfnet_l1.ra2_in1k_vaiq +eca_nfnet_l1.ra2_in1k_train_vaiq +eca_nfnet_l2.ra3_in1k_vaiq +eca_nfnet_l2.ra3_in1k_train_vaiq +ecaresnet101d_vaiq +ecaresnet101d_pruned_vaiq +ecaresnet101d_pruned_test_vaiq +ecaresnet101d_test_vaiq +ecaresnet26t_vaiq +ecaresnet26t_train_vaiq +eca_resnet33ts.ra2_in1k_vaiq +eca_resnet33ts.ra2_in1k_train_vaiq +ecaresnet50d_vaiq +ecaresnet50d_pruned_vaiq +ecaresnet50d_pruned_test_vaiq +ecaresnet50d_test_vaiq +ecaresnet50t_vaiq +ecaresnet50t_train_vaiq +ecaresnetlight_vaiq +ecaresnetlight_test_vaiq +eca_resnext26ts.ch_in1k_vaiq +eca_resnext26ts.ch_in1k_train_vaiq +efficientnet_b0.ra_in1k_vaiq +efficientnet_b1.ft_in1k_vaiq +efficientnet_b1.ft_in1k_train_vaiq +efficientnet_b2.ra_in1k_vaiq +efficientnet_b2.ra_in1k_train_vaiq +efficientnet_b3.ra2_in1k_vaiq +efficientnet_b3.ra2_in1k_train_vaiq +efficientnet_b4.ra2_in1k_vaiq +efficientnet_b4.ra2_in1k_train_vaiq +efficientnet_el_pruned.in1k_vaiq +efficientnet_el.ra_in1k_vaiq +efficientnet_em.ra2_in1k_vaiq +efficientnet_es_pruned.in1k_vaiq +efficientnet_es.ra_in1k_vaiq +efficientnet_lite0.ra_in1k_vaiq +efficientnetv2_rw_m.agc_in1k_vaiq +efficientnetv2_rw_m.agc_in1k_train_vaiq +efficientnetv2_rw_s.ra2_in1k_vaiq +efficientnetv2_rw_s.ra2_in1k_train_vaiq +efficientnetv2_rw_t.ra2_in1k_vaiq +efficientnetv2_rw_t.ra2_in1k_train_vaiq +ens_adv_inception_resnet_v2_vaiq +ese_vovnet19b_dw_vaiq +ese_vovnet19b_dw_test_vaiq +ese_vovnet39b_vaiq +ese_vovnet39b_test_vaiq +fbnetc_100.rmsp_in1k_vaiq +fbnetv3_b.ra2_in1k_vaiq +fbnetv3_b.ra2_in1k_train_vaiq +fbnetv3_d.ra2_in1k_vaiq +fbnetv3_d.ra2_in1k_train_vaiq +fbnetv3_g.ra2_in1k_vaiq +fbnetv3_g.ra2_in1k_train_vaiq +gc_efficientnetv2_rw_t.agc_in1k_vaiq +gc_efficientnetv2_rw_t.agc_in1k_train_vaiq +gcresnet33ts.ra2_in1k_vaiq +gcresnet33ts.ra2_in1k_train_vaiq +gcresnet50t.ra2_in1k_vaiq +gcresnet50t.ra2_in1k_train_vaiq +gcresnext26ts.ch_in1k_vaiq +gcresnext26ts.ch_in1k_train_vaiq +gcresnext50ts.ch_in1k_vaiq +gcresnext50ts.ch_in1k_train_vaiq +gernet_l.idstcv_in1k_vaiq +gernet_m.idstcv_in1k_vaiq +gernet_s.idstcv_in1k_vaiq +ghostnet_100_vaiq +gluon_inception_v3_vaiq +gluon_resnet101_v1b_vaiq +gluon_resnet101_v1c_vaiq +gluon_resnet101_v1d_vaiq +gluon_resnet101_v1s_vaiq +gluon_resnet152_v1b_vaiq +gluon_resnet152_v1c_vaiq +gluon_resnet152_v1d_vaiq +gluon_resnet152_v1s_vaiq +gluon_resnet18_v1b_vaiq +gluon_resnet34_v1b_vaiq +gluon_resnet50_v1b_vaiq +gluon_resnet50_v1c_vaiq +gluon_resnet50_v1d_vaiq +gluon_resnet50_v1s_vaiq +gluon_resnext101_32x4d_vaiq +gluon_resnext101_64x4d_vaiq +gluon_resnext50_32x4d_vaiq +gluon_senet154_vaiq +gluon_seresnext101_32x4d_vaiq +gluon_seresnext101_64x4d_vaiq +gluon_seresnext50_32x4d_vaiq +hardcorenas_a_vaiq +hardcorenas_b_vaiq +hardcorenas_c_vaiq +hardcorenas_d_vaiq +hardcorenas_e_vaiq +hardcorenas_f_vaiq +hrnet_w18_vaiq +hrnet_w18_small_vaiq +hrnet_w18_small_v2_vaiq +hrnet_w30_vaiq +hrnet_w32_vaiq +hrnet_w40_vaiq +hrnet_w44_vaiq +hrnet_w48_vaiq +hrnet_w64_vaiq +ig_resnext101_32x16d_vaiq +ig_resnext101_32x32d_vaiq +ig_resnext101_32x8d_vaiq \ No newline at end of file diff --git a/alt_e2eshark/onnx_tests/models/external_lists/vai-int8-p0p1-shard2.txt b/alt_e2eshark/onnx_tests/models/external_lists/vai-int8-p0p1-shard2.txt new file mode 100644 index 00000000..d54e08de --- /dev/null +++ b/alt_e2eshark/onnx_tests/models/external_lists/vai-int8-p0p1-shard2.txt @@ -0,0 +1,162 @@ +inception_resnet_v2_vaiq +inception_v3_vaiq +inception_v4_vaiq +lcnet_050.ra2_in1k_vaiq +lcnet_075.ra2_in1k_vaiq +lcnet_100.ra2_in1k_vaiq +legacy_senet154_vaiq +legacy_seresnet101_vaiq +legacy_seresnet152_vaiq +legacy_seresnet18_vaiq +legacy_seresnet34_vaiq +legacy_seresnet50_vaiq +legacy_seresnext101_32x4d_vaiq +legacy_seresnext26_32x4d_vaiq +legacy_seresnext50_32x4d_vaiq +mixnet_l.ft_in1k_vaiq +mixnet_m.ft_in1k_vaiq +mixnet_s.ft_in1k_vaiq +mixnet_xl.ra_in1k_vaiq +mnasnet_100.rmsp_in1k_vaiq +mnasnet_small.lamb_in1k_vaiq +mobilenetv2_050.lamb_in1k_vaiq +mobilenetv2_100.ra_in1k_vaiq +mobilenetv2_110d.ra_in1k_vaiq +mobilenetv2_120d.ra_in1k_vaiq +mobilenetv2_140.ra_in1k_vaiq +mobilenetv3_large_100.miil_in21k_ft_in1k_vaiq +mobilenetv3_large_100.ra_in1k_vaiq +mobilenetv3_rw.rmsp_in1k_vaiq +mobilenetv3_small_050.lamb_in1k_vaiq +mobilenetv3_small_075.lamb_in1k_vaiq +mobilenetv3_small_100.lamb_in1k_vaiq +nfnet_l0.ra2_in1k_vaiq +nfnet_l0.ra2_in1k_train_vaiq +nf_regnet_b1.ra2_in1k_vaiq +nf_regnet_b1.ra2_in1k_train_vaiq +nf_resnet50.ra2_in1k_vaiq +nf_resnet50.ra2_in1k_train_vaiq +regnetv_040.ra3_in1k_vaiq +regnetv_040.ra3_in1k_train_vaiq +regnetv_064.ra3_in1k_vaiq +regnetv_064.ra3_in1k_train_vaiq +regnetx_002.pycls_in1k_vaiq +regnetx_004.pycls_in1k_vaiq +regnetx_004_tv.tv2_in1k_vaiq +regnetx_006.pycls_in1k_vaiq +regnetx_008.pycls_in1k_vaiq +regnetx_008.tv2_in1k_vaiq +regnetx_016.pycls_in1k_vaiq +regnetx_016.tv2_in1k_vaiq +regnetx_032.pycls_in1k_vaiq +regnetx_032.tv2_in1k_vaiq +regnetx_040.pycls_in1k_vaiq +regnetx_064.pycls_in1k_vaiq +regnetx_080.pycls_in1k_vaiq +regnetx_080.tv2_in1k_vaiq +regnetx_120.pycls_in1k_vaiq +regnetx_160.pycls_in1k_vaiq +regnetx_160.tv2_in1k_vaiq +regnetx_320.pycls_in1k_vaiq +regnetx_320.tv2_in1k_vaiq +regnety_002.pycls_in1k_vaiq +regnety_004.pycls_in1k_vaiq +regnety_004.tv2_in1k_vaiq +regnety_006.pycls_in1k_vaiq +regnety_008.pycls_in1k_vaiq +regnety_008_tv.tv2_in1k_vaiq +regnety_016.pycls_in1k_vaiq +regnety_016.tv2_in1k_vaiq +regnety_032.pycls_in1k_vaiq +regnety_032.ra_in1k_vaiq +regnety_032.ra_in1k_train_vaiq +regnety_032.tv2_in1k_vaiq +regnety_040.pycls_in1k_vaiq +regnety_040.ra3_in1k_vaiq +regnety_040.ra3_in1k_train_vaiq +regnety_064.pycls_in1k_vaiq +regnety_064.ra3_in1k_vaiq +regnety_064.ra3_in1k_train_vaiq +regnety_080.pycls_in1k_vaiq +regnety_080.ra3_in1k_vaiq +regnety_080.ra3_in1k_train_vaiq +regnety_080_tv.tv2_in1k_vaiq +regnety_120.pycls_in1k_vaiq +regnety_120.sw_in12k_ft_in1k_vaiq +regnety_120.sw_in12k_ft_in1k_train_vaiq +regnety_160.lion_in12k_ft_in1k_vaiq +regnety_160.lion_in12k_ft_in1k_train_vaiq +regnety_160.pycls_in1k_vaiq +regnety_160.swag_ft_in1k_vaiq +regnety_160.swag_lc_in1k_vaiq +regnety_160.sw_in12k_ft_in1k_vaiq +regnety_160.sw_in12k_ft_in1k_train_vaiq +regnety_160.tv2_in1k_vaiq +regnety_320.pycls_in1k_vaiq +regnety_320.seer_ft_in1k_vaiq +regnety_320.swag_ft_in1k_vaiq +regnety_320.swag_lc_in1k_vaiq +regnety_320.tv2_in1k_vaiq +regnety_640.seer_ft_in1k_vaiq +regnetz_040_h.ra3_in1k_vaiq +regnetz_040_h.ra3_in1k_train_vaiq +regnetz_040.ra3_in1k_vaiq +regnetz_040.ra3_in1k_train_vaiq +regnetz_b16.ra3_in1k_vaiq +regnetz_b16.ra3_in1k_train_vaiq +regnetz_c16_evos.ch_in1k_vaiq +regnetz_c16_evos.ch_in1k_train_vaiq +regnetz_c16.ra3_in1k_vaiq +regnetz_c16.ra3_in1k_train_vaiq +regnetz_d32.ra3_in1k_vaiq +regnetz_d32.ra3_in1k_train_vaiq +regnetz_d8_evos.ch_in1k_vaiq +regnetz_d8_evos.ch_in1k_train_vaiq +regnetz_d8.ra3_in1k_vaiq +regnetz_d8.ra3_in1k_train_vaiq +regnetz_e8.ra3_in1k_vaiq +regnetz_e8.ra3_in1k_train_vaiq +repvgg_a2.rvgg_in1k_vaiq +repvgg_b0.rvgg_in1k_vaiq +repvgg_b1g4.rvgg_in1k_vaiq +repvgg_b1.rvgg_in1k_vaiq +repvgg_b2g4.rvgg_in1k_vaiq +repvgg_b2.rvgg_in1k_vaiq +repvgg_b3g4.rvgg_in1k_vaiq +repvgg_b3.rvgg_in1k_vaiq +res2net101_26w_4s_vaiq +res2net50_14w_8s_vaiq +res2net50_26w_4s_vaiq +res2net50_26w_6s_vaiq +res2net50_26w_8s_vaiq +res2net50_48w_2s_vaiq +res2next50_vaiq +resmlp_12_224.fb_distilled_in1k_vaiq +resmlp_12_224.fb_in1k_vaiq +resmlp_24_224.fb_distilled_in1k_vaiq +resmlp_24_224.fb_in1k_vaiq +resmlp_36_224.fb_distilled_in1k_vaiq +resmlp_36_224.fb_in1k_vaiq +resmlp_big_24_224.fb_distilled_in1k_vaiq +resnest101e_vaiq +resnest14d_vaiq +resnest26d_vaiq +resnest50d_1s4x24d_vaiq +resnest50d_4s2x40d_vaiq +resnest50d_vaiq +resnet101d_vaiq +resnet101d_train_vaiq +resnet101_vaiq +resnet101_test_vaiq +resnet10t_vaiq +resnet10t_train_vaiq +resnet14t_vaiq +resnet14t_train_vaiq +resnet152d_vaiq +resnet152d_train_vaiq +resnet152_vaiq +resnet152_test_vaiq +resnet18d_vaiq +resnet18d_test_vaiq +resnet18_vaiq +resnet18_test_vaiq \ No newline at end of file diff --git a/alt_e2eshark/onnx_tests/models/external_lists/vai-int8-p0p1-shard3.txt b/alt_e2eshark/onnx_tests/models/external_lists/vai-int8-p0p1-shard3.txt new file mode 100644 index 00000000..f14b32a7 --- /dev/null +++ b/alt_e2eshark/onnx_tests/models/external_lists/vai-int8-p0p1-shard3.txt @@ -0,0 +1,162 @@ +resnet200d_vaiq +resnet200d_train_vaiq +resnet26d_vaiq +resnet26d_test_vaiq +resnet26_vaiq +resnet26_test_vaiq +resnet26t_vaiq +resnet26t_test_vaiq +resnet32ts.ra2_in1k_vaiq +resnet32ts.ra2_in1k_train_vaiq +resnet33ts.ra2_in1k_vaiq +resnet33ts.ra2_in1k_train_vaiq +resnet34d_vaiq +resnet34d_test_vaiq +resnet34_vaiq +resnet34_test_vaiq +resnet50d_vaiq +resnet50d_test_vaiq +resnet50_gn_vaiq +resnet50_gn_test_vaiq +resnet50_vaiq +resnet50_test_vaiq +resnet51q.ra2_in1k_vaiq +resnet51q.ra2_in1k_train_vaiq +resnet61q.ra2_in1k_vaiq +resnet61q.ra2_in1k_train_vaiq +resnetaa50_vaiq +resnetaa50_train_vaiq +resnetblur50_vaiq +resnetblur50_test_vaiq +resnetrs101_vaiq +resnetrs101_train_vaiq +resnetrs152_vaiq +resnetrs152_train_vaiq +resnetrs200_vaiq +resnetrs200_train_vaiq +resnetrs50_vaiq +resnetrs50_train_vaiq +resnetv2_101.a1h_in1k_vaiq +resnetv2_101.a1h_in1k_train_vaiq +resnetv2_101x1_bit.goog_in21k_ft_in1k_vaiq +resnetv2_152x2_bit.goog_in21k_ft_in1k_vaiq +resnetv2_152x2_bit.goog_teacher_in21k_ft_in1k_384_vaiq +resnetv2_152x2_bit.goog_teacher_in21k_ft_in1k_vaiq +resnetv2_50.a1h_in1k_vaiq +resnetv2_50.a1h_in1k_train_vaiq +resnetv2_50d_evos.ah_in1k_vaiq +resnetv2_50d_evos.ah_in1k_train_vaiq +resnetv2_50d_gn.ah_in1k_vaiq +resnetv2_50d_gn.ah_in1k_train_vaiq +resnetv2_50x1_bit.goog_distilled_in1k_vaiq +resnetv2_50x1_bit.goog_in21k_ft_in1k_vaiq +resnetv2_50x3_bit.goog_in21k_ft_in1k_vaiq +resnext101_32x8d_vaiq +resnext101_64x4d_vaiq +resnext101_64x4d_train_vaiq +resnext26ts.ra2_in1k_vaiq +resnext26ts.ra2_in1k_train_vaiq +resnext50_32x4d_vaiq +resnext50_32x4d_test_vaiq +resnext50d_32x4d_vaiq +resnext50d_32x4d_test_vaiq +rexnet_100.nav_in1k_vaiq +rexnet_130.nav_in1k_vaiq +rexnet_150.nav_in1k_vaiq +rexnet_200.nav_in1k_vaiq +rexnet_300.nav_in1k_vaiq +rexnetr_200.sw_in12k_ft_in1k_vaiq +rexnetr_200.sw_in12k_ft_in1k_train_vaiq +rexnetr_300.sw_in12k_ft_in1k_vaiq +rexnetr_300.sw_in12k_ft_in1k_train_vaiq +selecsls42b_vaiq +selecsls60b_vaiq +selecsls60_vaiq +semnasnet_075.rmsp_in1k_vaiq +semnasnet_100.rmsp_in1k_vaiq +sequencer2d_m_vaiq +sequencer2d_s_vaiq +seresnet152d_vaiq +seresnet152d_train_vaiq +seresnet33ts.ra2_in1k_vaiq +seresnet33ts.ra2_in1k_train_vaiq +seresnet50_vaiq +seresnet50_test_vaiq +seresnext101_32x8d_vaiq +seresnext101_32x8d_train_vaiq +seresnext101d_32x8d_vaiq +seresnext101d_32x8d_train_vaiq +seresnext26d_32x4d_vaiq +seresnext26d_32x4d_test_vaiq +seresnext26t_32x4d_vaiq +seresnext26t_32x4d_test_vaiq +seresnext26ts.ch_in1k_vaiq +seresnext26ts.ch_in1k_train_vaiq +seresnext50_32x4d_vaiq +seresnext50_32x4d_test_vaiq +seresnextaa101d_32x8d_vaiq +seresnextaa101d_32x8d_test_vaiq +skresnet18_vaiq +skresnet34_vaiq +skresnext50_32x4d_vaiq +spnasnet_100.rmsp_in1k_vaiq +ssl_resnet18_vaiq +ssl_resnet50_vaiq +ssl_resnext101_32x16d_vaiq +ssl_resnext101_32x4d_vaiq +ssl_resnext101_32x8d_vaiq +ssl_resnext50_32x4d_vaiq +swsl_resnet18_vaiq +swsl_resnet50_vaiq +swsl_resnext101_32x16d_vaiq +swsl_resnext101_32x4d_vaiq +swsl_resnext101_32x8d_vaiq +swsl_resnext50_32x4d_vaiq +tf_efficientnet_el.in1k_vaiq +tf_efficientnet_em.in1k_vaiq +tf_efficientnet_es.in1k_vaiq +tf_efficientnet_lite0.in1k_vaiq +tf_efficientnet_lite1.in1k_vaiq +tf_efficientnet_lite2.in1k_vaiq +tf_efficientnet_lite3.in1k_vaiq +tf_efficientnet_lite4.in1k_vaiq +tf_efficientnetv2_b0.in1k_vaiq +tf_efficientnetv2_b0.in1k_train_vaiq +tf_efficientnetv2_b1.in1k_vaiq +tf_efficientnetv2_b1.in1k_train_vaiq +tf_efficientnetv2_b2.in1k_vaiq +tf_efficientnetv2_b2.in1k_train_vaiq +tf_efficientnetv2_b3.in1k_vaiq +tf_efficientnetv2_b3.in1k_train_vaiq +tf_efficientnetv2_b3.in21k_ft_in1k_vaiq +tf_efficientnetv2_b3.in21k_ft_in1k_train_vaiq +tf_inception_v3_vaiq +tf_mobilenetv3_large_minimal_100.in1k_vaiq +tf_mobilenetv3_small_075.in1k_vaiq +tf_mobilenetv3_small_100.in1k_vaiq +tf_mobilenetv3_small_minimal_100.in1k_vaiq +tinynet_a.in1k_vaiq +tinynet_d.in1k_vaiq +tv_densenet121_vaiq +tv_resnet101_vaiq +tv_resnet152_vaiq +tv_resnet34_vaiq +tv_resnet50_vaiq +tv_resnext50_32x4d_vaiq +vgg11_bn_vaiq +vgg11_vaiq +vgg13_bn_vaiq +vgg13_vaiq +vgg16_bn_vaiq +vgg16_vaiq +vgg19_bn_vaiq +vgg19_vaiq +wide_resnet101_2_vaiq +wide_resnet50_2_vaiq +wide_resnet50_2_test_vaiq +xception41_vaiq +xception41p_vaiq +xception65_vaiq +xception65p_vaiq +xception71_vaiq +xception_vaiq \ No newline at end of file diff --git a/alt_e2eshark/onnx_tests/models/external_lists/vai-vision-int8.txt b/alt_e2eshark/onnx_tests/models/external_lists/vai-vision-int8.txt new file mode 100644 index 00000000..87b777ac --- /dev/null +++ b/alt_e2eshark/onnx_tests/models/external_lists/vai-vision-int8.txt @@ -0,0 +1,78 @@ +DarkNet53_vaiq +dla169_vaiq +efficientnet_b0.ra_in1k_vaiq +EfficientNet_b0_vaiq +efficientnet_b1.ft_in1k_vaiq +EfficientNet_b1_vaiq +efficientnet_b2.ra_in1k_vaiq +EfficientNet_b2_vaiq +efficientnet_b3.ra2_in1k_vaiq +EfficientNet_b3_vaiq +efficientnet_b4.ra2_in1k_vaiq +EfficientNet_b4_vaiq +efficientnet_b5.sw_in12k_vaiq +EfficientNet_b5_vaiq +EfficientNet_b6_vaiq +EfficientNet_b7_vaiq +efficientnet_el_pruned.in1k_vaiq +efficientnet_el.ra_in1k_vaiq +efficientnet_em.ra2_in1k_vaiq +efficientnet_es_pruned.in1k_vaiq +efficientnet_es.ra_in1k_vaiq +efficientnet_lite0.ra_in1k_vaiq +EfficientNet_v2_l_vaiq +EfficientNet_v2_m_vaiq +efficientnetv2_rw_m.agc_in1k_vaiq +efficientnetv2_rw_s.ra2_in1k_vaiq +efficientnetv2_rw_t.ra2_in1k_vaiq +EfficientNet_v2_s_vaiq +fbnetc_100.rmsp_in1k_vaiq +gernet_l.idstcv_in1k_vaiq +gernet_m.idstcv_in1k_vaiq +gernet_s.idstcv_in1k_vaiq +GoogLeNet_vaiq +inception_v3.tf_in1k_vaiq +Inception_v3_vaiq +inception_v4.tf_in1k_vaiq +lcnet_050.ra2_in1k_vaiq +lcnet_075.ra2_in1k_vaiq +lcnet_100.ra2_in1k_vaiq +mnasnet_100.rmsp_in1k_vaiq +mnasnet_small.lamb_in1k_vaiq +mobilenetv2_050.lamb_in1k_vaiq +mobilenetv2_100.ra_in1k_vaiq +mobilenetv2_110d.ra_in1k_vaiq +mobilenetv2_120d.ra_in1k_vaiq +mobilenetv2_140.ra_in1k_vaiq +MobileNetV2_vaiq +mobilenetv3_large_100.ra_in1k_vaiq +MobileNetV3_large_vaiq +mobilenetv3_small_050.lamb_in1k_vaiq +mobilenetv3_small_075.lamb_in1k_vaiq +mobilenetv3_small_100.lamb_in1k_vaiq +MobileNetV3_small_vaiq +ResNet101_vaiq +ResNet152_vaiq +ResNet18_vaiq +resnet32ts.ra2_in1k_vaiq +resnet33ts.ra2_in1k_vaiq +ResNet34_vaiq +resnet50.a1_in1k_vaiq +ResNet50_vaiq +RRDB_ESRGAN_pro_vaiq +RRDB_ESRGAN_vaiq +SqueezeNet_1_0_vaiq +SqueezeNet_1_1_vaiq +VGG11_bn_vaiq +VGG11_vaiq +VGG13_bn_vaiq +VGG13_vaiq +VGG16_bn_vaiq +VGG16_vaiq +VGG19_bn_vaiq +VGG19_vaiq +WideResNet_101_2_vaiq +WideResNet_50_2_vaiq +YoloNetV3_vaiq +Yolov8m_vaiq +Yolov8n_vaiq \ No newline at end of file diff --git a/alt_e2eshark/onnx_tests/models/vision_models.py b/alt_e2eshark/onnx_tests/models/vision_models.py index e8d170e4..cd1aee84 100644 --- a/alt_e2eshark/onnx_tests/models/vision_models.py +++ b/alt_e2eshark/onnx_tests/models/vision_models.py @@ -23,8 +23,11 @@ def apply_postprocessing(self, output: TestTensors): return TestTensors(processed_outputs) # this will run resnet50_vaiq_int8 without post-processing -register_test(AzureDownloadableModel, "resnet50_vaiq_int8") -register_test(AzureDownloadableModel, "ResNet50_vaiq") +# register_test(AzureDownloadableModel, "resnet50_vaiq_int8") +# register_test(AzureDownloadableModel, "ResNet50_vaiq") +# register_test(AzureDownloadableModel, "ResNet152") +# register_test(AzureDownloadableModel, "ResNet152_vaiq") +# register_test(AzureDownloadableModel, "ResNet152_vaiq_int8") # this will run the same model, but with post-processing constructor0 = get_sibling_constructor(ImageClassificationModel, AzureDownloadableModel, "resnet50_vaiq_int8") diff --git a/alt_e2eshark/run.py b/alt_e2eshark/run.py index 368ddd77..c7eb4505 100644 --- a/alt_e2eshark/run.py +++ b/alt_e2eshark/run.py @@ -11,7 +11,7 @@ import argparse import re import logging -from typing import List +from typing import List, Literal, Optional # append alt_e2eshark dir to path to allow importing without explicit pythonpath management TEST_DIR = str(Path(__file__).parent) @@ -28,6 +28,7 @@ # import backends from e2e_testing.backends import SimpleIREEBackend, OnnxrtIreeEpBackend +from e2e_testing.storage import load_test_txt_file, load_json_dict from utils.report import generate_report, save_dict ALL_STAGES = [ @@ -41,7 +42,7 @@ "postprocessing", ] -def get_tests(groups, test_filter): +def get_tests(groups: Literal["all", "combinations", "operators"], test_filter: Optional[str], testsfile: Optional[str]) -> List[str]: """imports tests based on groups and test_filter specification""" combinations = True if groups == "all" or groups == "combinations" else False models = True if groups == "all" or groups == "models" else False @@ -57,12 +58,18 @@ def get_tests(groups, test_filter): if operators: from onnx_tests.operators import model + pre_test_list = GLOBAL_TEST_LIST + + if testsfile: + test_names = load_test_txt_file(testsfile) + pre_test_list = [t for t in GLOBAL_TEST_LIST if t.unique_name in test_names] + if test_filter: test_list = [ - test for test in GLOBAL_TEST_LIST if re.match(test_filter, test.unique_name) + test for test in pre_test_list if re.match(test_filter, test.unique_name) ] else: - test_list = GLOBAL_TEST_LIST + test_list = pre_test_list return test_list @@ -84,7 +91,7 @@ def main(args): raise NotImplementedError(f"unsupported mode: {args.mode}") # get test list - test_list = get_tests(args.groups, args.test_filter) + test_list = get_tests(args.groups, args.test_filter, args.testsfile) #setup test stages stages = ALL_STAGES @@ -376,10 +383,10 @@ def _get_argparse(): "--test-filter", help="Run given specific test(s) only", ) - # parser.add_argument( - # "--testsfile", - # help="A file with lists of tests (starting with framework name) to run", - # ) + parser.add_argument( + "--testsfile", + help="A file with lists of test names to run", + ) # test tolerance parser.add_argument(