Fix setup failures in Huggingface model tests (nod-ai#415)

Updates HF cache locations to respect CACHE_DIR, and fixes a number of
setup failures in the HF Model tests.

Also prunes the shard lists to remove some models that Optimum does not
have proper ONNX exports for.

alt_e2eshark/base_requirements.txt

@@ -11,11 +11,4 @@ huggingface-hub
 sentencepiece
 accelerate
 auto-gptq
-optimum
-sentence_transformers
 azure-storage-blob
-compressed_tensors
-fugashi
-tensorflow
-tf-keras
-timm

alt_e2eshark/e2e_testing/test_configs/onnxconfig.py

@@ -285,4 +285,4 @@ def benchmark(self, artifact: str, inputs: TestTensors, repetitions: int = 5, *,
         if mean_stats["name"] != f"BM_{func_name}/process_time/real_time_mean":
             raise ValueError("Name of benchmark item is unexpected")
         time = mean_stats["real_time"]
-        return time
+        return time

alt_e2eshark/hf_requirements.txt

@@ -0,0 +1,6 @@
+optimum[exporters]
+compressed_tensors
+sentence_transformers
+fugashi
+timm
+unidic-lite

alt_e2eshark/onnx_tests/helper_classes.py

@@ -3,12 +3,11 @@
 # Licensed under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-import os
+
 import requests
 import tarfile
 import shutil
 import yaml
-import subprocess
 import onnx
 import onnxruntime
 
@@ -22,8 +21,18 @@
     get_sample_inputs_for_onnx_model
 )
 
+# Checking if CACHE_DIR is set here will allow us to redefine
+# HF_HOME and HUGGINGFACE_HUB_CACHE without requiring them to
+# be set at shell level.
+import os
+parent_cache_dir = os.getenv("CACHE_DIR")
+
+os.environ['HF_HOME'] = "" if parent_cache_dir is None else parent_cache_dir
+os.environ['HUGGINGFACE_HUB_CACHE'] = "" if parent_cache_dir is None else parent_cache_dir
+
+
 try:
-    import optimum.exporters.onnx as opt_onnx
+    import optimum.exporters.onnx as exporter
 except ImportError:
     print("Failed to import ONNX Exporter module from optimum. Please install through `pip install optimum[exporters]`.")
 
@@ -42,12 +51,15 @@ def __init__(self, full_model_path, task_name, name, onnx_model_path):
             raise RuntimeError(
                 "Please specify a cache directory path in the CACHE_DIR environment variable for storing large model files."
             )
-        self.cache_dir = os.path.join(parent_cache_dir, name)
+        # Appending the test name just adds one redundant level of nesting to the cache dir.
+        # Use the value of the CACHE_DIR directly, the segregation of distinct models should
+        # be handled by Huggingface itself.
+        self.cache_dir = parent_cache_dir
         super().__init__(name, onnx_model_path, opset_version)
 
     def export_model(self):
         model_dir = str(Path(self.model).parent)
-        opt_onnx.main_export(
+        exporter.main_export(
             self.model_repo_path,
             output=model_dir,
             task=self.task,

alt_e2eshark/onnx_tests/models/external_lists/hf-model-paths/hf-feature-extraction-model-list.txt

@@ -11,67 +11,45 @@ sentence-transformers/hf_bert-base-nli-mean-tokens
 sentence-transformers/hf_all-distilroberta-v1
 BAAI/hf_bge-large-en-v1.5
 mixedbread-ai/hf_mxbai-embed-large-v1
-Alibaba-NLP/hf_gte-large-en-v1.5
 BAAI/hf_bge-base-en-v1.5
-BAAI/hf_bge-m3
 facebook/hf_bart-base
-unslothai/hf_1
 ntu-spml/hf_distilhubert
 cointegrated/hf_rubert-tiny
 sentence-transformers/hf_paraphrase-multilingual-mpnet-base-v2
-nomic-ai/hf_nomic-embed-text-v1.5
-BAAI/hf_bge-reranker-large
 BAAI/hf_bge-large-zh-v1.5
 cambridgeltl/hf_SapBERT-from-PubMedBERT-fulltext
 sentence-transformers/hf_multi-qa-mpnet-base-dot-v1
-intfloat/hf_multilingual-e5-large
-unslothai/hf_other
 dmis-lab/hf_biobert-v1.1
 YituTech/hf_conv-bert-base
-kyutai/hf_mimi
 WhereIsAI/hf_UAE-Large-V1
 sentence-transformers/hf_all-roberta-large-v1
-Alibaba-NLP/hf_gte-multilingual-base
-dunzhang/hf_stella_en_400M_v5
-stepfun-ai/hf_GOT-OCR2_0
 avsolatorio/hf_GIST-small-Embedding-v0
 jhgan/hf_ko-sroberta-multitask
 sentence-transformers/hf_msmarco-distilbert-dot-v5
 T-Systems-onsite/hf_cross-en-de-roberta-sentence-transformer
-google/hf_canine-c
 ai-forever/hf_sbert_large_nlu_ru
 cointegrated/hf_rubert-tiny2
 indobenchmark/hf_indobert-base-p1
-jinaai/hf_jina-embeddings-v3
 sentence-transformers/hf_distilbert-base-nli-stsb-mean-tokens
 nvidia/hf_dragon-multiturn-context-encoder
 nvidia/hf_dragon-multiturn-query-encoder
 sentence-transformers/hf_distiluse-base-multilingual-cased-v1
 BAAI/hf_bge-small-en
 sentence-transformers/hf_stsb-xlm-r-multilingual
 cambridgeltl/hf_SapBERT-UMLS-2020AB-all-lang-from-XLMR
-unslothai/hf_repeat
 sentence-transformers/hf_stsb-roberta-base
 sentence-transformers/hf_roberta-base-nli-mean-tokens
 BAAI/hf_bge-large-en
 Tochka-AI/hf_ruRoPEBert-e5-base-2k
-unslothai/hf_vram-24
 sentence-transformers/hf_distiluse-base-multilingual-cased-v2
-facebook/hf_encodec_24khz
 sentence-transformers/hf_msmarco-distilbert-base-v4
 avsolatorio/hf_GIST-Embedding-v0
-nomic-ai/hf_nomic-embed-text-v1
-unslothai/hf_vram-16
 sentence-transformers/hf_msmarco-distilbert-base-tas-b
 sentence-transformers/hf_paraphrase-mpnet-base-v2
 microsoft/hf_wavlm-base-plus
 avsolatorio/hf_GIST-large-Embedding-v0
 Supabase/hf_gte-small
-owkin/hf_phikon
-Alibaba-NLP/hf_gte-base-en-v1.5
 sentence-transformers/hf_paraphrase-MiniLM-L3-v2
-unslothai/hf_vram-80
-unslothai/hf_vram-48
 sentence-transformers/hf_LaBSE
 Snowflake/hf_snowflake-arctic-embed-m
 jinaai/hf_jina-embeddings-v2-small-en
@@ -81,18 +59,11 @@ sentence-transformers/hf_msmarco-MiniLM-L6-cos-v5
 monologg/hf_kobert
 sentence-transformers/hf_paraphrase-xlm-r-multilingual-v1
 lewtun/hf_tiny-random-mt5
-internlm/hf_internlm-xcomposer2d5-clip
 BAAI/hf_llm-embedder
 sentence-transformers/hf_multi-qa-distilbert-cos-v1
 cointegrated/hf_LaBSE-en-ru
-facebook/hf_w2v-bert-2.0
 hkunlp/hf_instructor-large
-rinna/hf_japanese-cloob-vit-b-16
 emrecan/hf_bert-base-turkish-cased-mean-nli-stsb-tr
 sentence-transformers/hf_msmarco-distilbert-cos-v5
-microsoft/hf_xclip-large-patch14
-unslothai/hf_colab
-microsoft/hf_unixcoder-base
 sentence-transformers/hf_distilbert-base-nli-mean-tokens
-intfloat/hf_multilingual-e5-large-instruct
 DeepPavlov/hf_rubert-base-cased

alt_e2eshark/onnx_tests/models/external_lists/hf-model-paths/hf-fill-mask-model-list.txt

@@ -1,4 +1,3 @@
-FacebookAI/hf_xlm-roberta-large
 google-bert/hf_bert-base-uncased
 FacebookAI/hf_roberta-base
 distilbert/hf_distilbert-base-uncased
@@ -13,7 +12,6 @@ microsoft/hf_deberta-base
 emilyalsentzer/hf_Bio_ClinicalBERT
 neuralmind/hf_bert-base-portuguese-cased
 microsoft/hf_deberta-v3-large
-microsoft/hf_infoxlm-large
 distilbert/hf_distilroberta-base
 google-bert/hf_bert-large-uncased
 facebook/hf_esm2_t36_3B_UR50D
@@ -34,7 +32,6 @@ naver/hf_splade-cocondenser-ensembledistil
 Rostlab/hf_prot_bert
 ku-nlp/hf_deberta-v2-base-japanese-char-wwm
 facebook/hf_esm2_t12_35M_UR50D
-facebook/hf_esm2_t33_650M_UR50D
 nlpaueb/hf_legal-bert-base-uncased
 distilbert/hf_distilbert-base-multilingual-cased
 prithivida/hf_Splade_PP_en_v1
@@ -44,46 +41,32 @@ vinai/hf_phobert-base-v2
 distilbert/hf_distilbert-base-cased
 tohoku-nlp/hf_bert-base-japanese-whole-word-masking
 microsoft/hf_BiomedNLP-BiomedBERT-base-uncased-abstract-fulltext
-yikuan8/hf_Clinical-Longformer
 neulab/hf_codebert-python
 neulab/hf_codebert-java
 pdelobelle/hf_robbert-v2-dutch-base
 facebook/hf_esm2_t6_8M_UR50D
 neuralmind/hf_bert-large-portuguese-cased
 hfl/hf_chinese-roberta-wwm-ext
-microsoft/hf_BiomedVLP-CXR-BERT-specialized
 sentence-transformers/hf_multi-qa-mpnet-base-cos-v1
 microsoft/hf_codebert-base-mlm
-google/hf_electra-small-generator
 vinai/hf_phobert-base
 ufal/hf_robeczech-base
 tohoku-nlp/hf_bert-base-japanese-char-v2
-kuleshov-group/hf_caduceus-ps_seqlen-131k_d_model-256_n_layer-16
-microsoft/hf_graphcodebert-base
 dmis-lab/hf_biobert-base-cased-v1.2
 kykim/hf_bert-kor-base
 microsoft/hf_deberta-v3-xsmall
 dbmdz/hf_bert-base-italian-xxl-uncased
 DeepChem/hf_ChemBERTa-77M-MLM
 vinai/hf_bertweet-base
 UBC-NLP/hf_ARBERTv2
-westlake-repl/hf_SaProt_650M_AF2
-microsoft/hf_deberta-v2-xlarge
-westlake-repl/hf_SaProt_650M_PDB
 amine/hf_bert-base-5lang-cased
-ibm/hf_MoLFormer-XL-both-10pct
 naver/hf_efficient-splade-VI-BT-large-doc
 naver/hf_efficient-splade-VI-BT-large-query
 ALINEAR/hf_albert-japanese-v2
 airesearch/hf_wangchanberta-base-att-spm-uncased
 google-bert/hf_bert-base-german-cased
-alger-ia/hf_dziribert
-wietsedv/hf_bert-base-dutch-cased
-multimolecule/hf_rinalmo
 facebook/hf_esm2_t30_150M_UR50D
 sentence-transformers/hf_all-mpnet-base-v1
 ai4bharat/hf_IndicBERTv2-MLM-only
 dbmdz/hf_bert-base-italian-xxl-cased
-aubmindlab/hf_bert-base-arabert
-facebook/hf_xlm-roberta-xl
 cahya/hf_bert-base-indonesian-1.5G

alt_e2eshark/onnx_tests/models/external_lists/hf-model-paths/hf-image-classification-model-list.txt

@@ -9,7 +9,6 @@ microsoft/hf_beit-base-patch16-224-pt22k-ft22k
 timm/hf_mobilenetv3_large_100.ra_in1k
 nateraw/hf_vit-age-classifier
 rizvandwiki/hf_gender-classification
-AdamCodd/hf_vit-base-nsfw-detector
 trpakov/hf_vit-face-expression
 timm/hf_efficientnet_b0.ra_in1k
 timm/hf_vgg16.tv_in1k
@@ -18,22 +17,14 @@ timm/hf_resnet34.a1_in1k
 apple/hf_mobilevit-small
 timm/hf_tf_mobilenetv3_large_minimal_100.in1k
 timm/hf_vit_base_patch16_224.augreg2_in21k_ft_in1k
-microsoft/hf_swinv2-tiny-patch4-window16-256
-google/hf_vit-large-patch32-384
 google/hf_mobilenet_v1_0.75_192
-timm/hf_resnet50_gn.a1h_in1k
 timm/hf_resnext50_32x4d.fb_swsl_ig1b_ft_in1k
 timm/hf_vit_base_patch16_224.augreg_in21k
 timm/hf_swin_base_patch4_window7_224.ms_in22k_ft_in1k
-timm/hf_convnext_xxlarge.clip_laion2b_soup_ft_in1k
-timm/hf_vit_base_patch16_384.augreg_in21k_ft_in1k
-google/hf_vit-base-patch16-384
-cledoux42/hf_Ethnicity_Test_v003
 timm/hf_efficientnet_b3.ra2_in1k
 timm/hf_tf_mobilenetv3_small_minimal_100.in1k
 amrul-hzz/hf_watermark_detector
 timm/hf_mobilenetv3_large_100.miil_in21k_ft_in1k
-timm/hf_vit_base_r50_s16_384.orig_in21k_ft_in1k
 timm/hf_vit_base_patch8_224.augreg2_in21k_ft_in1k
 microsoft/hf_resnet-18
 timm/hf_resnet101.a1h_in1k
@@ -42,7 +33,6 @@ facebook/hf_deit-base-patch16-224
 timm/hf_wide_resnet50_2.racm_in1k
 timm/hf_vit_small_patch16_224.augreg_in21k_ft_in1k
 timm/hf_inception_resnet_v2.tf_in1k
-robertsw/hf_dino_base_H100
 timm/hf_eva_large_patch14_196.in22k_ft_in22k_in1k
 timm/hf_densenet121.ra_in1k
 timm/hf_mobilenetv2_100.ra_in1k
@@ -56,16 +46,13 @@ timm/hf_vgg19.tv_in1k
 timm/hf_lcnet_050.ra2_in1k
 google/hf_mobilenet_v2_1.0_224
 sharmajai901/hf_UL_base_classification
-timm/hf_vit_tiny_patch16_224.augreg_in21k
 timm/hf_tf_efficientnetv2_s.in21k
 timm/hf_convnext_large_mlp.clip_laion2b_soup_ft_in12k_in1k_320
 timm/hf_convnext_base.fb_in22k_ft_in1k
 nvidia/hf_mit-b0
 pittawat/hf_vit-base-uppercase-english-characters
 timm/hf_inception_v3.tf_adv_in1k
 timm/hf_vit_tiny_patch16_224.augreg_in21k_ft_in1k
-microsoft/hf_dit-large-finetuned-rvlcdip
-timm/hf_mobilevit_s.cvnets_in1k
 nvidia/hf_mit-b5
 timm/hf_inception_v3.tv_in1k
 facebook/hf_deit-tiny-patch16-224
@@ -77,24 +64,8 @@ timm/hf_regnety_002.pycls_in1k
 timm/hf_pnasnet5large.tf_in1k
 timm/hf_hrnet_w18.ms_aug_in1k
 google/hf_vit-large-patch16-224
-nateraw/hf_food
 timm/hf_deit_base_distilled_patch16_224.fb_in1k
-timm/hf_ghostnet_100.in1k
-timm/hf_repvgg_a2.rvgg_in1k
 timm/hf_beit_base_patch16_224.in22k_ft_in22k_in1k
-timm/hf_mnasnet_100.rmsp_in1k
-timm/hf_dm_nfnet_f0.dm_in1k
 timm/hf_cspdarknet53.ra_in1k
 timm/hf_vit_large_patch16_224.augreg_in21k_ft_in1k
-timm/hf_pit_b_224.in1k
-timm/hf_resnest101e.in1k
-timm/hf_dla102.in1k
-Kaludi/hf_food-category-classification-v2.0
-timm/hf_dpn107.mx_in1k
-timm/hf_fbnetc_100.rmsp_in1k
-timm/hf_res2net101_26w_4s.in1k
-timm/hf_spnasnet_100.rmsp_in1k
-timm/hf_res2next50.in1k
-timm/hf_inception_v3.gluon_in1k
-timm/hf_visformer_small.in1k
 timm/hf_xcit_tiny_24_p8_384.fb_dist_in1k