diff --git a/tests/model_hub_tests/models_hub_common/utils.py b/tests/model_hub_tests/models_hub_common/utils.py index 74ed5b26406027..81b04926c1a7ed 100644 --- a/tests/model_hub_tests/models_hub_common/utils.py +++ b/tests/model_hub_tests/models_hub_common/utils.py @@ -31,6 +31,28 @@ def get_models_list(file_name: str): return models +def get_skipped_model_links(filename: str): + links = set() + if not os.path.exists(filename): + return links + with open(filename) as f: + for model_info in f: + model_info = model_info.strip() + model_name, model_link = model_info.split(',') + links.add(model_link) + return links + + +def get_models_list_not_skipped(model_list_file: str, skip_list_file: str): + skipped_links = get_skipped_model_links(skip_list_file) + not_skipped_models = [] + for model_name, model_link, mark, reason in get_models_list(model_list_file): + if model_link in skipped_links: + continue + not_skipped_models.append((model_name, model_link, mark, reason)) + return not_skipped_models + + def compare_two_tensors(ov_res, fw_res, eps): is_ok = True if not np.allclose(ov_res, fw_res, atol=eps, rtol=eps, equal_nan=True): diff --git a/tests/model_hub_tests/performance_tests/nightly_models.skip b/tests/model_hub_tests/performance_tests/nightly_models.skip new file mode 100644 index 00000000000000..f3c899a646efc6 --- /dev/null +++ b/tests/model_hub_tests/performance_tests/nightly_models.skip @@ -0,0 +1,245 @@ +universal-sentence-encoder,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/universal-sentence-encoder/versions/2 +universal-sentence-encoder-multilingual,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/multilingual/versions/2 +universal-sentence-encoder-large,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/large/versions/2 +bert_en_uncased_preprocess,https://www.kaggle.com/models/tensorflow/bert/frameworks/tensorFlow2/variations/en-uncased-preprocess/versions/3 +tf2-preview/mobilenet_v2/feature_vector,https://www.kaggle.com/models/google/mobilenet-v2/frameworks/tensorFlow2/variations/tf2-preview-feature-vector/versions/4 +nnlm-en-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/en-dim128/versions/1 +elmo,https://www.kaggle.com/models/google/elmo/frameworks/tensorFlow1/variations/elmo/versions/3 +universal-sentence-encoder-multilingual-large,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/multilingual-large/versions/2 +spice,https://www.kaggle.com/models/google/spice/frameworks/tensorFlow1/variations/spice/versions/2 +tf2-preview/nnlm-en-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-en-dim128/versions/1 +universal-sentence-encoder-lite,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow1/variations/lite/versions/2 +tf2-preview/mobilenet_v2/classification,https://www.kaggle.com/models/google/mobilenet-v2/frameworks/tensorFlow2/variations/tf2-preview-classification/versions/4 +i3d-kinetics-400,https://www.kaggle.com/models/deepmind/i3d-kinetics/frameworks/tensorFlow1/variations/400/versions/1 +tf2-preview/gnews-swivel-20dim,https://www.kaggle.com/models/google/gnews-swivel/frameworks/tensorFlow2/variations/tf2-preview-20dim/versions/1 +faster_rcnn/openimages_v4/inception_resnet_v2,https://www.kaggle.com/models/google/faster-rcnn-inception-resnet-v2/frameworks/tensorFlow1/variations/faster-rcnn-openimages-v4-inception-resnet-v2/versions/1 +openimages_v4/ssd/mobilenet_v2,https://www.kaggle.com/models/google/mobilenet-v2/frameworks/tensorFlow1/variations/openimages-v4-ssd-mobilenet-v2/versions/1 +universal-sentence-encoder-xling-many,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow1/variations/xling-many/versions/1 +nnlm-en-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/en-dim128-with-normalization/versions/1 +nnlm-en-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/en-dim50/versions/1 +universal-sentence-encoder-cmlm/multilingual-preprocess,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/cmlm-multilingual-preprocess/versions/2 +vggish,https://www.kaggle.com/models/google/vggish/frameworks/tensorFlow2/variations/vggish/versions/1 +tf2-preview/inception_v3/feature_vector,https://www.kaggle.com/models/google/inception-v3/frameworks/tensorFlow2/variations/tf2-preview-feature-vector/versions/4 +efficientdet/lite0/detection,https://www.kaggle.com/models/tensorflow/efficientdet/frameworks/tensorFlow2/variations/lite0-detection/versions/1 +nnlm-ja-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/ja-dim128/versions/1 +universal-sentence-encoder-multilingual-qa,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/multilingual-qa/versions/2 +nnlm-ja-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/ja-dim128-with-normalization/versions/1 +nnlm-en-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/en-dim50-with-normalization/versions/1 +universal-sentence-encoder-qa,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/qa/versions/2 +biggan-deep-256,https://www.kaggle.com/models/deepmind/biggan/frameworks/tensorFlow1/variations/deep-256/versions/1 +efficientdet/lite2/detection,https://www.kaggle.com/models/tensorflow/efficientdet/frameworks/tensorFlow2/variations/lite2-detection/versions/1 +delf,https://www.kaggle.com/models/google/delf/frameworks/tensorFlow1/variations/delf/versions/1 +tf2-preview/gnews-swivel-20dim-with-oov,https://www.kaggle.com/models/google/gnews-swivel/frameworks/tensorFlow2/variations/tf2-preview-20dim-with-oov/versions/1 +tf2-preview/nnlm-es-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-es-dim50-with-normalization/versions/1 +enformer,https://www.kaggle.com/models/deepmind/enformer/frameworks/tensorFlow2/variations/enformer/versions/1 +albert_en_base,https://www.kaggle.com/models/tensorflow/albert/frameworks/tensorFlow2/variations/en-base/versions/3 +nonsemantic-speech-benchmark/trill-distilled,https://www.kaggle.com/models/google/nonsemantic-speech-benchmark/frameworks/tensorFlow2/variations/trill-distilled/versions/3 +biggan-256,https://www.kaggle.com/models/deepmind/biggan/frameworks/tensorFlow1/variations/256/versions/2 +nnlm-de-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/de-dim50-with-normalization/versions/1 +bert_multi_cased_preprocess,https://www.kaggle.com/models/tensorflow/bert/frameworks/tensorFlow2/variations/multi-cased-preprocess/versions/3 +tfgan/eval/inception,https://www.kaggle.com/models/tensorflow/inception/frameworks/tensorFlow1/variations/tfgan-eval-inception/versions/1 +Wiki-words-250,https://www.kaggle.com/models/google/wiki-words/frameworks/tensorFlow2/variations/250/versions/1 +tfgan/eval/mnist/logits,https://www.kaggle.com/models/tensorflow/mnist/frameworks/tensorFlow1/variations/logits/versions/1 +albert_en_preprocess,https://www.kaggle.com/models/tensorflow/albert/frameworks/tensorFlow2/variations/en-preprocess/versions/3 +object_detection/mobile_object_localizer_v1,https://www.kaggle.com/models/google/mobile-object-localizer-v1/frameworks/tensorFlow1/variations/object-detection-mobile-object-localizer-v1/versions/1 +efficientdet/lite3/detection,https://www.kaggle.com/models/tensorflow/efficientdet/frameworks/tensorFlow2/variations/lite3-detection/versions/1 +tf2-preview/inception_v3/classification,https://www.kaggle.com/models/google/inception-v3/frameworks/tensorFlow2/variations/tf2-preview-classification/versions/4 +random-nnlm-en-dim128,https://www.kaggle.com/models/google/random-nnlm-en/frameworks/tensorFlow1/variations/dim128/versions/1 +biggan-deep-512,https://www.kaggle.com/models/deepmind/biggan/frameworks/tensorFlow1/variations/deep-512/versions/1 +tweening_conv3d_bair,https://www.kaggle.com/models/google/tweening-conv3d/frameworks/tensorFlow1/variations/bair/versions/1 +bert_en_cased_preprocess,https://www.kaggle.com/models/tensorflow/bert/frameworks/tensorFlow2/variations/en-cased-preprocess/versions/3 +albert_en_xxlarge,https://www.kaggle.com/models/tensorflow/albert/frameworks/tensorFlow2/variations/en-xxlarge/versions/3 +MuRIL_preprocess,https://www.kaggle.com/models/google/muril/frameworks/tensorFlow2/variations/preprocess/versions/1 +efficientdet/lite3x/detection,https://www.kaggle.com/models/tensorflow/efficientdet/frameworks/tensorFlow2/variations/lite3x-detection/versions/1 +midas/v2,https://www.kaggle.com/models/intel/midas/frameworks/tensorFlow1/variations/v2/versions/2 +nnlm-ja-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/ja-dim50/versions/1 +electra_small,https://www.kaggle.com/models/google/electra/frameworks/tensorFlow2/variations/small/versions/2 +nonsemantic-speech-benchmark/trill,https://www.kaggle.com/models/google/nonsemantic-speech-benchmark/frameworks/tensorFlow2/variations/trill/versions/3 +bigbigan-resnet50,https://www.kaggle.com/models/deepmind/bigbigan/frameworks/tensorFlow1/variations/resnet50/versions/1 +Wiki-words-250-with-normalization,https://www.kaggle.com/models/google/wiki-words/frameworks/tensorFlow2/variations/250-with-normalization/versions/1 +sentence-t5/st5-base,https://www.kaggle.com/models/google/sentence-t5/frameworks/tensorFlow2/variations/st5-base/versions/1 +bert_zh_preprocess,https://www.kaggle.com/models/tensorflow/bert/frameworks/tensorFlow2/variations/zh-preprocess/versions/3 +cropnet/classifier/cassava_disease_V1,https://www.kaggle.com/models/google/cropnet/frameworks/tensorFlow2/variations/classifier-cassava-disease-v1/versions/1 +midas/v2_1_small,https://www.kaggle.com/models/intel/midas/frameworks/tensorFlow1/variations/v2-1-small/versions/1 +i3d-kinetics-600,https://www.kaggle.com/models/deepmind/i3d-kinetics/frameworks/tensorFlow1/variations/600/versions/1 +nnlm-de-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/de-dim50/versions/1 +musiq/ava,https://www.kaggle.com/models/google/musiq/frameworks/tensorFlow2/variations/ava/versions/1 +nnlm-es-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/es-dim128/versions/1 +mil-nce/s3d,https://www.kaggle.com/models/deepmind/mil-nce/frameworks/tensorFlow1/variations/s3d/versions/1 +Wiki-words-500,https://www.kaggle.com/models/google/wiki-words/frameworks/tensorFlow2/variations/500/versions/1 +biggan-deep-128,https://www.kaggle.com/models/deepmind/biggan/frameworks/tensorFlow1/variations/deep-128/versions/1 +nnlm-es-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/es-dim128-with-normalization/versions/1 +vit_s16_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-s16-fe/versions/1 +zh_segmentation,https://www.kaggle.com/models/google/zh-segmentation/frameworks/tensorFlow1/variations/zh-segmentation/versions/1 +wiki40b-lm-es,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/es/versions/1 +nnlm-es-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/es-dim50-with-normalization/versions/1 +cord-19/swivel-128d,https://www.kaggle.com/models/tensorflow/cord-19-swivel-128d/frameworks/tensorFlow2/variations/cord-19-swivel-128d/versions/2 +centernet/resnet101v1_fpn_512x512,https://www.kaggle.com/models/tensorflow/centernet-resnet/frameworks/tensorFlow2/variations/101v1-fpn-512x512/versions/1 +movinet/a0/stream/kinetics-600/classification,https://www.kaggle.com/models/google/movinet/frameworks/tensorFlow2/variations/a0-stream-kinetics-600-classification/versions/2 +boundless/quarter,https://www.kaggle.com/models/google/boundless/frameworks/tensorFlow1/variations/quarter/versions/1 +gtr/gtr-base,https://www.kaggle.com/models/google/gtr/frameworks/tensorFlow2/variations/gtr-base/versions/1 +LAReQA/mBERT_En_En,https://www.kaggle.com/models/google/lareqa/frameworks/tensorFlow1/variations/mbert-en-en/versions/1 +small_bert/bert_uncased_L-2_H-128_A-2,https://www.kaggle.com/models/google/bert/frameworks/tensorFlow1/variations/bert-uncased-l-2-h-128-a-2/versions/2 +distilbert_en_uncased_preprocess,https://www.kaggle.com/models/jeongukjae/distilbert/frameworks/tensorFlow2/variations/en-uncased-preprocess/versions/2 +movinet/a2/stream/kinetics-600/classification,https://www.kaggle.com/models/google/movinet/frameworks/tensorFlow2/variations/a2-stream-kinetics-600-classification/versions/2 +nnlm-zh-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/zh-dim50/versions/1 +nnlm-id-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/id-dim128-with-normalization/versions/1 +humpback_whale,https://www.kaggle.com/models/google/humpback-whale/frameworks/tensorFlow2/variations/humpback-whale/versions/1 +nnlm-id-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/id-dim50/versions/1 +nonsemantic-speech-benchmark/frill,https://www.kaggle.com/models/google/nonsemantic-speech-benchmark/frameworks/tensorFlow2/variations/frill/versions/1 +vit_s16_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-s16-classification/versions/1 +roberta_en_cased_preprocess,https://www.kaggle.com/models/kaggle/roberta/frameworks/tensorFlow2/variations/en-cased-preprocess/versions/1 +tf2-preview/nnlm-ko-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-ko-dim128-with-normalization/versions/1 +nnlm-id-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/id-dim128/versions/1 +distilbert_multi_cased_preprocess,https://www.kaggle.com/models/jeongukjae/distilbert/frameworks/tensorFlow2/variations/multi-cased-preprocess/versions/2 +nnlm-de-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/de-dim128/versions/1 +bertseq2seq/roberta24_gigaword,https://www.kaggle.com/models/google/bertseq2seq/frameworks/tensorFlow1/variations/roberta24-gigaword/versions/1 +vit_b8_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-b8-fe/versions/1 +bertseq2seq/roberta24_cnndm,https://www.kaggle.com/models/google/bertseq2seq/frameworks/tensorFlow1/variations/roberta24-cnndm/versions/1 +universal-sentence-encoder-xling/en-de,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow1/variations/xling-en-de/versions/1 +smaller_LaBSE_15lang_preprocess,https://www.kaggle.com/models/kaggle/smaller-labse-15lang/frameworks/tensorFlow2/variations/preprocess/versions/1 +vgg19-block5-conv2-unpooling-decoder,https://www.kaggle.com/models/emilutz/vgg19-unpooling-encoder-decoder/frameworks/tensorFlow2/variations/vgg19-block5-conv2-unpooling-decoder/versions/1 +tf2-preview/nnlm-en-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-en-dim50-with-normalization/versions/1 +cropnet/feature_vector/concat,https://www.kaggle.com/models/google/cropnet/frameworks/tensorFlow2/variations/feature-vector-concat/versions/1 +nnlm-zh-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/zh-dim128/versions/1 +tf2-preview/nnlm-ja-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-ja-dim128-with-normalization/versions/1 +LAReQA/mBERT_X_Y,https://www.kaggle.com/models/google/lareqa/frameworks/tensorFlow1/variations/mbert-x-y/versions/1 +nnlm-es-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/es-dim50/versions/1 +nnlm-zh-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/zh-dim128-with-normalization/versions/1 +sentence-t5/st5-large,https://www.kaggle.com/models/google/sentence-t5/frameworks/tensorFlow2/variations/st5-large/versions/1 +mmv/s3d,https://www.kaggle.com/models/deepmind/mmv/frameworks/tensorFlow1/variations/s3d/versions/1 +ulmfit/en/sp35k_uncased,https://www.kaggle.com/models/edrone/ulmfit/frameworks/tensorFlow2/variations/en-sp35k-uncased/versions/1 +bertseq2seq/roberta24_wikisplit,https://www.kaggle.com/models/google/bertseq2seq/frameworks/tensorFlow1/variations/roberta24-wikisplit/versions/1 +mmv/tsm-resnet50x2,https://www.kaggle.com/models/deepmind/mmv/frameworks/tensorFlow1/variations/tsm-resnet50x2/versions/1 +cropnet/feature_vector/imagenet,https://www.kaggle.com/models/google/cropnet/frameworks/tensorFlow2/variations/feature-vector-imagenet/versions/1 +nnlm-ko-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/ko-dim50-with-normalization/versions/1 +nnlm-zh-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/zh-dim50-with-normalization/versions/1 +tf2-preview/nnlm-de-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-de-dim128-with-normalization/versions/1 +wiki40b-lm-zh-cn,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/zh-cn/versions/1 +HRNet/coco-hrnetv2-w48,https://www.kaggle.com/models/google/hrnet/frameworks/tensorFlow2/variations/hrnet-coco-hrnetv2-w48/versions/1 +german-mbmelgan,https://www.kaggle.com/models/myusufs/german-mbmelgan/frameworks/tensorFlow2/variations/german-mbmelgan/versions/1 +wiki40b-lm-fr,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/fr/versions/1 +german-tacotron2,https://www.kaggle.com/models/myusufs/german-tacotron2/frameworks/tensorFlow2/variations/german-tacotron2/versions/1 +gtr/gtr-large,https://www.kaggle.com/models/google/gtr/frameworks/tensorFlow2/variations/gtr-large/versions/1 +trillsson1,https://www.kaggle.com/models/google/trillsson/frameworks/tensorFlow2/variations/1/versions/1 +tf2-preview/nnlm-de-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-de-dim50-with-normalization/versions/1 +vit_b8_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-b8-classification/versions/1 +universal-sentence-encoder-xling/en-es,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow1/variations/xling-en-es/versions/1 +mil-nce/i3d,https://www.kaggle.com/models/deepmind/mil-nce/frameworks/tensorFlow1/variations/i3d/versions/1 +vit_l16_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-l16-fe/versions/1 +vit_r50_l32_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-r50-l32-fe/versions/1 +imagenet/efficientnet_v2_imagenet21k_ft1k_b2/feature_vector,https://www.kaggle.com/models/google/efficientnet-v2/frameworks/tensorFlow2/variations/imagenet21k-ft1k-b2-feature-vector/versions/1 +tf2-preview/nnlm-ko-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-ko-dim50/versions/1 +wav2vec2-xlsr-53,https://www.kaggle.com/models/kaggle/wav2vec2/frameworks/tensorFlow2/variations/xlsr-53/versions/1 +movinet/a5/stream/kinetics-600/classification,https://www.kaggle.com/models/google/movinet/frameworks/tensorFlow2/variations/a5-stream-kinetics-600-classification/versions/2 +vit_r26_s32_lightaug_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-r26-s32-lightaug-fe/versions/1 +mmt/architecture_image-q-24,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/architecture-image-q-24/versions/1 +wiki40b-lm-nl,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/nl/versions/1 +tf2-preview/nnlm-zh-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-zh-dim50/versions/1 +compare_gan/model_9_celebahq128_resnet19,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-9-celebahq128-resnet19/versions/1 +tf2-preview/nnlm-zh-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-zh-dim128/versions/1 +tweening_conv3d_kth,https://www.kaggle.com/models/google/tweening-conv3d/frameworks/tensorFlow1/variations/kth/versions/1 +LAReQA/mBERT_X_X_mono,https://www.kaggle.com/models/google/lareqa/frameworks/tensorFlow1/variations/mbert-x-x-mono/versions/1 +tf2-preview/nnlm-ja-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-ja-dim50/versions/1 +nnlm-ko-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/ko-dim50/versions/1 +compare_gan/model_12_cifar10_resnet_cifar,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-12-cifar10-resnet-cifar/versions/1 +spiral/default-fluid-gansn-celebahq64-gen-19steps,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-fluid-gansn-celebahq64-gen-19steps/versions/1 +compare_gan/s3gan_5_128x128,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/s3gan-5-128x128/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent5,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent5/versions/1 +image_augmentation/crop_rotate_color,https://www.kaggle.com/models/google/image-augmentation/frameworks/tensorFlow1/variations/crop-rotate-color/versions/1 +tf2-preview/nnlm-id-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-id-dim50-with-normalization/versions/1 +tf2-preview/nnlm-es-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-es-dim128/versions/1 +wiki40b-lm-sr,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/sr/versions/1 +tf2-preview/nnlm-es-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-es-dim50/versions/1 +wiki40b-lm-ko,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/ko/versions/1 +mmt/baseline_baseline,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/baseline-baseline/versions/1 +boundless/three_quarter,https://www.kaggle.com/models/google/boundless/frameworks/tensorFlow1/variations/three-quarter/versions/1 +musiq/koniq-10k,https://www.kaggle.com/models/google/musiq/frameworks/tensorFlow2/variations/koniq-10k/versions/1 +relative-patch-location,https://www.kaggle.com/models/vtab/relative-patch-location/frameworks/tensorFlow1/variations/relative-patch-location/versions/1 +ulmfit/en/sp35k_cased,https://www.kaggle.com/models/edrone/ulmfit/frameworks/tensorFlow2/variations/en-sp35k-cased/versions/1 +sentence-t5/st5-3b,https://www.kaggle.com/models/google/sentence-t5/frameworks/tensorFlow2/variations/st5-3b/versions/1 +bertseq2seq/bert24_en_de,https://www.kaggle.com/models/google/bertseq2seq/frameworks/tensorFlow1/variations/bert24-en-de/versions/1 +vit_b32_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-b32-fe/versions/1 +vit_b32_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-b32-classification/versions/1 +wiki40b-lm-ca,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/ca/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent3,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent3/versions/1 +ulmfit/pl/sp50k_uncased,https://www.kaggle.com/models/edrone/ulmfit/frameworks/tensorFlow2/variations/pl-sp50k-uncased/versions/1 +mixer_b16_i1k_fe,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b16-i1k-fe/versions/1 +mmt/loss_single-modality-contrastive1024,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/loss-single-modality-contrastive1024/versions/1 +wiki40b-lm-tr,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/tr/versions/1 +mixer_b16_sam_fe,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b16-sam-fe/versions/1 +logit_reconstruction/robust,https://www.kaggle.com/models/google/logit-reconstruction/frameworks/tensorFlow1/variations/robust/versions/1 +mmt/baseline_baseline-no-bert-transfer,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/baseline-baseline-no-bert-transfer/versions/1 +nnlm-ko-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/ko-dim128/versions/1 +vit_l16_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-l16-classification/versions/1 +wiki40b-lm-lt,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/lt/versions/1 +wiki40b-lm-pl,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/pl/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent8,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent8/versions/1 +HRNet/ade20k-hrnetv2-w48,https://www.kaggle.com/models/google/hrnet/frameworks/tensorFlow2/variations/hrnet-ade20k-hrnetv2-w48/versions/1 +mmt/architecture_image-q-12,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/architecture-image-q-12/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent9,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent9/versions/1 +answer_equivalence/bem,https://www.kaggle.com/models/google/bert/frameworks/tensorFlow2/variations/answer-equivalence-bem/versions/1 +wiki40b-lm-id,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/id/versions/1 +distilbert_en_cased_preprocess,https://www.kaggle.com/models/jeongukjae/distilbert/frameworks/tensorFlow2/variations/en-cased-preprocess/versions/2 +wiki40b-lm-ar,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/ar/versions/1 +compare_gan/model_2_celebahq128_resnet19,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-2-celebahq128-resnet19/versions/1 +vgg19-block4-conv2-unpooling-decoder,https://www.kaggle.com/models/emilutz/vgg19-unpooling-encoder-decoder/frameworks/tensorFlow2/variations/vgg19-block4-conv2-unpooling-decoder/versions/1 +compare_gan/ssgan_128x128,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/ssgan-128x128/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent4,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent4/versions/1 +image_augmentation/crop_color,https://www.kaggle.com/models/google/image-augmentation/frameworks/tensorFlow1/variations/crop-color/versions/1 +wiki40b-lm-fa,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/fa/versions/1 +vit_r26_s32_medaug_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-r26-s32-medaug-classification/versions/1 +compare_gan/model_10_lsun_bedroom_resnet19,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-10-lsun-bedroom-resnet19/versions/1 +mmt/data_cc,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/data-cc/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent1,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent1/versions/1 +image_augmentation/nas_imagenet,https://www.kaggle.com/models/google/image-augmentation/frameworks/tensorFlow1/variations/nas-imagenet/versions/1 +mixer_b16_i21k_fe,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b16-i21k-fe/versions/1 +nonsemantic-speech-benchmark/trillsson1,https://www.kaggle.com/models/google/trillsson/frameworks/tensorFlow2/variations/nonsemantic-speech-benchmark-trillsson1/versions/1 +trillsson5,https://www.kaggle.com/models/google/trillsson/frameworks/tensorFlow2/variations/5/versions/1 +logit_reconstruction/inceptionv3,https://www.kaggle.com/models/google/logit-reconstruction/frameworks/tensorFlow1/variations/inceptionv3/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent6,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent6/versions/1 +tf2-preview/nnlm-id-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-id-dim128/versions/1 +LEALLA/LEALLA-large,https://www.kaggle.com/models/google/lealla/frameworks/tensorFlow2/variations/lealla-large/versions/1 +compare_gan/model_13_cifar10_resnet_cifar,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-13-cifar10-resnet-cifar/versions/1 +vit_b16_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-b16-classification/versions/1 +compare_gan/model_11_cifar10_resnet_cifar,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-11-cifar10-resnet-cifar/versions/1 +wiki40b-lm-fi,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/fi/versions/1 +vit_r26_s32_medaug_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-r26-s32-medaug-fe/versions/1 +image_augmentation/flipx_crop_rotate_color,https://www.kaggle.com/models/google/image-augmentation/frameworks/tensorFlow1/variations/flipx-crop-rotate-color/versions/1 +circularnet_3,https://www.kaggle.com/models/google/circularnet/frameworks/tensorFlow2/variations/3/versions/1 +random-nnlm-en-dim50,https://www.kaggle.com/models/google/random-nnlm-en/frameworks/tensorFlow1/variations/dim50/versions/1 +mixer_b16_i21k_classification,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b16-i21k-classification/versions/1 +mixer_b16_sam_classification,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b16-sam-classification/versions/1 +wav2vec2-robust,https://www.kaggle.com/models/kaggle/wav2vec2/frameworks/tensorFlow2/variations/robust/versions/1 +wiki40b-lm-lv,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/lv/versions/1 +sentence-t5/st5-11b,https://www.kaggle.com/models/google/sentence-t5/frameworks/tensorFlow2/variations/st5-11b/versions/1 +gtr/gtr-xxl,https://www.kaggle.com/models/google/gtr/frameworks/tensorFlow2/variations/gtr-xxl/versions/1 +mmt/data_mscoco,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/data-mscoco/versions/1 +vit_r26_s32_lightaug_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-r26-s32-lightaug-classification/versions/1 +compare_gan/s3gan_10_256x256,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/s3gan-10-256x256/versions/1 +mixer_b16_i1k_classification,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b16-i1k-classification/versions/1 +HRNet/msegpcontext-hrnetv2-w48,https://www.kaggle.com/models/google/hrnet/frameworks/tensorFlow2/variations/hrnet-msegpcontext-hrnetv2-w48/versions/1 +mmt/baseline-ft_baseline-no-bert-transfer,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/baseline-ft-baseline-no-bert-transfer/versions/1 +wiki40b-lm-el,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/el/versions/1 +wiki40b-lm-ro,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/ro/versions/1 +wiki40b-lm-ms,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/ms/versions/1 +wiki40b-lm-th,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/th/versions/1 +mmt/data_sbu,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/data-sbu/versions/1 +mmt/data_combined-instance,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/data-combined-instance/versions/1 +vila/image,https://www.kaggle.com/models/google/vila/frameworks/tensorFlow2/variations/image/versions/1 +mmt/architecture_vilbert-4block,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/architecture-vilbert-4block/versions/1 +mmt/architecture-ft_language-q-12,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/architecture-ft-language-q-12/versions/1 +mmt/architecture_single-modality,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/architecture-single-modality/versions/1 +compare_gan/model_4_lsun_bedroom_resnet19,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-4-lsun-bedroom-resnet19/versions/1 +movinet/a3/stream/kinetics-600/classification,https://www.kaggle.com/models/google/movinet/frameworks/tensorFlow2/variations/a3-stream-kinetics-600-classification/versions/2 +mmt/architecture_vilbert-2block,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/architecture-vilbert-2block/versions/1 +jigsaw,https://www.kaggle.com/models/vtab/jigsaw/frameworks/tensorFlow1/variations/jigsaw/versions/1 +tf2-preview/nnlm-zh-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-zh-dim50-with-normalization/versions/1 +wiki40b-lm-de,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/de/versions/1 +mixer_b32_sam_fe,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b32-sam-fe/versions/1 +mmt/data-ft_sbu,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/data-ft-sbu/versions/1 +mmt/data_cc-with-bert,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/data-cc-with-bert/versions/1 +tf2-preview/nnlm-id-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-id-dim50/versions/1 +vgg19-block2-conv2-unpooling-decoder,https://www.kaggle.com/models/emilutz/vgg19-unpooling-encoder-decoder/frameworks/tensorFlow2/variations/vgg19-block2-conv2-unpooling-decoder/versions/1 +Wiki-words-500-with-normalization,https://www.kaggle.com/models/google/wiki-words/frameworks/tensorFlow2/variations/500-with-normalization/versions/1 +wiki40b-lm-zh-tw,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/zh-tw/versions/1 diff --git a/tests/model_hub_tests/performance_tests/test_tf_hub_perfomance_model.py b/tests/model_hub_tests/performance_tests/test_tf_hub_perfomance_model.py index 82f9be6a189f85..c8a80f25266646 100644 --- a/tests/model_hub_tests/performance_tests/test_tf_hub_perfomance_model.py +++ b/tests/model_hub_tests/performance_tests/test_tf_hub_perfomance_model.py @@ -12,6 +12,7 @@ from models_hub_common.constants import tf_hub_cache_dir from models_hub_common.test_performance_model import TestModelPerformance from models_hub_common.utils import get_models_list +from models_hub_common.utils import get_models_list_not_skipped from models_hub_common.utils import cleanup_dir @@ -50,7 +51,8 @@ def test_convert_model_precommit(self, model_name, model_link, mark, reason, ie_ self.run(model_name, model_link, ie_device) @pytest.mark.parametrize("model_name,model_link,mark,reason", - get_models_list(os.path.join(os.path.dirname(__file__), "nightly_models"))) + get_models_list_not_skipped(os.path.join(os.path.dirname(__file__), "nightly_models"), + os.path.join(os.path.dirname(__file__), "nightly_models.skip"))) @pytest.mark.nightly def test_convert_model_all_models(self, model_name, model_link, mark, reason, ie_device): assert mark is None or mark == 'skip', "Incorrect test case: {}, {}".format(model_name, model_link)