diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py index 7779883a84..2fb96a2365 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py @@ -105,7 +105,7 @@ class ROxfordHardI2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/r-oxford-hard", "revision": "f20b30211b7ba3fc64a02bd83998fe75f3023719", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py index ed49982fe5..e625258053 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py @@ -105,7 +105,7 @@ class RParisHardI2IRetrieval(AbsTaskAny2AnyRetrieval): "path": "JamieSJS/r-paris-hard", "revision": "fd121b6592fe946616fa85116703b94a4c61fd63", }, - type="Retrieval", + type="Any2AnyRetrieval", category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], diff --git a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py index 0e00980428..63b0b622ad 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py @@ -13,6 +13,7 @@ class Caltech101Classification(AbsTaskImageClassification): "path": "HuggingFaceM4/Caltech-101", "name": "with_background_category", "revision": "851374102055782c84f89b1b4e9d128a6568847b", + "trust_remote_code": True, }, type="ImageClassification", category="i2t", diff --git a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py index 3db3ef2d8c..aac18c7e57 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py @@ -12,6 +12,7 @@ class FGVCAircraftClassification(AbsTaskImageClassification): dataset={ "path": "HuggingFaceM4/FGVC-Aircraft", "revision": "91860adfc9a09aabca5cddb5247442109b38e213", + "trust_remote_code": True, }, type="ImageClassification", category="i2t", diff --git a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py index 5e9b9a86af..0956e0d1c3 100644 --- a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py @@ -11,7 +11,7 @@ class MNISTClassification(AbsTaskImageClassification): reference="https://en.wikipedia.org/wiki/MNIST_database", dataset={ "path": "ylecun/mnist", - "revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "revision": "77f3279092a1c1579b2250db8eafed0ad422088c", }, type="ImageClassification", category="i2t", diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py index 1a02997aec..057d631784 100644 --- a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -15,6 +15,7 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification): "path": "HuggingFaceM4/pascal_voc", "name": "voc2007_main", "revision": "dbafdb9e1506c9c419c5c4672e409463cd21ba50", + "trust_remote_code": True, }, type="ImageMultilabelClassification", category="i2t", diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py b/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py index 3137ad510d..fe89f06d42 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/CLEVR.py @@ -8,7 +8,7 @@ class CLEVR(AbsTaskZeroshotClassification): metadata = TaskMetadata( - name="CLEVR", + name="CLEVRZeroShot", description="CLEVR closest object distance identification task.", reference="https://openaccess.thecvf.com/content_cvpr_2017/html/Johnson_CLEVR_A_Diagnostic_CVPR_2017_paper.html", dataset={ @@ -61,7 +61,7 @@ def get_candidate_labels(self) -> list[str]: class CLEVRCount(AbsTaskZeroshotClassification): metadata = TaskMetadata( - name="CLEVRCount", + name="CLEVRCountZeroShot", description="CLEVR count objects task.", reference="https://openaccess.thecvf.com/content_cvpr_2017/html/Johnson_CLEVR_A_Diagnostic_CVPR_2017_paper.html", dataset={ diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py b/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py index ab7ca141cb..00bfdac874 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/Caltech101.py @@ -15,6 +15,7 @@ class Caltech101Classification(AbsTaskZeroshotClassification): "path": "HuggingFaceM4/Caltech-101", "name": "with_background_category", "revision": "851374102055782c84f89b1b4e9d128a6568847b", + "trust_remote_code": True, }, type="ZeroShotClassification", category="i2t", diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py b/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py index c15e0b6d4b..b973c85607 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/FGVCAircraft.py @@ -14,6 +14,7 @@ class FGVCAircraftClassification(AbsTaskZeroshotClassification): dataset={ "path": "HuggingFaceM4/FGVC-Aircraft", "revision": "91860adfc9a09aabca5cddb5247442109b38e213", + "trust_remote_code": True, }, type="ZeroShotClassification", category="i2t", diff --git a/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py b/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py index 253fa938ac..c274e72d61 100644 --- a/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py +++ b/mteb/tasks/Image/ZeroshotClassification/eng/MNIST.py @@ -13,7 +13,7 @@ class MNISTClassification(AbsTaskZeroshotClassification): reference="https://en.wikipedia.org/wiki/MNIST_database", dataset={ "path": "ylecun/mnist", - "revision": "b06aab39e05f7bcd9635d18ed25d06eae523c574", + "revision": "77f3279092a1c1579b2250db8eafed0ad422088c", }, type="ZeroShotClassification", category="i2t", diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index 8b00d64e9d..cf05096996 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -9,7 +9,6 @@ "royokong/e5-v", "BAAI/bge-visualized-base", "BAAI/bge-visualized-m3", - "google/siglip-so400m-patch14-384", "kakaobrain/align-base", "jinaai/jina-clip-v1", "nomic-ai/nomic-embed-vision-v1.5", @@ -26,6 +25,7 @@ "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "nyu-visionx/moco-v3-vit-b", "nyu-visionx/moco-v3-vit-l", + # "google/siglip-so400m-patch14-384",# haven't pushed ]: model = mteb.get_model(model_name) tasks = mteb.get_tasks(