From cdfbe1fbf8880aa0b31eac18f58cd7c077d94eb8 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Tue, 5 Nov 2024 09:22:10 +0000 Subject: [PATCH 1/7] fix align & winoground --- mteb/models/align_models.py | 6 +++++- .../Image/ImageTextPairClassification/Winoground.py | 3 +-- scripts/run_mieb.py | 9 ++++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/mteb/models/align_models.py b/mteb/models/align_models.py index a47e199209..28f3966ff7 100644 --- a/mteb/models/align_models.py +++ b/mteb/models/align_models.py @@ -54,9 +54,13 @@ def get_image_embeddings( image_outputs = self.model.get_image_features(**inputs) all_image_embeddings.append(image_outputs.cpu()) else: - with torch.no_grad(): + with torch.no_grad(): for i in tqdm(range(0, len(images), batch_size)): batch_images = images[i : i + batch_size] + batch_images = [ + img.convert('RGB') if isinstance(img, Image.Image) and img.mode != 'RGB' else img + for img in batch_images + ] inputs = self.processor( images=batch_images, return_tensors="pt", padding=True ) diff --git a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py index 0b8a8bedd7..f6727c0997 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py +++ b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py @@ -16,8 +16,7 @@ class Winoground(AbsTaskImageTextPairClassification): reference="https://openaccess.thecvf.com/content/CVPR2022/html/Thrush_Winoground_Probing_Vision_and_Language_Models_for_Visio-Linguistic_Compositionality_CVPR_2022_paper", dataset={ "path": "facebook/winoground", - "revision": "521ec2ba6f9a5d7380f7cca5a7b44aea5c1d677c", - "trust_remote_code": True, + "revision": "b400e173549071916ad1b3d449293bc8d8b4b763" }, type="ImageTextPairClassification", category="i2t", diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index 676fd966a7..9638686b44 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -68,8 +68,15 @@ ] ) # get i-only tasks for i-only models. + if ("moco" in model_name) or ("dinov2" in model_name): tasks = [task for task in tasks if "t" not in task.metadata.category] - + image_clustering_and_cls_tasks = mteb.get_tasks( + task_types=[ + "ImageClustering", + "ImageClassification", + ] + ) + tasks+=image_clustering_and_cls_tasks evaluation = mteb.MTEB(tasks=tasks) results = evaluation.run(model, output_folder="results-mieb-final") From 4d580088c4d095f111d920b060da68534f2284e0 Mon Sep 17 00:00:00 2001 From: gowitheflow-1998 Date: Tue, 5 Nov 2024 09:51:01 +0000 Subject: [PATCH 2/7] lint --- mteb/models/align_models.py | 6 ++++-- mteb/tasks/Image/ImageTextPairClassification/Winoground.py | 2 +- scripts/run_mieb.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mteb/models/align_models.py b/mteb/models/align_models.py index 28f3966ff7..e8f55c64ba 100644 --- a/mteb/models/align_models.py +++ b/mteb/models/align_models.py @@ -54,11 +54,13 @@ def get_image_embeddings( image_outputs = self.model.get_image_features(**inputs) all_image_embeddings.append(image_outputs.cpu()) else: - with torch.no_grad(): + with torch.no_grad(): for i in tqdm(range(0, len(images), batch_size)): batch_images = images[i : i + batch_size] batch_images = [ - img.convert('RGB') if isinstance(img, Image.Image) and img.mode != 'RGB' else img + img.convert("RGB") + if isinstance(img, Image.Image) and img.mode != "RGB" + else img for img in batch_images ] inputs = self.processor( diff --git a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py index f6727c0997..6169182286 100644 --- a/mteb/tasks/Image/ImageTextPairClassification/Winoground.py +++ b/mteb/tasks/Image/ImageTextPairClassification/Winoground.py @@ -16,7 +16,7 @@ class Winoground(AbsTaskImageTextPairClassification): reference="https://openaccess.thecvf.com/content/CVPR2022/html/Thrush_Winoground_Probing_Vision_and_Language_Models_for_Visio-Linguistic_Compositionality_CVPR_2022_paper", dataset={ "path": "facebook/winoground", - "revision": "b400e173549071916ad1b3d449293bc8d8b4b763" + "revision": "b400e173549071916ad1b3d449293bc8d8b4b763", }, type="ImageTextPairClassification", category="i2t", diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index 9638686b44..7a2df7bfa3 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -77,6 +77,6 @@ "ImageClassification", ] ) - tasks+=image_clustering_and_cls_tasks + tasks += image_clustering_and_cls_tasks evaluation = mteb.MTEB(tasks=tasks) results = evaluation.run(model, output_folder="results-mieb-final") From 2f25f58ffadadb01b00760dd1d3a8f84c8c7dc06 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 6 Nov 2024 08:34:32 +0000 Subject: [PATCH 3/7] Convert task category to i2i for tasks that only calls image encode --- mteb/tasks/Image/Clustering/eng/CIFAR.py | 2 +- mteb/tasks/Image/Clustering/eng/ImageNet.py | 2 +- mteb/tasks/Image/Clustering/eng/TinyImageNet.py | 2 +- .../Image/ImageClassification/eng/BirdsnapClassification.py | 2 +- mteb/tasks/Image/ImageClassification/eng/CIFAR.py | 2 +- .../Image/ImageClassification/eng/Caltech101Classification.py | 2 +- .../Image/ImageClassification/eng/Country211Classification.py | 2 +- mteb/tasks/Image/ImageClassification/eng/DTDClassification.py | 2 +- .../Image/ImageClassification/eng/EuroSATClassification.py | 2 +- .../Image/ImageClassification/eng/FER2013Classification.py | 2 +- .../Image/ImageClassification/eng/FGVCAircraftClassification.py | 2 +- .../Image/ImageClassification/eng/Food101Classification.py | 2 +- mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py | 2 +- mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py | 2 +- mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py | 2 +- .../ImageClassification/eng/OxfordFlowersClassification.py | 2 +- .../Image/ImageClassification/eng/OxfordPetsClassification.py | 2 +- .../ImageClassification/eng/PatchCamelyonClassification.py | 2 +- .../Image/ImageClassification/eng/RESISC45Classification.py | 2 +- mteb/tasks/Image/ImageClassification/eng/STL10Classification.py | 2 +- .../tasks/Image/ImageClassification/eng/SUN397Classification.py | 2 +- .../Image/ImageClassification/eng/StanfordCarsClassification.py | 2 +- .../tasks/Image/ImageClassification/eng/UCF101Classification.py | 2 +- .../Image/ImageMultilabelClassification/eng/PascalVOC2007.py | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/mteb/tasks/Image/Clustering/eng/CIFAR.py b/mteb/tasks/Image/Clustering/eng/CIFAR.py index 2bde390661..f9d08b684a 100644 --- a/mteb/tasks/Image/Clustering/eng/CIFAR.py +++ b/mteb/tasks/Image/Clustering/eng/CIFAR.py @@ -14,7 +14,7 @@ class CIFAR10Clustering(AbsTaskImageClustering): "revision": "0b2714987fa478483af9968de7c934580d0bb9a2", }, type="ImageClustering", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="nmi", diff --git a/mteb/tasks/Image/Clustering/eng/ImageNet.py b/mteb/tasks/Image/Clustering/eng/ImageNet.py index a2377e2334..aa0ab5720b 100644 --- a/mteb/tasks/Image/Clustering/eng/ImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/ImageNet.py @@ -14,7 +14,7 @@ class ImageNetDog15Clustering(AbsTaskImageClustering): "revision": "bfb6ad3b2109d26c9daddf14f98d315daa35ee72", }, type="ImageClustering", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="nmi", diff --git a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py index a5760b6b9c..d49ebbfde6 100644 --- a/mteb/tasks/Image/Clustering/eng/TinyImageNet.py +++ b/mteb/tasks/Image/Clustering/eng/TinyImageNet.py @@ -14,7 +14,7 @@ class TinyImageNet(AbsTaskImageClustering): "revision": "5a77092c28e51558c5586e9c5eb71a7e17a5e43f", }, type="ImageClustering", - category="s2s", + category="i2i", eval_splits=["valid"], eval_langs=["eng-Latn"], main_score="nmi", diff --git a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py index 2b08169aae..2e11094b09 100644 --- a/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/BirdsnapClassification.py @@ -14,7 +14,7 @@ class BirdsnapClassification(AbsTaskImageClassification): "revision": "fd23015508be94f0b5b59d61630e4ea2536509e4", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py index 5b4d096783..abed2ad617 100644 --- a/mteb/tasks/Image/ImageClassification/eng/CIFAR.py +++ b/mteb/tasks/Image/ImageClassification/eng/CIFAR.py @@ -14,7 +14,7 @@ class CIFAR10Classification(AbsTaskImageClassification): "revision": "0b2714987fa478483af9968de7c934580d0bb9a2", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py index 76843f50ba..30112cdf1d 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Caltech101Classification.py @@ -16,7 +16,7 @@ class Caltech101Classification(AbsTaskImageClassification): "trust_remote_code": True, }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py b/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py index 14427cd530..b73f895595 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Country211Classification.py @@ -14,7 +14,7 @@ class Country211Classification(AbsTaskImageClassification): "revision": "1699f138f0558342a1cbf99f7cf36b4361bb5ebc", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py index 48a6ca5243..eb7360f088 100644 --- a/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/DTDClassification.py @@ -14,7 +14,7 @@ class DTDClassification(AbsTaskImageClassification): "revision": "d2afa97d9f335b1a6b3b09c637aef667f98f966e", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py index 588cce89aa..5cac334c3d 100644 --- a/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/EuroSATClassification.py @@ -14,7 +14,7 @@ class EuroSATClassification(AbsTaskImageClassification): "revision": "b4e28552cd5f3932b6abc37eb20d3e84901ad728", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py index 81c2fc5857..074e92529a 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FER2013Classification.py @@ -14,7 +14,7 @@ class FER2013Classification(AbsTaskImageClassification): "revision": "9399b94167523fe5c40b3a857e24ef931ee4395b", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py index 2971faf863..8b2a41bd50 100644 --- a/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/FGVCAircraftClassification.py @@ -15,7 +15,7 @@ class FGVCAircraftClassification(AbsTaskImageClassification): "trust_remote_code": True, }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py index a0dee80ad0..1bbe8e106b 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/Food101Classification.py @@ -14,7 +14,7 @@ class Food101Classification(AbsTaskImageClassification): "revision": "e06acf2a88084f04bce4d4a525165d68e0a36c38", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["validation"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py b/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py index 4a77c47e98..3244b47dc8 100644 --- a/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/GTSRBClassification.py @@ -14,7 +14,7 @@ class GTSRBClassification(AbsTaskImageClassification): "revision": "1c13eff0803d2b02c9dc8dfe85e67770b3f0f3c5", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py b/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py index ff91119015..bed879d282 100644 --- a/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py +++ b/mteb/tasks/Image/ImageClassification/eng/Imagenet1k.py @@ -14,7 +14,7 @@ class Imagenet1kClassification(AbsTaskImageClassification): "revision": "b24c7a5a3ef12df09089055d1795e2ce7c7e7397", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py index 7e4b81f3f6..8230938a14 100644 --- a/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/MNISTClassification.py @@ -14,7 +14,7 @@ class MNISTClassification(AbsTaskImageClassification): "revision": "77f3279092a1c1579b2250db8eafed0ad422088c", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py index 316107534f..7f607d6aac 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordFlowersClassification.py @@ -14,7 +14,7 @@ class OxfordFlowersClassification(AbsTaskImageClassification): "revision": "a37b1891609c0376fa81eced756e7863e1bd873b", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py index 39620326ab..28a2357d5c 100644 --- a/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/OxfordPetsClassification.py @@ -14,7 +14,7 @@ class OxfordPetsClassification(AbsTaskImageClassification): "revision": "557b480fae8d69247be74d9503b378a09425096f", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py b/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py index e2518dc221..27508c8c17 100644 --- a/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/PatchCamelyonClassification.py @@ -14,7 +14,7 @@ class PatchCamelyonClassification(AbsTaskImageClassification): "revision": "502695fe1a141108650e3c5b91c8b5e0ff84ed49", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py index b14d31cdc0..7fa7cd5d3d 100644 --- a/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/RESISC45Classification.py @@ -14,7 +14,7 @@ class RESISC45Classification(AbsTaskImageClassification): "revision": "fe12fc5f1b7606543b0355eda392f1ddc54625c6", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py index 7acd6bb0eb..11ea833477 100644 --- a/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/STL10Classification.py @@ -14,7 +14,7 @@ class STL10Classification(AbsTaskImageClassification): "revision": "49ae7f94508f7feae62baf836db284306eab0b0f", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py index 4d0f987564..b4b5a8b931 100644 --- a/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/SUN397Classification.py @@ -14,7 +14,7 @@ class SUN397Classification(AbsTaskImageClassification): "revision": "7e6af6a2499ad708618be868e1471eac0aca1168", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py index f9836e4b05..74fa5e92b8 100644 --- a/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py +++ b/mteb/tasks/Image/ImageClassification/eng/StanfordCarsClassification.py @@ -14,7 +14,7 @@ class StanfordCarsClassification(AbsTaskImageClassification): "revision": "09ffe9bc7864d3f1e851529e5c4b7e05601a04fb", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py b/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py index 68d2cb74b7..dc4021b490 100644 --- a/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py +++ b/mteb/tasks/Image/ImageClassification/eng/UCF101Classification.py @@ -18,7 +18,7 @@ class UCF101Classification(AbsTaskImageClassification): "revision": "1098eed48f2929443f47c39f3b5c814e16369c11", }, type="ImageClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", diff --git a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py index bd5d0aad6d..4ceae17ff9 100644 --- a/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py +++ b/mteb/tasks/Image/ImageMultilabelClassification/eng/PascalVOC2007.py @@ -18,7 +18,7 @@ class VOC2007Classification(AbsTaskImageMultilabelClassification): "trust_remote_code": True, }, type="ImageMultilabelClassification", - category="i2t", + category="i2i", eval_splits=["test"], eval_langs=["eng-Latn"], main_score="accuracy", From 3ab90f11498c8c212cc958e634f3dbd48a3935b0 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 6 Nov 2024 08:54:06 +0000 Subject: [PATCH 4/7] update categories should include img cls, clustering, and multi label clf --- scripts/run_mieb.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index 7a2df7bfa3..71d292e5e6 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -71,12 +71,5 @@ if ("moco" in model_name) or ("dinov2" in model_name): tasks = [task for task in tasks if "t" not in task.metadata.category] - image_clustering_and_cls_tasks = mteb.get_tasks( - task_types=[ - "ImageClustering", - "ImageClassification", - ] - ) - tasks += image_clustering_and_cls_tasks evaluation = mteb.MTEB(tasks=tasks) results = evaluation.run(model, output_folder="results-mieb-final") From 362e951aa16eefe2f21605ca367d1e851742fe2c Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 6 Nov 2024 08:55:39 +0000 Subject: [PATCH 5/7] no op --- scripts/run_mieb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index 71d292e5e6..2fa6d52dc5 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -68,7 +68,6 @@ ] ) # get i-only tasks for i-only models. - if ("moco" in model_name) or ("dinov2" in model_name): tasks = [task for task in tasks if "t" not in task.metadata.category] evaluation = mteb.MTEB(tasks=tasks) From a86ab8bc93ffdd23eb3d1384b88726b657b140f1 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 6 Nov 2024 08:56:11 +0000 Subject: [PATCH 6/7] no op --- scripts/run_mieb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index 2fa6d52dc5..226cb70045 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -70,5 +70,6 @@ # get i-only tasks for i-only models. if ("moco" in model_name) or ("dinov2" in model_name): tasks = [task for task in tasks if "t" not in task.metadata.category] + evaluation = mteb.MTEB(tasks=tasks) results = evaluation.run(model, output_folder="results-mieb-final") From ecc65d06c43c0b8e0ae00b721cb8c76947fcdcad Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Wed, 6 Nov 2024 09:01:00 +0000 Subject: [PATCH 7/7] make lint --- scripts/run_mieb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_mieb.py b/scripts/run_mieb.py index 226cb70045..676fd966a7 100644 --- a/scripts/run_mieb.py +++ b/scripts/run_mieb.py @@ -70,6 +70,6 @@ # get i-only tasks for i-only models. if ("moco" in model_name) or ("dinov2" in model_name): tasks = [task for task in tasks if "t" not in task.metadata.category] - + evaluation = mteb.MTEB(tasks=tasks) results = evaluation.run(model, output_folder="results-mieb-final")