feat: Add MIEB and MIEB-lite as benchmarks

mteb/abstasks/TaskMetadata.py

@@ -105,13 +105,16 @@
 MIEB_TASK_TYPE = (
     "Any2AnyMultiChoice",
     "Any2AnyRetrieval",
-    "Any2TextMutipleChoice",
+    "Any2AnyMultilingualRetrieval",
+    "VisionCentric",
     "ImageClustering",
     "ImageClassification",
     "ImageMultilabelClassification",
-    "ImageTextPairClassification",
-    "VisualSTS",
+    "DocumentUnderstanding",
+    "VisualSTS(eng)",
+    "VisualSTS(multi)",
     "ZeroShotClassification",
+    "Compositionality",
 )
 
 TASK_TYPE = (

mteb/benchmarks/benchmarks.py

@@ -1402,6 +1402,253 @@
 }""",
 )
 
+MIEB_common_tasks = [
+    # Image Classification
+    "Birdsnap",
+    "Caltech101",
+    "CIFAR10",
+    "CIFAR100",
+    "Country211",
+    "DTD",
+    "EuroSAT",
+    "FER2013",
+    "FGVCAircraft",
+    "Food101Classification",
+    "GTSRB",
+    "Imagenet1k",
+    "MNIST",
+    "OxfordFlowersClassification",
+    "OxfordPets",
+    "PatchCamelyon",
+    "RESISC45",
+    "StanfordCars",
+    "STL10",
+    "SUN397",
+    "UCF101",
+    # ImageMultiLabelClassification
+    "VOC2007",
+    # Clustering
+    "CIFAR10Clustering",
+    "CIFAR100Clustering",
+    "ImageNetDog15Clustering",
+    "ImageNet10Clustering",
+    "TinyImageNetClustering",
+    # ZeroShotClassification
+    "BirdsnapZeroShot",
+    "Caltech101ZeroShot",
+    "CIFAR10ZeroShot",
+    "CIFAR100ZeroShot",
+    "CLEVRZeroShot",
+    "CLEVRCountZeroShot",
+    "Country211ZeroShot",
+    "DTDZeroShot",
+    "EuroSATZeroShot",
+    "FER2013ZeroShot",
+    "FGVCAircraftZeroShot",
+    "Food101ZeroShot",
+    "GTSRBZeroShot",
+    "Imagenet1kZeroShot",
+    "MNISTZeroShot",
+    "OxfordPetsZeroShot",
+    "PatchCamelyonZeroShot",
+    "RenderedSST2",
+    "RESISC45ZeroShot",
+    "StanfordCarsZeroShot",
+    "STL10ZeroShot",
+    "SUN397ZeroShot",
+    "UCF101ZeroShot",
+    # Any2TextMutipleChoice
+    "CVBenchCount",
+    "CVBenchRelation",
+    "CVBenchDepth",
+    "CVBenchDistance",
+    # Any2AnyMultipleChoice
+    "BLINKIT2IMultiChoice",
+    "BLINKIT2TMultiChoice",
+    "ImageCoDeT2IMultiChoice",
+    # ImageTextPairClassification
+    "AROCocoOrder",
+    "AROFlickrOrder",
+    "AROVisualAttribution",
+    "AROVisualRelation",
+    "SugarCrepe",
+    "Winoground",
+    # VisualSTS
+    "STS12VisualSTS",
+    "STS13VisualSTS",
+    "STS14VisualSTS",
+    "STS15VisualSTS",
+    "STS16VisualSTS",
+    # Any2AnyRetrieval
+    "BLINKIT2IRetrieval",
+    "BLINKIT2TRetrieval",
+    "CIRRIT2IRetrieval",
+    "CUB200I2IRetrieval",
+    "EDIST2ITRetrieval",
+    "Fashion200kI2TRetrieval",
+    "Fashion200kT2IRetrieval",
+    "FashionIQIT2IRetrieval",
+    "Flickr30kI2TRetrieval",
+    "Flickr30kT2IRetrieval",
+    "FORBI2IRetrieval",
+    "GLDv2I2IRetrieval",
+    "GLDv2I2TRetrieval",
+    "HatefulMemesI2TRetrieval",
+    "HatefulMemesT2IRetrieval",
+    "ImageCoDeT2IRetrieval",
+    "ImageCoDeT2IMultiChoice",
+    "InfoSeekIT2ITRetrieval",
+    "InfoSeekIT2TRetrieval",
+    "MemotionI2TRetrieval",
+    "MemotionT2IRetrieval",
+    "METI2IRetrieval",
+    "MSCOCOI2TRetrieval",
+    "MSCOCOT2IRetrieval",
+    "NIGHTSI2IRetrieval",
+    "OVENIT2ITRetrieval",
+    "OVENIT2TRetrieval",
+    "ROxfordEasyI2IMultiChoice",
+    "ROxfordMediumI2IMultiChoice",
+    "ROxfordHardI2IMultiChoice",
+    "RP2kI2IRetrieval",
+    "RParisEasyI2IMultiChoice",
+    "RParisMediumI2IMultiChoice",
+    "RParisHardI2IMultiChoice",
+    "SciMMIRI2TRetrieval",
+    "SciMMIRT2IRetrieval",
+    "SketchyI2IRetrieval",
+    "SOPI2IRetrieval",
+    "StanfordCarsI2IRetrieval",
+    "TUBerlinT2IRetrieval",
+    "VidoreArxivQARetrieval",
+    "VidoreDocVQARetrieval",
+    "VidoreInfoVQARetrieval",
+    "VidoreTabfquadRetrieval",
+    "VidoreTatdqaRetrieval",
+    "VidoreShiftProjectRetrieval",
+    "VidoreSyntheticDocQAAIRetrieval",
+    "VidoreSyntheticDocQAEnergyRetrieval",
+    "VidoreSyntheticDocQAGovernmentReportsRetrieval",
+    "VidoreSyntheticDocQAHealthcareIndustryRetrieval",
+    "VisualNewsI2TRetrieval",
+    "VisualNewsT2IRetrieval",
+    "VizWizIT2TRetrieval",
+    "VQA2IT2TRetrieval",
+    "WebQAT2ITRetrieval",
+    "WebQAT2TRetrieval",
+]
+
+MIEB_ENG = Benchmark(
+    name="MIEB(eng)",
+    tasks=get_tasks(
+        tasks=MIEB_common_tasks
+        + [
+            "VisualSTS17Eng",
+            "VisualSTS-b-Eng",
+        ],
+    ),
+    description="""MIEB(eng) is a comprehensive image embeddings benchmark, spanning 8 task types, covering 126 tasks. 
+    In addition to image classification (zero shot and linear probing), clustering, retrieval, MIEB includes tasks in compositionality evaluation, 
+    document undestanding, visual STS, and CV-centric tasks.""",
+    reference="",
+    contacts=["gowitheflow-1998", "isaac-chung"],
+    citation="",
+)
+
+MIEB_MULTILINGUAL = Benchmark(
+    name="MIEB(Multilingual)",
+    tasks=get_tasks(
+        tasks=MIEB_common_tasks
+        + [
+            "WITT2IRetrieval",
+            "XFlickr30kCoT2IRetrieval",
+            "XM3600T2IRetrieval",
+            "VisualSTS17Multilingual",
+            "VisualSTS-b-Multilingual",
+        ],
+    ),
+    description="""MIEB(Multilingual) is a comprehensive image embeddings benchmark, spanning 8 task types, covering 129 tasks and a total of 39 languages. 
+    In addition to image classification (zero shot and linear probing), clustering, retrieval, MIEB includes tasks in compositionality evaluation, 
+    document undestanding, visual STS, and CV-centric tasks. This benchmark consists of MIEB(eng) + 3 multilingual retrieval 
+    datasets + the multilingual parts of VisualSTS-b and VisualSTS-16.""",
+    reference="",
+    contacts=["gowitheflow-1998", "isaac-chung"],
+    citation="",
+)
+
+MIEB_LITE = Benchmark(
+    name="MIEB(lite)",
+    tasks=get_tasks(
+        tasks=[
+            # Image Classification
+            "Country211",
+            "DTD",
+            "EuroSAT",
+            "GTSRB",
+            "OxfordPets",
+            "PatchCamelyon",
+            "RESISC45",
+            "SUN397",
+            # Clustering
+            "ImageNetDog15Clustering",
+            "TinyImageNetClustering",
+            # ZeroShotClassification
+            "CIFAR100ZeroShot",
+            "Country211ZeroShot",
+            "FER2013ZeroShot",
+            "FGVCAircraftZeroShot",
+            "Food101ZeroShot",
+            "OxfordPetsZeroShot",
+            "StanfordCarsZeroShot",
+            # Any2TextMutipleChoice
+            "CVBenchCount",
+            "CVBenchRelation",
+            "CVBenchDepth",
+            "CVBenchDistance",
+            # Any2AnyMultipleChoice
+            "BLINKIT2IMultiChoice",
+            "ImageCoDeT2IMultiChoice",
+            # ImageTextPairClassification
+            "AROCocoOrder",
+            "AROFlickrOrder",
+            "AROVisualAttribution",
+            "AROVisualRelation",
+            "Winoground",
+            # VisualSTS
+            "STS13VisualSTS",
+            "STS15VisualSTS",
+            "STS17MultilingualVisualSTS",
+            "STSBenchmarkMultilingualVisualSTS",
+            # Any2AnyRetrieval
+            "CIRRIT2IRetrieval",
+            "CUB200I2IRetrieval",
+            "Fashion200kI2TRetrieval",
+            "HatefulMemesI2TRetrieval",
+            "InfoSeekIT2TRetrieval",
+            "NIGHTSI2IRetrieval",
+            "OVENIT2TRetrieval",
+            "RP2kI2IRetrieval",
+            "VidoreDocVQARetrieval",
+            "VidoreInfoVQARetrieval",
+            "VidoreTabfquadRetrieval",
+            "VidoreTatdqaRetrieval",
+            "VidoreShiftProjectRetrieval",
+            "VidoreSyntheticDocQAAIRetrieval",
+            "VisualNewsI2TRetrieval",
+            "VQA2IT2TRetrieval",
+            "WebQAT2ITRetrieval",
+            "WITT2IRetrieval",
+            "XM3600T2IRetrieval",
+        ],
+    ),
+    description="""MIEB(lite) is a comprehensive image embeddings benchmark, spanning 8 task types, covering 51 tasks. 
+    This is a lite version of MIEB(Multilingual), designed to be run at a fraction of the cost while maintaining
+    relative rank of models.""",
+    reference="",
+    contacts=["gowitheflow-1998", "isaac-chung"],
+    citation="",
+)
+
 BUILT_MTEB = Benchmark(
     name="BuiltBench(eng)",
     tasks=get_tasks(

mteb/leaderboard/app.py

@@ -6,7 +6,7 @@
 import tempfile
 import time
 from pathlib import Path
-from typing import Literal
+from typing import Literal, get_args
 from urllib.parse import urlencode
 
 import cachetools
@@ -15,7 +15,9 @@
 from gradio_rangeslider import RangeSlider
 
 import mteb
+from mteb.abstasks.TaskMetadata import TASK_DOMAIN, TASK_TYPE
 from mteb.benchmarks.benchmarks import MTEB_multilingual
+from mteb.languages import ISO_TO_LANGUAGE
 from mteb.leaderboard.figures import performance_size_plot, radar_chart
 from mteb.leaderboard.table import scores_to_tables
 
@@ -43,20 +45,6 @@
 We also thank the following companies which provide API credits to evaluate their models: [OpenAI](https://openai.com/), [Voyage AI](https://www.voyageai.com/)
 """
 
-MMTEB_TASK_TYPES = [  # TEMPORARY FIX: when adding MIEB to the leaderboard, this can probably be replaced with TASK_TYPE
-    "BitextMining",
-    "Classification",
-    "MultilabelClassification",
-    "Clustering",
-    "PairClassification",
-    "Reranking",
-    "Retrieval",
-    "STS",
-    "Summarization",
-    "InstructionRetrieval",
-    "Speed",
-]
-
 
 ALL_MODELS = {meta.name for meta in mteb.get_model_metas()}
 
@@ -238,21 +226,21 @@ def filter_models(
     info="Select one of our expert-selected benchmarks from MTEB publications.",
 )
 lang_select = gr.Dropdown(
-    all_results.languages,
+    ISO_TO_LANGUAGE,
     value=sorted(default_results.languages),
     multiselect=True,
     label="Language",
     info="Select languages to include.",
 )
 type_select = gr.Dropdown(
-    all_results.task_types,
-    value=sorted(MMTEB_TASK_TYPES),
+    sorted(get_args(TASK_TYPE)),
+    value=sorted(default_results.task_types),
     multiselect=True,
     label="Task Type",
     info="Select task types to include.",
 )
 domain_select = gr.Dropdown(
-    all_results.domains,
+    sorted(get_args(TASK_DOMAIN)),
     value=sorted(default_results.domains),
     multiselect=True,
     label="Domain",

mteb/leaderboard/figures.py

@@ -1,10 +1,14 @@
 from __future__ import annotations
 
+from typing import get_args
+
 import numpy as np
 import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
 
+from mteb.abstasks.TaskMetadata import TASK_TYPE
+
 
 def text_plot(text: str):
     """Returns empty scatter plot with text added, this can be great for error messages."""
@@ -56,6 +60,10 @@ def parse_float(value) -> float:
     "GritLM-7B",
     "LaBSE",
     "multilingual-e5-large-instruct",
+    "EVA02-CLIP-bigE-14-plus",
+    "voyage-multimodal-3",
+    "e5-v",
+    "VLM2Vec-Full",
 ]
 
 
@@ -165,21 +173,10 @@ def performance_size_plot(df: pd.DataFrame) -> go.Figure:
 
 
 TOP_N = 5
-task_types = [
-    "BitextMining",
-    "Classification",
-    "MultilabelClassification",
-    "Clustering",
-    "PairClassification",
-    "Reranking",
-    "Retrieval",
-    "STS",
-    "Summarization",
-    # "InstructionRetrieval",
-    # Not displayed, because the scores are negative,
-    # doesn't work well with the radar chart.
-    "Speed",
-]
+task_types = sorted(get_args(TASK_TYPE))
+task_types.remove("InstructionRetrieval")
+# Not displayed, because the scores are negative,
+# doesn't work well with the radar chart.
 
 line_colors = [
     "#EE4266",