dataset: add MixBench mixed-modality retrieval

mteb/_create_dataloaders.py

@@ -54,21 +54,23 @@ def _create_dataloader_from_texts(
 
 
 def _corpus_to_dict(
-    row: dict[str, str],
-) -> dict[str, str]:
-    text = (
-        (row["title"] + " " + row["text"]).strip()
-        if "title" in row and len(row["title"]) > 0
-        else row["text"].strip()
-    )
+    row: dict[str, Any],
+) -> dict[str, Any]:
+    body = row["text"]
+    title = row.get("title")
+    if title:
+        text = f"{title} {body or ''}".strip()
+    elif body is not None:
+        text = body.strip()
+    else:
+        text = None
     new_row = {
         "id": row["id"],
         "text": text,
-        "body": row["text"],
+        "body": body or "",
     }
-    # dataloaders can't handle None
-    if "title" in row and row["title"] is not None and len(row["title"]) > 0:
-        new_row["title"] = row["title"]
+    if title:
+        new_row["title"] = title
     return new_row
 
 
@@ -212,7 +214,7 @@ def _prepare_image_dataset(
 def _custom_collate_fn(batch: list[dict[str, Any]]) -> BatchedInput:
     """Custom collate function for DataLoader.
 
-    - For the "image", "conversation" key, leave the images as a list (to avoid stacking errors).
+    - For modality and conversation keys, leave values as a list.
     - For other keys, use the default collate.
 
     Args:
@@ -224,6 +226,7 @@ def _custom_collate_fn(batch: list[dict[str, Any]]) -> BatchedInput:
     collated = {}
     for key in batch[0]:
         if key in (  # noqa: PLR6201
+            "text",  # mixed-modality rows can have no text
             "image",  # images can be with different sizes
             "conversation",  # conversations are lists of varying lengths
             "audio",  # audio can have different lengths

mteb/abstasks/retrieval.py

@@ -532,13 +532,21 @@ def _calculate_descriptive_statistics_from_split(  # noqa: PLR0914
         # Build corpus col_inputs — text needs special mapping from the corpus dict format.
         corpus_col_inputs: dict[Modalities, list[Any]] = {}
         if "text" in corpus_modalities:
-            corpus_col_inputs["text"] = corpus.map(_corpus_to_dict)["text"]
+            corpus_col_inputs["text"] = [
+                text for text in corpus.map(_corpus_to_dict)["text"] if text is not None
+            ]
         if "image" in corpus_modalities:
-            corpus_col_inputs["image"] = corpus["image"]
+            corpus_col_inputs["image"] = [
+                image for image in corpus["image"] if image is not None
+            ]
         if "audio" in corpus_modalities:
-            corpus_col_inputs["audio"] = corpus["audio"]
+            corpus_col_inputs["audio"] = [
+                audio for audio in corpus["audio"] if audio is not None
+            ]
         if "video" in corpus_modalities:
-            corpus_col_inputs["video"] = corpus["video"]
+            corpus_col_inputs["video"] = [
+                video for video in corpus["video"] if video is not None
+            ]
 
         # Build queries col_inputs — text may need instruction/conversation transformations.
         queries_col_inputs: dict[Modalities, list[Any]] = {}
@@ -548,13 +556,21 @@ def _calculate_descriptive_statistics_from_split(  # noqa: PLR0914
                 queries_ = _combine_queries_with_instruction_text(queries_)
             if isinstance(queries_["text"][0], dict | list):
                 queries_ = queries_.map(_convert_conv_history_to_query)
-            queries_col_inputs["text"] = queries_["text"]
+            queries_col_inputs["text"] = [
+                text for text in queries_["text"] if text is not None
+            ]
         if "image" in queries_modalities:
-            queries_col_inputs["image"] = queries["image"]
+            queries_col_inputs["image"] = [
+                image for image in queries["image"] if image is not None
+            ]
         if "audio" in queries_modalities:
-            queries_col_inputs["audio"] = queries["audio"]
+            queries_col_inputs["audio"] = [
+                audio for audio in queries["audio"] if audio is not None
+            ]
         if "video" in queries_modalities:
-            queries_col_inputs["video"] = queries["video"]
+            queries_col_inputs["video"] = [
+                video for video in queries["video"] if video is not None
+            ]
 
         corpus_stats = calculate_single_input_modality_statistics(
             corpus_col_inputs, max_workers=num_proc
@@ -579,7 +595,7 @@ def _calculate_descriptive_statistics_from_split(  # noqa: PLR0914
             else None
         )
 
-        return RetrievalDescriptiveStatistics(
+        statistics = RetrievalDescriptiveStatistics(
             num_samples=num_documents + num_queries,
             num_queries=num_queries,
             num_documents=num_documents,
@@ -595,6 +611,35 @@ def _calculate_descriptive_statistics_from_split(  # noqa: PLR0914
             relevant_docs_statistics=relevant_docs_statistics,
             top_ranked_statistics=top_ranked_statistics,
         )
+        has_sparse_modalities = any(
+            len(values) != num_documents for values in corpus_col_inputs.values()
+        ) or any(len(values) != num_queries for values in queries_col_inputs.values())
+        if has_sparse_modalities:
+            statistics["num_documents_with_text"] = len(
+                corpus_col_inputs.get("text", [])
+            )
+            statistics["num_documents_with_image"] = len(
+                corpus_col_inputs.get("image", [])
+            )
+            statistics["num_documents_with_audio"] = len(
+                corpus_col_inputs.get("audio", [])
+            )
+            statistics["num_documents_with_video"] = len(
+                corpus_col_inputs.get("video", [])
+            )
+            statistics["num_queries_with_text"] = len(
+                queries_col_inputs.get("text", [])
+            )
+            statistics["num_queries_with_image"] = len(
+                queries_col_inputs.get("image", [])
+            )
+            statistics["num_queries_with_audio"] = len(
+                queries_col_inputs.get("audio", [])
+            )
+            statistics["num_queries_with_video"] = len(
+                queries_col_inputs.get("video", [])
+            )
+        return statistics
 
     def _push_dataset_to_hub(
         self,

mteb/benchmarks/benchmarks/__init__.py

@@ -26,6 +26,7 @@
     MIEB_IMG,
     MIEB_LITE,
     MIEB_MULTILINGUAL,
+    MIXBENCH,
     MTEB_DEU,
     MTEB_EN,
     MTEB_ENG_CLASSIC,
@@ -108,6 +109,7 @@
     "MIEB_IMG",
     "MIEB_LITE",
     "MIEB_MULTILINGUAL",
+    "MIXBENCH",
     "MTEB_DEU",
     "MTEB_EN",
     "MTEB_ENG_CLASSIC",

mteb/benchmarks/benchmarks/benchmarks.py

@@ -3571,3 +3571,28 @@
 }
 """,
 )
+
+MIXBENCH = Benchmark(
+    name="MixBench",
+    tasks=get_tasks(
+        tasks=[
+            "MixBenchMSCOCO",
+            "MixBenchGoogleWIT",
+            "MixBenchVisualNews",
+            "MixBenchOVEN",
+        ],
+    ),
+    description="Mixed-modality retrieval over heterogeneous corpora containing text-only, image-only, and image-and-text documents.",
+    reference="https://arxiv.org/abs/2507.19054",
+    citation=r"""
+@article{li2025closing,
+  author = {Li, Binxu and Zhang, Yuhui and Wang, Xiaohan and Liang, Weixin and Schmidt, Ludwig and Yeung-Levy, Serena},
+  journal = {arXiv preprint arXiv:2507.19054},
+  title = {Closing the Modality Gap for Mixed Modality Search},
+  url = {https://arxiv.org/abs/2507.19054},
+  year = {2025},
+}
+""",
+    contacts=["yuhui-zh15"],
+    aggregations=(BenchmarkAggregation.MEAN_TASK,),
+)

mteb/descriptive_stats/Image/Any2AnyRetrieval/MixBenchGoogleWIT.json

@@ -0,0 +1,52 @@
+{
+    "test": {
+        "num_samples": 5421,
+        "num_queries": 1000,
+        "num_documents": 4421,
+        "num_documents_with_text": 2948,
+        "num_documents_with_image": 2948,
+        "num_documents_with_audio": 0,
+        "num_documents_with_video": 0,
+        "num_queries_with_text": 1000,
+        "num_queries_with_image": 0,
+        "num_queries_with_audio": 0,
+        "num_queries_with_video": 0,
+        "number_of_characters": 1701096,
+        "documents_text_statistics": {
+            "total_text_length": 1643947,
+            "min_text_length": 4,
+            "average_text_length": 557.648236092266,
+            "max_text_length": 1199,
+            "unique_texts": 2924
+        },
+        "documents_image_statistics": {
+            "min_image_width": 102,
+            "average_image_width": 1909.6132971506106,
+            "max_image_width": 13433,
+            "min_image_height": 100,
+            "average_image_height": 1601.9684531886026,
+            "max_image_height": 9909,
+            "unique_images": 2948
+        },
+        "documents_audio_statistics": null,
+        "documents_video_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 57149,
+            "min_text_length": 3,
+            "average_text_length": 57.149,
+            "max_text_length": 418,
+            "unique_texts": 999
+        },
+        "queries_image_statistics": null,
+        "queries_audio_statistics": null,
+        "queries_video_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 1000
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Image/Any2AnyRetrieval/MixBenchMSCOCO.json

@@ -0,0 +1,52 @@
+{
+    "test": {
+        "num_samples": 1968,
+        "num_queries": 984,
+        "num_documents": 984,
+        "num_documents_with_text": 656,
+        "num_documents_with_image": 656,
+        "num_documents_with_audio": 0,
+        "num_documents_with_video": 0,
+        "num_queries_with_text": 984,
+        "num_queries_with_image": 0,
+        "num_queries_with_audio": 0,
+        "num_queries_with_video": 0,
+        "number_of_characters": 275373,
+        "documents_text_statistics": {
+            "total_text_length": 224276,
+            "min_text_length": 114,
+            "average_text_length": 341.8841463414634,
+            "max_text_length": 652,
+            "unique_texts": 656
+        },
+        "documents_image_statistics": {
+            "min_image_width": 229,
+            "average_image_width": 575.905487804878,
+            "max_image_width": 640,
+            "min_image_height": 191,
+            "average_image_height": 481.2012195121951,
+            "max_image_height": 640,
+            "unique_images": 656
+        },
+        "documents_audio_statistics": null,
+        "documents_video_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 51097,
+            "min_text_length": 29,
+            "average_text_length": 51.927845528455286,
+            "max_text_length": 156,
+            "unique_texts": 984
+        },
+        "queries_image_statistics": null,
+        "queries_audio_statistics": null,
+        "queries_video_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 984,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 984
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Image/Any2AnyRetrieval/MixBenchOVEN.json

@@ -0,0 +1,60 @@
+{
+    "test": {
+        "num_samples": 2000,
+        "num_queries": 1000,
+        "num_documents": 1000,
+        "num_documents_with_text": 667,
+        "num_documents_with_image": 667,
+        "num_documents_with_audio": 0,
+        "num_documents_with_video": 0,
+        "num_queries_with_text": 1000,
+        "num_queries_with_image": 1000,
+        "num_queries_with_audio": 0,
+        "num_queries_with_video": 0,
+        "number_of_characters": 438400,
+        "documents_text_statistics": {
+            "total_text_length": 408213,
+            "min_text_length": 20,
+            "average_text_length": 612.0134932533733,
+            "max_text_length": 724,
+            "unique_texts": 422
+        },
+        "documents_image_statistics": {
+            "min_image_width": 256,
+            "average_image_width": 359.71664167916043,
+            "max_image_width": 757,
+            "min_image_height": 256,
+            "average_image_height": 271.19790104947526,
+            "max_image_height": 502,
+            "unique_images": 440
+        },
+        "documents_audio_statistics": null,
+        "documents_video_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 30187,
+            "min_text_length": 19,
+            "average_text_length": 30.187,
+            "max_text_length": 58,
+            "unique_texts": 139
+        },
+        "queries_image_statistics": {
+            "min_image_width": 256,
+            "average_image_width": 335.365,
+            "max_image_width": 1796,
+            "min_image_height": 256,
+            "average_image_height": 271.978,
+            "max_image_height": 650,
+            "unique_images": 1000
+        },
+        "queries_audio_statistics": null,
+        "queries_video_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 1000,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 1000
+        },
+        "top_ranked_statistics": null
+    }
+}