dataset: add Mars-VL-Pairs bidirectional retrieval

mteb/descriptive_stats/Image/Any2AnyRetrieval/MarsVLPairsI2TRetrieval.json

@@ -0,0 +1,38 @@
+{
+    "test": {
+        "num_samples": 4494,
+        "num_queries": 2247,
+        "num_documents": 2247,
+        "number_of_characters": 348132,
+        "documents_text_statistics": {
+            "total_text_length": 348132,
+            "min_text_length": 34,
+            "average_text_length": 154.93190921228305,
+            "max_text_length": 357,
+            "unique_texts": 2247
+        },
+        "documents_image_statistics": null,
+        "documents_audio_statistics": null,
+        "documents_video_statistics": null,
+        "queries_text_statistics": null,
+        "queries_image_statistics": {
+            "min_image_width": 72,
+            "average_image_width": 857.351134846462,
+            "max_image_width": 37971,
+            "min_image_height": 40,
+            "average_image_height": 692.183800623053,
+            "max_image_height": 10000,
+            "unique_images": 2247
+        },
+        "queries_audio_statistics": null,
+        "queries_video_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 2247,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 2247
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/descriptive_stats/Image/Any2AnyRetrieval/MarsVLPairsT2IRetrieval.json

@@ -0,0 +1,38 @@
+{
+    "test": {
+        "num_samples": 4494,
+        "num_queries": 2247,
+        "num_documents": 2247,
+        "number_of_characters": 348132,
+        "documents_text_statistics": null,
+        "documents_image_statistics": {
+            "min_image_width": 72,
+            "average_image_width": 857.351134846462,
+            "max_image_width": 37971,
+            "min_image_height": 40,
+            "average_image_height": 692.183800623053,
+            "max_image_height": 10000,
+            "unique_images": 2247
+        },
+        "documents_audio_statistics": null,
+        "documents_video_statistics": null,
+        "queries_text_statistics": {
+            "total_text_length": 348132,
+            "min_text_length": 34,
+            "average_text_length": 154.93190921228305,
+            "max_text_length": 357,
+            "unique_texts": 2247
+        },
+        "queries_image_statistics": null,
+        "queries_audio_statistics": null,
+        "queries_video_statistics": null,
+        "relevant_docs_statistics": {
+            "num_relevant_docs": 2247,
+            "min_relevant_docs_per_query": 1,
+            "average_relevant_docs_per_query": 1.0,
+            "max_relevant_docs_per_query": 1,
+            "unique_relevant_docs": 2247
+        },
+        "top_ranked_statistics": null
+    }
+}

mteb/tasks/retrieval/eng/__init__.py

@@ -220,6 +220,7 @@
 )
 from .lotte_retrieval import LoTTERetrieval
 from .macs import MACSA2TRetrieval, MACST2ARetrieval
+from .mars_vl_pairs import MarsVLPairsI2TRetrieval, MarsVLPairsT2IRetrieval
 from .medical_qa_retrieval import MedicalQARetrieval
 from .memotion_i2t_retrieval import MemotionI2TRetrieval
 from .memotion_t2i_retrieval import MemotionT2IRetrieval
@@ -656,6 +657,8 @@
     "MSMARCOv2",
     "MSVDT2VRetrieval",
     "MSVDV2TRetrieval",
+    "MarsVLPairsI2TRetrieval",
+    "MarsVLPairsT2IRetrieval",
     "MedicalQARetrieval",
     "MemBench",
     "MemGovern",

mteb/tasks/retrieval/eng/mars_vl_pairs.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+from statistics import fmean
+from typing import TYPE_CHECKING, Literal
+
+from datasets import load_dataset
+from typing_extensions import override
+
+from mteb._evaluators.retrieval_metrics import mrr
+from mteb.abstasks.retrieval import AbsTaskRetrieval
+from mteb.abstasks.retrieval_dataset_loaders import RetrievalSplitData
+from mteb.abstasks.task_metadata import TaskMetadata
+
+if TYPE_CHECKING:
+    from mteb.types import RelevantDocumentsType
+
+_DATASET_PATH = "Cerru02/Mars-VL-Pairs-MTEB"
+_DATASET_REVISION = "f0084ab0ba2f584b15dc72a82502b38ee490f58d"
+_FROZEN_PAIRS = 2_247
+_REFERENCE = "https://arxiv.org/abs/2602.13961"
+_BIBTEX = r"""
+@article{wang2026marsretrieval,
+  author = {Wang, Shuoyuan and Wang, Yiran and Wei, Hongxin},
+  journal = {arXiv preprint arXiv:2602.13961},
+  title = {MarsRetrieval: Benchmarking Vision-Language Models for Planetary-Scale Geospatial Retrieval on Mars},
+  year = {2026},
+}
+"""
+_DESCRIPTION = (
+    "Mars-VL-Pairs is Task 1 of MarsRetrieval, a planetary-science benchmark "
+    "covering Mars imagery from global orbital mosaics to rover-scale views. "
+    "The source has 2,287 one-to-one image-caption pairs; this reproducibility "
+    "release freezes 2,247 pairs after 38 unavailable images and two "
+    "resize-equivalent image pairs were removed. It uses the expert-validated "
+    "refined captions from the paper's main evaluation. "
+)
+
+
+def _load_mars_vl_pairs(
+    task: AbsTaskRetrieval,
+    direction: Literal["t2i", "i2t"],
+    num_proc: int | None,
+) -> None:
+    if task.data_loaded:
+        return
+
+    pairs = load_dataset(
+        task.metadata.dataset["path"],
+        revision=task.metadata.dataset["revision"],
+        split="test",
+        num_proc=num_proc,
+    )
+    if len(pairs) != _FROZEN_PAIRS:
+        raise ValueError(f"Expected {_FROZEN_PAIRS} frozen pairs, found {len(pairs)}")
+
+    ids = [str(key) for key in pairs["key"]]
+
+    text = (
+        pairs.select_columns(["refined_caption"])
+        .rename_column("refined_caption", "text")
+        .add_column("id", ids)
+        .select_columns(["id", "text"])
+    )
+    images = (
+        pairs.select_columns(["image"])
+        .add_column("id", ids)
+        .select_columns(["id", "image"])
+    )
+    queries, corpus = (text, images) if direction == "t2i" else (images, text)
+    qrels = {pair_id: {pair_id: 1} for pair_id in ids}
+
+    task.dataset = {
+        "default": {
+            "test": RetrievalSplitData(
+                queries=queries,
+                corpus=corpus,
+                relevant_docs=qrels,
+                top_ranked=None,
+            )
+        }
+    }
+    task.data_loaded = True
+
+
+class _MarsVLPairsRetrieval(AbsTaskRetrieval):
+    _top_k = _FROZEN_PAIRS
+
+    @override
+    def task_specific_scores(
+        self,
+        scores: dict[str, dict[str, float]],
+        qrels: RelevantDocumentsType,
+        results: dict[str, dict[str, float]],
+        hf_split: str,
+        hf_subset: str,
+    ) -> dict[str, float]:
+        full_gallery_mrr = mrr(qrels, results, (_FROZEN_PAIRS,))[f"MRR@{_FROZEN_PAIRS}"]
+        return {f"mrr_at_{_FROZEN_PAIRS}": fmean(full_gallery_mrr)}
+
+
+class MarsVLPairsT2IRetrieval(_MarsVLPairsRetrieval):
+    metadata = TaskMetadata(
+        name="MarsVLPairsT2IRetrieval",
+        description=_DESCRIPTION
+        + "Given a scientific caption, retrieve its paired Mars image from the "
+        "full frozen gallery.",
+        reference=_REFERENCE,
+        dataset={"path": _DATASET_PATH, "revision": _DATASET_REVISION},
+        type="Any2AnyRetrieval",
+        category="t2i",
+        modalities=["text", "image"],
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="mrr_at_2247",
+        date=("2026-02-15", "2026-02-15"),
+        domains=["Academic", "Nature", "Scene", "Web"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="LM-generated and reviewed",
+        dialect=[],
+        sample_creation="multiple",
+        bibtex_citation=_BIBTEX,
+        prompt={
+            "query": "Retrieve the Mars image that matches this scientific description."
+        },
+        is_beta=True,
+    )
+
+    def load_data(self, num_proc: int | None = None, **kwargs) -> None:
+        _load_mars_vl_pairs(self, "t2i", num_proc)
+
+
+class MarsVLPairsI2TRetrieval(_MarsVLPairsRetrieval):
+    metadata = TaskMetadata(
+        name="MarsVLPairsI2TRetrieval",
+        description=_DESCRIPTION
+        + "Given a Mars image, retrieve its paired scientific caption from the "
+        "full frozen gallery.",
+        reference=_REFERENCE,
+        dataset={"path": _DATASET_PATH, "revision": _DATASET_REVISION},
+        type="Any2AnyRetrieval",
+        category="i2t",
+        modalities=["image", "text"],
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="mrr_at_2247",
+        date=("2026-02-15", "2026-02-15"),
+        domains=["Academic", "Nature", "Scene", "Web"],
+        task_subtypes=["Image Text Retrieval"],
+        license="cc-by-4.0",
+        annotations_creators="LM-generated and reviewed",
+        dialect=[],
+        sample_creation="multiple",
+        bibtex_citation=_BIBTEX,
+        prompt={
+            "query": "Retrieve the scientific caption that describes this Mars image."
+        },
+        is_beta=True,
+    )
+
+    def load_data(self, num_proc: int | None = None, **kwargs) -> None:
+        _load_mars_vl_pairs(self, "i2t", num_proc)

scripts/data/mars_vl_pairs/README.md

@@ -0,0 +1,43 @@
+# Mars-VL-Pairs construction
+
+This directory freezes Task 1 of
+[MarsRetrieval](https://github.com/ml-stat-Sustech/MarsRetrieval), replacing its
+mutable web image URLs with validated image bytes while preserving the original
+one-to-one order and provenance.
+
+The builder pins `SUSTech/Mars-VL-Pairs`, downloads every image with retries,
+validates it with Pillow, records redirects and failures, computes byte/pixel
+hashes and dimensions, checks URL/caption/media duplicates, and writes a JSON
+audit report. Both MTEB directions are derived from the same frozen pair table.
+
+```bash
+/path/to/python scripts/data/mars_vl_pairs/create_data.py \
+  --work-dir /tmp/mars_vl_pairs_mteb \
+  --archive-recovery
+
+# Upload only after reviewing audit_summary.json and audit_rows.jsonl.
+/path/to/python scripts/data/mars_vl_pairs/create_data.py \
+  --work-dir /tmp/mars_vl_pairs_mteb \
+  --archive-recovery \
+  --allow-missing \
+  --repo-id Cerru02/Mars-VL-Pairs-MTEB \
+  --push
+```
+
+If direct downloads fail, `--archive-recovery` also upgrades HTTP URLs to HTTPS
+and attempts exact-URL Wayback snapshots, including captures stored with an
+incorrect MIME type. HTTPS certificate verification is always enabled.
+Recovered rows retain the original URL and recovery method. The script stops
+on missing or exact duplicate media by default; `--allow-missing` and
+`--allow-duplicate-images` require an explicit, documented decision.
+
+Manual review of every image pair with dHash distance at most two found two
+resize-equivalent groups: source rows 460/907 and 1446/2098. The builder
+deterministically excludes the lower-resolution rows 907 and 2098 because their
+different captions would otherwise create ambiguous one-positive qrels. The
+other near-hash alerts are visibly different scenes.
+
+The source dataset declares CC-BY-4.0. Its images were selected from web-scale
+corpora and come from many external domains, so the builder preserves the
+source URL and does not treat the dataset-level declaration as a substitute for
+the original source's rights information.