From 7222be8e7696ff0964f71c2130c8275f549859de Mon Sep 17 00:00:00 2001 From: tommasocerruti Date: Tue, 11 Aug 2026 03:23:06 +0200 Subject: [PATCH 1/9] dataset: add Mars-VL-Pairs retrieval --- .../MarsVLPairsI2TRetrieval.json | 38 + .../MarsVLPairsT2IRetrieval.json | 38 + mteb/tasks/retrieval/eng/__init__.py | 3 + mteb/tasks/retrieval/eng/mars_vl_pairs.py | 148 ++++ scripts/data/mars_vl_pairs/README.md | 44 ++ scripts/data/mars_vl_pairs/create_data.py | 702 ++++++++++++++++++ 6 files changed, 973 insertions(+) create mode 100644 mteb/descriptive_stats/Image/Any2AnyRetrieval/MarsVLPairsI2TRetrieval.json create mode 100644 mteb/descriptive_stats/Image/Any2AnyRetrieval/MarsVLPairsT2IRetrieval.json create mode 100644 mteb/tasks/retrieval/eng/mars_vl_pairs.py create mode 100644 scripts/data/mars_vl_pairs/README.md create mode 100644 scripts/data/mars_vl_pairs/create_data.py diff --git a/mteb/descriptive_stats/Image/Any2AnyRetrieval/MarsVLPairsI2TRetrieval.json b/mteb/descriptive_stats/Image/Any2AnyRetrieval/MarsVLPairsI2TRetrieval.json new file mode 100644 index 0000000000..1f9bc279b5 --- /dev/null +++ b/mteb/descriptive_stats/Image/Any2AnyRetrieval/MarsVLPairsI2TRetrieval.json @@ -0,0 +1,38 @@ +{ + "test": { + "num_samples": 4494, + "num_queries": 2247, + "num_documents": 2247, + "number_of_characters": 348132, + "documents_text_statistics": { + "total_text_length": 348132, + "min_text_length": 34, + "average_text_length": 154.93190921228305, + "max_text_length": 357, + "unique_texts": 2247 + }, + "documents_image_statistics": null, + "documents_audio_statistics": null, + "documents_video_statistics": null, + "queries_text_statistics": null, + "queries_image_statistics": { + "min_image_width": 72, + "average_image_width": 857.351134846462, + "max_image_width": 37971, + "min_image_height": 40, + "average_image_height": 692.183800623053, + "max_image_height": 10000, + "unique_images": 2247 + }, + "queries_audio_statistics": null, + "queries_video_statistics": null, + "relevant_docs_statistics": { + "num_relevant_docs": 2247, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 2247 + }, + "top_ranked_statistics": null + } +} \ No newline at end of file diff --git a/mteb/descriptive_stats/Image/Any2AnyRetrieval/MarsVLPairsT2IRetrieval.json b/mteb/descriptive_stats/Image/Any2AnyRetrieval/MarsVLPairsT2IRetrieval.json new file mode 100644 index 0000000000..f80c4fa87f --- /dev/null +++ b/mteb/descriptive_stats/Image/Any2AnyRetrieval/MarsVLPairsT2IRetrieval.json @@ -0,0 +1,38 @@ +{ + "test": { + "num_samples": 4494, + "num_queries": 2247, + "num_documents": 2247, + "number_of_characters": 348132, + "documents_text_statistics": null, + "documents_image_statistics": { + "min_image_width": 72, + "average_image_width": 857.351134846462, + "max_image_width": 37971, + "min_image_height": 40, + "average_image_height": 692.183800623053, + "max_image_height": 10000, + "unique_images": 2247 + }, + "documents_audio_statistics": null, + "documents_video_statistics": null, + "queries_text_statistics": { + "total_text_length": 348132, + "min_text_length": 34, + "average_text_length": 154.93190921228305, + "max_text_length": 357, + "unique_texts": 2247 + }, + "queries_image_statistics": null, + "queries_audio_statistics": null, + "queries_video_statistics": null, + "relevant_docs_statistics": { + "num_relevant_docs": 2247, + "min_relevant_docs_per_query": 1, + "average_relevant_docs_per_query": 1.0, + "max_relevant_docs_per_query": 1, + "unique_relevant_docs": 2247 + }, + "top_ranked_statistics": null + } +} \ No newline at end of file diff --git a/mteb/tasks/retrieval/eng/__init__.py b/mteb/tasks/retrieval/eng/__init__.py index 30cef096da..b5430e6cda 100644 --- a/mteb/tasks/retrieval/eng/__init__.py +++ b/mteb/tasks/retrieval/eng/__init__.py @@ -220,6 +220,7 @@ ) from .lotte_retrieval import LoTTERetrieval from .macs import MACSA2TRetrieval, MACST2ARetrieval +from .mars_vl_pairs import MarsVLPairsI2TRetrieval, MarsVLPairsT2IRetrieval from .medical_qa_retrieval import MedicalQARetrieval from .memotion_i2t_retrieval import MemotionI2TRetrieval from .memotion_t2i_retrieval import MemotionT2IRetrieval @@ -656,6 +657,8 @@ "MSMARCOv2", "MSVDT2VRetrieval", "MSVDV2TRetrieval", + "MarsVLPairsI2TRetrieval", + "MarsVLPairsT2IRetrieval", "MedicalQARetrieval", "MemBench", "MemGovern", diff --git a/mteb/tasks/retrieval/eng/mars_vl_pairs.py b/mteb/tasks/retrieval/eng/mars_vl_pairs.py new file mode 100644 index 0000000000..2e7fffb4b4 --- /dev/null +++ b/mteb/tasks/retrieval/eng/mars_vl_pairs.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +from typing import Literal + +from datasets import load_dataset + +from mteb.abstasks.retrieval import AbsTaskRetrieval +from mteb.abstasks.retrieval_dataset_loaders import RetrievalSplitData +from mteb.abstasks.task_metadata import TaskMetadata + +_DATASET_PATH = "Cerru02/Mars-VL-Pairs-MTEB" +_DATASET_REVISION = "f0084ab0ba2f584b15dc72a82502b38ee490f58d" +_FROZEN_PAIRS = 2_247 +_REFERENCE = "https://arxiv.org/abs/2602.13961" +_BIBTEX = r""" +@article{wang2026marsretrieval, + author = {Wang, Shuoyuan and Wang, Yiran and Wei, Hongxin}, + journal = {arXiv preprint arXiv:2602.13961}, + title = {MarsRetrieval: Benchmarking Vision-Language Models for Planetary-Scale Geospatial Retrieval on Mars}, + year = {2026}, +} +""" +_DESCRIPTION = ( + "Mars-VL-Pairs is Task 1 of MarsRetrieval, a planetary-science benchmark " + "covering Mars imagery from global orbital mosaics to rover-scale views. " + "The source has 2,287 one-to-one image-caption pairs; this reproducibility " + "release freezes 2,247 pairs after 38 unavailable images and two " + "resize-equivalent image pairs were removed. It uses the expert-validated " + "refined captions from the paper's main evaluation. " +) + + +def _load_mars_vl_pairs( + task: AbsTaskRetrieval, + direction: Literal["t2i", "i2t"], + num_proc: int | None, +) -> None: + if task.data_loaded: + return + + pairs = load_dataset( + task.metadata.dataset["path"], + revision=task.metadata.dataset["revision"], + split="test", + num_proc=num_proc, + ) + if len(pairs) != _FROZEN_PAIRS: + raise ValueError(f"Expected {_FROZEN_PAIRS} frozen pairs, found {len(pairs)}") + + keys = pairs["key"] + query_ids = [f"query-{key}" for key in keys] + corpus_ids = [f"corpus-{key}" for key in keys] + + text = ( + pairs.select_columns(["refined_caption"]) + .rename_column("refined_caption", "text") + .add_column("id", query_ids if direction == "t2i" else corpus_ids) + .select_columns(["id", "text"]) + ) + images = ( + pairs.select_columns(["image"]) + .add_column("id", corpus_ids if direction == "t2i" else query_ids) + .select_columns(["id", "image"]) + ) + queries, corpus = (text, images) if direction == "t2i" else (images, text) + qrels = { + query_id: {corpus_id: 1} for query_id, corpus_id in zip(query_ids, corpus_ids) + } + + task.dataset = { + "default": { + "test": RetrievalSplitData( + queries=queries, + corpus=corpus, + relevant_docs=qrels, + top_ranked=None, + ) + } + } + task.data_loaded = True + + +class MarsVLPairsT2IRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="MarsVLPairsT2IRetrieval", + description=_DESCRIPTION + + "Given a scientific caption, retrieve its paired Mars image from the " + "full frozen gallery.", + reference=_REFERENCE, + dataset={"path": _DATASET_PATH, "revision": _DATASET_REVISION}, + type="Any2AnyRetrieval", + category="t2i", + modalities=["text", "image"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="mrr_at_2247", + date=("2026-02-15", "2026-02-15"), + domains=["Academic", "Nature", "Scene", "Web"], + task_subtypes=["Image Text Retrieval"], + license="cc-by-4.0", + annotations_creators="LM-generated and reviewed", + dialect=[], + sample_creation="multiple", + bibtex_citation=_BIBTEX, + prompt={ + "query": "Retrieve the Mars image that matches this scientific description." + }, + is_beta=True, + ) + k_values = (1, 3, 5, 10, 20, 100, 1000, _FROZEN_PAIRS) + _top_k = _FROZEN_PAIRS + + def load_data(self, num_proc: int | None = None, **kwargs) -> None: + _load_mars_vl_pairs(self, "t2i", num_proc) + + +class MarsVLPairsI2TRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="MarsVLPairsI2TRetrieval", + description=_DESCRIPTION + + "Given a Mars image, retrieve its paired scientific caption from the " + "full frozen gallery.", + reference=_REFERENCE, + dataset={"path": _DATASET_PATH, "revision": _DATASET_REVISION}, + type="Any2AnyRetrieval", + category="i2t", + modalities=["image", "text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="mrr_at_2247", + date=("2026-02-15", "2026-02-15"), + domains=["Academic", "Nature", "Scene", "Web"], + task_subtypes=["Image Text Retrieval"], + license="cc-by-4.0", + annotations_creators="LM-generated and reviewed", + dialect=[], + sample_creation="multiple", + bibtex_citation=_BIBTEX, + prompt={ + "query": "Retrieve the scientific caption that describes this Mars image." + }, + is_beta=True, + ) + k_values = (1, 3, 5, 10, 20, 100, 1000, _FROZEN_PAIRS) + _top_k = _FROZEN_PAIRS + + def load_data(self, num_proc: int | None = None, **kwargs) -> None: + _load_mars_vl_pairs(self, "i2t", num_proc) diff --git a/scripts/data/mars_vl_pairs/README.md b/scripts/data/mars_vl_pairs/README.md new file mode 100644 index 0000000000..01693d4442 --- /dev/null +++ b/scripts/data/mars_vl_pairs/README.md @@ -0,0 +1,44 @@ +# Mars-VL-Pairs construction + +This directory freezes Task 1 of +[MarsRetrieval](https://github.com/ml-stat-Sustech/MarsRetrieval), replacing its +mutable web image URLs with validated image bytes while preserving the original +one-to-one order and provenance. + +The builder pins `SUSTech/Mars-VL-Pairs`, downloads every image with retries, +validates it with Pillow, records redirects and failures, computes byte/pixel +hashes and dimensions, checks URL/caption/media duplicates, and writes a JSON +audit report. Both MTEB directions are derived from the same frozen pair table. + +```bash +/path/to/python scripts/data/mars_vl_pairs/create_data.py \ + --work-dir /tmp/mars_vl_pairs_mteb \ + --archive-recovery + +# Upload only after reviewing audit_summary.json and audit_rows.jsonl. +/path/to/python scripts/data/mars_vl_pairs/create_data.py \ + --work-dir /tmp/mars_vl_pairs_mteb \ + --archive-recovery \ + --allow-missing \ + --repo-id Cerru02/Mars-VL-Pairs-MTEB \ + --push +``` + +If direct downloads fail, `--archive-recovery` also attempts scheme variants +and exact-URL Wayback snapshots, including captures stored with an incorrect +MIME type. It may retry the exact source URL without TLS verification after a +certificate failure; this is labeled in `recovery_method`. Recovered rows +retain the original URL and recovery method. The script stops on missing or +exact duplicate media by default; `--allow-missing` and +`--allow-duplicate-images` require an explicit, documented decision. + +Manual review of every image pair with dHash distance at most two found two +resize-equivalent groups: source rows 460/907 and 1446/2098. The builder +deterministically excludes the lower-resolution rows 907 and 2098 because their +different captions would otherwise create ambiguous one-positive qrels. The +other near-hash alerts are visibly different scenes. + +The source dataset declares CC-BY-4.0. Its images were selected from web-scale +corpora and come from many external domains, so the builder preserves the +source URL and does not treat the dataset-level declaration as a substitute for +the original source's rights information. diff --git a/scripts/data/mars_vl_pairs/create_data.py b/scripts/data/mars_vl_pairs/create_data.py new file mode 100644 index 0000000000..06735b8fc9 --- /dev/null +++ b/scripts/data/mars_vl_pairs/create_data.py @@ -0,0 +1,702 @@ +#!/usr/bin/env python3 +"""Audit and freeze SUSTech/Mars-VL-Pairs with embedded image bytes. + +The output is one ordered pair table. MTEB derives both text-to-image and +image-to-text retrieval tasks from this exact table so attrition cannot diverge +between directions. +""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import re +import shutil +import statistics +import threading +import warnings +from collections import Counter, defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass +from io import BytesIO +from pathlib import Path +from typing import Any +from urllib.parse import quote, urlparse + +import requests +from datasets import Dataset, DatasetDict, Image, load_dataset +from huggingface_hub import DatasetCard, HfApi, create_repo, get_token +from PIL import Image as PILImage +from PIL import ImageOps +from requests.adapters import HTTPAdapter +from tqdm import tqdm +from urllib3.util.retry import Retry +from urllib3.exceptions import InsecureRequestWarning + +SOURCE = "SUSTech/Mars-VL-Pairs" +SOURCE_REVISION = "1cac8885e481256d3752ad0d3a0f8f9681c5f206" +EXPECTED_PAIRS = 2_287 +REQUIRED_COLUMNS = ("key", "image_url", "ori_caption", "refined_caption") +# Manual review of all dHash-distance <= 2 pairs found two resize-equivalent +# images with different captions. Retain the higher-resolution source member so +# every frozen query still has exactly one unambiguous positive. +PRACTICAL_DUPLICATE_GROUPS = ((460, 907), (1446, 2098)) +EXCLUDED_PRACTICAL_DUPLICATE_INDICES = frozenset({907, 2098}) +USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36" +) +IMAGE_EXTENSIONS = { + "AVIF": ".avif", + "BMP": ".bmp", + "GIF": ".gif", + "JPEG": ".jpg", + "PNG": ".png", + "TIFF": ".tiff", + "WEBP": ".webp", +} +_THREAD_LOCAL = threading.local() + + +@dataclass +class AuditRow: + index: int + key: str + source_url: str + ok: bool + attempts: list[str] + resolved_url: str | None = None + status_code: int | None = None + redirect_chain: list[dict[str, Any]] | None = None + recovery_method: str = "direct" + error: str | None = None + local_path: str | None = None + byte_sha256: str | None = None + pixel_sha256: str | None = None + dhash: str | None = None + width: int | None = None + height: int | None = None + image_format: str | None = None + mode: str | None = None + file_size: int | None = None + + +def _session(retries: int) -> requests.Session: + cache_key = f"session_{retries}" + session = getattr(_THREAD_LOCAL, cache_key, None) + if session is None: + retry = Retry( + total=retries, + connect=retries, + read=retries, + status=retries, + backoff_factor=0.75, + status_forcelist=(408, 425, 429, 500, 502, 503, 504), + allowed_methods=frozenset({"GET"}), + ) + session = requests.Session() + session.headers.update( + { + "User-Agent": USER_AGENT, + "Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + } + ) + session.mount("http://", HTTPAdapter(max_retries=retry)) + session.mount("https://", HTTPAdapter(max_retries=retry)) + setattr(_THREAD_LOCAL, cache_key, session) + return session + + +def _normalise_text(value: str) -> str: + return " ".join(value.split()).casefold() + + +def _safe_stem(index: int, key: str) -> str: + clean_key = re.sub(r"[^A-Za-z0-9._-]+", "_", key).strip("._") or "row" + return f"{index:04d}-{clean_key}" + + +def _dhash(image: PILImage.Image) -> str: + gray = ImageOps.exif_transpose(image).convert("L").resize((9, 8)) + pixels = list(gray.tobytes()) + value = 0 + for y in range(8): + for x in range(8): + value = (value << 1) | (pixels[y * 9 + x] > pixels[y * 9 + x + 1]) + return f"{value:016x}" + + +def _validate_image(content: bytes) -> dict[str, Any]: + with PILImage.open(BytesIO(content)) as check: + check.verify() + with PILImage.open(BytesIO(content)) as image: + image.load() + width, height = image.size + image_format = (image.format or "").upper() + if width <= 0 or height <= 0: + raise ValueError(f"invalid dimensions {width}x{height}") + rgb = ImageOps.exif_transpose(image).convert("RGB") + pixel_header = f"{rgb.width}x{rgb.height}:RGB:".encode() + pixel_sha256 = hashlib.sha256(pixel_header + rgb.tobytes()).hexdigest() + return { + "width": width, + "height": height, + "format": image_format, + "mode": image.mode, + "pixel_sha256": pixel_sha256, + "dhash": _dhash(image), + "extension": IMAGE_EXTENSIONS.get(image_format, ".img"), + } + + +def _fetch_bytes( + url: str, + *, + retries: int, + connect_timeout: float, + read_timeout: float, + max_bytes: int, + verify_tls: bool = True, +) -> tuple[bytes, requests.Response]: + parsed = urlparse(url) + with warnings.catch_warnings(): + if not verify_tls: + warnings.simplefilter("ignore", InsecureRequestWarning) + response = _session(retries).get( + url, + allow_redirects=True, + headers={"Referer": f"{parsed.scheme}://{parsed.netloc}/"}, + stream=True, + timeout=(connect_timeout, read_timeout), + verify=verify_tls, + ) + response.raise_for_status() + chunks: list[bytes] = [] + size = 0 + for chunk in response.iter_content(chunk_size=1024 * 256): + if not chunk: + continue + size += len(chunk) + if size > max_bytes: + raise ValueError(f"response exceeds {max_bytes} bytes") + chunks.append(chunk) + if not chunks: + raise ValueError("empty response") + return b"".join(chunks), response + + +def _wayback_candidates(url: str, retries: int) -> list[str]: + cdx_url = ( + "https://web.archive.org/cdx/search/cdx?" + f"url={quote(url, safe='')}&output=json&fl=timestamp,original,statuscode,mimetype" + "&filter=statuscode:200&collapse=digest&limit=10" + "&sort=reverse" + ) + try: + response = _session(retries).get(cdx_url, timeout=(20, 60)) + response.raise_for_status() + rows = response.json() + except (requests.RequestException, ValueError): + return [] + candidates = [] + if isinstance(rows, list) and len(rows) >= 2: + for row in rows[1:]: + if not isinstance(row, list) or len(row) < 2: + continue + timestamp, original = row[0], row[1] + candidates.append(f"https://web.archive.org/web/{timestamp}id_/{original}") + + availability_url = ( + f"https://archive.org/wayback/available?url={quote(url, safe='')}" + ) + try: + response = _session(retries).get(availability_url, timeout=(20, 60)) + response.raise_for_status() + closest = response.json().get("archived_snapshots", {}).get("closest", {}) + snapshot_url = closest.get("url") if closest.get("available") else None + if snapshot_url: + raw_url = re.sub(r"(/web/\d+)(?:[a-z_]+)?/", r"\1id_/", snapshot_url) + candidates.append(raw_url) + except (requests.RequestException, ValueError, AttributeError): + pass + return candidates + + +def _wikimedia_original(url: str) -> str | None: + """Return the underlying MediaWiki file URL for a thumbnail URL.""" + parsed = urlparse(url) + marker = "/wikipedia/commons/thumb/" + if parsed.netloc.lower() != "upload.wikimedia.org" or marker not in parsed.path: + return None + prefix, remainder = parsed.path.split(marker, 1) + parts = remainder.split("/") + if len(parts) < 4: + return None + original_path = "/".join(parts[:-1]) + return parsed._replace( + path=f"{prefix}/wikipedia/commons/{original_path}", query="", fragment="" + ).geturl() + + +def _recovery_candidates(source_url: str, retries: int) -> list[tuple[str, str, bool]]: + direct_variants = [source_url] + unescaped = html.unescape(source_url) + if unescaped != source_url: + direct_variants.append(unescaped) + + candidates: list[tuple[str, str, bool]] = [] + for variant in direct_variants: + candidates.append( + ( + variant, + "direct" if variant == source_url else "html-unescape", + True, + ) + ) + parsed = urlparse(variant) + alternate_scheme = "https" if parsed.scheme == "http" else "http" + candidates.append( + ( + parsed._replace(scheme=alternate_scheme).geturl(), + f"scheme-{alternate_scheme}", + True, + ) + ) + if parsed.scheme == "https": + candidates.append((variant, "direct-tls-unverified", False)) + for variant in direct_variants: + candidates.extend( + (candidate, "wayback-exact-url", True) + for candidate in _wayback_candidates(variant, retries) + ) + wikimedia_original = _wikimedia_original(unescaped) + if wikimedia_original: + candidates.append((wikimedia_original, "wikimedia-original-file", True)) + return candidates + + +def _download_row( + index: int, + row: dict[str, Any], + image_dir: Path, + *, + retries: int, + connect_timeout: float, + read_timeout: float, + max_bytes: int, + archive_recovery: bool, +) -> AuditRow: + source_url = row["image_url"] + result = AuditRow( + index=index, + key=str(row["key"]), + source_url=source_url, + ok=False, + attempts=[], + ) + candidates = ( + _recovery_candidates(source_url, retries) + if archive_recovery + else [(source_url, "direct", True)] + ) + + errors: list[str] = [] + attempted_candidates: set[tuple[str, bool]] = set() + for candidate, recovery_method, verify_tls in candidates: + candidate_key = (candidate, verify_tls) + if candidate_key in attempted_candidates: + continue + attempted_candidates.add(candidate_key) + attempt_label = candidate if verify_tls else f"{candidate} [TLS unverified]" + result.attempts.append(attempt_label) + try: + content, response = _fetch_bytes( + candidate, + retries=retries, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + max_bytes=max_bytes, + verify_tls=verify_tls, + ) + image_info = _validate_image(content) + extension = image_info.pop("extension") + path = image_dir / f"{_safe_stem(index, result.key)}{extension}" + path.write_bytes(content) + result.ok = True + result.resolved_url = response.url + result.status_code = response.status_code + result.redirect_chain = [ + {"status_code": item.status_code, "url": item.url} + for item in response.history + ] + result.recovery_method = recovery_method + result.local_path = str(path) + result.byte_sha256 = hashlib.sha256(content).hexdigest() + result.file_size = len(content) + result.width = image_info["width"] + result.height = image_info["height"] + result.image_format = image_info["format"] + result.mode = image_info["mode"] + result.pixel_sha256 = image_info["pixel_sha256"] + result.dhash = image_info["dhash"] + return result + except Exception as exc: # errors are preserved in the audit manifest + errors.append(f"{attempt_label}: {type(exc).__name__}: {exc}") + result.error = " | ".join(errors) + return result + + +def _duplicate_groups(values: list[str]) -> list[list[int]]: + groups: dict[str, list[int]] = defaultdict(list) + for index, value in enumerate(values): + groups[value].append(index) + return [indices for indices in groups.values() if len(indices) > 1] + + +def _near_dhash_pairs( + rows: list[AuditRow], max_distance: int = 2 +) -> list[dict[str, Any]]: + values = [(row.index, int(row.dhash, 16)) for row in rows if row.dhash] + pairs = [] + for offset, (left_index, left_hash) in enumerate(values): + for right_index, right_hash in values[offset + 1 :]: + distance = (left_hash ^ right_hash).bit_count() + if distance <= max_distance: + pairs.append( + { + "left_index": left_index, + "right_index": right_index, + "distance": distance, + } + ) + return pairs + + +def _write_json(path: Path, value: Any) -> None: + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def _write_jsonl(path: Path, values: list[dict[str, Any]]) -> None: + with path.open("w", encoding="utf-8") as handle: + for value in values: + handle.write(json.dumps(value, sort_keys=True) + "\n") + + +def _load_reusable_audit( + work_dir: Path, source_rows: list[dict[str, Any]] +) -> list[AuditRow | None]: + """Reuse validated successes so a recovery pass only touches failed URLs.""" + manifest_path = work_dir / "audit_rows.jsonl" + reusable: list[AuditRow | None] = [None] * len(source_rows) + if not manifest_path.exists(): + return reusable + manifest = [ + AuditRow(**json.loads(line)) + for line in manifest_path.read_text().splitlines() + if line.strip() + ] + for row in manifest: + if not row.ok or row.index >= len(source_rows) or row.local_path is None: + continue + source_row = source_rows[row.index] + if ( + row.key != str(source_row["key"]) + or row.source_url != source_row["image_url"] + ): + continue + path = Path(row.local_path) + if not path.is_file() or row.byte_sha256 is None: + continue + if hashlib.sha256(path.read_bytes()).hexdigest() != row.byte_sha256: + continue + reusable[row.index] = row + print(f"Reusing {sum(row is not None for row in reusable)} validated downloads") + return reusable + + +def _build_summary( + source_rows: list[dict[str, Any]], audit_rows: list[AuditRow] +) -> dict[str, Any]: + recovered = [row for row in audit_rows if row.ok] + widths = [row.width for row in recovered if row.width is not None] + heights = [row.height for row in recovered if row.height is not None] + areas = [width * height for width, height in zip(widths, heights)] + source_domains = Counter( + urlparse(row["image_url"]).netloc.lower().removeprefix("www.") + for row in source_rows + ) + byte_groups = _duplicate_groups([row.byte_sha256 or "" for row in recovered]) + pixel_groups = _duplicate_groups([row.pixel_sha256 or "" for row in recovered]) + return { + "source": SOURCE, + "source_revision": SOURCE_REVISION, + "expected_rows": EXPECTED_PAIRS, + "source_rows": len(source_rows), + "recovered_rows": len(recovered), + "missing_rows": len(audit_rows) - len(recovered), + "frozen_rows": len(recovered) + - sum(audit_rows[index].ok for index in EXCLUDED_PRACTICAL_DUPLICATE_INDICES), + "practical_duplicate_groups": [ + list(group) for group in PRACTICAL_DUPLICATE_GROUPS + ], + "excluded_practical_duplicate_indices": sorted( + EXCLUDED_PRACTICAL_DUPLICATE_INDICES + ), + "failed_indices": [row.index for row in audit_rows if not row.ok], + "source_url_duplicates": _duplicate_groups( + [row["image_url"] for row in source_rows] + ), + "key_duplicates": _duplicate_groups([str(row["key"]) for row in source_rows]), + "original_caption_duplicates": _duplicate_groups( + [_normalise_text(row["ori_caption"]) for row in source_rows] + ), + "refined_caption_duplicates": _duplicate_groups( + [_normalise_text(row["refined_caption"]) for row in source_rows] + ), + "byte_duplicate_groups": byte_groups, + "pixel_duplicate_groups": pixel_groups, + "near_dhash_pairs_distance_le_2": _near_dhash_pairs(recovered), + "redirected_rows": sum(bool(row.redirect_chain) for row in recovered), + "recovery_methods": dict(Counter(row.recovery_method for row in recovered)), + "formats": dict(Counter(row.image_format for row in recovered)), + "modes": dict(Counter(row.mode for row in recovered)), + "source_schemes": dict( + Counter(urlparse(row["image_url"]).scheme for row in source_rows) + ), + "source_domain_count": len(source_domains), + "source_domains": dict(source_domains.most_common()), + "total_image_bytes": sum(row.file_size or 0 for row in recovered), + "dimensions": { + "min_width": min(widths, default=None), + "max_width": max(widths, default=None), + "median_width": statistics.median(widths) if widths else None, + "min_height": min(heights, default=None), + "max_height": max(heights, default=None), + "median_height": statistics.median(heights) if heights else None, + "min_pixels": min(areas, default=None), + "max_pixels": max(areas, default=None), + "median_pixels": statistics.median(areas) if areas else None, + }, + } + + +def _make_dataset( + source_rows: list[dict[str, Any]], audit_rows: list[AuditRow] +) -> DatasetDict: + audit_by_index = {row.index: row for row in audit_rows if row.ok} + records = [] + for index, source_row in enumerate(source_rows): + if index in EXCLUDED_PRACTICAL_DUPLICATE_INDICES: + continue + audit = audit_by_index.get(index) + if audit is None: + continue + records.append( + { + "key": str(source_row["key"]), + "image": audit.local_path, + "ori_caption": source_row["ori_caption"], + "refined_caption": source_row["refined_caption"], + "source_url": source_row["image_url"], + "resolved_url": audit.resolved_url or "", + "source_domain": urlparse(source_row["image_url"]) + .netloc.lower() + .removeprefix("www."), + "recovery_method": audit.recovery_method, + "content_sha256": audit.byte_sha256, + "pixel_sha256": audit.pixel_sha256, + "width": audit.width, + "height": audit.height, + "image_format": audit.image_format, + "file_size": audit.file_size, + } + ) + dataset = Dataset.from_list(records).cast_column("image", Image()) + return DatasetDict({"test": dataset}) + + +def _card_appendix(summary: dict[str, Any]) -> str: + return f""" + +# Mars-VL-Pairs MTEB frozen media release + +This is a reproducibility-focused mirror of Task 1 from MarsRetrieval. It embeds +the image bytes used by the benchmark instead of downloading mutable web URLs +at MTEB evaluation time. The original row order, key, original caption, refined +caption, source URL, resolved URL, hashes, dimensions, and recovery method are +preserved. MTEB uses `refined_caption`, matching the paper's main evaluation. + +## Construction and integrity + +- Source: `{SOURCE}` at `{SOURCE_REVISION}` +- Expected source pairs: {summary["expected_rows"]} +- Recovered media rows: {summary["recovered_rows"]} +- Missing pairs: {summary["missing_rows"]} +- Evaluation pairs after duplicate review: {summary["frozen_rows"]} +- Excluded resize-equivalent rows: {summary["excluded_practical_duplicate_indices"]} +- Redirected downloads: {summary["redirected_rows"]} +- Exact byte duplicate groups: {len(summary["byte_duplicate_groups"])} +- Decoded-pixel duplicate groups: {len(summary["pixel_duplicate_groups"])} +- Source domains: {summary["source_domain_count"]} + +Both image-to-text and text-to-image tasks use this same frozen pair table. +Every row has exactly one paired refined caption and image. + +## License and provenance + +The source dataset declares CC-BY-4.0. The MarsRetrieval paper states that Task +1 candidates were selected from DataComp-1B and Relation-2B, and the retained +images resolve to many external web domains. The dataset-level license metadata +may not supersede rights or attribution requirements attached to each original +web image. The `source_url` and `source_domain` fields are retained so users can +inspect provenance and applicable source terms. + +## Citation + +```bibtex +@article{{wang2026marsretrieval, + title={{MarsRetrieval: Benchmarking Vision-Language Models for Planetary-Scale Geospatial Retrieval on Mars}}, + author={{Wang, Shuoyuan and Wang, Yiran and Wei, Hongxin}}, + journal={{arXiv preprint arXiv:2602.13961}}, + year={{2026}} +}} +``` +""" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-revision", default=SOURCE_REVISION) + parser.add_argument("--repo-id", default="Cerru02/Mars-VL-Pairs-MTEB") + parser.add_argument( + "--work-dir", type=Path, default=Path("/tmp/mars_vl_pairs_mteb") + ) + parser.add_argument("--workers", type=int, default=24) + parser.add_argument("--retries", type=int, default=3) + parser.add_argument("--connect-timeout", type=float, default=20) + parser.add_argument("--read-timeout", type=float, default=60) + parser.add_argument("--max-image-mb", type=int, default=128) + parser.add_argument("--archive-recovery", action="store_true") + parser.add_argument("--allow-missing", action="store_true") + parser.add_argument("--allow-duplicate-images", action="store_true") + parser.add_argument("--redownload-all", action="store_true") + parser.add_argument("--push", action="store_true") + args = parser.parse_args() + + if args.source_revision != SOURCE_REVISION: + raise SystemExit( + f"Refusing unreviewed source revision {args.source_revision}; " + f"expected {SOURCE_REVISION}" + ) + + args.work_dir.mkdir(parents=True, exist_ok=True) + image_dir = args.work_dir / "images" + image_dir.mkdir(parents=True, exist_ok=True) + + source = load_dataset( + SOURCE, + revision=args.source_revision, + split="train", + ) + missing_columns = set(REQUIRED_COLUMNS) - set(source.column_names) + if missing_columns: + raise SystemExit(f"Missing source columns: {sorted(missing_columns)}") + if len(source) != EXPECTED_PAIRS: + raise SystemExit(f"Expected {EXPECTED_PAIRS} rows, found {len(source)}") + source_rows = source.select_columns(REQUIRED_COLUMNS).to_list() + for index, row in enumerate(source_rows): + for column in REQUIRED_COLUMNS: + if not isinstance(row[column], str) or not row[column].strip(): + raise SystemExit(f"Empty {column} at source row {index}") + + futures = {} + audit_rows = ( + [None] * len(source_rows) + if args.redownload_all + else _load_reusable_audit(args.work_dir, source_rows) + ) + with ThreadPoolExecutor(max_workers=args.workers) as executor: + for index, row in enumerate(source_rows): + if audit_rows[index] is not None: + continue + future = executor.submit( + _download_row, + index, + row, + image_dir, + retries=args.retries, + connect_timeout=args.connect_timeout, + read_timeout=args.read_timeout, + max_bytes=args.max_image_mb * 1024 * 1024, + archive_recovery=args.archive_recovery, + ) + futures[future] = index + with tqdm(total=len(futures), desc="download and validate") as progress: + for future in as_completed(futures): + index = futures[future] + audit_rows[index] = future.result() + progress.update(1) + + completed_rows = [row for row in audit_rows if row is not None] + if len(completed_rows) != len(source_rows): + raise RuntimeError("Audit did not return every source row") + summary = _build_summary(source_rows, completed_rows) + _write_json(args.work_dir / "audit_summary.json", summary) + _write_jsonl( + args.work_dir / "audit_rows.jsonl", + [asdict(row) for row in completed_rows], + ) + print(json.dumps(summary, indent=2, sort_keys=True)) + + if summary["missing_rows"] and not args.allow_missing: + raise SystemExit( + "Some images are missing. Review audit_rows.jsonl, then retry with " + "--archive-recovery or explicitly accept deterministic attrition with " + "--allow-missing." + ) + if ( + summary["byte_duplicate_groups"] or summary["pixel_duplicate_groups"] + ) and not args.allow_duplicate_images: + raise SystemExit( + "Duplicate image content would make one-positive qrels ambiguous. " + "Review audit_summary.json before using --allow-duplicate-images." + ) + if summary["refined_caption_duplicates"]: + raise SystemExit( + "Normalized refined-caption duplicates would make text queries ambiguous." + ) + + dataset = _make_dataset(source_rows, completed_rows) + export_dir = args.work_dir / "dataset" + if export_dir.exists(): + shutil.rmtree(export_dir) + dataset.save_to_disk(export_dir) + print(f"Saved {len(dataset['test'])} frozen pairs to {export_dir}") + + if not args.push: + return + token = get_token() + if not token: + raise SystemExit("No authenticated Hugging Face token found") + api = HfApi(token=token) + identity = api.whoami()["name"] + namespace = args.repo_id.split("/", 1)[0] + if identity.casefold() != namespace.casefold(): + raise SystemExit( + f"Authenticated as {identity}, refusing to push to namespace {namespace}" + ) + create_repo(args.repo_id, repo_type="dataset", token=token, exist_ok=True) + dataset.push_to_hub(args.repo_id, token=token, max_shard_size="1GB") + card = DatasetCard.load(args.repo_id, token=token) + DatasetCard(str(card).rstrip() + _card_appendix(summary)).push_to_hub( + args.repo_id, token=token + ) + revision = api.dataset_info(args.repo_id).sha + print(f"Pushed https://huggingface.co/datasets/{args.repo_id}") + print(f"Immutable revision: {revision}") + + +if __name__ == "__main__": + main() From 4e7909fab9cafb9ef0bbff1eb65672ed9a00e44d Mon Sep 17 00:00:00 2001 From: tommasocerruti Date: Thu, 13 Aug 2026 00:58:47 +0200 Subject: [PATCH 2/9] fix: address Mars-VL-Pairs review feedback --- mteb/tasks/retrieval/eng/mars_vl_pairs.py | 44 +++++++++++------- tests/test_tasks/test_mars_vl_pairs.py | 54 +++++++++++++++++++++++ 2 files changed, 83 insertions(+), 15 deletions(-) create mode 100644 tests/test_tasks/test_mars_vl_pairs.py diff --git a/mteb/tasks/retrieval/eng/mars_vl_pairs.py b/mteb/tasks/retrieval/eng/mars_vl_pairs.py index 2e7fffb4b4..50e3185ab0 100644 --- a/mteb/tasks/retrieval/eng/mars_vl_pairs.py +++ b/mteb/tasks/retrieval/eng/mars_vl_pairs.py @@ -1,13 +1,19 @@ from __future__ import annotations -from typing import Literal +from statistics import fmean +from typing import TYPE_CHECKING, Literal from datasets import load_dataset +from typing_extensions import override +from mteb._evaluators.retrieval_metrics import mrr from mteb.abstasks.retrieval import AbsTaskRetrieval from mteb.abstasks.retrieval_dataset_loaders import RetrievalSplitData from mteb.abstasks.task_metadata import TaskMetadata +if TYPE_CHECKING: + from mteb.types import RelevantDocumentsType + _DATASET_PATH = "Cerru02/Mars-VL-Pairs-MTEB" _DATASET_REVISION = "f0084ab0ba2f584b15dc72a82502b38ee490f58d" _FROZEN_PAIRS = 2_247 @@ -47,25 +53,21 @@ def _load_mars_vl_pairs( if len(pairs) != _FROZEN_PAIRS: raise ValueError(f"Expected {_FROZEN_PAIRS} frozen pairs, found {len(pairs)}") - keys = pairs["key"] - query_ids = [f"query-{key}" for key in keys] - corpus_ids = [f"corpus-{key}" for key in keys] + ids = [str(key) for key in pairs["key"]] text = ( pairs.select_columns(["refined_caption"]) .rename_column("refined_caption", "text") - .add_column("id", query_ids if direction == "t2i" else corpus_ids) + .add_column("id", ids) .select_columns(["id", "text"]) ) images = ( pairs.select_columns(["image"]) - .add_column("id", corpus_ids if direction == "t2i" else query_ids) + .add_column("id", ids) .select_columns(["id", "image"]) ) queries, corpus = (text, images) if direction == "t2i" else (images, text) - qrels = { - query_id: {corpus_id: 1} for query_id, corpus_id in zip(query_ids, corpus_ids) - } + qrels = {pair_id: {pair_id: 1} for pair_id in ids} task.dataset = { "default": { @@ -80,7 +82,23 @@ def _load_mars_vl_pairs( task.data_loaded = True -class MarsVLPairsT2IRetrieval(AbsTaskRetrieval): +class _MarsVLPairsRetrieval(AbsTaskRetrieval): + _top_k = _FROZEN_PAIRS + + @override + def task_specific_scores( + self, + scores: dict[str, dict[str, float]], + qrels: RelevantDocumentsType, + results: dict[str, dict[str, float]], + hf_split: str, + hf_subset: str, + ) -> dict[str, float]: + full_gallery_mrr = mrr(qrels, results, (_FROZEN_PAIRS,))[f"MRR@{_FROZEN_PAIRS}"] + return {f"mrr_at_{_FROZEN_PAIRS}": fmean(full_gallery_mrr)} + + +class MarsVLPairsT2IRetrieval(_MarsVLPairsRetrieval): metadata = TaskMetadata( name="MarsVLPairsT2IRetrieval", description=_DESCRIPTION @@ -107,14 +125,12 @@ class MarsVLPairsT2IRetrieval(AbsTaskRetrieval): }, is_beta=True, ) - k_values = (1, 3, 5, 10, 20, 100, 1000, _FROZEN_PAIRS) - _top_k = _FROZEN_PAIRS def load_data(self, num_proc: int | None = None, **kwargs) -> None: _load_mars_vl_pairs(self, "t2i", num_proc) -class MarsVLPairsI2TRetrieval(AbsTaskRetrieval): +class MarsVLPairsI2TRetrieval(_MarsVLPairsRetrieval): metadata = TaskMetadata( name="MarsVLPairsI2TRetrieval", description=_DESCRIPTION @@ -141,8 +157,6 @@ class MarsVLPairsI2TRetrieval(AbsTaskRetrieval): }, is_beta=True, ) - k_values = (1, 3, 5, 10, 20, 100, 1000, _FROZEN_PAIRS) - _top_k = _FROZEN_PAIRS def load_data(self, num_proc: int | None = None, **kwargs) -> None: _load_mars_vl_pairs(self, "i2t", num_proc) diff --git a/tests/test_tasks/test_mars_vl_pairs.py b/tests/test_tasks/test_mars_vl_pairs.py new file mode 100644 index 0000000000..79634d1d71 --- /dev/null +++ b/tests/test_tasks/test_mars_vl_pairs.py @@ -0,0 +1,54 @@ +from unittest.mock import patch + +import pytest +from datasets import Dataset + +from mteb.tasks.retrieval.eng.mars_vl_pairs import ( + MarsVLPairsI2TRetrieval, + MarsVLPairsT2IRetrieval, +) + + +@pytest.mark.parametrize("task_cls", [MarsVLPairsT2IRetrieval, MarsVLPairsI2TRetrieval]) +def test_mars_vl_pairs_uses_shared_pair_ids(task_cls): + pairs = Dataset.from_dict( + { + "key": ["first", "second"], + "refined_caption": ["First caption", "Second caption"], + "image": [None, None], + } + ) + task = task_cls() + + with ( + patch("mteb.tasks.retrieval.eng.mars_vl_pairs._FROZEN_PAIRS", 2), + patch( + "mteb.tasks.retrieval.eng.mars_vl_pairs.load_dataset", + return_value=pairs, + ), + ): + task.load_data() + + split = task.dataset["default"]["test"] + assert split["queries"]["id"] == ["first", "second"] + assert split["corpus"]["id"] == ["first", "second"] + assert split["relevant_docs"] == { + "first": {"first": 1}, + "second": {"second": 1}, + } + + +@pytest.mark.parametrize("task_cls", [MarsVLPairsT2IRetrieval, MarsVLPairsI2TRetrieval]) +def test_mars_vl_pairs_computes_full_gallery_mrr_separately(task_cls): + task = task_cls() + qrels = {"first": {"first": 1}, "second": {"second": 1}} + results = { + "first": {"first": 0.9, "second": 0.1}, + "second": {"first": 0.9, "second": 0.8}, + } + + scores = task.task_specific_scores({}, qrels, results, "test", "default") + + assert task.k_values == (1, 3, 5, 10, 20, 100, 1000) + assert task._top_k == 2_247 + assert scores == {"mrr_at_2247": pytest.approx(0.75)} From ab96ce4f656d35caddafe21ccee613af7522e462 Mon Sep 17 00:00:00 2001 From: tommasocerruti Date: Thu, 13 Aug 2026 01:29:09 +0200 Subject: [PATCH 3/9] fix: require TLS verification for Mars recovery --- scripts/data/mars_vl_pairs/README.md | 11 ++-- scripts/data/mars_vl_pairs/create_data.py | 63 +++++++++-------------- 2 files changed, 30 insertions(+), 44 deletions(-) diff --git a/scripts/data/mars_vl_pairs/README.md b/scripts/data/mars_vl_pairs/README.md index 01693d4442..3baff2d5c0 100644 --- a/scripts/data/mars_vl_pairs/README.md +++ b/scripts/data/mars_vl_pairs/README.md @@ -24,12 +24,11 @@ audit report. Both MTEB directions are derived from the same frozen pair table. --push ``` -If direct downloads fail, `--archive-recovery` also attempts scheme variants -and exact-URL Wayback snapshots, including captures stored with an incorrect -MIME type. It may retry the exact source URL without TLS verification after a -certificate failure; this is labeled in `recovery_method`. Recovered rows -retain the original URL and recovery method. The script stops on missing or -exact duplicate media by default; `--allow-missing` and +If direct downloads fail, `--archive-recovery` also upgrades HTTP URLs to HTTPS +and attempts exact-URL Wayback snapshots, including captures stored with an +incorrect MIME type. HTTPS certificate verification is always enabled. +Recovered rows retain the original URL and recovery method. The script stops +on missing or exact duplicate media by default; `--allow-missing` and `--allow-duplicate-images` require an explicit, documented decision. Manual review of every image pair with dHash distance at most two found two diff --git a/scripts/data/mars_vl_pairs/create_data.py b/scripts/data/mars_vl_pairs/create_data.py index 06735b8fc9..531dc7304a 100644 --- a/scripts/data/mars_vl_pairs/create_data.py +++ b/scripts/data/mars_vl_pairs/create_data.py @@ -16,7 +16,6 @@ import shutil import statistics import threading -import warnings from collections import Counter, defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import asdict, dataclass @@ -33,7 +32,6 @@ from requests.adapters import HTTPAdapter from tqdm import tqdm from urllib3.util.retry import Retry -from urllib3.exceptions import InsecureRequestWarning SOURCE = "SUSTech/Mars-VL-Pairs" SOURCE_REVISION = "1cac8885e481256d3752ad0d3a0f8f9681c5f206" @@ -158,20 +156,16 @@ def _fetch_bytes( connect_timeout: float, read_timeout: float, max_bytes: int, - verify_tls: bool = True, ) -> tuple[bytes, requests.Response]: parsed = urlparse(url) - with warnings.catch_warnings(): - if not verify_tls: - warnings.simplefilter("ignore", InsecureRequestWarning) - response = _session(retries).get( - url, - allow_redirects=True, - headers={"Referer": f"{parsed.scheme}://{parsed.netloc}/"}, - stream=True, - timeout=(connect_timeout, read_timeout), - verify=verify_tls, - ) + response = _session(retries).get( + url, + allow_redirects=True, + headers={"Referer": f"{parsed.scheme}://{parsed.netloc}/"}, + stream=True, + timeout=(connect_timeout, read_timeout), + verify=True, + ) response.raise_for_status() chunks: list[bytes] = [] size = 0 @@ -240,40 +234,36 @@ def _wikimedia_original(url: str) -> str | None: ).geturl() -def _recovery_candidates(source_url: str, retries: int) -> list[tuple[str, str, bool]]: +def _recovery_candidates(source_url: str, retries: int) -> list[tuple[str, str]]: direct_variants = [source_url] unescaped = html.unescape(source_url) if unescaped != source_url: direct_variants.append(unescaped) - candidates: list[tuple[str, str, bool]] = [] + candidates: list[tuple[str, str]] = [] for variant in direct_variants: candidates.append( ( variant, "direct" if variant == source_url else "html-unescape", - True, ) ) parsed = urlparse(variant) - alternate_scheme = "https" if parsed.scheme == "http" else "http" - candidates.append( - ( - parsed._replace(scheme=alternate_scheme).geturl(), - f"scheme-{alternate_scheme}", - True, + if parsed.scheme == "http": + candidates.append( + ( + parsed._replace(scheme="https").geturl(), + "scheme-https", + ) ) - ) - if parsed.scheme == "https": - candidates.append((variant, "direct-tls-unverified", False)) for variant in direct_variants: candidates.extend( - (candidate, "wayback-exact-url", True) + (candidate, "wayback-exact-url") for candidate in _wayback_candidates(variant, retries) ) wikimedia_original = _wikimedia_original(unescaped) if wikimedia_original: - candidates.append((wikimedia_original, "wikimedia-original-file", True)) + candidates.append((wikimedia_original, "wikimedia-original-file")) return candidates @@ -299,18 +289,16 @@ def _download_row( candidates = ( _recovery_candidates(source_url, retries) if archive_recovery - else [(source_url, "direct", True)] + else [(source_url, "direct")] ) errors: list[str] = [] - attempted_candidates: set[tuple[str, bool]] = set() - for candidate, recovery_method, verify_tls in candidates: - candidate_key = (candidate, verify_tls) - if candidate_key in attempted_candidates: + attempted_candidates: set[str] = set() + for candidate, recovery_method in candidates: + if candidate in attempted_candidates: continue - attempted_candidates.add(candidate_key) - attempt_label = candidate if verify_tls else f"{candidate} [TLS unverified]" - result.attempts.append(attempt_label) + attempted_candidates.add(candidate) + result.attempts.append(candidate) try: content, response = _fetch_bytes( candidate, @@ -318,7 +306,6 @@ def _download_row( connect_timeout=connect_timeout, read_timeout=read_timeout, max_bytes=max_bytes, - verify_tls=verify_tls, ) image_info = _validate_image(content) extension = image_info.pop("extension") @@ -343,7 +330,7 @@ def _download_row( result.dhash = image_info["dhash"] return result except Exception as exc: # errors are preserved in the audit manifest - errors.append(f"{attempt_label}: {type(exc).__name__}: {exc}") + errors.append(f"{candidate}: {type(exc).__name__}: {exc}") result.error = " | ".join(errors) return result From c429a1af30c9d25b46774d09c285b2bfb6ead16d Mon Sep 17 00:00:00 2001 From: tommasocerruti Date: Thu, 13 Aug 2026 11:33:33 +0200 Subject: [PATCH 4/9] fix: use standard metrics for full-gallery MRR --- mteb/tasks/retrieval/eng/mars_vl_pairs.py | 21 ++------------------- tests/test_tasks/test_mars_vl_pairs.py | 12 ++---------- 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/mteb/tasks/retrieval/eng/mars_vl_pairs.py b/mteb/tasks/retrieval/eng/mars_vl_pairs.py index 50e3185ab0..daa8f8cb53 100644 --- a/mteb/tasks/retrieval/eng/mars_vl_pairs.py +++ b/mteb/tasks/retrieval/eng/mars_vl_pairs.py @@ -1,19 +1,13 @@ from __future__ import annotations -from statistics import fmean -from typing import TYPE_CHECKING, Literal +from typing import Literal from datasets import load_dataset -from typing_extensions import override -from mteb._evaluators.retrieval_metrics import mrr from mteb.abstasks.retrieval import AbsTaskRetrieval from mteb.abstasks.retrieval_dataset_loaders import RetrievalSplitData from mteb.abstasks.task_metadata import TaskMetadata -if TYPE_CHECKING: - from mteb.types import RelevantDocumentsType - _DATASET_PATH = "Cerru02/Mars-VL-Pairs-MTEB" _DATASET_REVISION = "f0084ab0ba2f584b15dc72a82502b38ee490f58d" _FROZEN_PAIRS = 2_247 @@ -83,20 +77,9 @@ def _load_mars_vl_pairs( class _MarsVLPairsRetrieval(AbsTaskRetrieval): + k_values = (1, 3, 5, 10, 20, 100, 1000, _FROZEN_PAIRS) _top_k = _FROZEN_PAIRS - @override - def task_specific_scores( - self, - scores: dict[str, dict[str, float]], - qrels: RelevantDocumentsType, - results: dict[str, dict[str, float]], - hf_split: str, - hf_subset: str, - ) -> dict[str, float]: - full_gallery_mrr = mrr(qrels, results, (_FROZEN_PAIRS,))[f"MRR@{_FROZEN_PAIRS}"] - return {f"mrr_at_{_FROZEN_PAIRS}": fmean(full_gallery_mrr)} - class MarsVLPairsT2IRetrieval(_MarsVLPairsRetrieval): metadata = TaskMetadata( diff --git a/tests/test_tasks/test_mars_vl_pairs.py b/tests/test_tasks/test_mars_vl_pairs.py index 79634d1d71..20c5881ff0 100644 --- a/tests/test_tasks/test_mars_vl_pairs.py +++ b/tests/test_tasks/test_mars_vl_pairs.py @@ -39,16 +39,8 @@ def test_mars_vl_pairs_uses_shared_pair_ids(task_cls): @pytest.mark.parametrize("task_cls", [MarsVLPairsT2IRetrieval, MarsVLPairsI2TRetrieval]) -def test_mars_vl_pairs_computes_full_gallery_mrr_separately(task_cls): +def test_mars_vl_pairs_uses_full_gallery_metric_cutoff(task_cls): task = task_cls() - qrels = {"first": {"first": 1}, "second": {"second": 1}} - results = { - "first": {"first": 0.9, "second": 0.1}, - "second": {"first": 0.9, "second": 0.8}, - } - - scores = task.task_specific_scores({}, qrels, results, "test", "default") - assert task.k_values == (1, 3, 5, 10, 20, 100, 1000) + assert task.k_values == (1, 3, 5, 10, 20, 100, 1000, 2_247) assert task._top_k == 2_247 - assert scores == {"mrr_at_2247": pytest.approx(0.75)} From f761ff94718853e3d86cd7fed74bd1fffc14ecf0 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:56:06 +0300 Subject: [PATCH 5/9] fix top k calculation --- mteb/abstasks/retrieval.py | 5 ++++- mteb/tasks/retrieval/eng/mars_vl_pairs.py | 9 ++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/mteb/abstasks/retrieval.py b/mteb/abstasks/retrieval.py index 899a602c30..31d06ae1ba 100644 --- a/mteb/abstasks/retrieval.py +++ b/mteb/abstasks/retrieval.py @@ -100,13 +100,16 @@ class AbsTaskRetrieval(AbsTask): ignore_identical_ids: bool = False abstask_prompt = "Retrieve text based on user query." k_values: Sequence[int] = (1, 3, 5, 10, 20, 100, 1000) - _top_k: int = max(k_values) dataset: dict[str, dict[str, RetrievalSplitData]] _support_cross_encoder: bool = True _support_search: bool = True _previous_results_model_meta: dict[str, Any] | None = None skip_first_result: bool = False + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._top_k = max(self.k_values) + def convert_v1_dataset_format_to_v2( self, num_proc: int | None, diff --git a/mteb/tasks/retrieval/eng/mars_vl_pairs.py b/mteb/tasks/retrieval/eng/mars_vl_pairs.py index daa8f8cb53..0628f8d607 100644 --- a/mteb/tasks/retrieval/eng/mars_vl_pairs.py +++ b/mteb/tasks/retrieval/eng/mars_vl_pairs.py @@ -76,12 +76,9 @@ def _load_mars_vl_pairs( task.data_loaded = True -class _MarsVLPairsRetrieval(AbsTaskRetrieval): +class MarsVLPairsT2IRetrieval(AbsTaskRetrieval): k_values = (1, 3, 5, 10, 20, 100, 1000, _FROZEN_PAIRS) - _top_k = _FROZEN_PAIRS - -class MarsVLPairsT2IRetrieval(_MarsVLPairsRetrieval): metadata = TaskMetadata( name="MarsVLPairsT2IRetrieval", description=_DESCRIPTION @@ -113,7 +110,9 @@ def load_data(self, num_proc: int | None = None, **kwargs) -> None: _load_mars_vl_pairs(self, "t2i", num_proc) -class MarsVLPairsI2TRetrieval(_MarsVLPairsRetrieval): +class MarsVLPairsI2TRetrieval(AbsTaskRetrieval): + k_values = (1, 3, 5, 10, 20, 100, 1000, _FROZEN_PAIRS) + metadata = TaskMetadata( name="MarsVLPairsI2TRetrieval", description=_DESCRIPTION From c5040146131b662a02e3ada7affbe390087590e6 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:04:03 +0300 Subject: [PATCH 6/9] fix test --- mteb/mocks/mock_tasks/retrieval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mteb/mocks/mock_tasks/retrieval.py b/mteb/mocks/mock_tasks/retrieval.py index a4cd21dbd1..88ac78fe82 100644 --- a/mteb/mocks/mock_tasks/retrieval.py +++ b/mteb/mocks/mock_tasks/retrieval.py @@ -104,7 +104,7 @@ def instruction_retrieval_datasplit() -> RetrievalSplitData: class MockRetrievalTask(AbsTaskRetrieval): - _top_k = 2 + k_values = (1, 2) expected_stats = { "val": { "num_samples": 4, From 9cca1061dbccf7dd185d24b8f6d9ad96005db5fe Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:16:56 +0300 Subject: [PATCH 7/9] remove k values from mock --- mteb/mocks/mock_tasks/retrieval.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mteb/mocks/mock_tasks/retrieval.py b/mteb/mocks/mock_tasks/retrieval.py index 88ac78fe82..050240849a 100644 --- a/mteb/mocks/mock_tasks/retrieval.py +++ b/mteb/mocks/mock_tasks/retrieval.py @@ -104,7 +104,6 @@ def instruction_retrieval_datasplit() -> RetrievalSplitData: class MockRetrievalTask(AbsTaskRetrieval): - k_values = (1, 2) expected_stats = { "val": { "num_samples": 4, From f4a993891adeb5d80f2aac7816a864644d626b1c Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:33:31 +0300 Subject: [PATCH 8/9] fix index --- mteb/mocks/mock_tasks/retrieval.py | 1 - .../search_encoder_index/search_indexes/faiss_search_index.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mteb/mocks/mock_tasks/retrieval.py b/mteb/mocks/mock_tasks/retrieval.py index 050240849a..55b5512675 100644 --- a/mteb/mocks/mock_tasks/retrieval.py +++ b/mteb/mocks/mock_tasks/retrieval.py @@ -191,7 +191,6 @@ def load_data(self, num_proc: int | None = None, **kwargs) -> None: class MockRetrievalDialogTask(AbsTaskRetrieval): - _top_k = 1 expected_stats = { "val": { "num_samples": 4, diff --git a/mteb/models/search_encoder_index/search_indexes/faiss_search_index.py b/mteb/models/search_encoder_index/search_indexes/faiss_search_index.py index 346d202ce6..39d05022ca 100644 --- a/mteb/models/search_encoder_index/search_indexes/faiss_search_index.py +++ b/mteb/models/search_encoder_index/search_indexes/faiss_search_index.py @@ -114,7 +114,8 @@ def search( query_idx_to_id=query_idx_to_id, ) else: - sim_array, id_array = self.index.search(embeddings_f32, top_k) + capped_top_k = min(top_k, self.index.ntotal) + sim_array, id_array = self.index.search(embeddings_f32, capped_top_k) similarities = sim_array.tolist() ids = id_array.tolist() From 34f517be82d0e083963df391fb358c79857e0df7 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Thu, 13 Aug 2026 19:25:29 +0300 Subject: [PATCH 9/9] remove files --- scripts/data/mars_vl_pairs/README.md | 43 ------------------------ tests/test_tasks/test_mars_vl_pairs.py | 46 -------------------------- 2 files changed, 89 deletions(-) delete mode 100644 scripts/data/mars_vl_pairs/README.md delete mode 100644 tests/test_tasks/test_mars_vl_pairs.py diff --git a/scripts/data/mars_vl_pairs/README.md b/scripts/data/mars_vl_pairs/README.md deleted file mode 100644 index 3baff2d5c0..0000000000 --- a/scripts/data/mars_vl_pairs/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Mars-VL-Pairs construction - -This directory freezes Task 1 of -[MarsRetrieval](https://github.com/ml-stat-Sustech/MarsRetrieval), replacing its -mutable web image URLs with validated image bytes while preserving the original -one-to-one order and provenance. - -The builder pins `SUSTech/Mars-VL-Pairs`, downloads every image with retries, -validates it with Pillow, records redirects and failures, computes byte/pixel -hashes and dimensions, checks URL/caption/media duplicates, and writes a JSON -audit report. Both MTEB directions are derived from the same frozen pair table. - -```bash -/path/to/python scripts/data/mars_vl_pairs/create_data.py \ - --work-dir /tmp/mars_vl_pairs_mteb \ - --archive-recovery - -# Upload only after reviewing audit_summary.json and audit_rows.jsonl. -/path/to/python scripts/data/mars_vl_pairs/create_data.py \ - --work-dir /tmp/mars_vl_pairs_mteb \ - --archive-recovery \ - --allow-missing \ - --repo-id Cerru02/Mars-VL-Pairs-MTEB \ - --push -``` - -If direct downloads fail, `--archive-recovery` also upgrades HTTP URLs to HTTPS -and attempts exact-URL Wayback snapshots, including captures stored with an -incorrect MIME type. HTTPS certificate verification is always enabled. -Recovered rows retain the original URL and recovery method. The script stops -on missing or exact duplicate media by default; `--allow-missing` and -`--allow-duplicate-images` require an explicit, documented decision. - -Manual review of every image pair with dHash distance at most two found two -resize-equivalent groups: source rows 460/907 and 1446/2098. The builder -deterministically excludes the lower-resolution rows 907 and 2098 because their -different captions would otherwise create ambiguous one-positive qrels. The -other near-hash alerts are visibly different scenes. - -The source dataset declares CC-BY-4.0. Its images were selected from web-scale -corpora and come from many external domains, so the builder preserves the -source URL and does not treat the dataset-level declaration as a substitute for -the original source's rights information. diff --git a/tests/test_tasks/test_mars_vl_pairs.py b/tests/test_tasks/test_mars_vl_pairs.py deleted file mode 100644 index 20c5881ff0..0000000000 --- a/tests/test_tasks/test_mars_vl_pairs.py +++ /dev/null @@ -1,46 +0,0 @@ -from unittest.mock import patch - -import pytest -from datasets import Dataset - -from mteb.tasks.retrieval.eng.mars_vl_pairs import ( - MarsVLPairsI2TRetrieval, - MarsVLPairsT2IRetrieval, -) - - -@pytest.mark.parametrize("task_cls", [MarsVLPairsT2IRetrieval, MarsVLPairsI2TRetrieval]) -def test_mars_vl_pairs_uses_shared_pair_ids(task_cls): - pairs = Dataset.from_dict( - { - "key": ["first", "second"], - "refined_caption": ["First caption", "Second caption"], - "image": [None, None], - } - ) - task = task_cls() - - with ( - patch("mteb.tasks.retrieval.eng.mars_vl_pairs._FROZEN_PAIRS", 2), - patch( - "mteb.tasks.retrieval.eng.mars_vl_pairs.load_dataset", - return_value=pairs, - ), - ): - task.load_data() - - split = task.dataset["default"]["test"] - assert split["queries"]["id"] == ["first", "second"] - assert split["corpus"]["id"] == ["first", "second"] - assert split["relevant_docs"] == { - "first": {"first": 1}, - "second": {"second": 1}, - } - - -@pytest.mark.parametrize("task_cls", [MarsVLPairsT2IRetrieval, MarsVLPairsI2TRetrieval]) -def test_mars_vl_pairs_uses_full_gallery_metric_cutoff(task_cls): - task = task_cls() - - assert task.k_values == (1, 3, 5, 10, 20, 100, 1000, 2_247) - assert task._top_k == 2_247