diff --git a/mteb/benchmarks/benchmarks.py b/mteb/benchmarks/benchmarks.py index 1924ff9ddf..270a0ac045 100644 --- a/mteb/benchmarks/benchmarks.py +++ b/mteb/benchmarks/benchmarks.py @@ -1963,3 +1963,39 @@ } """, ) + +VISUAL_DOCUMENT_RETRIEVAL = Benchmark( + name="VisualDocumentRetrieval", + display_name="VisualDocumentRetrieval", + icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-gui-picture.svg", + tasks=get_tasks( + tasks=[ + # v1 + "VidoreArxivQARetrieval", + "VidoreDocVQARetrieval", + "VidoreInfoVQARetrieval", + "VidoreTabfquadRetrieval", + "VidoreTatdqaRetrieval", + "VidoreShiftProjectRetrieval", + "VidoreSyntheticDocQAAIRetrieval", + "VidoreSyntheticDocQAEnergyRetrieval", + "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + # v2 + "Vidore2ESGReportsRetrieval", + "Vidore2EconomicsReportsRetrieval", + "Vidore2BioMedicalLecturesRetrieval", + "Vidore2ESGReportsHLRetrieval", + ], + ), + description="A benchmark for evaluating visual document retrieval, combining ViDoRe v1 and v2.", + reference="https://arxiv.org/abs/2407.01449", + citation=r""" +@article{mace2025vidorev2, + author = {Macé, Quentin and Loison António and Faysse, Manuel}, + journal = {arXiv preprint arXiv:2505.17166}, + title = {ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval}, + year = {2025}, +} +""", +) diff --git a/mteb/leaderboard/benchmark_selector.py b/mteb/leaderboard/benchmark_selector.py index 8ab9134244..22657df31d 100644 --- a/mteb/leaderboard/benchmark_selector.py +++ b/mteb/leaderboard/benchmark_selector.py @@ -31,6 +31,7 @@ class MenuEntry: "MIEB(eng)", "MIEB(lite)", "MIEB(Img)", + "VisualDocumentRetrieval", ] ), True,