dataset: Add JinaVDR

mteb/benchmarks/benchmarks.py

@@ -2151,3 +2151,66 @@
 """,
     contacts=["BaoLocPham"],
 )
+
+JINA_VDR = Benchmark(
+    name="JinaVDR",
+    display_name="Jina Visual Document Retrieval",
+    tasks=get_tasks(
+        tasks=[
+            "JinaVDRMedicalPrescriptionsRetrieval",
+            "JinaVDRStanfordSlideRetrieval",
+            "JinaVDRDonutVQAISynHMPRetrieval",
+            "JinaVDRTableVQARetrieval",
+            "JinaVDRChartQARetrieval",
+            "JinaVDRTQARetrieval",
+            "JinaVDROpenAINewsRetrieval",
+            "JinaVDREuropeanaDeNewsRetrieval",
+            "JinaVDREuropeanaEsNewsRetrieval",
+            "JinaVDREuropeanaItScansRetrieval",
+            "JinaVDREuropeanaNlLegalRetrieval",
+            "JinaVDRHindiGovVQARetrieval",
+            "JinaVDRAutomobileCatelogRetrieval",
+            "JinaVDRBeveragesCatalogueRetrieval",
+            "JinaVDRRamensBenchmarkRetrieval",
+            "JinaVDRJDocQARetrieval",
+            "JinaVDRHungarianDocQARetrieval",
+            "JinaVDRArabicChartQARetrieval",
+            "JinaVDRArabicInfographicsVQARetrieval",
+            "JinaVDROWIDChartsRetrieval",
+            "JinaVDRMPMQARetrieval",
+            "JinaVDRJina2024YearlyBookRetrieval",
+            "JinaVDRWikimediaCommonsMapsRetrieval",
+            "JinaVDRPlotQARetrieval",
+            "JinaVDRMMTabRetrieval",
+            "JinaVDRCharXivOCRRetrieval",
+            "JinaVDRStudentEnrollmentSyntheticRetrieval",
+            "JinaVDRGitHubReadmeRetrieval",
+            "JinaVDRTweetStockSyntheticsRetrieval",
+            "JinaVDRAirbnbSyntheticRetrieval",
+            "JinaVDRShanghaiMasterPlanRetrieval",
+            "JinaVDRWikimediaCommonsDocumentsRetrieval",
+            "JinaVDREuropeanaFrNewsRetrieval",
+            "JinaVDRDocQAHealthcareIndustryRetrieval",
+            "JinaVDRDocQAAI",
+            "JinaVDRShiftProjectRetrieval",
+            "JinaVDRTatQARetrieval",
+            "JinaVDRInfovqaRetrieval",
+            "JinaVDRDocVQARetrieval",
+            "JinaVDRDocQAGovReportRetrieval",
+            "JinaVDRTabFQuadRetrieval",
+            "JinaVDRDocQAEnergyRetrieval",
+            "JinaVDRArxivQARetrieval",
+        ],
+    ),
+    description="Multilingual, domain-diverse and layout-rich document retrieval benchmark.",
+    reference="https://arxiv.org/abs/2506.18902",
+    citation=r"""@misc{günther2025jinaembeddingsv4universalembeddingsmultimodal,
+  archiveprefix = {arXiv},
+  author = {Michael Günther and Saba Sturua and Mohammad Kalim Akram and Isabelle Mohr and Andrei Ungureanu and Bo Wang and Sedigheh Eslami and Scott Martens and Maximilian Werk and Nan Wang and Han Xiao},
+  eprint = {2506.18902},
+  primaryclass = {cs.AI},
+  title = {jina-embeddings-v4: Universal Embeddings for Multimodal Multilingual Retrieval},
+  url = {https://arxiv.org/abs/2506.18902},
+  year = {2025},
+}""",
+)

mteb/custom_validators.py

@@ -25,6 +25,7 @@
         "cc-by-4.0",
         "cc-by-sa-3.0",
         "cc-by-sa-4.0",
+        "cc-by-nc-3.0",
         "cc-by-nc-4.0",
         "cc-by-nc-sa-3.0",
         "cc-by-nc-sa-4.0",

mteb/leaderboard/benchmark_selector.py

@@ -36,6 +36,7 @@ class MenuEntry:
                         "MIEB(lite)",
                         "MIEB(Img)",
                         "VisualDocumentRetrieval",
+                        "JinaVDR",
                     ]
                 ),
             ),

mteb/models/colpali_models.py

@@ -131,7 +131,7 @@ def calculate_probs(self, text_embeddings, image_embeddings):
         return scores.softmax(dim=-1)
 
     def similarity(self, a, b):
-        return self.processor.score(a, b)
+        return self.processor.score(a, b, **self.processor_kwargs)
 
 
 class ColPaliWrapper(ColPaliEngineWrapper):

mteb/models/jina_models.py

@@ -341,9 +341,7 @@ def get_text_embeddings(
             return self.model.encode_text(
                 texts=texts,
                 batch_size=batch_size,
-                return_multivector=True
-                if task_type.startswith("DocumentUnderstanding")
-                else False,
+                return_multivector=self.vector_type == "multi_vector",
                 prompt_name=prompt_name_param,
                 task=base_task,
                 return_numpy=return_numpy,
@@ -389,7 +387,7 @@ def get_image_embeddings(
             images=all_images,
             batch_size=batch_size,
             max_pixels=max_pixels,
-            return_multivector=True,
+            return_multivector=self.vector_type == "multi_vector",
             task=base_task,
             return_numpy=return_numpy,
         )

mteb/tasks/Image/Any2AnyRetrieval/__init__.py

@@ -46,6 +46,7 @@
 from .eng.VQA2IT2TRetrieval import *
 from .eng.WebQAT2ITRetrieval import *
 from .eng.WebQAT2TRetrieval import *
+from .multilingual.JinaVDRBenchRetrieval import *
 from .multilingual.MIRACLVisionRetrieval import *
 from .multilingual.VdrMultilingualRetrieval import *
 from .multilingual.Vidore2BenchRetrieval import *