fix: no longer using same query text for all of BLINKIT2TMultiChoice

mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py

@@ -321,7 +321,19 @@ def _evaluate_subset(
         )
         scores = {
             **{f"ndcg_at_{k.split('@')[1]}": v for (k, v) in ndcg.items()},
+            **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()},
+            **{f"recall_at_{k.split('@')[1]}": v for (k, v) in recall.items()},
+            **{f"cv_recall_at_{k.split('@')[1]}": v for (k, v) in cv_recall.items()},
+            **{f"precision_at_{k.split('@')[1]}": v for (k, v) in precision.items()},
             **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()},
+            **{
+                k.replace("@", "_at_").replace("_P", "_precision").lower(): v
+                for k, v in naucs.items()
+            },
+            **{
+                k.replace("@", "_at_").replace("_P", "_precision").lower(): v
+                for k, v in naucs_mrr.items()
+            },
             "accuracy": recall["Recall@1"],
         }
         self._add_main_score(scores)

mteb/tasks/Image/Any2AnyMultiChoice/__init__.py

@@ -3,3 +3,5 @@
 from .eng.BLINKIT2IMultiChoice import *
 from .eng.BLINKIT2TMultiChoice import *
 from .eng.ImageCoDeT2IMultiChoice import *
+from .eng.ROxfordI2IMultiChoice import *
+from .eng.RParisI2IMultiChoice import *

mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py

@@ -11,7 +11,7 @@ class BLINKIT2IMultiChoice(AbsTaskAny2AnyMultiChoice):
         reference="https://arxiv.org/abs/2404.12390",
         dataset={
             "path": "JamieSJS/blink-it2i-multi",
-            "revision": "b7b46b72d1ed1fa44d25e2b9c4726afab4a7ce53",
+            "revision": "a9f994925551c14503d00d86f1307bac6e2ead6a",
             "trust_remote_code": True,
         },
         type="Any2AnyMultiChoice",

mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py

@@ -11,7 +11,7 @@ class BLINKIT2TMultiChoice(AbsTaskAny2AnyMultiChoice):
         reference="https://arxiv.org/abs/2404.12390",
         dataset={
             "path": "JamieSJS/blink-it2t-multi",
-            "revision": "ae713b03ae68e343f16c3bcdbd1b1ee760975d55",
+            "revision": "bc8f4c7f62450a4ceb737c8339061cf87aea42d5",
         },
         type="Any2AnyMultiChoice",
         category="it2t",

mteb/tasks/Image/Any2AnyMultiChoice/eng/ROxfordI2IMultiChoice.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+from mteb.abstasks.Image.AbsTaskAny2AnyMultiChoice import AbsTaskAny2AnyMultiChoice
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+
+class ROxfordEasyI2IMultiChoice(AbsTaskAny2AnyMultiChoice):
+    metadata = TaskMetadata(
+        name="ROxfordEasyI2IMultiChoice",
+        description="Retrieve photos of landmarks in Oxford, UK.",
+        reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html",
+        dataset={
+            "path": "JamieSJS/r-oxford-easy-multi",
+            "revision": "4c167c3ce529f19457c9b8e694258cc6cf8e7cc7",
+        },
+        type="Any2AnyMultiChoice",
+        category="i2i",
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="accuracy",
+        date=("2009-01-01", "2010-04-01"),
+        domains=["Web"],
+        task_subtypes=["Object recognition"],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["image"],
+        sample_creation="created",
+        bibtex_citation="""@inproceedings{radenovic2018revisiting,
+  title={Revisiting oxford and paris: Large-scale image MultiChoice benchmarking},
+  author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej},
+  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
+  pages={5706--5715},
+  year={2018}
+}
+        """,
+        descriptive_stats={
+            "n_samples": {"test": 70},
+            "avg_character_length": {
+                "test": {
+                    "average_document_length": 0.0,
+                    "average_query_length": 0.0,
+                    "num_documents": 516,
+                    "num_queries": 70,
+                    "average_relevant_docs_per_query": 43.3,
+                }
+            },
+        },
+    )
+    skip_first_result = False
+
+
+class ROxfordMediumI2IMultiChoice(AbsTaskAny2AnyMultiChoice):
+    metadata = TaskMetadata(
+        name="ROxfordMediumI2IMultiChoice",
+        description="Retrieve photos of landmarks in Oxford, UK.",
+        reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html",
+        dataset={
+            "path": "JamieSJS/r-oxford-medium-multi",
+            "revision": "83bd440268e200a4f60313070618e3f45000fa94",
+        },
+        type="Any2AnyMultiChoice",
+        category="i2i",
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="accuracy",
+        date=("2009-01-01", "2010-04-01"),
+        domains=["Web"],
+        task_subtypes=["Object recognition"],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["image"],
+        sample_creation="created",
+        bibtex_citation="""@inproceedings{radenovic2018revisiting,
+  title={Revisiting oxford and paris: Large-scale image MultiChoice benchmarking},
+  author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej},
+  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
+  pages={5706--5715},
+  year={2018}
+}
+        """,
+        descriptive_stats={
+            "n_samples": {"test": 70},
+            "avg_character_length": {
+                "test": {
+                    "average_document_length": 0.0,
+                    "average_query_length": 0.0,
+                    "num_documents": 788,
+                    "num_queries": 70,
+                    "average_relevant_docs_per_query": 78.9,
+                }
+            },
+        },
+    )
+    skip_first_result = False
+
+
+class ROxfordHardI2IMultiChoice(AbsTaskAny2AnyMultiChoice):
+    metadata = TaskMetadata(
+        name="ROxfordHardI2IMultiChoice",
+        description="Retrieve photos of landmarks in Oxford, UK.",
+        reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html",
+        dataset={
+            "path": "JamieSJS/r-oxford-hard-multi",
+            "revision": "fc7c4ae6655b1e6b132f3b262a359acef42dfce8",
+        },
+        type="Any2AnyMultiChoice",
+        category="i2i",
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="accuracy",
+        date=("2009-01-01", "2010-04-01"),
+        domains=["Web"],
+        task_subtypes=["Object recognition"],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["image"],
+        sample_creation="created",
+        bibtex_citation="""@inproceedings{radenovic2018revisiting,
+  title={Revisiting oxford and paris: Large-scale image MultiChoice benchmarking},
+  author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej},
+  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
+  pages={5706--5715},
+  year={2018}
+}
+        """,
+        descriptive_stats={
+            "n_samples": {"test": 70},
+            "avg_character_length": {
+                "test": {
+                    "average_document_length": 0.0,
+                    "average_query_length": 0.0,
+                    "num_documents": 685,
+                    "num_queries": 70,
+                    "average_relevant_docs_per_query": 35.7,
+                }
+            },
+        },
+    )
+    skip_first_result = False

mteb/tasks/Image/Any2AnyMultiChoice/eng/RParisI2IMultiChoice.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+from mteb.abstasks.Image.AbsTaskAny2AnyMultiChoice import AbsTaskAny2AnyMultiChoice
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+
+class RParisEasyI2IMultiChoice(AbsTaskAny2AnyMultiChoice):
+    metadata = TaskMetadata(
+        name="RParisEasyI2IMultiChoice",
+        description="Retrieve photos of landmarks in Paris, UK.",
+        reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html",
+        dataset={
+            "path": "JamieSJS/r-paris-easy-multi",
+            "revision": "db94b5afd0014ab8c978f20a0fbcc52da1612a08",
+        },
+        type="Any2AnyMultiChoice",
+        category="i2i",
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="accuracy",
+        date=("2009-01-01", "2010-04-01"),
+        domains=["Web"],
+        task_subtypes=["Object recognition"],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["image"],
+        sample_creation="created",
+        bibtex_citation="""@inproceedings{radenovic2018revisiting,
+  title={Revisiting paris and paris: Large-scale image MultiChoice benchmarking},
+  author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej},
+  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
+  pages={5706--5715},
+  year={2018}
+}
+        """,
+        descriptive_stats={
+            "n_samples": {"test": 70},
+            "avg_character_length": {
+                "test": {
+                    "average_document_length": 0.0,
+                    "average_query_length": 0.0,
+                    "num_documents": 516,
+                    "num_queries": 70,
+                    "average_relevant_docs_per_query": 43.3,
+                }
+            },
+        },
+    )
+    skip_first_result = False
+
+
+class RParisMediumI2IMultiChoice(AbsTaskAny2AnyMultiChoice):
+    metadata = TaskMetadata(
+        name="RParisMediumI2IMultiChoice",
+        description="Retrieve photos of landmarks in Paris, UK.",
+        reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html",
+        dataset={
+            "path": "JamieSJS/r-paris-medium-multi",
+            "revision": "372c79fc823e1cebc1d55f8e0039aa239285e177",
+        },
+        type="Any2AnyMultiChoice",
+        category="i2i",
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="accuracy",
+        date=("2009-01-01", "2010-04-01"),
+        domains=["Web"],
+        task_subtypes=["Object recognition"],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["image"],
+        sample_creation="created",
+        bibtex_citation="""@inproceedings{radenovic2018revisiting,
+  title={Revisiting paris and paris: Large-scale image MultiChoice benchmarking},
+  author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej},
+  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
+  pages={5706--5715},
+  year={2018}
+}
+        """,
+        descriptive_stats={
+            "n_samples": {"test": 70},
+            "avg_character_length": {
+                "test": {
+                    "average_document_length": 0.0,
+                    "average_query_length": 0.0,
+                    "num_documents": 788,
+                    "num_queries": 70,
+                    "average_relevant_docs_per_query": 78.9,
+                }
+            },
+        },
+    )
+    skip_first_result = False
+
+
+class RParisHardI2IMultiChoice(AbsTaskAny2AnyMultiChoice):
+    metadata = TaskMetadata(
+        name="RParisHardI2IMultiChoice",
+        description="Retrieve photos of landmarks in Paris, UK.",
+        reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html",
+        dataset={
+            "path": "JamieSJS/r-paris-hard-multi",
+            "revision": "4e5997e48fb2f2f8bf1c8973851dedeb17e09a83",
+        },
+        type="Any2AnyMultiChoice",
+        category="i2i",
+        eval_splits=["test"],
+        eval_langs=["eng-Latn"],
+        main_score="accuracy",
+        date=("2009-01-01", "2010-04-01"),
+        domains=["Web"],
+        task_subtypes=["Object recognition"],
+        license="not specified",
+        annotations_creators="derived",
+        dialect=[],
+        modalities=["image"],
+        sample_creation="created",
+        bibtex_citation="""@inproceedings{radenovic2018revisiting,
+  title={Revisiting paris and paris: Large-scale image MultiChoice benchmarking},
+  author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej},
+  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
+  pages={5706--5715},
+  year={2018}
+}
+        """,
+        descriptive_stats={
+            "n_samples": {"test": 70},
+            "avg_character_length": {
+                "test": {
+                    "average_document_length": 0.0,
+                    "average_query_length": 0.0,
+                    "num_documents": 685,
+                    "num_queries": 70,
+                    "average_relevant_docs_per_query": 35.7,
+                }
+            },
+        },
+    )
+    skip_first_result = False

mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py

@@ -11,7 +11,7 @@ class BLINKIT2IRetrieval(AbsTaskAny2AnyRetrieval):
         reference="https://arxiv.org/abs/2404.12390",
         dataset={
             "path": "JamieSJS/blink-it2i",
-            "revision": "359b66f11c25d19bc8f7108d98e660a5857f3d26",
+            "revision": "7a1a1330565faca9c1aeec6f5acfc64f21296753",
             "trust_remote_code": True,
         },
         type="Any2AnyRetrieval",

mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py

@@ -11,7 +11,7 @@ class BLINKIT2TRetrieval(AbsTaskAny2AnyRetrieval):
         reference="https://arxiv.org/abs/2404.12390",
         dataset={
             "path": "JamieSJS/blink-it2t",
-            "revision": "302cf2008f204285985099dcd46425b00356c610",
+            "revision": "c6470936de49d6d2ae5fc09612752c75175ce5b6",
             "trust_remote_code": True,
         },
         type="Any2AnyRetrieval",