diff --git a/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py b/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py index 4bef3f30b3..66a25c6619 100644 --- a/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py +++ b/mteb/abstasks/Image/AbsTaskAny2AnyMultiChoice.py @@ -321,7 +321,19 @@ def _evaluate_subset( ) scores = { **{f"ndcg_at_{k.split('@')[1]}": v for (k, v) in ndcg.items()}, + **{f"map_at_{k.split('@')[1]}": v for (k, v) in _map.items()}, + **{f"recall_at_{k.split('@')[1]}": v for (k, v) in recall.items()}, + **{f"cv_recall_at_{k.split('@')[1]}": v for (k, v) in cv_recall.items()}, + **{f"precision_at_{k.split('@')[1]}": v for (k, v) in precision.items()}, **{f"mrr_at_{k.split('@')[1]}": v for (k, v) in mrr.items()}, + **{ + k.replace("@", "_at_").replace("_P", "_precision").lower(): v + for k, v in naucs.items() + }, + **{ + k.replace("@", "_at_").replace("_P", "_precision").lower(): v + for k, v in naucs_mrr.items() + }, "accuracy": recall["Recall@1"], } self._add_main_score(scores) diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py b/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py index c818af7048..0e3b6d4505 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/__init__.py @@ -3,3 +3,5 @@ from .eng.BLINKIT2IMultiChoice import * from .eng.BLINKIT2TMultiChoice import * from .eng.ImageCoDeT2IMultiChoice import * +from .eng.ROxfordI2IMultiChoice import * +from .eng.RParisI2IMultiChoice import * diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py index 98b0a0120b..58db0c8c92 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2IMultiChoice.py @@ -11,7 +11,7 @@ class BLINKIT2IMultiChoice(AbsTaskAny2AnyMultiChoice): reference="https://arxiv.org/abs/2404.12390", dataset={ "path": "JamieSJS/blink-it2i-multi", - "revision": "b7b46b72d1ed1fa44d25e2b9c4726afab4a7ce53", + "revision": "a9f994925551c14503d00d86f1307bac6e2ead6a", "trust_remote_code": True, }, type="Any2AnyMultiChoice", diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py index 60f42b8b05..0a1dfcdc42 100644 --- a/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/BLINKIT2TMultiChoice.py @@ -11,7 +11,7 @@ class BLINKIT2TMultiChoice(AbsTaskAny2AnyMultiChoice): reference="https://arxiv.org/abs/2404.12390", dataset={ "path": "JamieSJS/blink-it2t-multi", - "revision": "ae713b03ae68e343f16c3bcdbd1b1ee760975d55", + "revision": "bc8f4c7f62450a4ceb737c8339061cf87aea42d5", }, type="Any2AnyMultiChoice", category="it2t", diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/ROxfordI2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ROxfordI2IMultiChoice.py new file mode 100644 index 0000000000..136848c128 --- /dev/null +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/ROxfordI2IMultiChoice.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyMultiChoice import AbsTaskAny2AnyMultiChoice +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class ROxfordEasyI2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="ROxfordEasyI2IMultiChoice", + description="Retrieve photos of landmarks in Oxford, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-oxford-easy-multi", + "revision": "4c167c3ce529f19457c9b8e694258cc6cf8e7cc7", + }, + type="Any2AnyMultiChoice", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image MultiChoice benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 516, + "num_queries": 70, + "average_relevant_docs_per_query": 43.3, + } + }, + }, + ) + skip_first_result = False + + +class ROxfordMediumI2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="ROxfordMediumI2IMultiChoice", + description="Retrieve photos of landmarks in Oxford, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-oxford-medium-multi", + "revision": "83bd440268e200a4f60313070618e3f45000fa94", + }, + type="Any2AnyMultiChoice", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image MultiChoice benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 788, + "num_queries": 70, + "average_relevant_docs_per_query": 78.9, + } + }, + }, + ) + skip_first_result = False + + +class ROxfordHardI2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="ROxfordHardI2IMultiChoice", + description="Retrieve photos of landmarks in Oxford, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-oxford-hard-multi", + "revision": "fc7c4ae6655b1e6b132f3b262a359acef42dfce8", + }, + type="Any2AnyMultiChoice", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image MultiChoice benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 685, + "num_queries": 70, + "average_relevant_docs_per_query": 35.7, + } + }, + }, + ) + skip_first_result = False diff --git a/mteb/tasks/Image/Any2AnyMultiChoice/eng/RParisI2IMultiChoice.py b/mteb/tasks/Image/Any2AnyMultiChoice/eng/RParisI2IMultiChoice.py new file mode 100644 index 0000000000..69da75118f --- /dev/null +++ b/mteb/tasks/Image/Any2AnyMultiChoice/eng/RParisI2IMultiChoice.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from mteb.abstasks.Image.AbsTaskAny2AnyMultiChoice import AbsTaskAny2AnyMultiChoice +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class RParisEasyI2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="RParisEasyI2IMultiChoice", + description="Retrieve photos of landmarks in Paris, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-paris-easy-multi", + "revision": "db94b5afd0014ab8c978f20a0fbcc52da1612a08", + }, + type="Any2AnyMultiChoice", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting paris and paris: Large-scale image MultiChoice benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 516, + "num_queries": 70, + "average_relevant_docs_per_query": 43.3, + } + }, + }, + ) + skip_first_result = False + + +class RParisMediumI2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="RParisMediumI2IMultiChoice", + description="Retrieve photos of landmarks in Paris, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-paris-medium-multi", + "revision": "372c79fc823e1cebc1d55f8e0039aa239285e177", + }, + type="Any2AnyMultiChoice", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting paris and paris: Large-scale image MultiChoice benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 788, + "num_queries": 70, + "average_relevant_docs_per_query": 78.9, + } + }, + }, + ) + skip_first_result = False + + +class RParisHardI2IMultiChoice(AbsTaskAny2AnyMultiChoice): + metadata = TaskMetadata( + name="RParisHardI2IMultiChoice", + description="Retrieve photos of landmarks in Paris, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-paris-hard-multi", + "revision": "4e5997e48fb2f2f8bf1c8973851dedeb17e09a83", + }, + type="Any2AnyMultiChoice", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting paris and paris: Large-scale image MultiChoice benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 685, + "num_queries": 70, + "average_relevant_docs_per_query": 35.7, + } + }, + }, + ) + skip_first_result = False diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py index 6b133f47c3..8202bb133c 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2IRetrieval.py @@ -11,7 +11,7 @@ class BLINKIT2IRetrieval(AbsTaskAny2AnyRetrieval): reference="https://arxiv.org/abs/2404.12390", dataset={ "path": "JamieSJS/blink-it2i", - "revision": "359b66f11c25d19bc8f7108d98e660a5857f3d26", + "revision": "7a1a1330565faca9c1aeec6f5acfc64f21296753", "trust_remote_code": True, }, type="Any2AnyRetrieval", diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py index da63f01df4..ff6ec42427 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py @@ -11,7 +11,7 @@ class BLINKIT2TRetrieval(AbsTaskAny2AnyRetrieval): reference="https://arxiv.org/abs/2404.12390", dataset={ "path": "JamieSJS/blink-it2t", - "revision": "302cf2008f204285985099dcd46425b00356c610", + "revision": "c6470936de49d6d2ae5fc09612752c75175ce5b6", "trust_remote_code": True, }, type="Any2AnyRetrieval", diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IMultiChoice.json new file mode 100644 index 0000000000..2b9fb1f38e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IMultiChoice.json @@ -0,0 +1,187 @@ +{ + "dataset_revision": "4c167c3ce529f19457c9b8e694258cc6cf8e7cc7", + "evaluation_time": 13.940337419509888, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.12011, + "cv_recall_at_1": 0.72857, + "cv_recall_at_10": 0.85714, + "cv_recall_at_100": 0.94286, + "cv_recall_at_1000": 0.97143, + "cv_recall_at_20": 0.87143, + "cv_recall_at_3": 0.78571, + "cv_recall_at_5": 0.84286, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.12011, + "map_at_1": 0.12011, + "map_at_10": 0.27377, + "map_at_100": 0.42059, + "map_at_1000": 0.48363, + "map_at_20": 0.32913, + "map_at_3": 0.1815, + "map_at_5": 0.21622, + "mrr_at_1": 0.7285714285714285, + "mrr_at_10": 0.7696598639455783, + "mrr_at_100": 0.7720894555929709, + "mrr_at_1000": 0.7721998445218422, + "mrr_at_20": 0.7705527210884354, + "mrr_at_3": 0.7547619047619047, + "mrr_at_5": 0.7676190476190478, + "nauc_cv_recall_at_1000_diff1": 0.24042950513538902, + "nauc_cv_recall_at_1000_max": -0.004668534080296818, + "nauc_cv_recall_at_1000_std": -0.5144724556489252, + "nauc_cv_recall_at_100_diff1": 0.22934173669467725, + "nauc_cv_recall_at_100_max": 0.3862044817927167, + "nauc_cv_recall_at_100_std": 0.06185807656395899, + "nauc_cv_recall_at_10_diff1": 0.46904761904761855, + "nauc_cv_recall_at_10_max": 0.45544217687074784, + "nauc_cv_recall_at_10_std": -0.11797862001943518, + "nauc_cv_recall_at_1_diff1": 0.37913486005089103, + "nauc_cv_recall_at_1_max": 0.6452758437491262, + "nauc_cv_recall_at_1_std": 0.13670553365209895, + "nauc_cv_recall_at_20_diff1": 0.4747302062186126, + "nauc_cv_recall_at_20_max": 0.38631264023934025, + "nauc_cv_recall_at_20_std": -0.1437653595469617, + "nauc_cv_recall_at_3_diff1": 0.2941673710904481, + "nauc_cv_recall_at_3_max": 0.5864074387151308, + "nauc_cv_recall_at_3_std": 0.08314454775993227, + "nauc_cv_recall_at_5_diff1": 0.48302290348453764, + "nauc_cv_recall_at_5_max": 0.4865876481597005, + "nauc_cv_recall_at_5_std": -0.012743962213706547, + "nauc_map_at_1000_diff1": 0.039056418726859325, + "nauc_map_at_1000_max": 0.3108646441695373, + "nauc_map_at_1000_std": 0.18387643315553479, + "nauc_map_at_100_diff1": 0.17039421636019683, + "nauc_map_at_100_max": 0.09905795512067436, + "nauc_map_at_100_std": -0.047296000166814706, + "nauc_map_at_10_diff1": 0.39318659870946504, + "nauc_map_at_10_max": -0.2567134684719764, + "nauc_map_at_10_std": -0.3372313247661854, + "nauc_map_at_1_diff1": 0.4052522690135478, + "nauc_map_at_1_max": -0.3133714030581606, + "nauc_map_at_1_std": -0.3279366241134995, + "nauc_map_at_20_diff1": 0.31128533550252463, + "nauc_map_at_20_max": -0.1668988523200872, + "nauc_map_at_20_std": -0.2950570027817496, + "nauc_map_at_3_diff1": 0.47350551731133683, + "nauc_map_at_3_max": -0.3190738829133001, + "nauc_map_at_3_std": -0.34746748342150613, + "nauc_map_at_5_diff1": 0.4297189568601358, + "nauc_map_at_5_max": -0.306734565163091, + "nauc_map_at_5_std": -0.3356128846263281, + "nauc_mrr_at_1000_diff1": 0.3765086703362948, + "nauc_mrr_at_1000_max": 0.6070793223986851, + "nauc_mrr_at_1000_std": 0.09762189861063993, + "nauc_mrr_at_100_diff1": 0.3765699744452178, + "nauc_mrr_at_100_max": 0.6072126617329957, + "nauc_mrr_at_100_std": 0.0978972735979146, + "nauc_mrr_at_10_diff1": 0.3788805046207156, + "nauc_mrr_at_10_max": 0.6082629986581378, + "nauc_mrr_at_10_std": 0.0963034659332654, + "nauc_mrr_at_1_diff1": 0.37913486005089103, + "nauc_mrr_at_1_max": 0.6452758437491262, + "nauc_mrr_at_1_std": 0.13670553365209895, + "nauc_mrr_at_20_diff1": 0.3787562524948725, + "nauc_mrr_at_20_max": 0.6062244307479707, + "nauc_mrr_at_20_std": 0.09613446360209521, + "nauc_mrr_at_3_diff1": 0.34401526617095446, + "nauc_mrr_at_3_max": 0.6246273302929221, + "nauc_mrr_at_3_std": 0.12242294784901903, + "nauc_mrr_at_5_diff1": 0.38107615384083393, + "nauc_mrr_at_5_max": 0.6101526396341715, + "nauc_mrr_at_5_std": 0.10525288237443857, + "nauc_ndcg_at_1000_diff1": 0.12762070579347834, + "nauc_ndcg_at_1000_max": 0.4065392036326618, + "nauc_ndcg_at_1000_std": 0.13600118977631057, + "nauc_ndcg_at_100_diff1": 0.08905540963562808, + "nauc_ndcg_at_100_max": 0.3340876494383974, + "nauc_ndcg_at_100_std": 0.16389116328814335, + "nauc_ndcg_at_10_diff1": 0.031817431716988705, + "nauc_ndcg_at_10_max": 0.48755851455947774, + "nauc_ndcg_at_10_std": 0.1880385361511959, + "nauc_ndcg_at_1_diff1": 0.37913486005089103, + "nauc_ndcg_at_1_max": 0.6452758437491262, + "nauc_ndcg_at_1_std": 0.13670553365209895, + "nauc_ndcg_at_20_diff1": 0.03212844503807327, + "nauc_ndcg_at_20_max": 0.40893860255098513, + "nauc_ndcg_at_20_std": 0.2191836350931408, + "nauc_ndcg_at_3_diff1": 0.10952929105103347, + "nauc_ndcg_at_3_max": 0.5868226539522446, + "nauc_ndcg_at_3_std": 0.20418399857463668, + "nauc_ndcg_at_5_diff1": 0.05132083453784389, + "nauc_ndcg_at_5_max": 0.5305326791254221, + "nauc_ndcg_at_5_std": 0.2280586632195748, + "nauc_precision_at_1000_diff1": -0.3414619911663539, + "nauc_precision_at_1000_max": 0.5560000366090037, + "nauc_precision_at_1000_std": 0.5508816734665237, + "nauc_precision_at_100_diff1": -0.3787961838502515, + "nauc_precision_at_100_max": 0.7160900928470757, + "nauc_precision_at_100_std": 0.6714675209553157, + "nauc_precision_at_10_diff1": -0.3066362781491847, + "nauc_precision_at_10_max": 0.6670893640902829, + "nauc_precision_at_10_std": 0.4078566027551021, + "nauc_precision_at_1_diff1": 0.37913486005089103, + "nauc_precision_at_1_max": 0.6452758437491262, + "nauc_precision_at_1_std": 0.13670553365209895, + "nauc_precision_at_20_diff1": -0.3569525663518488, + "nauc_precision_at_20_max": 0.6799005999378203, + "nauc_precision_at_20_std": 0.47997442491768877, + "nauc_precision_at_3_diff1": -0.022372098222603304, + "nauc_precision_at_3_max": 0.6144576517494506, + "nauc_precision_at_3_std": 0.2747400976264113, + "nauc_precision_at_5_diff1": -0.2176180661349585, + "nauc_precision_at_5_max": 0.6350255399229319, + "nauc_precision_at_5_std": 0.3850972309346714, + "nauc_recall_at_1000_diff1": 0.21587695470985352, + "nauc_recall_at_1000_max": 0.11425941926240299, + "nauc_recall_at_1000_std": 0.05546332909555691, + "nauc_recall_at_100_diff1": 0.21066338253500203, + "nauc_recall_at_100_max": -0.23122605362849036, + "nauc_recall_at_100_std": -0.17781012181323355, + "nauc_recall_at_10_diff1": 0.42259978636740364, + "nauc_recall_at_10_max": -0.34951875178863107, + "nauc_recall_at_10_std": -0.39276608756700837, + "nauc_recall_at_1_diff1": 0.4052522690135478, + "nauc_recall_at_1_max": -0.3133714030581606, + "nauc_recall_at_1_std": -0.3279366241134995, + "nauc_recall_at_20_diff1": 0.3429964665906505, + "nauc_recall_at_20_max": -0.32830343914659427, + "nauc_recall_at_20_std": -0.37518353692962664, + "nauc_recall_at_3_diff1": 0.4793068578557846, + "nauc_recall_at_3_max": -0.3280804979129421, + "nauc_recall_at_3_std": -0.3517807268312443, + "nauc_recall_at_5_diff1": 0.43717782790927456, + "nauc_recall_at_5_max": -0.36275023715800275, + "nauc_recall_at_5_std": -0.34949269884261674, + "ndcg_at_1": 0.72857, + "ndcg_at_10": 0.62501, + "ndcg_at_100": 0.61893, + "ndcg_at_1000": 0.69861, + "ndcg_at_20": 0.60364, + "ndcg_at_3": 0.64316, + "ndcg_at_5": 0.63774, + "precision_at_1": 0.72857, + "precision_at_10": 0.47286, + "precision_at_100": 0.19714, + "precision_at_1000": 0.0413, + "precision_at_20": 0.38857, + "precision_at_3": 0.57619, + "precision_at_5": 0.53429, + "recall_at_1": 0.12011, + "recall_at_10": 0.3365, + "recall_at_100": 0.67186, + "recall_at_1000": 0.915, + "recall_at_20": 0.43313, + "recall_at_3": 0.18464, + "recall_at_5": 0.24446 + } + ] + }, + "task_name": "ROxfordEasyI2IMultiChoice" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IMultiChoice.json new file mode 100644 index 0000000000..0c110c6ead --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IMultiChoice.json @@ -0,0 +1,187 @@ +{ + "dataset_revision": "fc7c4ae6655b1e6b132f3b262a359acef42dfce8", + "evaluation_time": 15.003882884979248, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.01148, + "cv_recall_at_1": 0.25714, + "cv_recall_at_10": 0.65714, + "cv_recall_at_100": 0.84286, + "cv_recall_at_1000": 0.92857, + "cv_recall_at_20": 0.68571, + "cv_recall_at_3": 0.45714, + "cv_recall_at_5": 0.57143, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.01148, + "map_at_1": 0.01148, + "map_at_10": 0.05335, + "map_at_100": 0.07243, + "map_at_1000": 0.08784, + "map_at_20": 0.0581, + "map_at_3": 0.03866, + "map_at_5": 0.04476, + "mrr_at_1": 0.2571428571428571, + "mrr_at_10": 0.38481859410430835, + "mrr_at_100": 0.39253963385655055, + "mrr_at_1000": 0.393048468885359, + "mrr_at_20": 0.38730777159348584, + "mrr_at_3": 0.3476190476190476, + "mrr_at_5": 0.3726190476190476, + "nauc_cv_recall_at_1000_diff1": 0.26685340802987884, + "nauc_cv_recall_at_1000_max": 0.07301587301587338, + "nauc_cv_recall_at_1000_std": -0.1830999066293196, + "nauc_cv_recall_at_100_diff1": 0.21040905445147495, + "nauc_cv_recall_at_100_max": 0.11148738971571162, + "nauc_cv_recall_at_100_std": 0.02419570448266679, + "nauc_cv_recall_at_10_diff1": 0.17949926917817913, + "nauc_cv_recall_at_10_max": 0.06129473336791022, + "nauc_cv_recall_at_10_std": 0.20811919468150294, + "nauc_cv_recall_at_1_diff1": 0.4553397252313951, + "nauc_cv_recall_at_1_max": -0.00898601253476105, + "nauc_cv_recall_at_1_std": 0.15209811978583004, + "nauc_cv_recall_at_20_diff1": 0.12448806313055567, + "nauc_cv_recall_at_20_max": 0.16716611727100145, + "nauc_cv_recall_at_20_std": 0.15480471481370445, + "nauc_cv_recall_at_3_diff1": 0.30826644662816005, + "nauc_cv_recall_at_3_max": -0.013483609631685515, + "nauc_cv_recall_at_3_std": 0.12409046583152049, + "nauc_cv_recall_at_5_diff1": 0.1614784394250518, + "nauc_cv_recall_at_5_max": -0.02459958932238193, + "nauc_cv_recall_at_5_std": 0.15406570841889153, + "nauc_map_at_1000_diff1": 0.31728785205951265, + "nauc_map_at_1000_max": -0.21424686509817786, + "nauc_map_at_1000_std": 0.0221906403447802, + "nauc_map_at_100_diff1": 0.31755174177151774, + "nauc_map_at_100_max": -0.19078874190305894, + "nauc_map_at_100_std": -0.023681508955016594, + "nauc_map_at_10_diff1": 0.2799136913282966, + "nauc_map_at_10_max": -0.1520026777231754, + "nauc_map_at_10_std": -0.07629896780635309, + "nauc_map_at_1_diff1": 0.4868843030130222, + "nauc_map_at_1_max": -0.15511847387665895, + "nauc_map_at_1_std": -0.02716611321363879, + "nauc_map_at_20_diff1": 0.2902764596993506, + "nauc_map_at_20_max": -0.12983705173414045, + "nauc_map_at_20_std": -0.04242912291533665, + "nauc_map_at_3_diff1": 0.2199130409925843, + "nauc_map_at_3_max": -0.1300575199918267, + "nauc_map_at_3_std": -0.12177697907382404, + "nauc_map_at_5_diff1": 0.2526425553708014, + "nauc_map_at_5_max": -0.1506194670949868, + "nauc_map_at_5_std": -0.11862027451443419, + "nauc_mrr_at_1000_diff1": 0.3597607815149087, + "nauc_mrr_at_1000_max": 0.008898477350336038, + "nauc_mrr_at_1000_std": 0.14259842374421733, + "nauc_mrr_at_100_diff1": 0.3595466281641209, + "nauc_mrr_at_100_max": 0.008904925581822543, + "nauc_mrr_at_100_std": 0.1429363715769275, + "nauc_mrr_at_10_diff1": 0.3577159847817333, + "nauc_mrr_at_10_max": 0.00585201676533247, + "nauc_mrr_at_10_std": 0.1493154319727965, + "nauc_mrr_at_1_diff1": 0.4553397252313951, + "nauc_mrr_at_1_max": -0.00898601253476105, + "nauc_mrr_at_1_std": 0.15209811978583004, + "nauc_mrr_at_20_diff1": 0.354629000714685, + "nauc_mrr_at_20_max": 0.013433224395754593, + "nauc_mrr_at_20_std": 0.14517705322833865, + "nauc_mrr_at_3_diff1": 0.38374642485486704, + "nauc_mrr_at_3_max": -0.007022664947542781, + "nauc_mrr_at_3_std": 0.13390513839165924, + "nauc_mrr_at_5_diff1": 0.3519534083953614, + "nauc_mrr_at_5_max": -0.004425335185700782, + "nauc_mrr_at_5_std": 0.14223014611578186, + "nauc_ndcg_at_1000_diff1": 0.3270873952168286, + "nauc_ndcg_at_1000_max": -0.23540909631297202, + "nauc_ndcg_at_1000_std": 0.056615205005261696, + "nauc_ndcg_at_100_diff1": 0.3574410299969898, + "nauc_ndcg_at_100_max": -0.2459841410933895, + "nauc_ndcg_at_100_std": -0.004941432018533064, + "nauc_ndcg_at_10_diff1": 0.27284914278868305, + "nauc_ndcg_at_10_max": -0.09860237927268649, + "nauc_ndcg_at_10_std": 0.17143394924414862, + "nauc_ndcg_at_1_diff1": 0.4553397252313951, + "nauc_ndcg_at_1_max": -0.00898601253476105, + "nauc_ndcg_at_1_std": 0.15209811978583004, + "nauc_ndcg_at_20_diff1": 0.24219624937954304, + "nauc_ndcg_at_20_max": -0.09686643301083671, + "nauc_ndcg_at_20_std": 0.1612277453319146, + "nauc_ndcg_at_3_diff1": 0.32690150790533234, + "nauc_ndcg_at_3_max": -0.047784701430556326, + "nauc_ndcg_at_3_std": 0.16012561788746316, + "nauc_ndcg_at_5_diff1": 0.31332748247111847, + "nauc_ndcg_at_5_max": -0.10699011157518731, + "nauc_ndcg_at_5_std": 0.15645669379446075, + "nauc_precision_at_1000_diff1": 0.0819429686636851, + "nauc_precision_at_1000_max": -0.1290271513175996, + "nauc_precision_at_1000_std": 0.2830467579249119, + "nauc_precision_at_100_diff1": 0.1086615495942521, + "nauc_precision_at_100_max": -0.22937147878224026, + "nauc_precision_at_100_std": 0.24534165503695654, + "nauc_precision_at_10_diff1": 0.23400746254303456, + "nauc_precision_at_10_max": -0.12882503413921087, + "nauc_precision_at_10_std": 0.24371550016348392, + "nauc_precision_at_1_diff1": 0.4553397252313951, + "nauc_precision_at_1_max": -0.00898601253476105, + "nauc_precision_at_1_std": 0.15209811978583004, + "nauc_precision_at_20_diff1": 0.12509716785790417, + "nauc_precision_at_20_max": -0.08597893607214858, + "nauc_precision_at_20_std": 0.3218130662392427, + "nauc_precision_at_3_diff1": 0.2960017153008923, + "nauc_precision_at_3_max": -0.05259235057482985, + "nauc_precision_at_3_std": 0.18596720508055797, + "nauc_precision_at_5_diff1": 0.2793466731531475, + "nauc_precision_at_5_max": -0.14856965124532454, + "nauc_precision_at_5_std": 0.18446594198853172, + "nauc_recall_at_1000_diff1": 0.23767465676853808, + "nauc_recall_at_1000_max": -0.21102175240994991, + "nauc_recall_at_1000_std": -0.1566324123319451, + "nauc_recall_at_100_diff1": 0.3128178199504678, + "nauc_recall_at_100_max": -0.2698286972760149, + "nauc_recall_at_100_std": -0.06876008993571361, + "nauc_recall_at_10_diff1": 0.19422816254098801, + "nauc_recall_at_10_max": -0.13751003581798485, + "nauc_recall_at_10_std": -0.07386552580291213, + "nauc_recall_at_1_diff1": 0.4868843030130222, + "nauc_recall_at_1_max": -0.15511847387665895, + "nauc_recall_at_1_std": -0.02716611321363879, + "nauc_recall_at_20_diff1": 0.19903944694380463, + "nauc_recall_at_20_max": -0.07262351965705542, + "nauc_recall_at_20_std": -0.04461273334236608, + "nauc_recall_at_3_diff1": 0.13345389226212073, + "nauc_recall_at_3_max": -0.1026168555200003, + "nauc_recall_at_3_std": -0.13519897599044758, + "nauc_recall_at_5_diff1": 0.16489673880783298, + "nauc_recall_at_5_max": -0.13503944945681817, + "nauc_recall_at_5_std": -0.13677876300364675, + "ndcg_at_1": 0.25714, + "ndcg_at_10": 0.19342, + "ndcg_at_100": 0.19552, + "ndcg_at_1000": 0.32441, + "ndcg_at_20": 0.17295, + "ndcg_at_3": 0.23726, + "ndcg_at_5": 0.21753, + "precision_at_1": 0.25714, + "precision_at_10": 0.15429, + "precision_at_100": 0.05786, + "precision_at_1000": 0.01964, + "precision_at_20": 0.10571, + "precision_at_3": 0.21905, + "precision_at_5": 0.19143, + "recall_at_1": 0.01148, + "recall_at_10": 0.10105, + "recall_at_100": 0.26388, + "recall_at_1000": 0.62162, + "recall_at_20": 0.11767, + "recall_at_3": 0.06125, + "recall_at_5": 0.07694 + } + ] + }, + "task_name": "ROxfordHardI2IMultiChoice" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IMultiChoice.json new file mode 100644 index 0000000000..2579f6ef68 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IMultiChoice.json @@ -0,0 +1,187 @@ +{ + "dataset_revision": "83bd440268e200a4f60313070618e3f45000fa94", + "evaluation_time": 14.132107257843018, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.04963, + "cv_recall_at_1": 0.77143, + "cv_recall_at_10": 0.87143, + "cv_recall_at_100": 0.94286, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.88571, + "cv_recall_at_3": 0.82857, + "cv_recall_at_5": 0.87143, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.04963, + "map_at_1": 0.04963, + "map_at_10": 0.15207, + "map_at_100": 0.2724, + "map_at_1000": 0.34306, + "map_at_20": 0.19547, + "map_at_3": 0.087, + "map_at_5": 0.11245, + "mrr_at_1": 0.7714285714285715, + "mrr_at_10": 0.81, + "mrr_at_100": 0.8126116140632085, + "mrr_at_1000": 0.8129819154176776, + "mrr_at_20": 0.8108928571428572, + "mrr_at_3": 0.8, + "mrr_at_5": 0.81, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.22934173669467725, + "nauc_cv_recall_at_100_max": 0.3862044817927167, + "nauc_cv_recall_at_100_std": 0.09862278244631179, + "nauc_cv_recall_at_10_diff1": 0.40127150336574263, + "nauc_cv_recall_at_10_max": 0.5091355914093374, + "nauc_cv_recall_at_10_std": -0.04840260711614463, + "nauc_cv_recall_at_1_diff1": 0.33182314761262127, + "nauc_cv_recall_at_1_max": 0.6395797185270868, + "nauc_cv_recall_at_1_std": 0.17595270226849136, + "nauc_cv_recall_at_20_diff1": 0.40009492168960564, + "nauc_cv_recall_at_20_max": 0.4383009017560512, + "nauc_cv_recall_at_20_std": -0.0693521594684388, + "nauc_cv_recall_at_3_diff1": 0.1894338380513495, + "nauc_cv_recall_at_3_max": 0.6075543120473995, + "nauc_cv_recall_at_3_std": 0.12532916392363366, + "nauc_cv_recall_at_5_diff1": 0.40127150336574263, + "nauc_cv_recall_at_5_max": 0.5091355914093374, + "nauc_cv_recall_at_5_std": -0.04840260711614463, + "nauc_map_at_1000_diff1": -0.13224573449005503, + "nauc_map_at_1000_max": 0.4227560714777927, + "nauc_map_at_1000_std": 0.31351087144984363, + "nauc_map_at_100_diff1": -0.02836497538559071, + "nauc_map_at_100_max": 0.19639808649260623, + "nauc_map_at_100_std": 0.04954005369447823, + "nauc_map_at_10_diff1": 0.25121232316392567, + "nauc_map_at_10_max": -0.15556227534649655, + "nauc_map_at_10_std": -0.2488378494247982, + "nauc_map_at_1_diff1": 0.272345967565727, + "nauc_map_at_1_max": -0.2643088054590121, + "nauc_map_at_1_std": -0.27101749212632714, + "nauc_map_at_20_diff1": 0.13842997076502075, + "nauc_map_at_20_max": -0.06672772445238266, + "nauc_map_at_20_std": -0.19809071582803736, + "nauc_map_at_3_diff1": 0.3224142774768474, + "nauc_map_at_3_max": -0.2326757410552881, + "nauc_map_at_3_std": -0.2675429832114887, + "nauc_map_at_5_diff1": 0.3167741526994269, + "nauc_map_at_5_max": -0.22646890969571978, + "nauc_map_at_5_std": -0.26391954191876693, + "nauc_mrr_at_1000_diff1": 0.3082925432291351, + "nauc_mrr_at_1000_max": 0.6087298059076731, + "nauc_mrr_at_1000_std": 0.128068145081543, + "nauc_mrr_at_100_diff1": 0.30825889635587544, + "nauc_mrr_at_100_max": 0.6080934109778549, + "nauc_mrr_at_100_std": 0.12852823805463925, + "nauc_mrr_at_10_diff1": 0.31165182357930454, + "nauc_mrr_at_10_max": 0.6097725826972008, + "nauc_mrr_at_10_std": 0.12443013146734498, + "nauc_mrr_at_1_diff1": 0.33182314761262127, + "nauc_mrr_at_1_max": 0.6395797185270868, + "nauc_mrr_at_1_std": 0.17595270226849136, + "nauc_mrr_at_20_diff1": 0.31120802031702144, + "nauc_mrr_at_20_max": 0.6073798478505396, + "nauc_mrr_at_20_std": 0.12435570379822256, + "nauc_mrr_at_3_diff1": 0.26938620863854473, + "nauc_mrr_at_3_max": 0.6255367517049751, + "nauc_mrr_at_3_std": 0.1537545556237145, + "nauc_mrr_at_5_diff1": 0.31165182357930454, + "nauc_mrr_at_5_max": 0.6097725826972008, + "nauc_mrr_at_5_std": 0.12443013146734498, + "nauc_ndcg_at_1000_diff1": 0.004789789538060326, + "nauc_ndcg_at_1000_max": 0.4304267036999359, + "nauc_ndcg_at_1000_std": 0.20516350588416282, + "nauc_ndcg_at_100_diff1": -0.0985051399330637, + "nauc_ndcg_at_100_max": 0.4047840878665377, + "nauc_ndcg_at_100_std": 0.3189929050444598, + "nauc_ndcg_at_10_diff1": -0.12943461724680122, + "nauc_ndcg_at_10_max": 0.5959856882638876, + "nauc_ndcg_at_10_std": 0.3322657775788708, + "nauc_ndcg_at_1_diff1": 0.33182314761262127, + "nauc_ndcg_at_1_max": 0.6395797185270868, + "nauc_ndcg_at_1_std": 0.17595270226849136, + "nauc_ndcg_at_20_diff1": -0.15227287405763856, + "nauc_ndcg_at_20_max": 0.577403391414749, + "nauc_ndcg_at_20_std": 0.36970889043336586, + "nauc_ndcg_at_3_diff1": 0.022411758177576204, + "nauc_ndcg_at_3_max": 0.575626107585026, + "nauc_ndcg_at_3_std": 0.24755287374915227, + "nauc_ndcg_at_5_diff1": -0.034454400308166065, + "nauc_ndcg_at_5_max": 0.5655036501426706, + "nauc_ndcg_at_5_std": 0.280337402240637, + "nauc_precision_at_1000_diff1": -0.1979455862208862, + "nauc_precision_at_1000_max": 0.49452629750592836, + "nauc_precision_at_1000_std": 0.5170286173407959, + "nauc_precision_at_100_diff1": -0.31426081091366387, + "nauc_precision_at_100_max": 0.6695443542749885, + "nauc_precision_at_100_std": 0.6261360571210748, + "nauc_precision_at_10_diff1": -0.2527118526563905, + "nauc_precision_at_10_max": 0.6263582372380011, + "nauc_precision_at_10_std": 0.3992628197721442, + "nauc_precision_at_1_diff1": 0.33182314761262127, + "nauc_precision_at_1_max": 0.6395797185270868, + "nauc_precision_at_1_std": 0.17595270226849136, + "nauc_precision_at_20_diff1": -0.31590964166066615, + "nauc_precision_at_20_max": 0.645482829549644, + "nauc_precision_at_20_std": 0.4668678815358941, + "nauc_precision_at_3_diff1": -0.05580043594090598, + "nauc_precision_at_3_max": 0.5759748123032211, + "nauc_precision_at_3_std": 0.2806167756518931, + "nauc_precision_at_5_diff1": -0.14313081953662518, + "nauc_precision_at_5_max": 0.5793499136946008, + "nauc_precision_at_5_std": 0.3373884715435296, + "nauc_recall_at_1000_diff1": 0.07995145169939738, + "nauc_recall_at_1000_max": 0.1842438166265837, + "nauc_recall_at_1000_std": 0.043545956333912506, + "nauc_recall_at_100_diff1": 0.03299050541578269, + "nauc_recall_at_100_max": -0.11358015050633144, + "nauc_recall_at_100_std": -0.03558993365235519, + "nauc_recall_at_10_diff1": 0.31258822262912317, + "nauc_recall_at_10_max": -0.21448543830771524, + "nauc_recall_at_10_std": -0.2702885296246692, + "nauc_recall_at_1_diff1": 0.272345967565727, + "nauc_recall_at_1_max": -0.2643088054590121, + "nauc_recall_at_1_std": -0.27101749212632714, + "nauc_recall_at_20_diff1": 0.20536080239790513, + "nauc_recall_at_20_max": -0.1828385547433498, + "nauc_recall_at_20_std": -0.23751709286812595, + "nauc_recall_at_3_diff1": 0.3339569204499953, + "nauc_recall_at_3_max": -0.23145861780156063, + "nauc_recall_at_3_std": -0.26721917195809763, + "nauc_recall_at_5_diff1": 0.3584675101320892, + "nauc_recall_at_5_max": -0.2639487514014815, + "nauc_recall_at_5_std": -0.2733945840450043, + "ndcg_at_1": 0.77143, + "ndcg_at_10": 0.60443, + "ndcg_at_100": 0.49539, + "ndcg_at_1000": 0.60683, + "ndcg_at_20": 0.54202, + "ndcg_at_3": 0.67149, + "ndcg_at_5": 0.65383, + "precision_at_1": 0.77143, + "precision_at_10": 0.53286, + "precision_at_100": 0.229, + "precision_at_1000": 0.05983, + "precision_at_20": 0.435, + "precision_at_3": 0.62857, + "precision_at_5": 0.6, + "recall_at_1": 0.04963, + "recall_at_10": 0.18302, + "recall_at_100": 0.4477, + "recall_at_1000": 0.76971, + "recall_at_20": 0.25156, + "recall_at_3": 0.09291, + "recall_at_5": 0.12898 + } + ] + }, + "task_name": "ROxfordMediumI2IMultiChoice" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IMultiChoice.json new file mode 100644 index 0000000000..f13cd60fba --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IMultiChoice.json @@ -0,0 +1,187 @@ +{ + "dataset_revision": "db94b5afd0014ab8c978f20a0fbcc52da1612a08", + "evaluation_time": 18.75471258163452, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.02814, + "cv_recall_at_1": 0.97143, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.02814, + "map_at_1": 0.02814, + "map_at_10": 0.14604, + "map_at_100": 0.613, + "map_at_1000": 0.71993, + "map_at_20": 0.24425, + "map_at_3": 0.06669, + "map_at_5": 0.09219, + "mrr_at_1": 0.9714285714285714, + "mrr_at_10": 0.9857142857142858, + "mrr_at_100": 0.9857142857142858, + "mrr_at_1000": 0.9857142857142858, + "mrr_at_20": 0.9857142857142858, + "mrr_at_3": 0.9857142857142858, + "mrr_at_5": 0.9857142857142858, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": 0.15289449112978365, + "nauc_cv_recall_at_1_max": 0.4960317460317484, + "nauc_cv_recall_at_1_std": 0.8692810457516342, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_map_at_1000_diff1": -0.5162392334753443, + "nauc_map_at_1000_max": 0.4031740557038615, + "nauc_map_at_1000_std": 0.41563960286590534, + "nauc_map_at_100_diff1": -0.23225006961960362, + "nauc_map_at_100_max": 0.3085695443992811, + "nauc_map_at_100_std": 0.12110491855395035, + "nauc_map_at_10_diff1": 0.5381554521493018, + "nauc_map_at_10_max": -0.10493092334419908, + "nauc_map_at_10_std": -0.2680755265314605, + "nauc_map_at_1_diff1": 0.8594916968198884, + "nauc_map_at_1_max": -0.24256237866427466, + "nauc_map_at_1_std": -0.4444380497243242, + "nauc_map_at_20_diff1": 0.3590188805543108, + "nauc_map_at_20_max": -0.013558846615686978, + "nauc_map_at_20_std": -0.16797583975226438, + "nauc_map_at_3_diff1": 0.7548580604028502, + "nauc_map_at_3_max": -0.2047628239453275, + "nauc_map_at_3_std": -0.4210073714397216, + "nauc_map_at_5_diff1": 0.6593773157320483, + "nauc_map_at_5_max": -0.18318576348363613, + "nauc_map_at_5_std": -0.3667442070438189, + "nauc_mrr_at_1000_diff1": 0.15289449112977452, + "nauc_mrr_at_1000_max": 0.49603174603173467, + "nauc_mrr_at_1000_std": 0.8692810457516342, + "nauc_mrr_at_100_diff1": 0.15289449112977452, + "nauc_mrr_at_100_max": 0.49603174603173467, + "nauc_mrr_at_100_std": 0.8692810457516342, + "nauc_mrr_at_10_diff1": 0.15289449112977452, + "nauc_mrr_at_10_max": 0.49603174603173467, + "nauc_mrr_at_10_std": 0.8692810457516342, + "nauc_mrr_at_1_diff1": 0.15289449112978365, + "nauc_mrr_at_1_max": 0.4960317460317484, + "nauc_mrr_at_1_std": 0.8692810457516342, + "nauc_mrr_at_20_diff1": 0.15289449112977452, + "nauc_mrr_at_20_max": 0.49603174603173467, + "nauc_mrr_at_20_std": 0.8692810457516342, + "nauc_mrr_at_3_diff1": 0.15289449112977452, + "nauc_mrr_at_3_max": 0.49603174603173467, + "nauc_mrr_at_3_std": 0.8692810457516342, + "nauc_mrr_at_5_diff1": 0.15289449112977452, + "nauc_mrr_at_5_max": 0.49603174603173467, + "nauc_mrr_at_5_std": 0.8692810457516342, + "nauc_ndcg_at_1000_diff1": -0.6430087042511464, + "nauc_ndcg_at_1000_max": 0.42655378925643106, + "nauc_ndcg_at_1000_std": 0.5044627676548901, + "nauc_ndcg_at_100_diff1": -0.5428592210544468, + "nauc_ndcg_at_100_max": 0.446739791744679, + "nauc_ndcg_at_100_std": 0.465672473136578, + "nauc_ndcg_at_10_diff1": -0.9208561880491332, + "nauc_ndcg_at_10_max": 0.5967384668371186, + "nauc_ndcg_at_10_std": 0.7022870489893225, + "nauc_ndcg_at_1_diff1": 0.15289449112978365, + "nauc_ndcg_at_1_max": 0.4960317460317484, + "nauc_ndcg_at_1_std": 0.8692810457516342, + "nauc_ndcg_at_20_diff1": -0.8251725302548746, + "nauc_ndcg_at_20_max": 0.5377716300435347, + "nauc_ndcg_at_20_std": 0.5566860440051336, + "nauc_ndcg_at_3_diff1": -0.9569729064943051, + "nauc_ndcg_at_3_max": 0.6190215986665834, + "nauc_ndcg_at_3_std": 0.9222091686879381, + "nauc_ndcg_at_5_diff1": -1.1226587238391097, + "nauc_ndcg_at_5_max": 0.5998868316343212, + "nauc_ndcg_at_5_std": 0.8858441072536013, + "nauc_precision_at_1000_diff1": -0.4777678980183267, + "nauc_precision_at_1000_max": 0.19411501983394858, + "nauc_precision_at_1000_std": 0.4317886018119784, + "nauc_precision_at_100_diff1": -0.6193565917597147, + "nauc_precision_at_100_max": 0.31610227934661556, + "nauc_precision_at_100_std": 0.41899007291320295, + "nauc_precision_at_10_diff1": -1.0942053142464947, + "nauc_precision_at_10_max": 0.57491910971664, + "nauc_precision_at_10_std": 0.7286596725169142, + "nauc_precision_at_1_diff1": 0.15289449112978365, + "nauc_precision_at_1_max": 0.4960317460317484, + "nauc_precision_at_1_std": 0.8692810457516342, + "nauc_precision_at_20_diff1": -1.006987444474548, + "nauc_precision_at_20_max": 0.5295046147956847, + "nauc_precision_at_20_std": 0.6468370068581858, + "nauc_precision_at_3_diff1": -1.188608776844072, + "nauc_precision_at_3_max": 0.6297333748314129, + "nauc_precision_at_3_std": 0.9273783587509071, + "nauc_precision_at_5_diff1": -1.304355295936774, + "nauc_precision_at_5_max": 0.5779142685336303, + "nauc_precision_at_5_std": 0.8799716519199388, + "nauc_recall_at_1000_diff1": -0.1671984561020124, + "nauc_recall_at_1000_max": 0.006203927434206601, + "nauc_recall_at_1000_std": 0.14154184625248917, + "nauc_recall_at_100_diff1": 0.07725532133003771, + "nauc_recall_at_100_max": 0.13911681142723956, + "nauc_recall_at_100_std": -0.13776713676284705, + "nauc_recall_at_10_diff1": 0.5586543206392892, + "nauc_recall_at_10_max": -0.1405893884569057, + "nauc_recall_at_10_std": -0.30419927459760987, + "nauc_recall_at_1_diff1": 0.8594916968198884, + "nauc_recall_at_1_max": -0.24256237866427466, + "nauc_recall_at_1_std": -0.4444380497243242, + "nauc_recall_at_20_diff1": 0.3955687201747704, + "nauc_recall_at_20_max": -0.084623109059799, + "nauc_recall_at_20_std": -0.24589251462208517, + "nauc_recall_at_3_diff1": 0.7237068204895556, + "nauc_recall_at_3_max": -0.21361841875189405, + "nauc_recall_at_3_std": -0.4269612949915014, + "nauc_recall_at_5_diff1": 0.6397655259145639, + "nauc_recall_at_5_max": -0.20024408587215523, + "nauc_recall_at_5_std": -0.38234727563640214, + "ndcg_at_1": 0.97143, + "ndcg_at_10": 0.92763, + "ndcg_at_100": 0.82979, + "ndcg_at_1000": 0.88963, + "ndcg_at_20": 0.90969, + "ndcg_at_3": 0.96255, + "ndcg_at_5": 0.95149, + "precision_at_1": 0.97143, + "precision_at_10": 0.89286, + "precision_at_100": 0.61286, + "precision_at_1000": 0.09256, + "precision_at_20": 0.85429, + "precision_at_3": 0.95714, + "precision_at_5": 0.93714, + "recall_at_1": 0.02814, + "recall_at_10": 0.15323, + "recall_at_100": 0.6921, + "recall_at_1000": 0.94803, + "recall_at_20": 0.26283, + "recall_at_3": 0.06802, + "recall_at_5": 0.09552 + } + ] + }, + "task_name": "RParisEasyI2IMultiChoice" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IMultiChoice.json new file mode 100644 index 0000000000..e6162419e8 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IMultiChoice.json @@ -0,0 +1,187 @@ +{ + "dataset_revision": "4e5997e48fb2f2f8bf1c8973851dedeb17e09a83", + "evaluation_time": 18.11166262626648, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.00907, + "cv_recall_at_1": 0.81429, + "cv_recall_at_10": 0.95714, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.97143, + "cv_recall_at_3": 0.9, + "cv_recall_at_5": 0.9, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.00907, + "map_at_1": 0.00907, + "map_at_10": 0.06456, + "map_at_100": 0.23858, + "map_at_1000": 0.33338, + "map_at_20": 0.10802, + "map_at_3": 0.0239, + "map_at_5": 0.03746, + "mrr_at_1": 0.8142857142857143, + "mrr_at_10": 0.8544047619047621, + "mrr_at_100": 0.8560692239858908, + "mrr_at_1000": 0.8560692239858908, + "mrr_at_20": 0.8555952380952383, + "mrr_at_3": 0.8476190476190477, + "mrr_at_5": 0.8476190476190477, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": 0.12651727357610024, + "nauc_cv_recall_at_10_max": 0.742452536570181, + "nauc_cv_recall_at_10_std": 0.5659819483348865, + "nauc_cv_recall_at_1_diff1": 0.07804020484598333, + "nauc_cv_recall_at_1_max": 0.60700909577314, + "nauc_cv_recall_at_1_std": 0.041771764885728854, + "nauc_cv_recall_at_20_diff1": -0.171335200746965, + "nauc_cv_recall_at_20_max": 0.6790382819794577, + "nauc_cv_recall_at_20_std": 0.3489729225023323, + "nauc_cv_recall_at_3_diff1": 0.12058156595971639, + "nauc_cv_recall_at_3_max": 0.6089102307589694, + "nauc_cv_recall_at_3_std": 0.36261171135120646, + "nauc_cv_recall_at_5_diff1": 0.12058156595971639, + "nauc_cv_recall_at_5_max": 0.6089102307589694, + "nauc_cv_recall_at_5_std": 0.36261171135120646, + "nauc_map_at_1000_diff1": -0.14730057025035195, + "nauc_map_at_1000_max": 0.35443918991459294, + "nauc_map_at_1000_std": 0.2592666345486196, + "nauc_map_at_100_diff1": -0.02007929644055961, + "nauc_map_at_100_max": 0.12631888830781082, + "nauc_map_at_100_std": -0.04309297943451873, + "nauc_map_at_10_diff1": 0.316050298469623, + "nauc_map_at_10_max": -0.13310958861916383, + "nauc_map_at_10_std": -0.3020901743993513, + "nauc_map_at_1_diff1": 0.5378248883416507, + "nauc_map_at_1_max": -0.13238109598299855, + "nauc_map_at_1_std": -0.2002514420418722, + "nauc_map_at_20_diff1": 0.21121038948324744, + "nauc_map_at_20_max": -0.06818178937655603, + "nauc_map_at_20_std": -0.2902145910447691, + "nauc_map_at_3_diff1": 0.45725171311121, + "nauc_map_at_3_max": -0.19287806549614692, + "nauc_map_at_3_std": -0.25735181540630836, + "nauc_map_at_5_diff1": 0.42067337504189933, + "nauc_map_at_5_max": -0.18761920119844047, + "nauc_map_at_5_std": -0.2766021627006926, + "nauc_mrr_at_1000_diff1": 0.11587681189811185, + "nauc_mrr_at_1000_max": 0.6097767662452036, + "nauc_mrr_at_1000_std": 0.08966689534414451, + "nauc_mrr_at_100_diff1": 0.11587681189811185, + "nauc_mrr_at_100_max": 0.6097767662452036, + "nauc_mrr_at_100_std": 0.08966689534414451, + "nauc_mrr_at_10_diff1": 0.12009817077420695, + "nauc_mrr_at_10_max": 0.6119594659009971, + "nauc_mrr_at_10_std": 0.09790880388517216, + "nauc_mrr_at_1_diff1": 0.07804020484598333, + "nauc_mrr_at_1_max": 0.60700909577314, + "nauc_mrr_at_1_std": 0.041771764885728854, + "nauc_mrr_at_20_diff1": 0.11488715765282757, + "nauc_mrr_at_20_max": 0.6097325059817074, + "nauc_mrr_at_20_std": 0.09010175972487022, + "nauc_mrr_at_3_diff1": 0.11806489062690863, + "nauc_mrr_at_3_max": 0.6070359281437123, + "nauc_mrr_at_3_std": 0.101521446902114, + "nauc_mrr_at_5_diff1": 0.11806489062690863, + "nauc_mrr_at_5_max": 0.6070359281437123, + "nauc_mrr_at_5_std": 0.101521446902114, + "nauc_ndcg_at_1000_diff1": 0.025969018525589978, + "nauc_ndcg_at_1000_max": 0.36217088530709435, + "nauc_ndcg_at_1000_std": 0.25309337496879, + "nauc_ndcg_at_100_diff1": -0.17968384395623355, + "nauc_ndcg_at_100_max": 0.4239707948301639, + "nauc_ndcg_at_100_std": 0.3667281666093518, + "nauc_ndcg_at_10_diff1": -0.2397422113161351, + "nauc_ndcg_at_10_max": 0.3812994195515857, + "nauc_ndcg_at_10_std": 0.04827273248313771, + "nauc_ndcg_at_1_diff1": 0.07804020484598333, + "nauc_ndcg_at_1_max": 0.60700909577314, + "nauc_ndcg_at_1_std": 0.041771764885728854, + "nauc_ndcg_at_20_diff1": -0.28132858513315473, + "nauc_ndcg_at_20_max": 0.4204685927011596, + "nauc_ndcg_at_20_std": 0.10721091043225721, + "nauc_ndcg_at_3_diff1": -0.09344111682157266, + "nauc_ndcg_at_3_max": 0.379692511012279, + "nauc_ndcg_at_3_std": 0.04006405355649845, + "nauc_ndcg_at_5_diff1": -0.12981305098158163, + "nauc_ndcg_at_5_max": 0.3878036317455262, + "nauc_ndcg_at_5_std": 0.028676296191065903, + "nauc_precision_at_1000_diff1": -0.42027963870829504, + "nauc_precision_at_1000_max": 0.5131562832922288, + "nauc_precision_at_1000_std": 0.6312618988329652, + "nauc_precision_at_100_diff1": -0.3568206317095141, + "nauc_precision_at_100_max": 0.5107076773140476, + "nauc_precision_at_100_std": 0.4725107993998001, + "nauc_precision_at_10_diff1": -0.2953003666954273, + "nauc_precision_at_10_max": 0.361267795513373, + "nauc_precision_at_10_std": 0.055085202761000866, + "nauc_precision_at_1_diff1": 0.07804020484598333, + "nauc_precision_at_1_max": 0.60700909577314, + "nauc_precision_at_1_std": 0.041771764885728854, + "nauc_precision_at_20_diff1": -0.3210375121807198, + "nauc_precision_at_20_max": 0.42343158585680635, + "nauc_precision_at_20_std": 0.13184322004366664, + "nauc_precision_at_3_diff1": -0.15055311012827152, + "nauc_precision_at_3_max": 0.3240710586307241, + "nauc_precision_at_3_std": 0.04530985723140268, + "nauc_precision_at_5_diff1": -0.1735214934469398, + "nauc_precision_at_5_max": 0.3602496967545344, + "nauc_precision_at_5_std": 0.026229993195467694, + "nauc_recall_at_1000_diff1": 0.29022911927159833, + "nauc_recall_at_1000_max": 0.2125654628634255, + "nauc_recall_at_1000_std": 0.28260953057103566, + "nauc_recall_at_100_diff1": 0.19480792831160149, + "nauc_recall_at_100_max": -0.003561859262334835, + "nauc_recall_at_100_std": -0.12640776822732094, + "nauc_recall_at_10_diff1": 0.35131981594744904, + "nauc_recall_at_10_max": -0.17286224306644377, + "nauc_recall_at_10_std": -0.3177220315856104, + "nauc_recall_at_1_diff1": 0.5378248883416507, + "nauc_recall_at_1_max": -0.13238109598299855, + "nauc_recall_at_1_std": -0.2002514420418722, + "nauc_recall_at_20_diff1": 0.276802747494477, + "nauc_recall_at_20_max": -0.13421078554955512, + "nauc_recall_at_20_std": -0.3214777307000227, + "nauc_recall_at_3_diff1": 0.4639750095588438, + "nauc_recall_at_3_max": -0.2163484225516375, + "nauc_recall_at_3_std": -0.2587991550499656, + "nauc_recall_at_5_diff1": 0.45052599994510506, + "nauc_recall_at_5_max": -0.20428456806687892, + "nauc_recall_at_5_std": -0.28400516448621627, + "ndcg_at_1": 0.81429, + "ndcg_at_10": 0.70636, + "ndcg_at_100": 0.50703, + "ndcg_at_1000": 0.62521, + "ndcg_at_20": 0.6597, + "ndcg_at_3": 0.76457, + "ndcg_at_5": 0.74394, + "precision_at_1": 0.81429, + "precision_at_10": 0.68143, + "precision_at_100": 0.39386, + "precision_at_1000": 0.10154, + "precision_at_20": 0.62643, + "precision_at_3": 0.75238, + "precision_at_5": 0.72857, + "recall_at_1": 0.00907, + "recall_at_10": 0.06983, + "recall_at_100": 0.33142, + "recall_at_1000": 0.70794, + "recall_at_20": 0.12346, + "recall_at_3": 0.02464, + "recall_at_5": 0.03943 + } + ] + }, + "task_name": "RParisHardI2IMultiChoice" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IMultiChoice.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IMultiChoice.json new file mode 100644 index 0000000000..229344b315 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IMultiChoice.json @@ -0,0 +1,187 @@ +{ + "dataset_revision": "372c79fc823e1cebc1d55f8e0039aa239285e177", + "evaluation_time": 17.42084789276123, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "accuracy": 0.00556, + "cv_recall_at_1": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.00556, + "map_at_1": 0.00556, + "map_at_10": 0.05258, + "map_at_100": 0.35085, + "map_at_1000": 0.57431, + "map_at_20": 0.1003, + "map_at_3": 0.01627, + "map_at_5": 0.02709, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": NaN, + "nauc_cv_recall_at_1_max": NaN, + "nauc_cv_recall_at_1_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_map_at_1000_diff1": -0.6053563345663608, + "nauc_map_at_1000_max": 0.1905977136179189, + "nauc_map_at_1000_std": 0.32921704776067334, + "nauc_map_at_100_diff1": -0.22879583528362765, + "nauc_map_at_100_max": -0.021084881609830797, + "nauc_map_at_100_std": 0.040852018876228476, + "nauc_map_at_10_diff1": 0.07357334898390998, + "nauc_map_at_10_max": -0.07446530454222411, + "nauc_map_at_10_std": 0.030841288385805625, + "nauc_map_at_1_diff1": 0.17749829551090057, + "nauc_map_at_1_max": -0.1265328999786591, + "nauc_map_at_1_std": -0.04029967299581773, + "nauc_map_at_20_diff1": 0.031681429904680244, + "nauc_map_at_20_max": -0.04579935466000076, + "nauc_map_at_20_std": 0.0600822959918182, + "nauc_map_at_3_diff1": 0.1040211006608559, + "nauc_map_at_3_max": -0.10556625686545555, + "nauc_map_at_3_std": 0.004385201213601456, + "nauc_map_at_5_diff1": 0.10349991026936102, + "nauc_map_at_5_max": -0.10412867049469088, + "nauc_map_at_5_std": 0.0066619256709396935, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": -0.48417134785087246, + "nauc_ndcg_at_1000_max": 0.18405070377983077, + "nauc_ndcg_at_1000_std": 0.31559100185273364, + "nauc_ndcg_at_100_diff1": -0.7053306583324787, + "nauc_ndcg_at_100_max": 0.3110977036980108, + "nauc_ndcg_at_100_std": 0.30984291735802094, + "nauc_ndcg_at_10_diff1": -1.0871954957472245, + "nauc_ndcg_at_10_max": 0.5259679929438033, + "nauc_ndcg_at_10_std": 0.6817427956237951, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": -0.8880065469302778, + "nauc_ndcg_at_20_max": 0.4873208891288291, + "nauc_ndcg_at_20_std": 0.561373434731217, + "nauc_ndcg_at_3_diff1": -1.4590964058433729, + "nauc_ndcg_at_3_max": 0.49010909822533216, + "nauc_ndcg_at_3_std": 0.911235002385481, + "nauc_ndcg_at_5_diff1": -1.4171604522672352, + "nauc_ndcg_at_5_max": 0.5204958630957071, + "nauc_ndcg_at_5_std": 0.9186533100767078, + "nauc_precision_at_1000_diff1": -0.38234481360274825, + "nauc_precision_at_1000_max": 0.24781406894305885, + "nauc_precision_at_1000_std": 0.26403818342266816, + "nauc_precision_at_100_diff1": -0.6128971684961878, + "nauc_precision_at_100_max": 0.31904847150594623, + "nauc_precision_at_100_std": 0.24419530291834549, + "nauc_precision_at_10_diff1": -1.0307736255745676, + "nauc_precision_at_10_max": 0.5403562988547558, + "nauc_precision_at_10_std": 0.650531072272575, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": -0.836691736727432, + "nauc_precision_at_20_max": 0.4856534423121902, + "nauc_precision_at_20_std": 0.5336271166968242, + "nauc_precision_at_3_diff1": -1.4785247432306272, + "nauc_precision_at_3_max": 0.5099595393713005, + "nauc_precision_at_3_std": 0.912854030501091, + "nauc_precision_at_5_diff1": -1.4262371615312837, + "nauc_precision_at_5_max": 0.5442577030812298, + "nauc_precision_at_5_std": 0.9215686274509778, + "nauc_recall_at_1000_diff1": -0.25506002472395295, + "nauc_recall_at_1000_max": 0.10322366296428723, + "nauc_recall_at_1000_std": 0.2469529706691035, + "nauc_recall_at_100_diff1": -0.13201603693344977, + "nauc_recall_at_100_max": -0.0624401543533192, + "nauc_recall_at_100_std": -0.018309376923327982, + "nauc_recall_at_10_diff1": 0.09463286260272151, + "nauc_recall_at_10_max": -0.08130077043564286, + "nauc_recall_at_10_std": 0.017648859246839073, + "nauc_recall_at_1_diff1": 0.17749829551090057, + "nauc_recall_at_1_max": -0.1265328999786591, + "nauc_recall_at_1_std": -0.04029967299581773, + "nauc_recall_at_20_diff1": 0.05416706502986838, + "nauc_recall_at_20_max": -0.05925356554363863, + "nauc_recall_at_20_std": 0.04363713977519275, + "nauc_recall_at_3_diff1": 0.11034320172647584, + "nauc_recall_at_3_max": -0.1059689684621208, + "nauc_recall_at_3_std": -0.0006734851708733446, + "nauc_recall_at_5_diff1": 0.11748996520995737, + "nauc_recall_at_5_max": -0.10649893198419375, + "nauc_recall_at_5_std": -0.0026859671591820122, + "ndcg_at_1": 1.0, + "ndcg_at_10": 0.95692, + "ndcg_at_100": 0.81534, + "ndcg_at_1000": 0.7841, + "ndcg_at_20": 0.93009, + "ndcg_at_3": 0.97638, + "ndcg_at_5": 0.975, + "precision_at_1": 1.0, + "precision_at_10": 0.94714, + "precision_at_100": 0.76243, + "precision_at_1000": 0.18964, + "precision_at_20": 0.91357, + "precision_at_3": 0.97143, + "precision_at_5": 0.97143, + "recall_at_1": 0.00556, + "recall_at_10": 0.05313, + "recall_at_100": 0.38141, + "recall_at_1000": 0.79881, + "recall_at_20": 0.10199, + "recall_at_3": 0.01632, + "recall_at_5": 0.02723 + } + ] + }, + "task_name": "RParisMediumI2IMultiChoice" +} \ No newline at end of file