diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py index d0fb86ba90..36d7434575 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/BLINKIT2TRetrieval.py @@ -11,7 +11,7 @@ class BLINKIT2TRetrieval(AbsTaskAny2AnyRetrieval): reference="https://arxiv.org/abs/2404.12390", dataset={ "path": "JamieSJS/blink-it2t", - "revision": "4ab83c87ac5b24e3b730f86d585671493a3a423c", + "revision": "302cf2008f204285985099dcd46425b00356c610", "trust_remote_code": True, }, type="Any2AnyRetrieval", @@ -40,7 +40,7 @@ class BLINKIT2TRetrieval(AbsTaskAny2AnyRetrieval): "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 3080, + "num_documents": 26, "num_queries": 1073, "average_relevant_docs_per_query": 1, } diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py index dc43e34e29..7779883a84 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/ROxfordI2IRetrieval.py @@ -4,14 +4,14 @@ from mteb.abstasks.TaskMetadata import TaskMetadata -class ROxfordI2IRetrieval(AbsTaskAny2AnyRetrieval): +class ROxfordEasyI2IRetrieval(AbsTaskAny2AnyRetrieval): metadata = TaskMetadata( - name="ROxfordI2IRetrieval", + name="ROxfordEasyI2IRetrieval", description="Retrieve photos of landmarks in Oxford, UK.", reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", dataset={ - "path": "JamieSJS/r-oxford", - "revision": "d8daad98b4e4896a7f7fa1b3485a22420242d4fc", + "path": "JamieSJS/r-oxford-easy", + "revision": "3f018eb7ad32218a5a4ebd704493e0834a265cf5", }, type="Any2AnyRetrieval", category="i2i", @@ -35,16 +35,108 @@ class ROxfordI2IRetrieval(AbsTaskAny2AnyRetrieval): } """, descriptive_stats={ - "n_samples": {"test": 3555537}, + "n_samples": {"test": 70}, "avg_character_length": { "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 5063, - "num_queries": 5063, - "average_relevant_docs_per_query": 702, + "num_documents": 4993, + "num_queries": 70, + "average_relevant_docs_per_query": 43.3, } }, }, ) - skip_first_result = True + skip_first_result = False + + +class ROxfordMediumI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="ROxfordMediumI2IRetrieval", + description="Retrieve photos of landmarks in Oxford, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-oxford-medium", + "revision": "3bd28e9c45e15f299117c634799f7035c4de2d31", + }, + type="Any2AnyRetrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image retrieval benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 4993, + "num_queries": 70, + "average_relevant_docs_per_query": 78.9, + } + }, + }, + ) + skip_first_result = False + + +class ROxfordHardI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="ROxfordHardI2IRetrieval", + description="Retrieve photos of landmarks in Oxford, UK.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-oxford-hard", + "revision": "f20b30211b7ba3fc64a02bd83998fe75f3023719", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image retrieval benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 4993, + "num_queries": 70, + "average_relevant_docs_per_query": 35.7, + } + }, + }, + ) + skip_first_result = False diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py index 258ec836c6..ed49982fe5 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/RParisI2IRetrieval.py @@ -4,14 +4,14 @@ from mteb.abstasks.TaskMetadata import TaskMetadata -class RParisI2IRetrieval(AbsTaskAny2AnyRetrieval): +class RParisEasyI2IRetrieval(AbsTaskAny2AnyRetrieval): metadata = TaskMetadata( - name="RParisI2IRetrieval", - description="Retrieve photos of landmarks in Paris.", - reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html", + name="RParisEasyI2IRetrieval", + description="Retrieve photos of landmarks in Paris, France.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", dataset={ - "path": "JamieSJS/r-paris", - "revision": "bafc3a08fdffd72558021ce3a41250833d7e0e88", + "path": "JamieSJS/r-paris-easy", + "revision": "a7293da8a341de665ee4dcb2f209281df342d80b", }, type="Any2AnyRetrieval", category="i2i", @@ -35,16 +35,108 @@ class RParisI2IRetrieval(AbsTaskAny2AnyRetrieval): } """, descriptive_stats={ - "n_samples": {"test": 6392}, + "n_samples": {"test": 70}, "avg_character_length": { "test": { "average_document_length": 0.0, "average_query_length": 0.0, - "num_documents": 6392, - "num_queries": 6392, - "average_relevant_docs_per_query": 734, + "num_documents": 6322, + "num_queries": 70, + "average_relevant_docs_per_query": 98.2, } }, }, ) - skip_first_result = True + skip_first_result = False + + +class RParisMediumI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="RParisMediumI2IRetrieval", + description="Retrieve photos of landmarks in Paris, France.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-paris-medium", + "revision": "900267b49003a086979e8d52f6942624236bfc34", + }, + type="Any2AnyRetrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image retrieval benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 6322, + "num_queries": 70, + "average_relevant_docs_per_query": 147.9, + } + }, + }, + ) + skip_first_result = False + + +class RParisHardI2IRetrieval(AbsTaskAny2AnyRetrieval): + metadata = TaskMetadata( + name="RParisHardI2IRetrieval", + description="Retrieve photos of landmarks in Paris, France.", + reference="https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html", + dataset={ + "path": "JamieSJS/r-paris-hard", + "revision": "fd121b6592fe946616fa85116703b94a4c61fd63", + }, + type="Retrieval", + category="i2i", + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="cv_recall_at_1", + date=("2009-01-01", "2010-04-01"), + domains=["Web"], + task_subtypes=["Object recognition"], + license="Not specified", + annotations_creators="derived", + dialect=[], + modalities=["image"], + sample_creation="created", + bibtex_citation="""@inproceedings{radenovic2018revisiting, + title={Revisiting oxford and paris: Large-scale image retrieval benchmarking}, + author={Radenovi{\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\v{r}}ej}, + booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, + pages={5706--5715}, + year={2018} +} + """, + descriptive_stats={ + "n_samples": {"test": 70}, + "avg_character_length": { + "test": { + "average_document_length": 0.0, + "average_query_length": 0.0, + "num_documents": 6322, + "num_queries": 70, + "average_relevant_docs_per_query": 35.7, + } + }, + }, + ) + skip_first_result = False diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json index d701facb1e..21d3b7f0dc 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/BLINKIT2TRetrieval.json @@ -1,184 +1,184 @@ { - "dataset_revision": "4ab83c87ac5b24e3b730f86d585671493a3a423c", - "evaluation_time": 13.768992900848389, + "dataset_revision": "302cf2008f204285985099dcd46425b00356c610", + "evaluation_time": 11.171209335327148, "kg_co2_emissions": null, "mteb_version": "1.12.90", "scores": { "test": [ { - "cv_recall_at_1": 0.00093, - "cv_recall_at_10": 0.00373, - "cv_recall_at_100": 0.06151, - "cv_recall_at_1000": 0.49301, - "cv_recall_at_20": 0.01025, - "cv_recall_at_3": 0.00093, - "cv_recall_at_5": 0.00093, + "cv_recall_at_1": 0.08947, + "cv_recall_at_10": 0.64678, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.96645, + "cv_recall_at_3": 0.2479, + "cv_recall_at_5": 0.37651, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.00209, - "map_at_1": 0.00093, - "map_at_10": 0.00139, - "map_at_100": 0.00274, - "map_at_1000": 0.00397, - "map_at_20": 0.00176, - "map_at_3": 0.00093, - "map_at_5": 0.00093, - "mrr_at_1": 0.0009319664492078285, - "mrr_at_10": 0.0013147383837039009, - "mrr_at_100": 0.002685356046665323, - "mrr_at_1000": 0.003948307638249348, - "mrr_at_20": 0.0016977247814228705, - "mrr_at_3": 0.0009319664492078285, - "mrr_at_5": 0.0009319664492078285, - "nauc_cv_recall_at_1000_diff1": 0.21618180421784144, - "nauc_cv_recall_at_1000_max": 0.1966391139887544, - "nauc_cv_recall_at_1000_std": 0.6300616078101204, - "nauc_cv_recall_at_100_diff1": 0.17443594463110845, - "nauc_cv_recall_at_100_max": 0.3594175633623019, - "nauc_cv_recall_at_100_std": 0.5893797181186331, - "nauc_cv_recall_at_10_diff1": -0.05990075938028686, - "nauc_cv_recall_at_10_max": 0.43603863739514936, - "nauc_cv_recall_at_10_std": 0.6462545358013111, - "nauc_cv_recall_at_1_diff1": -0.49952309145315554, - "nauc_cv_recall_at_1_max": 0.6610449563695936, - "nauc_cv_recall_at_1_std": 1.0, - "nauc_cv_recall_at_20_diff1": 0.1175713728405099, - "nauc_cv_recall_at_20_max": 0.36443381136528874, - "nauc_cv_recall_at_20_std": 0.3549079762661633, - "nauc_cv_recall_at_3_diff1": -0.49952309145315554, - "nauc_cv_recall_at_3_max": 0.6610449563695936, - "nauc_cv_recall_at_3_std": 1.0, - "nauc_cv_recall_at_5_diff1": -0.49952309145315554, - "nauc_cv_recall_at_5_max": 0.6610449563695936, - "nauc_cv_recall_at_5_std": 1.0, - "nauc_map_at_1000_diff1": 0.3461462162176906, - "nauc_map_at_1000_max": 0.30890589543493097, - "nauc_map_at_1000_std": 0.707200194236762, - "nauc_map_at_100_diff1": 0.4005511014650457, - "nauc_map_at_100_max": 0.30811121696833377, - "nauc_map_at_100_std": 0.7144966661138995, - "nauc_map_at_10_diff1": 0.5208347947487291, - "nauc_map_at_10_max": 0.206028460069979, - "nauc_map_at_10_std": 0.8872521734282164, - "nauc_map_at_1_diff1": 0.6610449563695936, - "nauc_map_at_1_max": 0.09839818901557038, - "nauc_map_at_1_std": 1.0, - "nauc_map_at_20_diff1": 0.4542718670585475, - "nauc_map_at_20_max": 0.26815935842012756, - "nauc_map_at_20_std": 0.7972838331802248, - "nauc_map_at_3_diff1": 0.6610449563695936, - "nauc_map_at_3_max": 0.09839818901557038, - "nauc_map_at_3_std": 1.0, - "nauc_map_at_5_diff1": 0.6610449563695936, - "nauc_map_at_5_max": 0.09839818901557038, - "nauc_map_at_5_std": 1.0, - "nauc_mrr_at_1000_diff1": -0.013155580619472381, - "nauc_mrr_at_1000_max": 0.4252494452276833, - "nauc_mrr_at_1000_std": 0.687428882142861, - "nauc_mrr_at_100_diff1": -0.05688061033793547, - "nauc_mrr_at_100_max": 0.4528834485316897, - "nauc_mrr_at_100_std": 0.6942429482097452, - "nauc_mrr_at_10_diff1": -0.3361389575479778, - "nauc_mrr_at_10_max": 0.5699031562786795, - "nauc_mrr_at_10_std": 0.8567106980461006, - "nauc_mrr_at_1_diff1": -0.49952309145315554, - "nauc_mrr_at_1_max": 0.6610449563695936, - "nauc_mrr_at_1_std": 1.0, - "nauc_mrr_at_20_diff1": -0.2128688275475949, - "nauc_mrr_at_20_max": 0.5173348447205537, - "nauc_mrr_at_20_std": 0.6977606046374574, - "nauc_mrr_at_3_diff1": -0.49952309145315554, - "nauc_mrr_at_3_max": 0.6610449563695936, - "nauc_mrr_at_3_std": 1.0, - "nauc_mrr_at_5_diff1": -0.49952309145315554, - "nauc_mrr_at_5_max": 0.6610449563695936, - "nauc_mrr_at_5_std": 1.0, - "nauc_ndcg_at_1000_diff1": 0.2267671022987118, - "nauc_ndcg_at_1000_max": 0.2471557214473064, - "nauc_ndcg_at_1000_std": 0.6348682667422092, - "nauc_ndcg_at_100_diff1": 0.2912265250539601, - "nauc_ndcg_at_100_max": 0.3559110476318641, - "nauc_ndcg_at_100_std": 0.5901042407967306, - "nauc_ndcg_at_10_diff1": 0.42096720717204206, - "nauc_ndcg_at_10_max": 0.2838485324864617, - "nauc_ndcg_at_10_std": 0.7948899807288888, - "nauc_ndcg_at_1_diff1": 0.6610449563695936, - "nauc_ndcg_at_1_max": 0.09839818901557038, - "nauc_ndcg_at_1_std": 1.0, - "nauc_ndcg_at_20_diff1": 0.3330912830350423, - "nauc_ndcg_at_20_max": 0.36298197701132934, - "nauc_ndcg_at_20_std": 0.6588880474018856, - "nauc_ndcg_at_3_diff1": 0.6610449563695936, - "nauc_ndcg_at_3_max": 0.09839818901557038, - "nauc_ndcg_at_3_std": 1.0, - "nauc_ndcg_at_5_diff1": 0.6610449563695936, - "nauc_ndcg_at_5_max": 0.09839818901557038, - "nauc_ndcg_at_5_std": 1.0, - "nauc_precision_at_1000_diff1": 0.21618180421784106, - "nauc_precision_at_1000_max": 0.19663911398875364, - "nauc_precision_at_1000_std": 0.6300616078101199, - "nauc_precision_at_100_diff1": 0.2633008669431452, - "nauc_precision_at_100_max": 0.35813105058108047, - "nauc_precision_at_100_std": 0.5625569512563295, - "nauc_precision_at_10_diff1": 0.31227200390917714, - "nauc_precision_at_10_max": 0.3688258547365643, - "nauc_precision_at_10_std": 0.6910567239150112, - "nauc_precision_at_1_diff1": 0.6610449563695936, - "nauc_precision_at_1_max": 0.09839818901557038, - "nauc_precision_at_1_std": 1.0, - "nauc_precision_at_20_diff1": 0.24987842391547985, - "nauc_precision_at_20_max": 0.4262053224877857, - "nauc_precision_at_20_std": 0.5596936671568136, - "nauc_precision_at_3_diff1": 0.6610449563695937, - "nauc_precision_at_3_max": 0.09839818901557038, - "nauc_precision_at_3_std": 1.0, - "nauc_precision_at_5_diff1": 0.6610449563695937, - "nauc_precision_at_5_max": 0.09839818901557036, - "nauc_precision_at_5_std": 1.0, - "nauc_recall_at_1000_diff1": 0.21618180421784144, - "nauc_recall_at_1000_max": 0.1966391139887544, - "nauc_recall_at_1000_std": 0.6300616078101204, - "nauc_recall_at_100_diff1": 0.2633008669431453, - "nauc_recall_at_100_max": 0.3581310505810804, - "nauc_recall_at_100_std": 0.5625569512563297, - "nauc_recall_at_10_diff1": 0.3122720039091771, - "nauc_recall_at_10_max": 0.3688258547365644, - "nauc_recall_at_10_std": 0.691056723915011, - "nauc_recall_at_1_diff1": 0.6610449563695936, - "nauc_recall_at_1_max": 0.09839818901557038, - "nauc_recall_at_1_std": 1.0, - "nauc_recall_at_20_diff1": 0.24987842391547993, - "nauc_recall_at_20_max": 0.4262053224877856, - "nauc_recall_at_20_std": 0.5596936671568133, - "nauc_recall_at_3_diff1": 0.6610449563695936, - "nauc_recall_at_3_max": 0.09839818901557038, - "nauc_recall_at_3_std": 1.0, - "nauc_recall_at_5_diff1": 0.6610449563695936, - "nauc_recall_at_5_max": 0.09839818901557038, - "nauc_recall_at_5_std": 1.0, - "ndcg_at_1": 0.00093, - "ndcg_at_10": 0.00209, - "ndcg_at_100": 0.0128, - "ndcg_at_1000": 0.06231, - "ndcg_at_20": 0.00349, - "ndcg_at_3": 0.00093, - "ndcg_at_5": 0.00093, - "precision_at_1": 0.00093, - "precision_at_10": 0.00047, - "precision_at_100": 0.00065, - "precision_at_1000": 0.00049, - "precision_at_20": 0.00051, - "precision_at_3": 0.00031, - "precision_at_5": 0.00019, - "recall_at_1": 0.00093, - "recall_at_10": 0.00466, - "recall_at_100": 0.06524, - "recall_at_1000": 0.49301, - "recall_at_20": 0.01025, - "recall_at_3": 0.00093, - "recall_at_5": 0.00093 + "main_score": 0.32202, + "map_at_1": 0.08947, + "map_at_10": 0.22464, + "map_at_100": 0.24864, + "map_at_1000": 0.24864, + "map_at_20": 0.24715, + "map_at_3": 0.16061, + "map_at_5": 0.18987, + "mrr_at_1": 0.08946877912395154, + "mrr_at_10": 0.2246397875708215, + "mrr_at_100": 0.24864178823066596, + "mrr_at_1000": 0.24864178823066596, + "mrr_at_20": 0.24714520392298733, + "mrr_at_3": 0.16060888474681598, + "mrr_at_5": 0.18987263125194162, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": 0.6643245257587762, + "nauc_cv_recall_at_10_max": 0.3394378418139761, + "nauc_cv_recall_at_10_std": 0.5631230256620064, + "nauc_cv_recall_at_1_diff1": 0.3501554820571643, + "nauc_cv_recall_at_1_max": 0.3599782655731798, + "nauc_cv_recall_at_1_std": 0.7041481187202615, + "nauc_cv_recall_at_20_diff1": 0.5587451940825555, + "nauc_cv_recall_at_20_max": 0.47434883444027837, + "nauc_cv_recall_at_20_std": 0.6875358512675467, + "nauc_cv_recall_at_3_diff1": 0.5321534332531633, + "nauc_cv_recall_at_3_max": 0.3540044845277292, + "nauc_cv_recall_at_3_std": 0.7076694823976849, + "nauc_cv_recall_at_5_diff1": 0.6570524014466579, + "nauc_cv_recall_at_5_max": 0.3001299949877772, + "nauc_cv_recall_at_5_std": 0.6030407237879231, + "nauc_map_at_1000_diff1": 0.4915782753126093, + "nauc_map_at_1000_max": 0.3523024760071231, + "nauc_map_at_1000_std": 0.6868659279458145, + "nauc_map_at_100_diff1": 0.4915782753126093, + "nauc_map_at_100_max": 0.3523024760071231, + "nauc_map_at_100_std": 0.6868659279458145, + "nauc_map_at_10_diff1": 0.507291819494973, + "nauc_map_at_10_max": 0.34661422520261387, + "nauc_map_at_10_std": 0.6748191769380557, + "nauc_map_at_1_diff1": 0.3501554820571643, + "nauc_map_at_1_max": 0.3599782655731798, + "nauc_map_at_1_std": 0.7041481187202615, + "nauc_map_at_20_diff1": 0.49198266429986276, + "nauc_map_at_20_max": 0.35311180756517996, + "nauc_map_at_20_std": 0.6868445762814864, + "nauc_map_at_3_diff1": 0.4648021929978972, + "nauc_map_at_3_max": 0.3663517861281038, + "nauc_map_at_3_std": 0.7205919836574776, + "nauc_map_at_5_diff1": 0.5087022522085294, + "nauc_map_at_5_max": 0.3467565269949981, + "nauc_map_at_5_std": 0.683363887437365, + "nauc_mrr_at_1000_diff1": 0.4915782753126093, + "nauc_mrr_at_1000_max": 0.3523024760071231, + "nauc_mrr_at_1000_std": 0.6868659279458145, + "nauc_mrr_at_100_diff1": 0.4915782753126093, + "nauc_mrr_at_100_max": 0.3523024760071231, + "nauc_mrr_at_100_std": 0.6868659279458145, + "nauc_mrr_at_10_diff1": 0.507291819494973, + "nauc_mrr_at_10_max": 0.34661422520261387, + "nauc_mrr_at_10_std": 0.6748191769380557, + "nauc_mrr_at_1_diff1": 0.3501554820571643, + "nauc_mrr_at_1_max": 0.3599782655731798, + "nauc_mrr_at_1_std": 0.7041481187202615, + "nauc_mrr_at_20_diff1": 0.49198266429986276, + "nauc_mrr_at_20_max": 0.35311180756517996, + "nauc_mrr_at_20_std": 0.6868445762814864, + "nauc_mrr_at_3_diff1": 0.4648021929978972, + "nauc_mrr_at_3_max": 0.3663517861281038, + "nauc_mrr_at_3_std": 0.7205919836574776, + "nauc_mrr_at_5_diff1": 0.5087022522085294, + "nauc_mrr_at_5_max": 0.3467565269949981, + "nauc_mrr_at_5_std": 0.683363887437365, + "nauc_ndcg_at_1000_diff1": 0.5054976057073902, + "nauc_ndcg_at_1000_max": 0.3504516506472678, + "nauc_ndcg_at_1000_std": 0.6795326611960923, + "nauc_ndcg_at_100_diff1": 0.5054976057073902, + "nauc_ndcg_at_100_max": 0.3504516506472678, + "nauc_ndcg_at_100_std": 0.6795326611960923, + "nauc_ndcg_at_10_diff1": 0.5529277638560192, + "nauc_ndcg_at_10_max": 0.3418676893611852, + "nauc_ndcg_at_10_std": 0.6438431211574437, + "nauc_ndcg_at_1_diff1": 0.3501554820571643, + "nauc_ndcg_at_1_max": 0.3599782655731798, + "nauc_ndcg_at_1_std": 0.7041481187202615, + "nauc_ndcg_at_20_diff1": 0.5075033561523076, + "nauc_ndcg_at_20_max": 0.35530204332453247, + "nauc_ndcg_at_20_std": 0.6798053401234159, + "nauc_ndcg_at_3_diff1": 0.48649313476516426, + "nauc_ndcg_at_3_max": 0.36277467680108344, + "nauc_ndcg_at_3_std": 0.7169907526506151, + "nauc_ndcg_at_5_diff1": 0.5565020379741223, + "nauc_ndcg_at_5_max": 0.33223528863871, + "nauc_ndcg_at_5_std": 0.6584725779305518, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": 0.6643245257587758, + "nauc_precision_at_10_max": 0.3394378418139761, + "nauc_precision_at_10_std": 0.563123025662006, + "nauc_precision_at_1_diff1": 0.3501554820571643, + "nauc_precision_at_1_max": 0.3599782655731798, + "nauc_precision_at_1_std": 0.7041481187202615, + "nauc_precision_at_20_diff1": 0.5587451940825576, + "nauc_precision_at_20_max": 0.47434883444027603, + "nauc_precision_at_20_std": 0.6875358512675491, + "nauc_precision_at_3_diff1": 0.5321534332531633, + "nauc_precision_at_3_max": 0.3540044845277291, + "nauc_precision_at_3_std": 0.7076694823976849, + "nauc_precision_at_5_diff1": 0.6570524014466578, + "nauc_precision_at_5_max": 0.30012999498777754, + "nauc_precision_at_5_std": 0.6030407237879232, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.6643245257587762, + "nauc_recall_at_10_max": 0.3394378418139761, + "nauc_recall_at_10_std": 0.5631230256620064, + "nauc_recall_at_1_diff1": 0.3501554820571643, + "nauc_recall_at_1_max": 0.3599782655731798, + "nauc_recall_at_1_std": 0.7041481187202615, + "nauc_recall_at_20_diff1": 0.5587451940825555, + "nauc_recall_at_20_max": 0.47434883444027837, + "nauc_recall_at_20_std": 0.6875358512675467, + "nauc_recall_at_3_diff1": 0.5321534332531633, + "nauc_recall_at_3_max": 0.3540044845277292, + "nauc_recall_at_3_std": 0.7076694823976849, + "nauc_recall_at_5_diff1": 0.6570524014466579, + "nauc_recall_at_5_max": 0.3001299949877772, + "nauc_recall_at_5_std": 0.6030407237879231, + "ndcg_at_1": 0.08947, + "ndcg_at_10": 0.32202, + "ndcg_at_100": 0.41063, + "ndcg_at_1000": 0.41063, + "ndcg_at_20": 0.40326, + "ndcg_at_3": 0.18308, + "ndcg_at_5": 0.23594, + "precision_at_1": 0.08947, + "precision_at_10": 0.06468, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.04832, + "precision_at_3": 0.08263, + "precision_at_5": 0.0753, + "recall_at_1": 0.08947, + "recall_at_10": 0.64678, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.96645, + "recall_at_3": 0.2479, + "recall_at_5": 0.37651 } ] }, diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json new file mode 100644 index 0000000000..319e3d389b --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordEasyI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "3f018eb7ad32218a5a4ebd704493e0834a265cf5", + "evaluation_time": 17.977893829345703, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.73529, + "cv_recall_at_10": 0.88235, + "cv_recall_at_100": 0.95588, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.89706, + "cv_recall_at_3": 0.79412, + "cv_recall_at_5": 0.82353, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.73529, + "map_at_1": 0.12315, + "map_at_10": 0.25156, + "map_at_100": 0.39829, + "map_at_1000": 0.46268, + "map_at_20": 0.30547, + "map_at_3": 0.16397, + "map_at_5": 0.20244, + "mrr_at_1": 0.7352941176470589, + "mrr_at_10": 0.7796977124183007, + "mrr_at_100": 0.782038983404294, + "mrr_at_1000": 0.7822887502278202, + "mrr_at_20": 0.7806168300653595, + "mrr_at_3": 0.7647058823529411, + "mrr_at_5": 0.7713235294117647, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.43205924601836393, + "nauc_cv_recall_at_100_max": 0.46045430774676144, + "nauc_cv_recall_at_100_std": 0.092928936942703, + "nauc_cv_recall_at_10_diff1": 0.5238482274120124, + "nauc_cv_recall_at_10_max": 0.5447795999610322, + "nauc_cv_recall_at_10_std": -0.06773626862669024, + "nauc_cv_recall_at_1_diff1": 0.12227878218725667, + "nauc_cv_recall_at_1_max": 0.6674736854281768, + "nauc_cv_recall_at_1_std": 0.14078948712021733, + "nauc_cv_recall_at_20_diff1": 0.5375955313486179, + "nauc_cv_recall_at_20_max": 0.46954147068219054, + "nauc_cv_recall_at_20_std": -0.09380403989753433, + "nauc_cv_recall_at_3_diff1": 0.032732730804488214, + "nauc_cv_recall_at_3_max": 0.6199038499058639, + "nauc_cv_recall_at_3_std": 0.1529241887407745, + "nauc_cv_recall_at_5_diff1": 0.05658962411663845, + "nauc_cv_recall_at_5_max": 0.5877996333799903, + "nauc_cv_recall_at_5_std": 0.07730142560650717, + "nauc_map_at_1000_diff1": -0.11043090289526158, + "nauc_map_at_1000_max": 0.3448001394797035, + "nauc_map_at_1000_std": 0.27617723792125437, + "nauc_map_at_100_diff1": -0.01003127759400255, + "nauc_map_at_100_max": 0.1251706076455483, + "nauc_map_at_100_std": 0.03701229955981713, + "nauc_map_at_10_diff1": 0.2139884419568207, + "nauc_map_at_10_max": -0.26177042855989124, + "nauc_map_at_10_std": -0.26537536804238954, + "nauc_map_at_1_diff1": 0.21399681007835022, + "nauc_map_at_1_max": -0.3465990811601411, + "nauc_map_at_1_std": -0.29206561782006085, + "nauc_map_at_20_diff1": 0.1442752699133972, + "nauc_map_at_20_max": -0.15930665035054287, + "nauc_map_at_20_std": -0.21656927646466867, + "nauc_map_at_3_diff1": 0.23213875592486613, + "nauc_map_at_3_max": -0.3123546266529488, + "nauc_map_at_3_std": -0.28815041773010014, + "nauc_map_at_5_diff1": 0.2555448109113392, + "nauc_map_at_5_max": -0.29279690087015, + "nauc_map_at_5_std": -0.2774467752271305, + "nauc_mrr_at_1000_diff1": 0.12342303172342535, + "nauc_mrr_at_1000_max": 0.6362159972261828, + "nauc_mrr_at_1000_std": 0.12097023869090642, + "nauc_mrr_at_100_diff1": 0.12414954187540776, + "nauc_mrr_at_100_max": 0.6357802506977609, + "nauc_mrr_at_100_std": 0.12039621076657632, + "nauc_mrr_at_10_diff1": 0.12909230332110294, + "nauc_mrr_at_10_max": 0.6368549836172475, + "nauc_mrr_at_10_std": 0.11895636170173347, + "nauc_mrr_at_1_diff1": 0.12227878218725667, + "nauc_mrr_at_1_max": 0.6674736854281768, + "nauc_mrr_at_1_std": 0.14078948712021733, + "nauc_mrr_at_20_diff1": 0.1279569336937743, + "nauc_mrr_at_20_max": 0.6347997662966113, + "nauc_mrr_at_20_std": 0.11886311296272292, + "nauc_mrr_at_3_diff1": 0.0819493669888214, + "nauc_mrr_at_3_max": 0.6460493610936507, + "nauc_mrr_at_3_std": 0.14625466858168346, + "nauc_mrr_at_5_diff1": 0.08926776001340989, + "nauc_mrr_at_5_max": 0.6409743370271209, + "nauc_mrr_at_5_std": 0.13086158199344464, + "nauc_ndcg_at_1000_diff1": 0.0019895005695910535, + "nauc_ndcg_at_1000_max": 0.45479785106891296, + "nauc_ndcg_at_1000_std": 0.24337684988960884, + "nauc_ndcg_at_100_diff1": -0.09459953169491665, + "nauc_ndcg_at_100_max": 0.3603949417192614, + "nauc_ndcg_at_100_std": 0.25977974826865124, + "nauc_ndcg_at_10_diff1": -0.13743096053058393, + "nauc_ndcg_at_10_max": 0.5086455404107587, + "nauc_ndcg_at_10_std": 0.2942246989065215, + "nauc_ndcg_at_1_diff1": 0.12227878218725667, + "nauc_ndcg_at_1_max": 0.6674736854281768, + "nauc_ndcg_at_1_std": 0.14078948712021733, + "nauc_ndcg_at_20_diff1": -0.12980804316788766, + "nauc_ndcg_at_20_max": 0.4455786946909776, + "nauc_ndcg_at_20_std": 0.31146671038942925, + "nauc_ndcg_at_3_diff1": -0.12622874187697183, + "nauc_ndcg_at_3_max": 0.6321053376737653, + "nauc_ndcg_at_3_std": 0.2970450461706155, + "nauc_ndcg_at_5_diff1": -0.1167444253887786, + "nauc_ndcg_at_5_max": 0.5830509606014207, + "nauc_ndcg_at_5_std": 0.3092017793979728, + "nauc_precision_at_1000_diff1": -0.20870904079910094, + "nauc_precision_at_1000_max": 0.553838513916762, + "nauc_precision_at_1000_std": 0.5530198613029336, + "nauc_precision_at_100_diff1": -0.35505246743099134, + "nauc_precision_at_100_max": 0.7356593773083951, + "nauc_precision_at_100_std": 0.6881226249275548, + "nauc_precision_at_10_diff1": -0.32094129231704527, + "nauc_precision_at_10_max": 0.6957334058235605, + "nauc_precision_at_10_std": 0.4986922302670923, + "nauc_precision_at_1_diff1": 0.12227878218725667, + "nauc_precision_at_1_max": 0.6674736854281768, + "nauc_precision_at_1_std": 0.14078948712021733, + "nauc_precision_at_20_diff1": -0.348557232862726, + "nauc_precision_at_20_max": 0.7349093491157456, + "nauc_precision_at_20_std": 0.5403176104598045, + "nauc_precision_at_3_diff1": -0.19598105078336095, + "nauc_precision_at_3_max": 0.6900728385079788, + "nauc_precision_at_3_std": 0.38284792684568514, + "nauc_precision_at_5_diff1": -0.24325694519449578, + "nauc_precision_at_5_max": 0.6894112316594818, + "nauc_precision_at_5_std": 0.43885055756284386, + "nauc_recall_at_1000_diff1": 0.1247114496786093, + "nauc_recall_at_1000_max": 0.1972100012754324, + "nauc_recall_at_1000_std": 0.45358985850344147, + "nauc_recall_at_100_diff1": -0.007219441799189107, + "nauc_recall_at_100_max": -0.2726450323976995, + "nauc_recall_at_100_std": -0.09774220148121496, + "nauc_recall_at_10_diff1": 0.2493200864593003, + "nauc_recall_at_10_max": -0.35624261159143006, + "nauc_recall_at_10_std": -0.3178591018768707, + "nauc_recall_at_1_diff1": 0.21399681007835022, + "nauc_recall_at_1_max": -0.3465990811601411, + "nauc_recall_at_1_std": -0.29206561782006085, + "nauc_recall_at_20_diff1": 0.17625014384578389, + "nauc_recall_at_20_max": -0.3372766383490987, + "nauc_recall_at_20_std": -0.30264233629183895, + "nauc_recall_at_3_diff1": 0.23996176062420918, + "nauc_recall_at_3_max": -0.3204556669767577, + "nauc_recall_at_3_std": -0.292832765895384, + "nauc_recall_at_5_diff1": 0.28900424501627087, + "nauc_recall_at_5_max": -0.31760291427784704, + "nauc_recall_at_5_std": -0.2855625586155051, + "ndcg_at_1": 0.73529, + "ndcg_at_10": 0.60178, + "ndcg_at_100": 0.61347, + "ndcg_at_1000": 0.69898, + "ndcg_at_20": 0.58929, + "ndcg_at_3": 0.62141, + "ndcg_at_5": 0.61571, + "precision_at_1": 0.73529, + "precision_at_10": 0.45, + "precision_at_100": 0.19691, + "precision_at_1000": 0.04228, + "precision_at_20": 0.37647, + "precision_at_3": 0.54412, + "precision_at_5": 0.51176, + "recall_at_1": 0.12315, + "recall_at_10": 0.32137, + "recall_at_100": 0.68206, + "recall_at_1000": 0.93425, + "recall_at_20": 0.43253, + "recall_at_3": 0.16748, + "recall_at_5": 0.22328 + } + ] + }, + "task_name": "ROxfordEasyI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json new file mode 100644 index 0000000000..f957d7315e --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordHardI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f20b30211b7ba3fc64a02bd83998fe75f3023719", + "evaluation_time": 17.18506622314453, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.04286, + "cv_recall_at_10": 0.44286, + "cv_recall_at_100": 0.77143, + "cv_recall_at_1000": 0.92857, + "cv_recall_at_20": 0.52857, + "cv_recall_at_3": 0.18571, + "cv_recall_at_5": 0.27143, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.04286, + "map_at_1": 0.00178, + "map_at_10": 0.02316, + "map_at_100": 0.03694, + "map_at_1000": 0.05036, + "map_at_20": 0.02814, + "map_at_3": 0.01285, + "map_at_5": 0.01752, + "mrr_at_1": 0.04285714285714286, + "mrr_at_10": 0.13929705215419497, + "mrr_at_100": 0.15254576393638014, + "mrr_at_1000": 0.15348328701607364, + "mrr_at_20": 0.14564255189255187, + "mrr_at_3": 0.1, + "mrr_at_5": 0.11857142857142858, + "nauc_cv_recall_at_1000_diff1": 0.47861811391223064, + "nauc_cv_recall_at_1000_max": 0.5808590102707742, + "nauc_cv_recall_at_1000_std": 0.09140989729224985, + "nauc_cv_recall_at_100_diff1": 0.5586401902191371, + "nauc_cv_recall_at_100_max": -0.12197159565580537, + "nauc_cv_recall_at_100_std": -0.46324143692564745, + "nauc_cv_recall_at_10_diff1": 0.33702203892735166, + "nauc_cv_recall_at_10_max": -0.2566107451434116, + "nauc_cv_recall_at_10_std": -0.2562935040214977, + "nauc_cv_recall_at_1_diff1": 0.5001341561577676, + "nauc_cv_recall_at_1_max": -0.12592791342455964, + "nauc_cv_recall_at_1_std": 0.13701815580001792, + "nauc_cv_recall_at_20_diff1": 0.33340522734848327, + "nauc_cv_recall_at_20_max": -0.27021038803160713, + "nauc_cv_recall_at_20_std": -0.2604850885276762, + "nauc_cv_recall_at_3_diff1": 0.42356508767210377, + "nauc_cv_recall_at_3_max": -0.30471252842923874, + "nauc_cv_recall_at_3_std": -0.2722848548580372, + "nauc_cv_recall_at_5_diff1": 0.37677403680683497, + "nauc_cv_recall_at_5_max": -0.30530641993804797, + "nauc_cv_recall_at_5_std": -0.2763347032980382, + "nauc_map_at_1000_diff1": 0.32492788395298233, + "nauc_map_at_1000_max": -0.264513294885872, + "nauc_map_at_1000_std": -0.06025413474178467, + "nauc_map_at_100_diff1": 0.33932455574204345, + "nauc_map_at_100_max": -0.3183944729391321, + "nauc_map_at_100_std": -0.11067793875024108, + "nauc_map_at_10_diff1": 0.3627322367418886, + "nauc_map_at_10_max": -0.270787009945954, + "nauc_map_at_10_std": -0.12449041958590679, + "nauc_map_at_1_diff1": 0.8349360037495761, + "nauc_map_at_1_max": 0.04030601005296298, + "nauc_map_at_1_std": 0.28342764489523986, + "nauc_map_at_20_diff1": 0.36471735689250456, + "nauc_map_at_20_max": -0.27443485632176895, + "nauc_map_at_20_std": -0.10351956211548237, + "nauc_map_at_3_diff1": 0.33740985231621823, + "nauc_map_at_3_max": -0.26557891525076527, + "nauc_map_at_3_std": -0.13603337759846046, + "nauc_map_at_5_diff1": 0.3822623605794258, + "nauc_map_at_5_max": -0.26535510732907475, + "nauc_map_at_5_std": -0.15188815608303466, + "nauc_mrr_at_1000_diff1": 0.42803033998914947, + "nauc_mrr_at_1000_max": -0.25330248029839103, + "nauc_mrr_at_1000_std": -0.1498777504287341, + "nauc_mrr_at_100_diff1": 0.428756525260202, + "nauc_mrr_at_100_max": -0.255106401697983, + "nauc_mrr_at_100_std": -0.15247667584774333, + "nauc_mrr_at_10_diff1": 0.4269277134986295, + "nauc_mrr_at_10_max": -0.26024547551332144, + "nauc_mrr_at_10_std": -0.15027887241014917, + "nauc_mrr_at_1_diff1": 0.5001341561577676, + "nauc_mrr_at_1_max": -0.12592791342455964, + "nauc_mrr_at_1_std": 0.13701815580001792, + "nauc_mrr_at_20_diff1": 0.4273548864254061, + "nauc_mrr_at_20_max": -0.2649857851418372, + "nauc_mrr_at_20_std": -0.150811901899745, + "nauc_mrr_at_3_diff1": 0.46234460520174797, + "nauc_mrr_at_3_max": -0.26846705418134004, + "nauc_mrr_at_3_std": -0.14951136379707816, + "nauc_mrr_at_5_diff1": 0.4315603993226756, + "nauc_mrr_at_5_max": -0.2751708043421038, + "nauc_mrr_at_5_std": -0.15676414005890574, + "nauc_ndcg_at_1000_diff1": 0.3031539661086623, + "nauc_ndcg_at_1000_max": -0.06324949257415059, + "nauc_ndcg_at_1000_std": 0.03291344536352667, + "nauc_ndcg_at_100_diff1": 0.3537878100611368, + "nauc_ndcg_at_100_max": -0.3440256642252378, + "nauc_ndcg_at_100_std": -0.13928638317122793, + "nauc_ndcg_at_10_diff1": 0.31423572757944224, + "nauc_ndcg_at_10_max": -0.27799943432453955, + "nauc_ndcg_at_10_std": -0.12000726018713354, + "nauc_ndcg_at_1_diff1": 0.5001341561577676, + "nauc_ndcg_at_1_max": -0.12592791342455964, + "nauc_ndcg_at_1_std": 0.13701815580001792, + "nauc_ndcg_at_20_diff1": 0.3527561180413498, + "nauc_ndcg_at_20_max": -0.29516361937414604, + "nauc_ndcg_at_20_std": -0.10951825882002067, + "nauc_ndcg_at_3_diff1": 0.3265974284434805, + "nauc_ndcg_at_3_max": -0.27320998825572146, + "nauc_ndcg_at_3_std": -0.12237689567430368, + "nauc_ndcg_at_5_diff1": 0.32240119227390085, + "nauc_ndcg_at_5_max": -0.30077766985994786, + "nauc_ndcg_at_5_std": -0.15147497522509035, + "nauc_precision_at_1000_diff1": -0.12639282009702774, + "nauc_precision_at_1000_max": 0.22826737103796205, + "nauc_precision_at_1000_std": 0.32628514757600596, + "nauc_precision_at_100_diff1": 0.2772154726137188, + "nauc_precision_at_100_max": -0.23028476826992986, + "nauc_precision_at_100_std": -0.1303180540104937, + "nauc_precision_at_10_diff1": 0.29107138713345343, + "nauc_precision_at_10_max": -0.30600784337992426, + "nauc_precision_at_10_std": -0.12021874830794935, + "nauc_precision_at_1_diff1": 0.5001341561577676, + "nauc_precision_at_1_max": -0.12592791342455964, + "nauc_precision_at_1_std": 0.13701815580001792, + "nauc_precision_at_20_diff1": 0.27727122281419164, + "nauc_precision_at_20_max": -0.3127822679663893, + "nauc_precision_at_20_std": -0.07951548670204237, + "nauc_precision_at_3_diff1": 0.3064711218627153, + "nauc_precision_at_3_max": -0.29553472432214506, + "nauc_precision_at_3_std": -0.17296643483519816, + "nauc_precision_at_5_diff1": 0.27289717372431765, + "nauc_precision_at_5_max": -0.33688627375325547, + "nauc_precision_at_5_std": -0.1943667406192727, + "nauc_recall_at_1000_diff1": 0.2612937021120264, + "nauc_recall_at_1000_max": -0.04177364682142833, + "nauc_recall_at_1000_std": -0.10225779216767208, + "nauc_recall_at_100_diff1": 0.23045925470740378, + "nauc_recall_at_100_max": -0.32266175361589855, + "nauc_recall_at_100_std": -0.10122639227831537, + "nauc_recall_at_10_diff1": 0.3214491777002914, + "nauc_recall_at_10_max": -0.2191819309348708, + "nauc_recall_at_10_std": -0.13294514318629588, + "nauc_recall_at_1_diff1": 0.8349360037495761, + "nauc_recall_at_1_max": 0.04030601005296298, + "nauc_recall_at_1_std": 0.28342764489523986, + "nauc_recall_at_20_diff1": 0.30304432643409973, + "nauc_recall_at_20_max": -0.2369864752715589, + "nauc_recall_at_20_std": -0.11319354561879752, + "nauc_recall_at_3_diff1": 0.2610862099372524, + "nauc_recall_at_3_max": -0.26808957441898906, + "nauc_recall_at_3_std": -0.1754514836876096, + "nauc_recall_at_5_diff1": 0.3659777743786138, + "nauc_recall_at_5_max": -0.23557478979416202, + "nauc_recall_at_5_std": -0.1749293090721626, + "ndcg_at_1": 0.04286, + "ndcg_at_10": 0.08519, + "ndcg_at_100": 0.12707, + "ndcg_at_1000": 0.26609, + "ndcg_at_20": 0.08785, + "ndcg_at_3": 0.07857, + "ndcg_at_5": 0.08201, + "precision_at_1": 0.04286, + "precision_at_10": 0.07571, + "precision_at_100": 0.03571, + "precision_at_1000": 0.01857, + "precision_at_20": 0.05714, + "precision_at_3": 0.08571, + "precision_at_5": 0.08, + "recall_at_1": 0.00178, + "recall_at_10": 0.07379, + "recall_at_100": 0.22471, + "recall_at_1000": 0.60888, + "recall_at_20": 0.09447, + "recall_at_3": 0.02931, + "recall_at_5": 0.04839 + } + ] + }, + "task_name": "ROxfordHardI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordI2IRetrieval.json deleted file mode 100644 index e47a943219..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "d8daad98b4e4896a7f7fa1b3485a22420242d4fc", - "evaluation_time": 29.71091365814209, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.80946, - "cv_recall_at_10": 0.99953, - "cv_recall_at_100": 1.0, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 1.0, - "cv_recall_at_3": 0.97636, - "cv_recall_at_5": 0.99291, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.80946, - "map_at_1": 0.00083, - "map_at_10": 0.00583, - "map_at_100": 0.04516, - "map_at_1000": 0.27358, - "map_at_20": 0.01067, - "map_at_3": 0.00211, - "map_at_5": 0.00324, - "mrr_at_1": 0.9985815602836879, - "mrr_at_10": 0.9992119779353821, - "mrr_at_100": 0.9992119779353821, - "mrr_at_1000": 0.9992119779353821, - "mrr_at_20": 0.9992119779353821, - "mrr_at_3": 0.9992119779353821, - "mrr_at_5": 0.9992119779353821, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": NaN, - "nauc_cv_recall_at_100_max": NaN, - "nauc_cv_recall_at_100_std": NaN, - "nauc_cv_recall_at_10_diff1": 0.5540638703677596, - "nauc_cv_recall_at_10_max": 0.7221878767269868, - "nauc_cv_recall_at_10_std": 0.869246700256431, - "nauc_cv_recall_at_1_diff1": -0.24158864035258856, - "nauc_cv_recall_at_1_max": -0.0420448021592233, - "nauc_cv_recall_at_1_std": 0.04484895202286055, - "nauc_cv_recall_at_20_diff1": NaN, - "nauc_cv_recall_at_20_max": NaN, - "nauc_cv_recall_at_20_std": NaN, - "nauc_cv_recall_at_3_diff1": -0.2813966714480976, - "nauc_cv_recall_at_3_max": -0.04571848249527183, - "nauc_cv_recall_at_3_std": 0.06606763479339506, - "nauc_cv_recall_at_5_diff1": -0.057737057928295976, - "nauc_cv_recall_at_5_max": 0.017855647720182007, - "nauc_cv_recall_at_5_std": 0.08427153819844212, - "nauc_map_at_1000_diff1": -0.3857995513654731, - "nauc_map_at_1000_max": 0.03808456962512866, - "nauc_map_at_1000_std": -0.06663520492889344, - "nauc_map_at_100_diff1": -0.34437629024420324, - "nauc_map_at_100_max": 0.020553049295469906, - "nauc_map_at_100_std": 0.0556790533768489, - "nauc_map_at_10_diff1": -0.18993364966776907, - "nauc_map_at_10_max": 0.006866799821313959, - "nauc_map_at_10_std": 0.13195237855420855, - "nauc_map_at_1_diff1": -0.05431107143472811, - "nauc_map_at_1_max": 0.016884860597154504, - "nauc_map_at_1_std": 0.11466554606849116, - "nauc_map_at_20_diff1": -0.2414863261116838, - "nauc_map_at_20_max": 0.008935096908909975, - "nauc_map_at_20_std": 0.11557906533166638, - "nauc_map_at_3_diff1": -0.12217591294530498, - "nauc_map_at_3_max": 0.008152934040333987, - "nauc_map_at_3_std": 0.13797559913063503, - "nauc_map_at_5_diff1": -0.13888462292383946, - "nauc_map_at_5_max": 0.007634414930511178, - "nauc_map_at_5_std": 0.13391202111166425, - "nauc_mrr_at_1000_diff1": 0.9476986801026335, - "nauc_mrr_at_1000_max": 1.0, - "nauc_mrr_at_1000_std": 0.9166563630181881, - "nauc_mrr_at_100_diff1": 0.9476986801026335, - "nauc_mrr_at_100_max": 1.0, - "nauc_mrr_at_100_std": 0.9166563630181881, - "nauc_mrr_at_10_diff1": 0.9476986801026335, - "nauc_mrr_at_10_max": 1.0, - "nauc_mrr_at_10_std": 0.9166563630181881, - "nauc_mrr_at_1_diff1": 0.9564155667521437, - "nauc_mrr_at_1_max": 1.0, - "nauc_mrr_at_1_std": 0.9073959589089956, - "nauc_mrr_at_20_diff1": 0.9476986801026335, - "nauc_mrr_at_20_max": 1.0, - "nauc_mrr_at_20_std": 0.9166563630181881, - "nauc_mrr_at_3_diff1": 0.9476986801026335, - "nauc_mrr_at_3_max": 1.0, - "nauc_mrr_at_3_std": 0.9166563630181881, - "nauc_mrr_at_5_diff1": 0.9476986801026335, - "nauc_mrr_at_5_max": 1.0, - "nauc_mrr_at_5_std": 0.9166563630181881, - "nauc_ndcg_at_1000_diff1": -0.29077931122633116, - "nauc_ndcg_at_1000_max": 0.014960510380277934, - "nauc_ndcg_at_1000_std": -0.12483981180901933, - "nauc_ndcg_at_100_diff1": -0.3609816949819758, - "nauc_ndcg_at_100_max": 0.021419925854651446, - "nauc_ndcg_at_100_std": -0.17265752707444715, - "nauc_ndcg_at_10_diff1": -0.27457750496652206, - "nauc_ndcg_at_10_max": 0.0008466224260104365, - "nauc_ndcg_at_10_std": -0.045841557226464144, - "nauc_ndcg_at_1_diff1": 0.9564155667521437, - "nauc_ndcg_at_1_max": 1.0, - "nauc_ndcg_at_1_std": 0.9073959589089956, - "nauc_ndcg_at_20_diff1": -0.3094417772089175, - "nauc_ndcg_at_20_max": 0.009714640004268442, - "nauc_ndcg_at_20_std": -0.08586776230182966, - "nauc_ndcg_at_3_diff1": -0.23757585938494089, - "nauc_ndcg_at_3_max": -0.016387555353346198, - "nauc_ndcg_at_3_std": 0.030386940725562046, - "nauc_ndcg_at_5_diff1": -0.23196460003691158, - "nauc_ndcg_at_5_max": -0.013062943721282708, - "nauc_ndcg_at_5_std": -0.010500172553175749, - "nauc_precision_at_1000_diff1": -0.17619733662463302, - "nauc_precision_at_1000_max": 0.012339801300641195, - "nauc_precision_at_1000_std": -0.1876530479926194, - "nauc_precision_at_100_diff1": -0.3626475300981416, - "nauc_precision_at_100_max": 0.025040872201569076, - "nauc_precision_at_100_std": -0.18897442356689456, - "nauc_precision_at_10_diff1": -0.2782683949086005, - "nauc_precision_at_10_max": 0.006150543325428016, - "nauc_precision_at_10_std": -0.05974378846400665, - "nauc_precision_at_1_diff1": 0.9564155667521437, - "nauc_precision_at_1_max": 1.0, - "nauc_precision_at_1_std": 0.9073959589089956, - "nauc_precision_at_20_diff1": -0.31218967700544853, - "nauc_precision_at_20_max": 0.015101101880288523, - "nauc_precision_at_20_std": -0.10187042490803115, - "nauc_precision_at_3_diff1": -0.2418495400445544, - "nauc_precision_at_3_max": -0.017073069963463458, - "nauc_precision_at_3_std": 0.024904433150907852, - "nauc_precision_at_5_diff1": -0.23061542055201162, - "nauc_precision_at_5_max": -0.013293477378375076, - "nauc_precision_at_5_std": -0.02217796284197703, - "nauc_recall_at_1000_diff1": -0.1758800893625114, - "nauc_recall_at_1000_max": 0.025501344233019, - "nauc_recall_at_1000_std": 0.042539963927428814, - "nauc_recall_at_100_diff1": -0.2486522818740174, - "nauc_recall_at_100_max": 0.019163648439019607, - "nauc_recall_at_100_std": 0.08018065043566762, - "nauc_recall_at_10_diff1": -0.1643890021643603, - "nauc_recall_at_10_max": 0.015612163534741345, - "nauc_recall_at_10_std": 0.12932048693738263, - "nauc_recall_at_1_diff1": -0.05431107143472811, - "nauc_recall_at_1_max": 0.016884860597154504, - "nauc_recall_at_1_std": 0.11466554606849116, - "nauc_recall_at_20_diff1": -0.19900512249205043, - "nauc_recall_at_20_max": 0.013811903907386002, - "nauc_recall_at_20_std": 0.11661397478238389, - "nauc_recall_at_3_diff1": -0.11946531860608983, - "nauc_recall_at_3_max": 0.009964580844462759, - "nauc_recall_at_3_std": 0.13655833592145109, - "nauc_recall_at_5_diff1": -0.1255934399523269, - "nauc_recall_at_5_max": 0.012044411079830297, - "nauc_recall_at_5_std": 0.1305906448336473, - "ndcg_at_1": 0.99858, - "ndcg_at_10": 0.82209, - "ndcg_at_100": 0.74273, - "ndcg_at_1000": 0.56583, - "ndcg_at_20": 0.79415, - "ndcg_at_3": 0.89211, - "ndcg_at_5": 0.8591, - "precision_at_1": 0.99858, - "precision_at_10": 0.79054, - "precision_at_100": 0.72617, - "precision_at_1000": 0.49807, - "precision_at_20": 0.76664, - "precision_at_3": 0.86383, - "precision_at_5": 0.82752, - "recall_at_1": 0.00083, - "recall_at_10": 0.00647, - "recall_at_100": 0.05838, - "recall_at_1000": 0.3868, - "recall_at_20": 0.01248, - "recall_at_3": 0.00214, - "recall_at_5": 0.00341 - } - ] - }, - "task_name": "ROxfordI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json new file mode 100644 index 0000000000..27241284b1 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/ROxfordMediumI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "3bd28e9c45e15f299117c634799f7035c4de2d31", + "evaluation_time": 17.523592710494995, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.75714, + "cv_recall_at_10": 0.87143, + "cv_recall_at_100": 0.94286, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.88571, + "cv_recall_at_3": 0.82857, + "cv_recall_at_5": 0.85714, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.75714, + "map_at_1": 0.04928, + "map_at_10": 0.14238, + "map_at_100": 0.25975, + "map_at_1000": 0.33, + "map_at_20": 0.1832, + "map_at_3": 0.08106, + "map_at_5": 0.10574, + "mrr_at_1": 0.7571428571428571, + "mrr_at_10": 0.7992857142857144, + "mrr_at_100": 0.8018753503269448, + "mrr_at_1000": 0.8022392070599913, + "mrr_at_20": 0.8001785714285715, + "mrr_at_3": 0.7904761904761906, + "mrr_at_5": 0.796904761904762, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": 0.15581232492997169, + "nauc_cv_recall_at_100_max": 0.2811624649859951, + "nauc_cv_recall_at_100_std": -0.12196545284780497, + "nauc_cv_recall_at_10_diff1": 0.394540014958862, + "nauc_cv_recall_at_10_max": 0.4610535313601873, + "nauc_cv_recall_at_10_std": -0.14937493321936154, + "nauc_cv_recall_at_1_diff1": 0.3099268419602683, + "nauc_cv_recall_at_1_max": 0.6277204689460957, + "nauc_cv_recall_at_1_std": 0.11781811503259924, + "nauc_cv_recall_at_20_diff1": 0.39261983863312777, + "nauc_cv_recall_at_20_max": 0.3849074513526345, + "nauc_cv_recall_at_20_std": -0.18147840531561454, + "nauc_cv_recall_at_3_diff1": 0.29657669519420665, + "nauc_cv_recall_at_3_max": 0.5705233706385782, + "nauc_cv_recall_at_3_std": 0.04756418696510771, + "nauc_cv_recall_at_5_diff1": 0.37162293488824066, + "nauc_cv_recall_at_5_max": 0.5234207968901848, + "nauc_cv_recall_at_5_std": -0.06068999028182738, + "nauc_map_at_1000_diff1": -0.14810539444535628, + "nauc_map_at_1000_max": 0.42639970063449095, + "nauc_map_at_1000_std": 0.35965851851847036, + "nauc_map_at_100_diff1": -0.04654719636323819, + "nauc_map_at_100_max": 0.20124568398625026, + "nauc_map_at_100_std": 0.09779920001336984, + "nauc_map_at_10_diff1": 0.19553982602379108, + "nauc_map_at_10_max": -0.16962211647877953, + "nauc_map_at_10_std": -0.2049718676191143, + "nauc_map_at_1_diff1": 0.19831336261662813, + "nauc_map_at_1_max": -0.26048526225990315, + "nauc_map_at_1_std": -0.24848660044643464, + "nauc_map_at_20_diff1": 0.11533311201293248, + "nauc_map_at_20_max": -0.06940468788280867, + "nauc_map_at_20_std": -0.15394999023265438, + "nauc_map_at_3_diff1": 0.22661360439380449, + "nauc_map_at_3_max": -0.24582246289343465, + "nauc_map_at_3_std": -0.24358060859761363, + "nauc_map_at_5_diff1": 0.2391073300885911, + "nauc_map_at_5_max": -0.21313902222536138, + "nauc_map_at_5_std": -0.22555952910665222, + "nauc_mrr_at_1000_diff1": 0.3118072480358942, + "nauc_mrr_at_1000_max": 0.5896434235555521, + "nauc_mrr_at_1000_std": 0.07565750084460038, + "nauc_mrr_at_100_diff1": 0.3114581767470047, + "nauc_mrr_at_100_max": 0.5888175740478946, + "nauc_mrr_at_100_std": 0.07590573329917173, + "nauc_mrr_at_10_diff1": 0.31501956699434985, + "nauc_mrr_at_10_max": 0.5906694492065899, + "nauc_mrr_at_10_std": 0.07282707502971797, + "nauc_mrr_at_1_diff1": 0.3099268419602683, + "nauc_mrr_at_1_max": 0.6277204689460957, + "nauc_mrr_at_1_std": 0.11781811503259924, + "nauc_mrr_at_20_diff1": 0.31460997067981183, + "nauc_mrr_at_20_max": 0.5883036453949262, + "nauc_mrr_at_20_std": 0.07253787758405468, + "nauc_mrr_at_3_diff1": 0.298829153668517, + "nauc_mrr_at_3_max": 0.6036890630335515, + "nauc_mrr_at_3_std": 0.09929911274966623, + "nauc_mrr_at_5_diff1": 0.31309074738187426, + "nauc_mrr_at_5_max": 0.5968819745150035, + "nauc_mrr_at_5_std": 0.08124058142418439, + "nauc_ndcg_at_1000_diff1": 0.023759962305708315, + "nauc_ndcg_at_1000_max": 0.42118350886771483, + "nauc_ndcg_at_1000_std": 0.21503403862781068, + "nauc_ndcg_at_100_diff1": -0.10602196843045252, + "nauc_ndcg_at_100_max": 0.4040222401408388, + "nauc_ndcg_at_100_std": 0.3422251986340049, + "nauc_ndcg_at_10_diff1": -0.14230825595892613, + "nauc_ndcg_at_10_max": 0.5717619317460108, + "nauc_ndcg_at_10_std": 0.35188240284777716, + "nauc_ndcg_at_1_diff1": 0.3099268419602683, + "nauc_ndcg_at_1_max": 0.6277204689460957, + "nauc_ndcg_at_1_std": 0.11781811503259924, + "nauc_ndcg_at_20_diff1": -0.15743789243142528, + "nauc_ndcg_at_20_max": 0.5724054251457065, + "nauc_ndcg_at_20_std": 0.3797961049903619, + "nauc_ndcg_at_3_diff1": 0.02676887075350004, + "nauc_ndcg_at_3_max": 0.5535721465155271, + "nauc_ndcg_at_3_std": 0.22786549855388236, + "nauc_ndcg_at_5_diff1": -0.04449400646171614, + "nauc_ndcg_at_5_max": 0.555861797739005, + "nauc_ndcg_at_5_std": 0.27833851228984907, + "nauc_precision_at_1000_diff1": -0.19203752528557633, + "nauc_precision_at_1000_max": 0.4830680560318716, + "nauc_precision_at_1000_std": 0.5059538376358517, + "nauc_precision_at_100_diff1": -0.29506043721080727, + "nauc_precision_at_100_max": 0.658086670179558, + "nauc_precision_at_100_std": 0.6166722456503947, + "nauc_precision_at_10_diff1": -0.2282600490658613, + "nauc_precision_at_10_max": 0.5965276467257976, + "nauc_precision_at_10_std": 0.42049443291187055, + "nauc_precision_at_1_diff1": 0.3099268419602683, + "nauc_precision_at_1_max": 0.6277204689460957, + "nauc_precision_at_1_std": 0.11781811503259924, + "nauc_precision_at_20_diff1": -0.2777369235938004, + "nauc_precision_at_20_max": 0.6422763272584597, + "nauc_precision_at_20_std": 0.46973331485105685, + "nauc_precision_at_3_diff1": -0.03905783426100687, + "nauc_precision_at_3_max": 0.548195217505554, + "nauc_precision_at_3_std": 0.2602422302381689, + "nauc_precision_at_5_diff1": -0.12812907648800725, + "nauc_precision_at_5_max": 0.5661902630852285, + "nauc_precision_at_5_std": 0.33254144330289526, + "nauc_recall_at_1000_diff1": 0.11552874226597364, + "nauc_recall_at_1000_max": 0.1991754113531379, + "nauc_recall_at_1000_std": 0.05667927260800616, + "nauc_recall_at_100_diff1": -0.01602892704072587, + "nauc_recall_at_100_max": -0.12257633928876718, + "nauc_recall_at_100_std": -0.01682143204953458, + "nauc_recall_at_10_diff1": 0.22298978148074566, + "nauc_recall_at_10_max": -0.22859971193805304, + "nauc_recall_at_10_std": -0.23715467924390157, + "nauc_recall_at_1_diff1": 0.19831336261662813, + "nauc_recall_at_1_max": -0.26048526225990315, + "nauc_recall_at_1_std": -0.24848660044643464, + "nauc_recall_at_20_diff1": 0.16037912600194174, + "nauc_recall_at_20_max": -0.1835918931237162, + "nauc_recall_at_20_std": -0.20833669696201074, + "nauc_recall_at_3_diff1": 0.2179234117693627, + "nauc_recall_at_3_max": -0.2525912771978034, + "nauc_recall_at_3_std": -0.24319572308223014, + "nauc_recall_at_5_diff1": 0.2530123469958082, + "nauc_recall_at_5_max": -0.22138719273185706, + "nauc_recall_at_5_std": -0.2215811361906131, + "ndcg_at_1": 0.75714, + "ndcg_at_10": 0.58385, + "ndcg_at_100": 0.48629, + "ndcg_at_1000": 0.59891, + "ndcg_at_20": 0.52663, + "ndcg_at_3": 0.6553, + "ndcg_at_5": 0.63094, + "precision_at_1": 0.75714, + "precision_at_10": 0.51286, + "precision_at_100": 0.227, + "precision_at_1000": 0.05964, + "precision_at_20": 0.42286, + "precision_at_3": 0.61429, + "precision_at_5": 0.57714, + "recall_at_1": 0.04928, + "recall_at_10": 0.17721, + "recall_at_100": 0.44393, + "recall_at_1000": 0.76857, + "recall_at_20": 0.24576, + "recall_at_3": 0.08507, + "recall_at_5": 0.11871 + } + ] + }, + "task_name": "ROxfordMediumI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json new file mode 100644 index 0000000000..bd4a6c53b9 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisEasyI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "a7293da8a341de665ee4dcb2f209281df342d80b", + "evaluation_time": 22.57354736328125, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.94286, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94286, + "map_at_1": 0.02765, + "map_at_10": 0.11901, + "map_at_100": 0.46776, + "map_at_1000": 0.58942, + "map_at_20": 0.19679, + "map_at_3": 0.05048, + "map_at_5": 0.07407, + "mrr_at_1": 0.9428571428571428, + "mrr_at_10": 0.9666666666666666, + "mrr_at_100": 0.9666666666666666, + "mrr_at_1000": 0.9666666666666666, + "mrr_at_20": 0.9666666666666666, + "mrr_at_3": 0.9666666666666666, + "mrr_at_5": 0.9666666666666666, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": -0.303338001867413, + "nauc_cv_recall_at_1_max": -0.11951447245564903, + "nauc_cv_recall_at_1_std": 0.38328664799253065, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_map_at_1000_diff1": -0.2774518254418231, + "nauc_map_at_1000_max": 0.25032079891265085, + "nauc_map_at_1000_std": 0.45081359384807485, + "nauc_map_at_100_diff1": -0.04907209740285267, + "nauc_map_at_100_max": 0.18686662832088577, + "nauc_map_at_100_std": 0.31839565681325593, + "nauc_map_at_10_diff1": 0.4545859442374117, + "nauc_map_at_10_max": -0.04495045809094941, + "nauc_map_at_10_std": 0.002947268581625076, + "nauc_map_at_1_diff1": 0.7049183369958152, + "nauc_map_at_1_max": -0.2616079702049637, + "nauc_map_at_1_std": -0.3618487407893112, + "nauc_map_at_20_diff1": 0.30890872824690435, + "nauc_map_at_20_max": 0.07581745603833642, + "nauc_map_at_20_std": 0.15307053171724933, + "nauc_map_at_3_diff1": 0.5935344781456238, + "nauc_map_at_3_max": -0.20146382202873225, + "nauc_map_at_3_std": -0.22821424999017603, + "nauc_map_at_5_diff1": 0.5703897004682528, + "nauc_map_at_5_max": -0.16308005302080678, + "nauc_map_at_5_std": -0.1620575175690372, + "nauc_mrr_at_1000_diff1": -0.39652527677737137, + "nauc_mrr_at_1000_max": -0.10590903027877008, + "nauc_mrr_at_1000_std": 0.452714419100977, + "nauc_mrr_at_100_diff1": -0.39652527677737137, + "nauc_mrr_at_100_max": -0.10590903027877008, + "nauc_mrr_at_100_std": 0.452714419100977, + "nauc_mrr_at_10_diff1": -0.39652527677737137, + "nauc_mrr_at_10_max": -0.10590903027877008, + "nauc_mrr_at_10_std": 0.452714419100977, + "nauc_mrr_at_1_diff1": -0.303338001867413, + "nauc_mrr_at_1_max": -0.11951447245564903, + "nauc_mrr_at_1_std": 0.38328664799253065, + "nauc_mrr_at_20_diff1": -0.39652527677737137, + "nauc_mrr_at_20_max": -0.10590903027877008, + "nauc_mrr_at_20_std": 0.452714419100977, + "nauc_mrr_at_3_diff1": -0.39652527677737137, + "nauc_mrr_at_3_max": -0.10590903027877008, + "nauc_mrr_at_3_std": 0.452714419100977, + "nauc_mrr_at_5_diff1": -0.39652527677737137, + "nauc_mrr_at_5_max": -0.10590903027877008, + "nauc_mrr_at_5_std": 0.452714419100977, + "nauc_ndcg_at_1000_diff1": -0.49032575238867226, + "nauc_ndcg_at_1000_max": 0.30827149571860624, + "nauc_ndcg_at_1000_std": 0.5422742306681069, + "nauc_ndcg_at_100_diff1": -0.2712801258824707, + "nauc_ndcg_at_100_max": 0.2469510949945573, + "nauc_ndcg_at_100_std": 0.4367311454283413, + "nauc_ndcg_at_10_diff1": -0.519586478031318, + "nauc_ndcg_at_10_max": 0.35401668386388874, + "nauc_ndcg_at_10_std": 0.5596598049318611, + "nauc_ndcg_at_1_diff1": -0.303338001867413, + "nauc_ndcg_at_1_max": -0.11951447245564903, + "nauc_ndcg_at_1_std": 0.38328664799253065, + "nauc_ndcg_at_20_diff1": -0.4606517361381654, + "nauc_ndcg_at_20_max": 0.38378869676003413, + "nauc_ndcg_at_20_std": 0.474186229263476, + "nauc_ndcg_at_3_diff1": -0.585149663762276, + "nauc_ndcg_at_3_max": 0.217548589813707, + "nauc_ndcg_at_3_std": 0.550167698282911, + "nauc_ndcg_at_5_diff1": -0.5267224367233376, + "nauc_ndcg_at_5_max": 0.2591401820511711, + "nauc_ndcg_at_5_std": 0.5702945720424282, + "nauc_precision_at_1000_diff1": -0.4499561707406712, + "nauc_precision_at_1000_max": 0.15634621233710536, + "nauc_precision_at_1000_std": 0.17441880422894795, + "nauc_precision_at_100_diff1": -0.4968893518170479, + "nauc_precision_at_100_max": 0.19447257428460527, + "nauc_precision_at_100_std": 0.25819757590889797, + "nauc_precision_at_10_diff1": -0.6167580528722199, + "nauc_precision_at_10_max": 0.3956822166454678, + "nauc_precision_at_10_std": 0.587829591941972, + "nauc_precision_at_1_diff1": -0.303338001867413, + "nauc_precision_at_1_max": -0.11951447245564903, + "nauc_precision_at_1_std": 0.38328664799253065, + "nauc_precision_at_20_diff1": -0.6031642916224456, + "nauc_precision_at_20_max": 0.4138166765495704, + "nauc_precision_at_20_std": 0.5161953717797159, + "nauc_precision_at_3_diff1": -0.6298090956856782, + "nauc_precision_at_3_max": 0.24751396267538517, + "nauc_precision_at_3_std": 0.557806449101913, + "nauc_precision_at_5_diff1": -0.5860255772185323, + "nauc_precision_at_5_max": 0.29785990954926145, + "nauc_precision_at_5_std": 0.5944106636660691, + "nauc_recall_at_1000_diff1": -0.12209326635132679, + "nauc_recall_at_1000_max": 0.008543724924431836, + "nauc_recall_at_1000_std": 0.25794589209932395, + "nauc_recall_at_100_diff1": 0.16704714994186, + "nauc_recall_at_100_max": 0.053397233537995845, + "nauc_recall_at_100_std": 0.10488570154536475, + "nauc_recall_at_10_diff1": 0.5240194164262459, + "nauc_recall_at_10_max": -0.03784601294172749, + "nauc_recall_at_10_std": -0.07182820315172336, + "nauc_recall_at_1_diff1": 0.7049183369958152, + "nauc_recall_at_1_max": -0.2616079702049637, + "nauc_recall_at_1_std": -0.3618487407893112, + "nauc_recall_at_20_diff1": 0.40266587722079766, + "nauc_recall_at_20_max": 0.03757413630827814, + "nauc_recall_at_20_std": 0.042791117828726286, + "nauc_recall_at_3_diff1": 0.6179795805128587, + "nauc_recall_at_3_max": -0.1966728040865525, + "nauc_recall_at_3_std": -0.2470220135910208, + "nauc_recall_at_5_diff1": 0.6188095385694616, + "nauc_recall_at_5_max": -0.1447272403059871, + "nauc_recall_at_5_std": -0.20646683990606104, + "ndcg_at_1": 0.94286, + "ndcg_at_10": 0.85179, + "ndcg_at_100": 0.71775, + "ndcg_at_1000": 0.83657, + "ndcg_at_20": 0.82388, + "ndcg_at_3": 0.8936, + "ndcg_at_5": 0.88343, + "precision_at_1": 0.94286, + "precision_at_10": 0.81429, + "precision_at_100": 0.519, + "precision_at_1000": 0.09099, + "precision_at_20": 0.77143, + "precision_at_3": 0.87619, + "precision_at_5": 0.86286, + "recall_at_1": 0.02765, + "recall_at_10": 0.1307, + "recall_at_100": 0.60631, + "recall_at_1000": 0.93409, + "recall_at_20": 0.22272, + "recall_at_3": 0.05209, + "recall_at_5": 0.07985 + } + ] + }, + "task_name": "RParisEasyI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json new file mode 100644 index 0000000000..5b15d0ae51 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisHardI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "fd121b6592fe946616fa85116703b94a4c61fd63", + "evaluation_time": 22.44923186302185, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.04286, + "cv_recall_at_10": 0.41429, + "cv_recall_at_100": 0.97143, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 0.6, + "cv_recall_at_3": 0.17143, + "cv_recall_at_5": 0.27143, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.04286, + "map_at_1": 0.00032, + "map_at_10": 0.00319, + "map_at_100": 0.04427, + "map_at_1000": 0.14044, + "map_at_20": 0.00602, + "map_at_3": 0.00096, + "map_at_5": 0.00146, + "mrr_at_1": 0.04285714285714286, + "mrr_at_10": 0.13515873015873012, + "mrr_at_100": 0.15938730320565236, + "mrr_at_1000": 0.15962520658618265, + "mrr_at_20": 0.14815883634769078, + "mrr_at_3": 0.0976190476190476, + "mrr_at_5": 0.11833333333333333, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": -0.6909430438842217, + "nauc_cv_recall_at_100_max": 0.6790382819794577, + "nauc_cv_recall_at_100_std": 0.4143323996265152, + "nauc_cv_recall_at_10_diff1": 0.2550034187718293, + "nauc_cv_recall_at_10_max": -0.26788387263688956, + "nauc_cv_recall_at_10_std": -0.3843993125496647, + "nauc_cv_recall_at_1_diff1": 0.3989804132009661, + "nauc_cv_recall_at_1_max": -0.12905822377247125, + "nauc_cv_recall_at_1_std": -0.401663536356319, + "nauc_cv_recall_at_20_diff1": 0.12233388457787381, + "nauc_cv_recall_at_20_max": -0.21169910528770744, + "nauc_cv_recall_at_20_std": -0.2534876818046898, + "nauc_cv_recall_at_3_diff1": 0.35100458047053923, + "nauc_cv_recall_at_3_max": -0.1396262752446388, + "nauc_cv_recall_at_3_std": -0.314985425775557, + "nauc_cv_recall_at_5_diff1": 0.35853258558905104, + "nauc_cv_recall_at_5_max": -0.1935699390601907, + "nauc_cv_recall_at_5_std": -0.4389893304719292, + "nauc_map_at_1000_diff1": -0.048721647577313475, + "nauc_map_at_1000_max": 0.02185883585100512, + "nauc_map_at_1000_std": -0.014590760062175558, + "nauc_map_at_100_diff1": 0.20175041133547802, + "nauc_map_at_100_max": -0.2999406736946551, + "nauc_map_at_100_std": -0.21513200915624464, + "nauc_map_at_10_diff1": 0.4340980160657004, + "nauc_map_at_10_max": -0.30137691467001876, + "nauc_map_at_10_std": -0.4166728188377905, + "nauc_map_at_1_diff1": 0.3225400054210336, + "nauc_map_at_1_max": -0.20455804791357643, + "nauc_map_at_1_std": -0.3502703918125217, + "nauc_map_at_20_diff1": 0.4482686549814925, + "nauc_map_at_20_max": -0.33767777626178874, + "nauc_map_at_20_std": -0.39261152175985364, + "nauc_map_at_3_diff1": 0.3906475942798504, + "nauc_map_at_3_max": -0.1421273626640729, + "nauc_map_at_3_std": -0.3164227766935927, + "nauc_map_at_5_diff1": 0.40607500232859556, + "nauc_map_at_5_max": -0.21636348183658996, + "nauc_map_at_5_std": -0.385395422874435, + "nauc_mrr_at_1000_diff1": 0.33380716561285934, + "nauc_mrr_at_1000_max": -0.19720559281432273, + "nauc_mrr_at_1000_std": -0.3739725633585593, + "nauc_mrr_at_100_diff1": 0.3320423802626735, + "nauc_mrr_at_100_max": -0.19584551091041755, + "nauc_mrr_at_100_std": -0.37267220068248286, + "nauc_mrr_at_10_diff1": 0.33810313585026747, + "nauc_mrr_at_10_max": -0.20898927155954197, + "nauc_mrr_at_10_std": -0.39317334419879707, + "nauc_mrr_at_1_diff1": 0.3989804132009661, + "nauc_mrr_at_1_max": -0.12905822377247125, + "nauc_mrr_at_1_std": -0.401663536356319, + "nauc_mrr_at_20_diff1": 0.3270365458084192, + "nauc_mrr_at_20_max": -0.206603016304459, + "nauc_mrr_at_20_std": -0.3731460382346558, + "nauc_mrr_at_3_diff1": 0.3569994314378864, + "nauc_mrr_at_3_max": -0.17174174325462235, + "nauc_mrr_at_3_std": -0.3561501867618589, + "nauc_mrr_at_5_diff1": 0.3571575774665211, + "nauc_mrr_at_5_max": -0.18834228874209263, + "nauc_mrr_at_5_std": -0.40200287994954503, + "nauc_ndcg_at_1000_diff1": -0.0743652757576749, + "nauc_ndcg_at_1000_max": 0.09391843454021216, + "nauc_ndcg_at_1000_std": 0.08425685683454673, + "nauc_ndcg_at_100_diff1": 0.023516077444591114, + "nauc_ndcg_at_100_max": -0.11892313686780007, + "nauc_ndcg_at_100_std": -0.16710001090586352, + "nauc_ndcg_at_10_diff1": 0.3882520721308202, + "nauc_ndcg_at_10_max": -0.2924404653976282, + "nauc_ndcg_at_10_std": -0.4189010025783995, + "nauc_ndcg_at_1_diff1": 0.3989804132009661, + "nauc_ndcg_at_1_max": -0.12905822377247125, + "nauc_ndcg_at_1_std": -0.401663536356319, + "nauc_ndcg_at_20_diff1": 0.3805388505094879, + "nauc_ndcg_at_20_max": -0.30519760291519316, + "nauc_ndcg_at_20_std": -0.3772566200501027, + "nauc_ndcg_at_3_diff1": 0.412201762045363, + "nauc_ndcg_at_3_max": -0.17116948069763432, + "nauc_ndcg_at_3_std": -0.3608861222611007, + "nauc_ndcg_at_5_diff1": 0.35696244084680845, + "nauc_ndcg_at_5_max": -0.25186160853169576, + "nauc_ndcg_at_5_std": -0.400509495428499, + "nauc_precision_at_1000_diff1": -0.22093427992384315, + "nauc_precision_at_1000_max": 0.2196784647696767, + "nauc_precision_at_1000_std": 0.04361074681539203, + "nauc_precision_at_100_diff1": -0.016149306619419775, + "nauc_precision_at_100_max": 0.034799312614065006, + "nauc_precision_at_100_std": -0.13882957890950748, + "nauc_precision_at_10_diff1": 0.38489780804914814, + "nauc_precision_at_10_max": -0.30500607571435273, + "nauc_precision_at_10_std": -0.41532331666205985, + "nauc_precision_at_1_diff1": 0.3989804132009661, + "nauc_precision_at_1_max": -0.12905822377247125, + "nauc_precision_at_1_std": -0.401663536356319, + "nauc_precision_at_20_diff1": 0.3692528574157947, + "nauc_precision_at_20_max": -0.30111599366142605, + "nauc_precision_at_20_std": -0.35015003540161194, + "nauc_precision_at_3_diff1": 0.428985570425912, + "nauc_precision_at_3_max": -0.1563930424168448, + "nauc_precision_at_3_std": -0.3449801647848643, + "nauc_precision_at_5_diff1": 0.34983601813446513, + "nauc_precision_at_5_max": -0.2741390952059421, + "nauc_precision_at_5_std": -0.4086645123951, + "nauc_recall_at_1000_diff1": -0.08562125010420926, + "nauc_recall_at_1000_max": 0.07813883934551541, + "nauc_recall_at_1000_std": 0.23218650354926193, + "nauc_recall_at_100_diff1": -0.01041195221018056, + "nauc_recall_at_100_max": -0.23261397534923414, + "nauc_recall_at_100_std": -0.05979942942574269, + "nauc_recall_at_10_diff1": 0.3361904724346995, + "nauc_recall_at_10_max": -0.28110372866797806, + "nauc_recall_at_10_std": -0.36784022778417585, + "nauc_recall_at_1_diff1": 0.3225400054210336, + "nauc_recall_at_1_max": -0.20455804791357643, + "nauc_recall_at_1_std": -0.3502703918125217, + "nauc_recall_at_20_diff1": 0.31944679632949374, + "nauc_recall_at_20_max": -0.32588194303425905, + "nauc_recall_at_20_std": -0.25546971593163326, + "nauc_recall_at_3_diff1": 0.36197730614062007, + "nauc_recall_at_3_max": -0.07376636861137077, + "nauc_recall_at_3_std": -0.2538555661579059, + "nauc_recall_at_5_diff1": 0.3711994561836934, + "nauc_recall_at_5_max": -0.1684006987187415, + "nauc_recall_at_5_std": -0.373838797717674, + "ndcg_at_1": 0.04286, + "ndcg_at_10": 0.08888, + "ndcg_at_100": 0.20767, + "ndcg_at_1000": 0.46304, + "ndcg_at_20": 0.0984, + "ndcg_at_3": 0.06137, + "ndcg_at_5": 0.07145, + "precision_at_1": 0.04286, + "precision_at_10": 0.10143, + "precision_at_100": 0.202, + "precision_at_1000": 0.09564, + "precision_at_20": 0.10857, + "precision_at_3": 0.06667, + "precision_at_5": 0.08, + "recall_at_1": 0.00032, + "recall_at_10": 0.0087, + "recall_at_100": 0.18965, + "recall_at_1000": 0.67973, + "recall_at_20": 0.01989, + "recall_at_3": 0.00187, + "recall_at_5": 0.00327 + } + ] + }, + "task_name": "RParisHardI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisI2IRetrieval.json deleted file mode 100644 index 0f309a07f3..0000000000 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisI2IRetrieval.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "dataset_revision": "bafc3a08fdffd72558021ce3a41250833d7e0e88", - "evaluation_time": 43.56014585494995, - "kg_co2_emissions": null, - "mteb_version": "1.12.90", - "scores": { - "test": [ - { - "cv_recall_at_1": 0.78348, - "cv_recall_at_10": 0.96558, - "cv_recall_at_100": 0.99812, - "cv_recall_at_1000": 1.0, - "cv_recall_at_20": 0.98217, - "cv_recall_at_3": 0.89972, - "cv_recall_at_5": 0.93367, - "hf_subset": "default", - "languages": [ - "eng-Latn" - ], - "main_score": 0.78348, - "map_at_1": 0.00183, - "map_at_10": 0.01259, - "map_at_100": 0.08554, - "map_at_1000": 0.22541, - "map_at_20": 0.02257, - "map_at_3": 0.0046, - "map_at_5": 0.00704, - "mrr_at_1": 0.9870150187734669, - "mrr_at_10": 0.9935075093867334, - "mrr_at_100": 0.9935075093867334, - "mrr_at_1000": 0.9935075093867334, - "mrr_at_20": 0.9935075093867334, - "mrr_at_3": 0.9935075093867334, - "mrr_at_5": 0.9935075093867334, - "nauc_cv_recall_at_1000_diff1": NaN, - "nauc_cv_recall_at_1000_max": NaN, - "nauc_cv_recall_at_1000_std": NaN, - "nauc_cv_recall_at_100_diff1": 0.12564420871970045, - "nauc_cv_recall_at_100_max": 0.3655746256621164, - "nauc_cv_recall_at_100_std": -0.2306365991736563, - "nauc_cv_recall_at_10_diff1": -0.5058662936618029, - "nauc_cv_recall_at_10_max": 0.09207458773253613, - "nauc_cv_recall_at_10_std": 0.4663743642051124, - "nauc_cv_recall_at_1_diff1": -0.4013157463788695, - "nauc_cv_recall_at_1_max": 0.013502025515305102, - "nauc_cv_recall_at_1_std": 0.32242775071432206, - "nauc_cv_recall_at_20_diff1": -0.5198855681306671, - "nauc_cv_recall_at_20_max": 0.10151852705481003, - "nauc_cv_recall_at_20_std": 0.41412564884036973, - "nauc_cv_recall_at_3_diff1": -0.5198700551458323, - "nauc_cv_recall_at_3_max": 0.025885833103837443, - "nauc_cv_recall_at_3_std": 0.4187563971498901, - "nauc_cv_recall_at_5_diff1": -0.5209006482350635, - "nauc_cv_recall_at_5_max": 0.051797269624278094, - "nauc_cv_recall_at_5_std": 0.4360707529546872, - "nauc_map_at_1000_diff1": -0.32518052685711313, - "nauc_map_at_1000_max": 0.0038005812075390727, - "nauc_map_at_1000_std": 0.43958508353436493, - "nauc_map_at_100_diff1": -0.324729620306574, - "nauc_map_at_100_max": 0.003712984860206863, - "nauc_map_at_100_std": 0.3530965583894132, - "nauc_map_at_10_diff1": -0.2865369016119553, - "nauc_map_at_10_max": 0.012519773293503903, - "nauc_map_at_10_std": 0.23475872496725028, - "nauc_map_at_1_diff1": -0.0855732613663, - "nauc_map_at_1_max": 0.023221460373481606, - "nauc_map_at_1_std": 0.03199412568851411, - "nauc_map_at_20_diff1": -0.3089685409009752, - "nauc_map_at_20_max": 0.008164069309369689, - "nauc_map_at_20_std": 0.28138880870717525, - "nauc_map_at_3_diff1": -0.21596259885289204, - "nauc_map_at_3_max": 0.026694429615608712, - "nauc_map_at_3_std": 0.13815328440691188, - "nauc_map_at_5_diff1": -0.2539278390954876, - "nauc_map_at_5_max": 0.01952941733527699, - "nauc_map_at_5_std": 0.18318190468395143, - "nauc_mrr_at_1000_diff1": 1.0, - "nauc_mrr_at_1000_max": -0.13028347793646336, - "nauc_mrr_at_1000_std": -0.28303491792776114, - "nauc_mrr_at_100_diff1": 1.0, - "nauc_mrr_at_100_max": -0.13028347793646336, - "nauc_mrr_at_100_std": -0.28303491792776114, - "nauc_mrr_at_10_diff1": 1.0, - "nauc_mrr_at_10_max": -0.13028347793646336, - "nauc_mrr_at_10_std": -0.28303491792776114, - "nauc_mrr_at_1_diff1": 1.0, - "nauc_mrr_at_1_max": -0.1302834779364721, - "nauc_mrr_at_1_std": -0.2830349179277583, - "nauc_mrr_at_20_diff1": 1.0, - "nauc_mrr_at_20_max": -0.13028347793646336, - "nauc_mrr_at_20_std": -0.28303491792776114, - "nauc_mrr_at_3_diff1": 1.0, - "nauc_mrr_at_3_max": -0.13028347793646336, - "nauc_mrr_at_3_std": -0.28303491792776114, - "nauc_mrr_at_5_diff1": 1.0, - "nauc_mrr_at_5_max": -0.13028347793646336, - "nauc_mrr_at_5_std": -0.28303491792776114, - "nauc_ndcg_at_1000_diff1": -0.3461037602745701, - "nauc_ndcg_at_1000_max": 0.01170706578078914, - "nauc_ndcg_at_1000_std": 0.40425946623874776, - "nauc_ndcg_at_100_diff1": -0.39901516979641344, - "nauc_ndcg_at_100_max": -0.00685883548003498, - "nauc_ndcg_at_100_std": 0.465350992380673, - "nauc_ndcg_at_10_diff1": -0.3955198632793018, - "nauc_ndcg_at_10_max": -0.0037363142445774643, - "nauc_ndcg_at_10_std": 0.38110734984115535, - "nauc_ndcg_at_1_diff1": 1.0, - "nauc_ndcg_at_1_max": -0.14486457971944697, - "nauc_ndcg_at_1_std": -0.2830349179277583, - "nauc_ndcg_at_20_diff1": -0.41278062763010503, - "nauc_ndcg_at_20_max": -0.004138702255930161, - "nauc_ndcg_at_20_std": 0.4191652248206659, - "nauc_ndcg_at_3_diff1": -0.34367743707797543, - "nauc_ndcg_at_3_max": 0.004103711367444123, - "nauc_ndcg_at_3_std": 0.3101278349486688, - "nauc_ndcg_at_5_diff1": -0.37388771810166976, - "nauc_ndcg_at_5_max": -0.004694760292318884, - "nauc_ndcg_at_5_std": 0.3429402757237829, - "nauc_precision_at_1000_diff1": -0.1804983224177905, - "nauc_precision_at_1000_max": -0.01148320196422333, - "nauc_precision_at_1000_std": 0.2504460959613102, - "nauc_precision_at_100_diff1": -0.3911848294276243, - "nauc_precision_at_100_max": -0.007414470176849031, - "nauc_precision_at_100_std": 0.4673613835957893, - "nauc_precision_at_10_diff1": -0.4052976521224889, - "nauc_precision_at_10_max": -0.00433906156493395, - "nauc_precision_at_10_std": 0.39212256495307285, - "nauc_precision_at_1_diff1": 1.0, - "nauc_precision_at_1_max": -0.14486457971944697, - "nauc_precision_at_1_std": -0.2830349179277583, - "nauc_precision_at_20_diff1": -0.4173913678934324, - "nauc_precision_at_20_max": -0.004543026048413097, - "nauc_precision_at_20_std": 0.4299497661264597, - "nauc_precision_at_3_diff1": -0.37619357586668295, - "nauc_precision_at_3_max": 0.006637408448532989, - "nauc_precision_at_3_std": 0.3255087096886432, - "nauc_precision_at_5_diff1": -0.393543490911112, - "nauc_precision_at_5_max": -0.0054501833188476565, - "nauc_precision_at_5_std": 0.3553468241787662, - "nauc_recall_at_1000_diff1": -0.28865460441970253, - "nauc_recall_at_1000_max": 0.02031269144932309, - "nauc_recall_at_1000_std": 0.3581111189760722, - "nauc_recall_at_100_diff1": -0.31737603693043515, - "nauc_recall_at_100_max": 0.010815413934469631, - "nauc_recall_at_100_std": 0.32072251393381573, - "nauc_recall_at_10_diff1": -0.2835809857944858, - "nauc_recall_at_10_max": 0.01631628873177338, - "nauc_recall_at_10_std": 0.2203863362341707, - "nauc_recall_at_1_diff1": -0.0855732613663, - "nauc_recall_at_1_max": 0.023221460373481606, - "nauc_recall_at_1_std": 0.03199412568851411, - "nauc_recall_at_20_diff1": -0.30410786586422606, - "nauc_recall_at_20_max": 0.012058890403941508, - "nauc_recall_at_20_std": 0.2609223499236706, - "nauc_recall_at_3_diff1": -0.22686364377869106, - "nauc_recall_at_3_max": 0.02838523233952593, - "nauc_recall_at_3_std": 0.13768932686394217, - "nauc_recall_at_5_diff1": -0.2568046605350692, - "nauc_recall_at_5_max": 0.02092009985156803, - "nauc_recall_at_5_std": 0.17610456507060399, - "ndcg_at_1": 0.98702, - "ndcg_at_10": 0.76647, - "ndcg_at_100": 0.59394, - "ndcg_at_1000": 0.44945, - "ndcg_at_20": 0.71423, - "ndcg_at_3": 0.8678, - "ndcg_at_5": 0.82287, - "precision_at_1": 0.98702, - "precision_at_10": 0.72226, - "precision_at_100": 0.5574, - "precision_at_1000": 0.25148, - "precision_at_20": 0.67042, - "precision_at_3": 0.83521, - "precision_at_5": 0.7832, - "recall_at_1": 0.00183, - "recall_at_10": 0.01349, - "recall_at_100": 0.10451, - "recall_at_1000": 0.40829, - "recall_at_20": 0.02511, - "recall_at_3": 0.00467, - "recall_at_5": 0.0073 - } - ] - }, - "task_name": "RParisI2IRetrieval" -} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json new file mode 100644 index 0000000000..700a4a2774 --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/RParisMediumI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "900267b49003a086979e8d52f6942624236bfc34", + "evaluation_time": 23.04814124107361, + "kg_co2_emissions": null, + "mteb_version": "1.12.90", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.98571, + "cv_recall_at_10": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.98571, + "map_at_1": 0.0055, + "map_at_10": 0.0505, + "map_at_100": 0.32128, + "map_at_1000": 0.53905, + "map_at_20": 0.09538, + "map_at_3": 0.01586, + "map_at_5": 0.02626, + "mrr_at_1": 0.9857142857142858, + "mrr_at_10": 0.9928571428571429, + "mrr_at_100": 0.9928571428571429, + "mrr_at_1000": 0.9928571428571429, + "mrr_at_20": 0.9928571428571429, + "mrr_at_3": 0.9928571428571429, + "mrr_at_5": 0.9928571428571429, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_1_diff1": 0.8692810457516342, + "nauc_cv_recall_at_1_max": -1.151727357609713, + "nauc_cv_recall_at_1_std": -0.5634920634920669, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_map_at_1000_diff1": -0.5202144222378728, + "nauc_map_at_1000_max": 0.11126568280766698, + "nauc_map_at_1000_std": 0.32813155264948524, + "nauc_map_at_100_diff1": -0.17082735069359992, + "nauc_map_at_100_max": -0.07951969633170079, + "nauc_map_at_100_std": 0.13798406208276, + "nauc_map_at_10_diff1": 0.08925396593997569, + "nauc_map_at_10_max": -0.078676032579074, + "nauc_map_at_10_std": 0.13021224342356594, + "nauc_map_at_1_diff1": 0.20034146659034022, + "nauc_map_at_1_max": -0.1488526539306114, + "nauc_map_at_1_std": 0.036837238597286596, + "nauc_map_at_20_diff1": 0.044029095752394776, + "nauc_map_at_20_max": -0.044115438949865464, + "nauc_map_at_20_std": 0.16589359171211718, + "nauc_map_at_3_diff1": 0.12936241106392424, + "nauc_map_at_3_max": -0.11171026414768116, + "nauc_map_at_3_std": 0.09557523635145118, + "nauc_map_at_5_diff1": 0.12090379123610429, + "nauc_map_at_5_max": -0.11656881579212829, + "nauc_map_at_5_std": 0.09921919672363319, + "nauc_mrr_at_1000_diff1": 0.8692810457516355, + "nauc_mrr_at_1000_max": -1.1517273576096931, + "nauc_mrr_at_1000_std": -0.5634920634920526, + "nauc_mrr_at_100_diff1": 0.8692810457516355, + "nauc_mrr_at_100_max": -1.1517273576096931, + "nauc_mrr_at_100_std": -0.5634920634920526, + "nauc_mrr_at_10_diff1": 0.8692810457516355, + "nauc_mrr_at_10_max": -1.1517273576096931, + "nauc_mrr_at_10_std": -0.5634920634920526, + "nauc_mrr_at_1_diff1": 0.8692810457516342, + "nauc_mrr_at_1_max": -1.151727357609713, + "nauc_mrr_at_1_std": -0.5634920634920669, + "nauc_mrr_at_20_diff1": 0.8692810457516355, + "nauc_mrr_at_20_max": -1.1517273576096931, + "nauc_mrr_at_20_std": -0.5634920634920526, + "nauc_mrr_at_3_diff1": 0.8692810457516355, + "nauc_mrr_at_3_max": -1.1517273576096931, + "nauc_mrr_at_3_std": -0.5634920634920526, + "nauc_mrr_at_5_diff1": 0.8692810457516355, + "nauc_mrr_at_5_max": -1.1517273576096931, + "nauc_mrr_at_5_std": -0.5634920634920526, + "nauc_ndcg_at_1000_diff1": -0.44820569134469457, + "nauc_ndcg_at_1000_max": 0.14278328260517373, + "nauc_ndcg_at_1000_std": 0.35909204639356485, + "nauc_ndcg_at_100_diff1": -0.6018888319473394, + "nauc_ndcg_at_100_max": 0.21192336225080224, + "nauc_ndcg_at_100_std": 0.25475321800547673, + "nauc_ndcg_at_10_diff1": -0.5815570895956712, + "nauc_ndcg_at_10_max": 0.2384364105122267, + "nauc_ndcg_at_10_std": 0.47588628797056887, + "nauc_ndcg_at_1_diff1": 0.8692810457516342, + "nauc_ndcg_at_1_max": -1.151727357609713, + "nauc_ndcg_at_1_std": -0.5634920634920669, + "nauc_ndcg_at_20_diff1": -0.5953841029416077, + "nauc_ndcg_at_20_max": 0.32157620990503083, + "nauc_ndcg_at_20_std": 0.47071964995241883, + "nauc_ndcg_at_3_diff1": -0.47333802126140007, + "nauc_ndcg_at_3_max": 0.13407004770022382, + "nauc_ndcg_at_3_std": 0.46679765020950414, + "nauc_ndcg_at_5_diff1": -0.5587389095148461, + "nauc_ndcg_at_5_max": 0.07339891244234566, + "nauc_ndcg_at_5_std": 0.47215237618630984, + "nauc_precision_at_1000_diff1": -0.3950971876695561, + "nauc_precision_at_1000_max": 0.24406161426957987, + "nauc_precision_at_1000_std": 0.12174947607128742, + "nauc_precision_at_100_diff1": -0.5472068296947269, + "nauc_precision_at_100_max": 0.23378022737023205, + "nauc_precision_at_100_std": 0.1829472692928344, + "nauc_precision_at_10_diff1": -0.5950517610712319, + "nauc_precision_at_10_max": 0.28672134365425656, + "nauc_precision_at_10_std": 0.48780862485384807, + "nauc_precision_at_1_diff1": 0.8692810457516342, + "nauc_precision_at_1_max": -1.151727357609713, + "nauc_precision_at_1_std": -0.5634920634920669, + "nauc_precision_at_20_diff1": -0.5947364591012498, + "nauc_precision_at_20_max": 0.35422908868857506, + "nauc_precision_at_20_std": 0.4705389149562903, + "nauc_precision_at_3_diff1": -0.510094916344916, + "nauc_precision_at_3_max": 0.22297297297297156, + "nauc_precision_at_3_std": 0.5162886100386108, + "nauc_precision_at_5_diff1": -0.6028283796740131, + "nauc_precision_at_5_max": 0.11490891658676922, + "nauc_precision_at_5_std": 0.504458293384468, + "nauc_recall_at_1000_diff1": -0.2460203893111549, + "nauc_recall_at_1000_max": 0.07681666757366377, + "nauc_recall_at_1000_std": 0.29402261311504047, + "nauc_recall_at_100_diff1": -0.10390008545301123, + "nauc_recall_at_100_max": -0.09447732649357599, + "nauc_recall_at_100_std": 0.0928830407002215, + "nauc_recall_at_10_diff1": 0.10590480631127752, + "nauc_recall_at_10_max": -0.08162621409653509, + "nauc_recall_at_10_std": 0.11847839880663995, + "nauc_recall_at_1_diff1": 0.20034146659034022, + "nauc_recall_at_1_max": -0.1488526539306114, + "nauc_recall_at_1_std": 0.036837238597286596, + "nauc_recall_at_20_diff1": 0.06586007946887822, + "nauc_recall_at_20_max": -0.05281307817884481, + "nauc_recall_at_20_std": 0.14970188120798883, + "nauc_recall_at_3_diff1": 0.13126830970729877, + "nauc_recall_at_3_max": -0.10596900293507791, + "nauc_recall_at_3_std": 0.09476793236811286, + "nauc_recall_at_5_diff1": 0.12983221018552543, + "nauc_recall_at_5_max": -0.11532911944953175, + "nauc_recall_at_5_std": 0.09212571294283788, + "ndcg_at_1": 0.98571, + "ndcg_at_10": 0.92841, + "ndcg_at_100": 0.77588, + "ndcg_at_1000": 0.7685, + "ndcg_at_20": 0.89894, + "ndcg_at_3": 0.95291, + "ndcg_at_5": 0.94991, + "precision_at_1": 0.98571, + "precision_at_10": 0.91571, + "precision_at_100": 0.721, + "precision_at_1000": 0.18663, + "precision_at_20": 0.88, + "precision_at_3": 0.94286, + "precision_at_5": 0.94286, + "recall_at_1": 0.0055, + "recall_at_10": 0.05165, + "recall_at_100": 0.36172, + "recall_at_1000": 0.79065, + "recall_at_20": 0.0987, + "recall_at_3": 0.01595, + "recall_at_5": 0.0266 + } + ] + }, + "task_name": "RParisMediumI2IRetrieval" +} \ No newline at end of file