diff --git a/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py b/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py index 051aa324e6..6f19a91f44 100644 --- a/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py +++ b/mteb/tasks/Image/Any2AnyRetrieval/eng/FORBI2IRetrieval.py @@ -11,7 +11,7 @@ class FORBI2I(AbsTaskAny2AnyRetrieval): reference="https://github.com/pxiangwu/FORB", dataset={ "path": "isaacchung/forb_retrieval", - "revision": "336607d5bcc853fb7f7276c2c9721d4b5b1ca8e4", + "revision": "26ab4bd972854becada339afc80f5f3ffc047e2b", }, type="Retrieval", category="i2i", diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Caltech101.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Caltech101.json new file mode 100644 index 0000000000..9cf71ae0fc --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Caltech101.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "851374102055782c84f89b1b4e9d128a6568847b", + "evaluation_time": 140.4232726097107, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "accuracy": 0.9374424720578567, + "f1": 0.8896507530328425, + "f1_weighted": 0.9381916904354753, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9374424720578567, + "scores_per_experiment": [ + { + "accuracy": 0.9316239316239316, + "f1": 0.8916896755795425, + "f1_weighted": 0.9320804382676573 + }, + { + "accuracy": 0.9421433267587114, + "f1": 0.8911440787847722, + "f1_weighted": 0.9432545002991353 + }, + { + "accuracy": 0.9349112426035503, + "f1": 0.8901121996677432, + "f1_weighted": 0.9358989633433914 + }, + { + "accuracy": 0.9373767258382643, + "f1": 0.8843712891021908, + "f1_weighted": 0.9381450174798306 + }, + { + "accuracy": 0.9411571334648258, + "f1": 0.8909365220299629, + "f1_weighted": 0.9415795327873617 + } + ] + } + ] + }, + "task_name": "Caltech101" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/FORBI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/FORBI2IRetrieval.json new file mode 100644 index 0000000000..4d3811487d --- /dev/null +++ b/results-mieb/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/FORBI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "26ab4bd972854becada339afc80f5f3ffc047e2b", + "evaluation_time": 1242.6785640716553, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.40136, + "cv_recall_at_10": 0.55162, + "cv_recall_at_100": 0.66423, + "cv_recall_at_1000": 0.7397, + "cv_recall_at_20": 0.58755, + "cv_recall_at_3": 0.48317, + "cv_recall_at_5": 0.51185, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.40136, + "map_at_1": 0.40023, + "map_at_10": 0.44883, + "map_at_100": 0.45322, + "map_at_1000": 0.45356, + "map_at_20": 0.45134, + "map_at_3": 0.43694, + "map_at_5": 0.4435, + "mrr_at_1": 0.40135849056603773, + "mrr_at_10": 0.44943953279424914, + "mrr_at_100": 0.4538207184180781, + "mrr_at_1000": 0.4541651229469619, + "mrr_at_20": 0.4519378833805158, + "mrr_at_3": 0.43757232704402443, + "mrr_at_5": 0.44411194968553264, + "nauc_cv_recall_at_1000_diff1": 0.9500334258065578, + "nauc_cv_recall_at_1000_max": -1.2528093194175578, + "nauc_cv_recall_at_1000_std": -1.1856652591596875, + "nauc_cv_recall_at_100_diff1": 0.877926997353908, + "nauc_cv_recall_at_100_max": -0.8870905020174381, + "nauc_cv_recall_at_100_std": -0.8732695918632056, + "nauc_cv_recall_at_10_diff1": 0.8177473807784005, + "nauc_cv_recall_at_10_max": -0.5466647529055957, + "nauc_cv_recall_at_10_std": -0.6554690765334921, + "nauc_cv_recall_at_1_diff1": 0.8246695670414056, + "nauc_cv_recall_at_1_max": -0.3034632255124627, + "nauc_cv_recall_at_1_std": -0.4858988248874936, + "nauc_cv_recall_at_20_diff1": 0.82479953127787, + "nauc_cv_recall_at_20_max": -0.6318860267460972, + "nauc_cv_recall_at_20_std": -0.7024579748752124, + "nauc_cv_recall_at_3_diff1": 0.8072308741555094, + "nauc_cv_recall_at_3_max": -0.4123387172398377, + "nauc_cv_recall_at_3_std": -0.5690279516462133, + "nauc_cv_recall_at_5_diff1": 0.8047845690821774, + "nauc_cv_recall_at_5_max": -0.46313010588875514, + "nauc_cv_recall_at_5_std": -0.5999518918827577, + "nauc_map_at_1000_diff1": 0.821123720578351, + "nauc_map_at_1000_max": -0.380874046553554, + "nauc_map_at_1000_std": -0.5404325379702113, + "nauc_map_at_100_diff1": 0.8209485062871567, + "nauc_map_at_100_max": -0.3800979301805527, + "nauc_map_at_100_std": -0.5397866691260815, + "nauc_map_at_10_diff1": 0.8195428750181966, + "nauc_map_at_10_max": -0.37103423612650216, + "nauc_map_at_10_std": -0.5343816263585759, + "nauc_map_at_1_diff1": 0.8270196566959886, + "nauc_map_at_1_max": -0.30320990154210953, + "nauc_map_at_1_std": -0.4833533184781571, + "nauc_map_at_20_diff1": 0.820009618850825, + "nauc_map_at_20_max": -0.3759310074719576, + "nauc_map_at_20_std": -0.5369871242494627, + "nauc_map_at_3_diff1": 0.8186425667711692, + "nauc_map_at_3_max": -0.3502082517227007, + "nauc_map_at_3_std": -0.5210663781673923, + "nauc_map_at_5_diff1": 0.8178707187282951, + "nauc_map_at_5_max": -0.36111543033237625, + "nauc_map_at_5_std": -0.5277312572627604, + "nauc_mrr_at_1000_diff1": 0.8198732381938787, + "nauc_mrr_at_1000_max": -0.3810706994001967, + "nauc_mrr_at_1000_std": -0.5418434797910091, + "nauc_mrr_at_100_diff1": 0.8196987253945341, + "nauc_mrr_at_100_max": -0.38029428979297347, + "nauc_mrr_at_100_std": -0.5411966208887677, + "nauc_mrr_at_10_diff1": 0.8182978327883684, + "nauc_mrr_at_10_max": -0.3712361113968832, + "nauc_mrr_at_10_std": -0.5358019360627169, + "nauc_mrr_at_1_diff1": 0.8246695670414056, + "nauc_mrr_at_1_max": -0.3034632255124627, + "nauc_mrr_at_1_std": -0.4858988248874936, + "nauc_mrr_at_20_diff1": 0.8187629322864929, + "nauc_mrr_at_20_max": -0.3761251367912585, + "nauc_mrr_at_20_std": -0.538391953833271, + "nauc_mrr_at_3_diff1": 0.8173376655828167, + "nauc_mrr_at_3_max": -0.35042983995133714, + "nauc_mrr_at_3_std": -0.5225353792328652, + "nauc_mrr_at_5_diff1": 0.8166387092255286, + "nauc_mrr_at_5_max": -0.3613393270069024, + "nauc_mrr_at_5_std": -0.5291442800691877, + "nauc_ndcg_at_1000_diff1": 0.8334868024040154, + "nauc_ndcg_at_1000_max": -0.48731436283957213, + "nauc_ndcg_at_1000_std": -0.6163695321444129, + "nauc_ndcg_at_100_diff1": 0.8281459653974711, + "nauc_ndcg_at_100_max": -0.4631929003672613, + "nauc_ndcg_at_100_std": -0.594729965146729, + "nauc_ndcg_at_10_diff1": 0.8186924353738807, + "nauc_ndcg_at_10_max": -0.4106178964102518, + "nauc_ndcg_at_10_std": -0.5618271856100127, + "nauc_ndcg_at_1_diff1": 0.8270196566959886, + "nauc_ndcg_at_1_max": -0.30320990154210953, + "nauc_ndcg_at_1_std": -0.4833533184781571, + "nauc_ndcg_at_20_diff1": 0.8203675518215456, + "nauc_ndcg_at_20_max": -0.42922604110746493, + "nauc_ndcg_at_20_std": -0.5718977180047882, + "nauc_ndcg_at_3_diff1": 0.8158251981916631, + "nauc_ndcg_at_3_max": -0.3659119877900009, + "nauc_ndcg_at_3_std": -0.5331616044670827, + "nauc_ndcg_at_5_diff1": 0.8145904125253537, + "nauc_ndcg_at_5_max": -0.38591697079578874, + "nauc_ndcg_at_5_std": -0.5454027961145099, + "nauc_precision_at_1000_diff1": 0.9500334258065563, + "nauc_precision_at_1000_max": -1.2528093194175558, + "nauc_precision_at_1000_std": -1.1856652591596857, + "nauc_precision_at_100_diff1": 0.877926997353908, + "nauc_precision_at_100_max": -0.8870905020174389, + "nauc_precision_at_100_std": -0.8732695918632062, + "nauc_precision_at_10_diff1": 0.817762219827385, + "nauc_precision_at_10_max": -0.5465388231157847, + "nauc_precision_at_10_std": -0.6552180490907682, + "nauc_precision_at_1_diff1": 0.8270196566959886, + "nauc_precision_at_1_max": -0.30320990154210953, + "nauc_precision_at_1_std": -0.4833533184781571, + "nauc_precision_at_20_diff1": 0.824799531277869, + "nauc_precision_at_20_max": -0.6318860267460973, + "nauc_precision_at_20_std": -0.7024579748752124, + "nauc_precision_at_3_diff1": 0.8075292153589316, + "nauc_precision_at_3_max": -0.4121216998390528, + "nauc_precision_at_3_std": -0.568597762738296, + "nauc_precision_at_5_diff1": 0.8047485091566745, + "nauc_precision_at_5_max": -0.46287592219959217, + "nauc_precision_at_5_std": -0.5997842327013363, + "nauc_recall_at_1000_diff1": 0.9500334258065578, + "nauc_recall_at_1000_max": -1.2528093194175578, + "nauc_recall_at_1000_std": -1.1856652591596875, + "nauc_recall_at_100_diff1": 0.877926997353908, + "nauc_recall_at_100_max": -0.8870905020174381, + "nauc_recall_at_100_std": -0.8732695918632056, + "nauc_recall_at_10_diff1": 0.8177622198273848, + "nauc_recall_at_10_max": -0.5465388231157855, + "nauc_recall_at_10_std": -0.6552180490907689, + "nauc_recall_at_1_diff1": 0.8270196566959886, + "nauc_recall_at_1_max": -0.30320990154210953, + "nauc_recall_at_1_std": -0.4833533184781571, + "nauc_recall_at_20_diff1": 0.82479953127787, + "nauc_recall_at_20_max": -0.6318860267460972, + "nauc_recall_at_20_std": -0.7024579748752124, + "nauc_recall_at_3_diff1": 0.8075292153589314, + "nauc_recall_at_3_max": -0.41212169983905295, + "nauc_recall_at_3_std": -0.5685977627382963, + "nauc_recall_at_5_diff1": 0.8047485091566744, + "nauc_recall_at_5_max": -0.4628759221995928, + "nauc_recall_at_5_std": -0.5997842327013364, + "ndcg_at_1": 0.40023, + "ndcg_at_10": 0.4735, + "ndcg_at_100": 0.49668, + "ndcg_at_1000": 0.50618, + "ndcg_at_20": 0.48261, + "ndcg_at_3": 0.44879, + "ndcg_at_5": 0.46062, + "precision_at_1": 0.40023, + "precision_at_10": 0.05515, + "precision_at_100": 0.00664, + "precision_at_1000": 0.00074, + "precision_at_20": 0.02938, + "precision_at_3": 0.16101, + "precision_at_5": 0.10235, + "recall_at_1": 0.40023, + "recall_at_10": 0.55155, + "recall_at_100": 0.66423, + "recall_at_1000": 0.7397, + "recall_at_20": 0.58755, + "recall_at_3": 0.48302, + "recall_at_5": 0.51177 + } + ] + }, + "task_name": "FORBI2IRetrieval" +} \ No newline at end of file diff --git a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json index e09859a559..9e86b31861 100644 --- a/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json +++ b/results-mieb/openai__clip-vit-base-patch32/3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268/FORBI2IRetrieval.json @@ -1,184 +1,184 @@ { - "dataset_revision": "ac01fba09e554b68ba4a79dc7ae45415e653a3aa", - "evaluation_time": 1184.2214772701263, + "dataset_revision": "26ab4bd972854becada339afc80f5f3ffc047e2b", + "evaluation_time": 1013.2197253704071, "kg_co2_emissions": null, - "mteb_version": "1.12.90", + "mteb_version": "1.14.21", "scores": { "test": [ { - "cv_recall_at_1": 0.00038, - "cv_recall_at_10": 0.00385, - "cv_recall_at_100": 0.03457, - "cv_recall_at_1000": 0.24649, - "cv_recall_at_20": 0.00823, - "cv_recall_at_3": 0.00083, - "cv_recall_at_5": 0.00166, + "cv_recall_at_1": 0.2997, + "cv_recall_at_10": 0.43985, + "cv_recall_at_100": 0.57849, + "cv_recall_at_1000": 0.70921, + "cv_recall_at_20": 0.47864, + "cv_recall_at_3": 0.37042, + "cv_recall_at_5": 0.40053, "hf_subset": "default", "languages": [ "eng-Latn" ], - "main_score": 0.00038, - "map_at_1": 0.00038, - "map_at_10": 0.00116, - "map_at_100": 0.00197, - "map_at_1000": 0.00259, - "map_at_20": 0.00144, - "map_at_3": 0.00065, - "map_at_5": 0.00088, - "mrr_at_1": 0.0003773584905660377, - "mrr_at_10": 0.0010433063791554353, - "mrr_at_100": 0.0018764402917165674, - "mrr_at_1000": 0.0024966567136127343, - "mrr_at_20": 0.0013360460673738318, - "mrr_at_3": 0.0005786163522012579, - "mrr_at_5": 0.0007635220125786163, - "nauc_cv_recall_at_1000_diff1": 0.22783742518067585, - "nauc_cv_recall_at_1000_max": -0.30094937055349125, - "nauc_cv_recall_at_1000_std": 0.01784167295863668, - "nauc_cv_recall_at_100_diff1": 0.1523855776857887, - "nauc_cv_recall_at_100_max": -0.2333491118159354, - "nauc_cv_recall_at_100_std": 0.07124160645261825, - "nauc_cv_recall_at_10_diff1": 0.07590081914171622, - "nauc_cv_recall_at_10_max": -0.13915413227271056, - "nauc_cv_recall_at_10_std": 0.11458514181095016, - "nauc_cv_recall_at_1_diff1": -0.0763616850013415, - "nauc_cv_recall_at_1_max": -0.2638046686342903, - "nauc_cv_recall_at_1_std": 0.037617386638046785, - "nauc_cv_recall_at_20_diff1": 0.07168369670369706, - "nauc_cv_recall_at_20_max": -0.1409353515014412, - "nauc_cv_recall_at_20_std": 0.12879237303781227, - "nauc_cv_recall_at_3_diff1": 0.08354269824621324, - "nauc_cv_recall_at_3_max": -0.2661170329536308, - "nauc_cv_recall_at_3_std": 0.02573359026270213, - "nauc_cv_recall_at_5_diff1": 0.05917506158987249, - "nauc_cv_recall_at_5_max": -0.23538307681049842, - "nauc_cv_recall_at_5_std": -0.008110349537771025, - "nauc_map_at_1000_diff1": 0.08679252942719358, - "nauc_map_at_1000_max": -0.19008245077835548, - "nauc_map_at_1000_std": 0.07439215236240897, - "nauc_map_at_100_diff1": 0.05299214493219657, - "nauc_map_at_100_max": -0.16858241572701993, - "nauc_map_at_100_std": 0.09152877376316754, - "nauc_map_at_10_diff1": -0.009168108240029638, - "nauc_map_at_10_max": -0.12428032980862758, - "nauc_map_at_10_std": 0.09071826395276378, - "nauc_map_at_1_diff1": -0.0763616850013415, - "nauc_map_at_1_max": -0.2638046686342903, - "nauc_map_at_1_std": 0.037617386638046785, - "nauc_map_at_20_diff1": 0.009578085831426258, - "nauc_map_at_20_max": -0.13374391534901683, - "nauc_map_at_20_std": 0.10562861158352993, - "nauc_map_at_3_diff1": -0.035427545355100974, - "nauc_map_at_3_max": -0.16112154547893748, - "nauc_map_at_3_std": 0.05967885079771333, - "nauc_map_at_5_diff1": -0.04279524182649304, - "nauc_map_at_5_max": -0.12683730006820387, - "nauc_map_at_5_std": 0.053172020254335796, - "nauc_mrr_at_1000_diff1": 0.11254472976600662, - "nauc_mrr_at_1000_max": -0.22511289948916516, - "nauc_mrr_at_1000_std": 0.06254278735292283, - "nauc_mrr_at_100_diff1": 0.0840199114831485, - "nauc_mrr_at_100_max": -0.2120464897313364, - "nauc_mrr_at_100_std": 0.07733222295763165, - "nauc_mrr_at_10_diff1": 0.04776673206413517, - "nauc_mrr_at_10_max": -0.20880049606815707, - "nauc_mrr_at_10_std": 0.061443566470517366, - "nauc_mrr_at_1_diff1": -0.0763616850013415, - "nauc_mrr_at_1_max": -0.2638046686342903, - "nauc_mrr_at_1_std": 0.037617386638046785, - "nauc_mrr_at_20_diff1": 0.05220974403818351, - "nauc_mrr_at_20_max": -0.19611442600889642, - "nauc_mrr_at_20_std": 0.08143751083549026, - "nauc_mrr_at_3_diff1": 0.034915598278134806, - "nauc_mrr_at_3_max": -0.26847563607517416, - "nauc_mrr_at_3_std": 0.02522135766031656, - "nauc_mrr_at_5_diff1": 0.03687636725458146, - "nauc_mrr_at_5_max": -0.2515841256355842, - "nauc_mrr_at_5_std": 0.010362513365887428, - "nauc_ndcg_at_1000_diff1": 0.20529190954097523, - "nauc_ndcg_at_1000_max": -0.27985057845510525, - "nauc_ndcg_at_1000_std": 0.028715467526608203, - "nauc_ndcg_at_100_diff1": 0.12489112426211123, - "nauc_ndcg_at_100_max": -0.21266051793182603, - "nauc_ndcg_at_100_std": 0.08036708473960381, - "nauc_ndcg_at_10_diff1": 0.011495883635102193, - "nauc_ndcg_at_10_max": -0.10236713608768004, - "nauc_ndcg_at_10_std": 0.11503781091876635, - "nauc_ndcg_at_1_diff1": -0.0763616850013415, - "nauc_ndcg_at_1_max": -0.2638046686342903, - "nauc_ndcg_at_1_std": 0.037617386638046785, - "nauc_ndcg_at_20_diff1": 0.04032171735248905, - "nauc_ndcg_at_20_max": -0.12785521817880435, - "nauc_ndcg_at_20_std": 0.13108451777376734, - "nauc_ndcg_at_3_diff1": -0.029332307317860733, - "nauc_ndcg_at_3_max": -0.1479246364175155, - "nauc_ndcg_at_3_std": 0.06371707260127392, - "nauc_ndcg_at_5_diff1": -0.044971001591395644, - "nauc_ndcg_at_5_max": -0.1026097612407452, - "nauc_ndcg_at_5_std": 0.05344340637950877, - "nauc_precision_at_1000_diff1": 0.22783742518067573, - "nauc_precision_at_1000_max": -0.30094937055349136, - "nauc_precision_at_1000_std": 0.017841672958636725, - "nauc_precision_at_100_diff1": 0.15238557768578878, - "nauc_precision_at_100_max": -0.23334911181593537, - "nauc_precision_at_100_std": 0.07124160645261826, - "nauc_precision_at_10_diff1": 0.03404639363855913, - "nauc_precision_at_10_max": -0.08089372393101929, - "nauc_precision_at_10_std": 0.14209820230748604, - "nauc_precision_at_1_diff1": -0.0763616850013415, - "nauc_precision_at_1_max": -0.2638046686342903, - "nauc_precision_at_1_std": 0.037617386638046785, - "nauc_precision_at_20_diff1": 0.06390467595189904, - "nauc_precision_at_20_max": -0.1269775837256385, - "nauc_precision_at_20_std": 0.14869868526965405, - "nauc_precision_at_3_diff1": -0.017729252234215863, - "nauc_precision_at_3_max": -0.12334110751068078, - "nauc_precision_at_3_std": 0.07159810943014612, - "nauc_precision_at_5_diff1": -0.04986634071689092, - "nauc_precision_at_5_max": -0.06921463991493515, - "nauc_precision_at_5_std": 0.053364338312017305, - "nauc_recall_at_1000_diff1": 0.22783742518067585, - "nauc_recall_at_1000_max": -0.30094937055349125, - "nauc_recall_at_1000_std": 0.01784167295863668, - "nauc_recall_at_100_diff1": 0.1523855776857887, - "nauc_recall_at_100_max": -0.2333491118159354, - "nauc_recall_at_100_std": 0.07124160645261825, - "nauc_recall_at_10_diff1": 0.034046393638559026, - "nauc_recall_at_10_max": -0.08089372393101936, - "nauc_recall_at_10_std": 0.1420982023074859, - "nauc_recall_at_1_diff1": -0.0763616850013415, - "nauc_recall_at_1_max": -0.2638046686342903, - "nauc_recall_at_1_std": 0.037617386638046785, - "nauc_recall_at_20_diff1": 0.06390467595189901, - "nauc_recall_at_20_max": -0.12697758372563853, - "nauc_recall_at_20_std": 0.14869868526965388, - "nauc_recall_at_3_diff1": -0.017729252234215977, - "nauc_recall_at_3_max": -0.12334110751068085, - "nauc_recall_at_3_std": 0.07159810943014594, - "nauc_recall_at_5_diff1": -0.049866340716890785, - "nauc_recall_at_5_max": -0.06921463991493515, - "nauc_recall_at_5_std": 0.053364338312017305, - "ndcg_at_1": 0.00038, - "ndcg_at_10": 0.00184, - "ndcg_at_100": 0.00747, - "ndcg_at_1000": 0.0321, - "ndcg_at_20": 0.00288, - "ndcg_at_3": 0.00074, - "ndcg_at_5": 0.00116, - "precision_at_1": 0.00038, - "precision_at_10": 0.00042, - "precision_at_100": 0.00035, - "precision_at_1000": 0.00025, - "precision_at_20": 0.00042, - "precision_at_3": 0.00033, - "precision_at_5": 0.00041, - "recall_at_1": 0.00038, - "recall_at_10": 0.00415, - "recall_at_100": 0.03457, - "recall_at_1000": 0.24649, - "recall_at_20": 0.0083, - "recall_at_3": 0.00098, - "recall_at_5": 0.00204 + "main_score": 0.2997, + "map_at_1": 0.29872, + "map_at_10": 0.34317, + "map_at_100": 0.34826, + "map_at_1000": 0.34882, + "map_at_20": 0.34587, + "map_at_3": 0.33092, + "map_at_5": 0.33784, + "mrr_at_1": 0.2996981132075472, + "mrr_at_10": 0.34371009284216936, + "mrr_at_100": 0.3488068311925895, + "mrr_at_1000": 0.3493644373926243, + "mrr_at_20": 0.3464172969474597, + "mrr_at_3": 0.3314716981132089, + "mrr_at_5": 0.33839622641509476, + "nauc_cv_recall_at_1000_diff1": 0.8843727921961846, + "nauc_cv_recall_at_1000_max": -1.1122059360557914, + "nauc_cv_recall_at_1000_std": -0.9675872247776091, + "nauc_cv_recall_at_100_diff1": 0.7785055286507491, + "nauc_cv_recall_at_100_max": -0.7106703205453045, + "nauc_cv_recall_at_100_std": -0.6128728702791083, + "nauc_cv_recall_at_10_diff1": 0.7465191137528097, + "nauc_cv_recall_at_10_max": -0.45743970495195085, + "nauc_cv_recall_at_10_std": -0.44231729089667093, + "nauc_cv_recall_at_1_diff1": 0.809005297864194, + "nauc_cv_recall_at_1_max": -0.2993329488342445, + "nauc_cv_recall_at_1_std": -0.32620741505098666, + "nauc_cv_recall_at_20_diff1": 0.7469876391343502, + "nauc_cv_recall_at_20_max": -0.5177087431978706, + "nauc_cv_recall_at_20_std": -0.4798695500368616, + "nauc_cv_recall_at_3_diff1": 0.7660283014554723, + "nauc_cv_recall_at_3_max": -0.37414600202429155, + "nauc_cv_recall_at_3_std": -0.3812954637897764, + "nauc_cv_recall_at_5_diff1": 0.7506021664679934, + "nauc_cv_recall_at_5_max": -0.40594245769432563, + "nauc_cv_recall_at_5_std": -0.40001908508400597, + "nauc_map_at_1000_diff1": 0.787800072995429, + "nauc_map_at_1000_max": -0.3566392040651437, + "nauc_map_at_1000_std": -0.36714262152547383, + "nauc_map_at_100_diff1": 0.7874967433485706, + "nauc_map_at_100_max": -0.355689418488149, + "nauc_map_at_100_std": -0.3663899874287131, + "nauc_map_at_10_diff1": 0.7868312493681175, + "nauc_map_at_10_max": -0.34775384038463175, + "nauc_map_at_10_std": -0.36125695985470363, + "nauc_map_at_1_diff1": 0.8126115518742145, + "nauc_map_at_1_max": -0.29913151762869594, + "nauc_map_at_1_std": -0.32523258687122936, + "nauc_map_at_20_diff1": 0.7869431680525095, + "nauc_map_at_20_max": -0.35188890045617494, + "nauc_map_at_20_std": -0.36378010134017036, + "nauc_map_at_3_diff1": 0.7911567236759326, + "nauc_map_at_3_max": -0.333149500020068, + "nauc_map_at_3_std": -0.35085539025987095, + "nauc_map_at_5_diff1": 0.787330722378602, + "nauc_map_at_5_max": -0.34076571407782574, + "nauc_map_at_5_std": -0.35536942851206027, + "nauc_mrr_at_1000_diff1": 0.7860249244782958, + "nauc_mrr_at_1000_max": -0.35671253205293696, + "nauc_mrr_at_1000_std": -0.36767791330321364, + "nauc_mrr_at_100_diff1": 0.7857232346858175, + "nauc_mrr_at_100_max": -0.35576342624777996, + "nauc_mrr_at_100_std": -0.3669254208694941, + "nauc_mrr_at_10_diff1": 0.7850630778152726, + "nauc_mrr_at_10_max": -0.347832581099885, + "nauc_mrr_at_10_std": -0.3617931109970315, + "nauc_mrr_at_1_diff1": 0.809005297864194, + "nauc_mrr_at_1_max": -0.2993329488342445, + "nauc_mrr_at_1_std": -0.32620741505098666, + "nauc_mrr_at_20_diff1": 0.7851670975927381, + "nauc_mrr_at_20_max": -0.3519692193968048, + "nauc_mrr_at_20_std": -0.36432365017895374, + "nauc_mrr_at_3_diff1": 0.7893467497060561, + "nauc_mrr_at_3_max": -0.3332399381221576, + "nauc_mrr_at_3_std": -0.35142740018662944, + "nauc_mrr_at_5_diff1": 0.7855592591112479, + "nauc_mrr_at_5_max": -0.34085078593741425, + "nauc_mrr_at_5_std": -0.3559324773796398, + "nauc_ndcg_at_1000_diff1": 0.7921287476026665, + "nauc_ndcg_at_1000_max": -0.4495754958173373, + "nauc_ndcg_at_1000_std": -0.4379750085804611, + "nauc_ndcg_at_100_diff1": 0.7823270946335801, + "nauc_ndcg_at_100_max": -0.41962827268592534, + "nauc_ndcg_at_100_std": -0.4109985043880115, + "nauc_ndcg_at_10_diff1": 0.7766582486969618, + "nauc_ndcg_at_10_max": -0.37418523977947854, + "nauc_ndcg_at_10_std": -0.38075972176781303, + "nauc_ndcg_at_1_diff1": 0.8126115518742145, + "nauc_ndcg_at_1_max": -0.29913151762869594, + "nauc_ndcg_at_1_std": -0.32523258687122936, + "nauc_ndcg_at_20_diff1": 0.776952347684857, + "nauc_ndcg_at_20_max": -0.38921670424824983, + "nauc_ndcg_at_20_std": -0.3900253543363332, + "nauc_ndcg_at_3_diff1": 0.7846298919901769, + "nauc_ndcg_at_3_max": -0.3438430556258442, + "nauc_ndcg_at_3_std": -0.35876507125063356, + "nauc_ndcg_at_5_diff1": 0.7779001958221061, + "nauc_ndcg_at_5_max": -0.35740366313832533, + "nauc_ndcg_at_5_std": -0.36678964402271136, + "nauc_precision_at_1000_diff1": 0.8843727921961839, + "nauc_precision_at_1000_max": -1.1122059360557912, + "nauc_precision_at_1000_std": -0.967587224777609, + "nauc_precision_at_100_diff1": 0.778505528650749, + "nauc_precision_at_100_max": -0.7106703205453052, + "nauc_precision_at_100_std": -0.6128728702791086, + "nauc_precision_at_10_diff1": 0.7465191137528097, + "nauc_precision_at_10_max": -0.4574397049519504, + "nauc_precision_at_10_std": -0.44231729089667066, + "nauc_precision_at_1_diff1": 0.8126115518742145, + "nauc_precision_at_1_max": -0.29913151762869594, + "nauc_precision_at_1_std": -0.32523258687122936, + "nauc_precision_at_20_diff1": 0.7471224397052061, + "nauc_precision_at_20_max": -0.5176185970858633, + "nauc_precision_at_20_std": -0.47970254372347515, + "nauc_precision_at_3_diff1": 0.7662084419673741, + "nauc_precision_at_3_max": -0.3741336169388402, + "nauc_precision_at_3_std": -0.3811257749890647, + "nauc_precision_at_5_diff1": 0.7506575148707284, + "nauc_precision_at_5_max": -0.40590652706796215, + "nauc_precision_at_5_std": -0.39985853688778367, + "nauc_recall_at_1000_diff1": 0.8843727921961846, + "nauc_recall_at_1000_max": -1.1122059360557914, + "nauc_recall_at_1000_std": -0.9675872247776091, + "nauc_recall_at_100_diff1": 0.7785055286507491, + "nauc_recall_at_100_max": -0.7106703205453045, + "nauc_recall_at_100_std": -0.6128728702791083, + "nauc_recall_at_10_diff1": 0.7465191137528097, + "nauc_recall_at_10_max": -0.45743970495195085, + "nauc_recall_at_10_std": -0.44231729089667093, + "nauc_recall_at_1_diff1": 0.8126115518742145, + "nauc_recall_at_1_max": -0.29913151762869594, + "nauc_recall_at_1_std": -0.32523258687122936, + "nauc_recall_at_20_diff1": 0.7471224397052065, + "nauc_recall_at_20_max": -0.5176185970858634, + "nauc_recall_at_20_std": -0.47970254372347554, + "nauc_recall_at_3_diff1": 0.7662084419673746, + "nauc_recall_at_3_max": -0.37413361693884, + "nauc_recall_at_3_std": -0.38112577498906464, + "nauc_recall_at_5_diff1": 0.750657514870729, + "nauc_recall_at_5_max": -0.4059065270679627, + "nauc_recall_at_5_std": -0.39985853688778417, + "ndcg_at_1": 0.29872, + "ndcg_at_10": 0.36632, + "ndcg_at_100": 0.39432, + "ndcg_at_1000": 0.41057, + "ndcg_at_20": 0.37613, + "ndcg_at_3": 0.34107, + "ndcg_at_5": 0.35351, + "precision_at_1": 0.29872, + "precision_at_10": 0.04398, + "precision_at_100": 0.00578, + "precision_at_1000": 0.00071, + "precision_at_20": 0.02393, + "precision_at_3": 0.12345, + "precision_at_5": 0.08009, + "recall_at_1": 0.29872, + "recall_at_10": 0.43985, + "recall_at_100": 0.57849, + "recall_at_1000": 0.70921, + "recall_at_20": 0.47857, + "recall_at_3": 0.37034, + "recall_at_5": 0.40045 } ] }, diff --git a/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/FORBI2IRetrieval.json b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/FORBI2IRetrieval.json new file mode 100644 index 0000000000..67200bf463 --- /dev/null +++ b/results-mieb/openai__clip-vit-large-patch14/32bd64288804d66eefd0ccbe215aa642df71cc41/FORBI2IRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "26ab4bd972854becada339afc80f5f3ffc047e2b", + "evaluation_time": 2148.242924451828, + "kg_co2_emissions": null, + "mteb_version": "1.14.21", + "scores": { + "test": [ + { + "cv_recall_at_1": 0.47955, + "cv_recall_at_10": 0.64309, + "cv_recall_at_100": 0.72468, + "cv_recall_at_1000": 0.75774, + "cv_recall_at_20": 0.67275, + "cv_recall_at_3": 0.56981, + "cv_recall_at_5": 0.60325, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.47955, + "map_at_1": 0.47826, + "map_at_10": 0.53199, + "map_at_100": 0.53545, + "map_at_1000": 0.53562, + "map_at_20": 0.53409, + "map_at_3": 0.51886, + "map_at_5": 0.52653, + "mrr_at_1": 0.47954716981132073, + "mrr_at_10": 0.5327790955375838, + "mrr_at_100": 0.536231063428097, + "mrr_at_1000": 0.5363963198824263, + "mrr_at_20": 0.5348666258075478, + "mrr_at_3": 0.5196729559748401, + "mrr_at_5": 0.527325786163518, + "nauc_cv_recall_at_1000_diff1": 0.9783622452451709, + "nauc_cv_recall_at_1000_max": -1.3700380127097909, + "nauc_cv_recall_at_1000_std": -1.3622593419555964, + "nauc_cv_recall_at_100_diff1": 0.9367206943403367, + "nauc_cv_recall_at_100_max": -1.1415729064069615, + "nauc_cv_recall_at_100_std": -1.1585751691509931, + "nauc_cv_recall_at_10_diff1": 0.8585716756688088, + "nauc_cv_recall_at_10_max": -0.7579079603295624, + "nauc_cv_recall_at_10_std": -0.8979314486065754, + "nauc_cv_recall_at_1_diff1": 0.8389554763575047, + "nauc_cv_recall_at_1_max": -0.3861612888474326, + "nauc_cv_recall_at_1_std": -0.6155125338004539, + "nauc_cv_recall_at_20_diff1": 0.8841302115144418, + "nauc_cv_recall_at_20_max": -0.8720455440695557, + "nauc_cv_recall_at_20_std": -0.9664298519627592, + "nauc_cv_recall_at_3_diff1": 0.8259560459790033, + "nauc_cv_recall_at_3_max": -0.546319243284802, + "nauc_cv_recall_at_3_std": -0.7634227610142231, + "nauc_cv_recall_at_5_diff1": 0.8299350588708171, + "nauc_cv_recall_at_5_max": -0.6287235649657446, + "nauc_cv_recall_at_5_std": -0.8161938597529815, + "nauc_map_at_1000_diff1": 0.840854892934761, + "nauc_map_at_1000_max": -0.4924278647794887, + "nauc_map_at_1000_std": -0.7022277605230981, + "nauc_map_at_100_diff1": 0.8407441848445416, + "nauc_map_at_100_max": -0.49192560404032804, + "nauc_map_at_100_std": -0.7017634806879928, + "nauc_map_at_10_diff1": 0.8385260397871099, + "nauc_map_at_10_max": -0.4821878658539431, + "nauc_map_at_10_std": -0.6956217111863439, + "nauc_map_at_1_diff1": 0.8418694895112901, + "nauc_map_at_1_max": -0.38442688222770177, + "nauc_map_at_1_std": -0.6125805636780283, + "nauc_map_at_20_diff1": 0.8399206633399918, + "nauc_map_at_20_max": -0.4878232150574702, + "nauc_map_at_20_std": -0.6989058410509833, + "nauc_map_at_3_diff1": 0.8341959402895627, + "nauc_map_at_3_max": -0.45174026022682956, + "nauc_map_at_3_std": -0.676617769201114, + "nauc_map_at_5_diff1": 0.8350923824381429, + "nauc_map_at_5_max": -0.46831637558370504, + "nauc_map_at_5_std": -0.6870541683615817, + "nauc_mrr_at_1000_diff1": 0.8393074209785932, + "nauc_mrr_at_1000_max": -0.4936213541118185, + "nauc_mrr_at_1000_std": -0.7041826758357784, + "nauc_mrr_at_100_diff1": 0.8391972194458248, + "nauc_mrr_at_100_max": -0.4931184490320341, + "nauc_mrr_at_100_std": -0.7037174874420373, + "nauc_mrr_at_10_diff1": 0.8369860440129665, + "nauc_mrr_at_10_max": -0.4833850308584579, + "nauc_mrr_at_10_std": -0.6975876349014124, + "nauc_mrr_at_1_diff1": 0.8389554763575047, + "nauc_mrr_at_1_max": -0.3861612888474326, + "nauc_mrr_at_1_std": -0.6155125338004539, + "nauc_mrr_at_20_diff1": 0.8383772686177878, + "nauc_mrr_at_20_max": -0.4890112340146167, + "nauc_mrr_at_20_std": -0.7008535231923642, + "nauc_mrr_at_3_diff1": 0.8326910963164327, + "nauc_mrr_at_3_max": -0.45292910686532045, + "nauc_mrr_at_3_std": -0.6786157396873091, + "nauc_mrr_at_5_diff1": 0.8335794976849904, + "nauc_mrr_at_5_max": -0.46951576427639513, + "nauc_mrr_at_5_std": -0.689014845411203, + "nauc_ndcg_at_1000_diff1": 0.8571833725542346, + "nauc_ndcg_at_1000_max": -0.6105533656454696, + "nauc_ndcg_at_1000_std": -0.7896000730053025, + "nauc_ndcg_at_100_diff1": 0.854211660299563, + "nauc_ndcg_at_100_max": -0.5955309613288091, + "nauc_ndcg_at_100_std": -0.7753517475583974, + "nauc_ndcg_at_10_diff1": 0.8421946492855936, + "nauc_ndcg_at_10_max": -0.5410015654361832, + "nauc_ndcg_at_10_std": -0.7393165425948717, + "nauc_ndcg_at_1_diff1": 0.8418694895112901, + "nauc_ndcg_at_1_max": -0.38442688222770177, + "nauc_ndcg_at_1_std": -0.6125805636780283, + "nauc_ndcg_at_20_diff1": 0.8475183312428272, + "nauc_ndcg_at_20_max": -0.563016555679603, + "nauc_ndcg_at_20_std": -0.7522312081819034, + "nauc_ndcg_at_3_diff1": 0.8321161349170787, + "nauc_ndcg_at_3_max": -0.47507879358661737, + "nauc_ndcg_at_3_std": -0.6980061182251038, + "nauc_ndcg_at_5_diff1": 0.8337229410549184, + "nauc_ndcg_at_5_max": -0.5059407346638893, + "nauc_ndcg_at_5_std": -0.7175685317009618, + "nauc_precision_at_1000_diff1": 0.9783622452451708, + "nauc_precision_at_1000_max": -1.370038012709789, + "nauc_precision_at_1000_std": -1.362259341955595, + "nauc_precision_at_100_diff1": 0.9367206943403371, + "nauc_precision_at_100_max": -1.141572906406963, + "nauc_precision_at_100_std": -1.1585751691509945, + "nauc_precision_at_10_diff1": 0.8585920210973836, + "nauc_precision_at_10_max": -0.7576550732857764, + "nauc_precision_at_10_std": -0.8975426783303393, + "nauc_precision_at_1_diff1": 0.8418694895112901, + "nauc_precision_at_1_max": -0.38442688222770177, + "nauc_precision_at_1_std": -0.6125805636780283, + "nauc_precision_at_20_diff1": 0.8841302115144408, + "nauc_precision_at_20_max": -0.8720455440695557, + "nauc_precision_at_20_std": -0.9664298519627592, + "nauc_precision_at_3_diff1": 0.8259800230617228, + "nauc_precision_at_3_max": -0.5459658712057691, + "nauc_precision_at_3_std": -0.7627160737347208, + "nauc_precision_at_5_diff1": 0.8298537192402428, + "nauc_precision_at_5_max": -0.6284652527439748, + "nauc_precision_at_5_std": -0.8158750015206129, + "nauc_recall_at_1000_diff1": 0.9783622452451709, + "nauc_recall_at_1000_max": -1.3700380127097909, + "nauc_recall_at_1000_std": -1.3622593419555964, + "nauc_recall_at_100_diff1": 0.9367206943403367, + "nauc_recall_at_100_max": -1.1415729064069615, + "nauc_recall_at_100_std": -1.1585751691509931, + "nauc_recall_at_10_diff1": 0.8585920210973839, + "nauc_recall_at_10_max": -0.7576550732857767, + "nauc_recall_at_10_std": -0.8975426783303397, + "nauc_recall_at_1_diff1": 0.8418694895112901, + "nauc_recall_at_1_max": -0.38442688222770177, + "nauc_recall_at_1_std": -0.6125805636780283, + "nauc_recall_at_20_diff1": 0.8841302115144418, + "nauc_recall_at_20_max": -0.8720455440695557, + "nauc_recall_at_20_std": -0.9664298519627592, + "nauc_recall_at_3_diff1": 0.8259800230617225, + "nauc_recall_at_3_max": -0.5459658712057688, + "nauc_recall_at_3_std": -0.76271607373472, + "nauc_recall_at_5_diff1": 0.829853719240243, + "nauc_recall_at_5_max": -0.628465252743975, + "nauc_recall_at_5_std": -0.8158750015206132, + "ndcg_at_1": 0.47826, + "ndcg_at_10": 0.55876, + "ndcg_at_100": 0.57601, + "ndcg_at_1000": 0.58025, + "ndcg_at_20": 0.56632, + "ndcg_at_3": 0.5319, + "ndcg_at_5": 0.54573, + "precision_at_1": 0.47826, + "precision_at_10": 0.0643, + "precision_at_100": 0.00725, + "precision_at_1000": 0.00076, + "precision_at_20": 0.03364, + "precision_at_3": 0.18986, + "precision_at_5": 0.12063, + "recall_at_1": 0.47826, + "recall_at_10": 0.64302, + "recall_at_100": 0.72468, + "recall_at_1000": 0.75774, + "recall_at_20": 0.67275, + "recall_at_3": 0.56958, + "recall_at_5": 0.60317 + } + ] + }, + "task_name": "FORBI2IRetrieval" +} \ No newline at end of file