diff --git a/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/Vidore2BioMedicalLecturesRetrieval.json b/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..eefcd056cc --- /dev/null +++ b/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.275, + "ndcg_at_3": 0.31977, + "ndcg_at_5": 0.33223, + "ndcg_at_10": 0.36138, + "ndcg_at_20": 0.39399, + "ndcg_at_100": 0.45052, + "ndcg_at_1000": 0.49212, + "map_at_1": 0.15094, + "map_at_3": 0.23984, + "map_at_5": 0.26226, + "map_at_10": 0.28585, + "map_at_20": 0.29904, + "map_at_100": 0.31089, + "map_at_1000": 0.31432, + "recall_at_1": 0.15094, + "recall_at_3": 0.30647, + "recall_at_5": 0.36965, + "recall_at_10": 0.45966, + "recall_at_20": 0.56439, + "recall_at_100": 0.79047, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.275, + "cv_recall_at_3": 0.48125, + "cv_recall_at_5": 0.5375, + "cv_recall_at_10": 0.63125, + "cv_recall_at_20": 0.73125, + "cv_recall_at_100": 0.91875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.275, + "precision_at_3": 0.21667, + "precision_at_5": 0.16, + "precision_at_10": 0.10812, + "precision_at_20": 0.07031, + "precision_at_100": 0.02156, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.275, + "mrr_at_3": 0.367708, + "mrr_at_5": 0.381146, + "mrr_at_10": 0.3938, + "mrr_at_20": 0.401066, + "mrr_at_100": 0.405711, + "mrr_at_1000": 0.406138, + "nauc_ndcg_at_1_max": -0.003812, + "nauc_ndcg_at_1_std": -0.198313, + "nauc_ndcg_at_1_diff1": 0.245915, + "nauc_ndcg_at_3_max": 0.011094, + "nauc_ndcg_at_3_std": -0.227441, + "nauc_ndcg_at_3_diff1": 0.313768, + "nauc_ndcg_at_5_max": 0.002221, + "nauc_ndcg_at_5_std": -0.216149, + "nauc_ndcg_at_5_diff1": 0.293572, + "nauc_ndcg_at_10_max": 9.5e-05, + "nauc_ndcg_at_10_std": -0.219019, + "nauc_ndcg_at_10_diff1": 0.252361, + "nauc_ndcg_at_20_max": 0.015593, + "nauc_ndcg_at_20_std": -0.208763, + "nauc_ndcg_at_20_diff1": 0.262865, + "nauc_ndcg_at_100_max": 0.013521, + "nauc_ndcg_at_100_std": -0.201938, + "nauc_ndcg_at_100_diff1": 0.257956, + "nauc_ndcg_at_1000_max": 0.02561, + "nauc_ndcg_at_1000_std": -0.192262, + "nauc_ndcg_at_1000_diff1": 0.262454, + "nauc_map_at_1_max": -0.012973, + "nauc_map_at_1_std": -0.089713, + "nauc_map_at_1_diff1": 0.257163, + "nauc_map_at_3_max": -0.020464, + "nauc_map_at_3_std": -0.191867, + "nauc_map_at_3_diff1": 0.314068, + "nauc_map_at_5_max": -0.008064, + "nauc_map_at_5_std": -0.19211, + "nauc_map_at_5_diff1": 0.288927, + "nauc_map_at_10_max": -0.000422, + "nauc_map_at_10_std": -0.200053, + "nauc_map_at_10_diff1": 0.260871, + "nauc_map_at_20_max": 0.00551, + "nauc_map_at_20_std": -0.197024, + "nauc_map_at_20_diff1": 0.262532, + "nauc_map_at_100_max": 0.007185, + "nauc_map_at_100_std": -0.195483, + "nauc_map_at_100_diff1": 0.265897, + "nauc_map_at_1000_max": 0.007618, + "nauc_map_at_1000_std": -0.194706, + "nauc_map_at_1000_diff1": 0.265057, + "nauc_recall_at_1_max": -0.012973, + "nauc_recall_at_1_std": -0.089713, + "nauc_recall_at_1_diff1": 0.257163, + "nauc_recall_at_3_max": -0.006791, + "nauc_recall_at_3_std": -0.218866, + "nauc_recall_at_3_diff1": 0.348221, + "nauc_recall_at_5_max": -0.015946, + "nauc_recall_at_5_std": -0.205086, + "nauc_recall_at_5_diff1": 0.280116, + "nauc_recall_at_10_max": 0.001073, + "nauc_recall_at_10_std": -0.205012, + "nauc_recall_at_10_diff1": 0.202595, + "nauc_recall_at_20_max": 0.000694, + "nauc_recall_at_20_std": -0.198519, + "nauc_recall_at_20_diff1": 0.176145, + "nauc_recall_at_100_max": -0.048379, + "nauc_recall_at_100_std": -0.160567, + "nauc_recall_at_100_diff1": 0.095955, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.003812, + "nauc_precision_at_1_std": -0.198313, + "nauc_precision_at_1_diff1": 0.245915, + "nauc_precision_at_3_max": 0.041325, + "nauc_precision_at_3_std": -0.236945, + "nauc_precision_at_3_diff1": 0.23466, + "nauc_precision_at_5_max": 0.074374, + "nauc_precision_at_5_std": -0.210462, + "nauc_precision_at_5_diff1": 0.164964, + "nauc_precision_at_10_max": 0.068934, + "nauc_precision_at_10_std": -0.211473, + "nauc_precision_at_10_diff1": 0.069522, + "nauc_precision_at_20_max": 0.094543, + "nauc_precision_at_20_std": -0.145224, + "nauc_precision_at_20_diff1": 0.096009, + "nauc_precision_at_100_max": 0.056717, + "nauc_precision_at_100_std": -0.083469, + "nauc_precision_at_100_diff1": 0.015211, + "nauc_precision_at_1000_max": 0.043366, + "nauc_precision_at_1000_std": 0.023843, + "nauc_precision_at_1000_diff1": -0.09637, + "nauc_cv_recall_at_1_max": -0.003812, + "nauc_cv_recall_at_1_std": -0.198313, + "nauc_cv_recall_at_1_diff1": 0.245915, + "nauc_cv_recall_at_3_max": 0.053388, + "nauc_cv_recall_at_3_std": -0.228148, + "nauc_cv_recall_at_3_diff1": 0.384433, + "nauc_cv_recall_at_5_max": 0.014851, + "nauc_cv_recall_at_5_std": -0.25501, + "nauc_cv_recall_at_5_diff1": 0.354137, + "nauc_cv_recall_at_10_max": 0.052781, + "nauc_cv_recall_at_10_std": -0.221138, + "nauc_cv_recall_at_10_diff1": 0.245521, + "nauc_cv_recall_at_20_max": 0.010187, + "nauc_cv_recall_at_20_std": -0.273344, + "nauc_cv_recall_at_20_diff1": 0.239649, + "nauc_cv_recall_at_100_max": -0.001867, + "nauc_cv_recall_at_100_std": -0.111183, + "nauc_cv_recall_at_100_diff1": -0.229333, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.003812, + "nauc_mrr_at_1_std": -0.198313, + "nauc_mrr_at_1_diff1": 0.245915, + "nauc_mrr_at_3_max": 0.021644, + "nauc_mrr_at_3_std": -0.209061, + "nauc_mrr_at_3_diff1": 0.307419, + "nauc_mrr_at_5_max": 0.011389, + "nauc_mrr_at_5_std": -0.215012, + "nauc_mrr_at_5_diff1": 0.298134, + "nauc_mrr_at_10_max": 0.014987, + "nauc_mrr_at_10_std": -0.212408, + "nauc_mrr_at_10_diff1": 0.284866, + "nauc_mrr_at_20_max": 0.012956, + "nauc_mrr_at_20_std": -0.213646, + "nauc_mrr_at_20_diff1": 0.284587, + "nauc_mrr_at_100_max": 0.012952, + "nauc_mrr_at_100_std": -0.211258, + "nauc_mrr_at_100_diff1": 0.281557, + "nauc_mrr_at_1000_max": 0.01282, + "nauc_mrr_at_1000_std": -0.211406, + "nauc_mrr_at_1000_diff1": 0.282213, + "main_score": 0.33223, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.2625, + "ndcg_at_3": 0.27851, + "ndcg_at_5": 0.30546, + "ndcg_at_10": 0.34962, + "ndcg_at_20": 0.37246, + "ndcg_at_100": 0.42589, + "ndcg_at_1000": 0.47099, + "map_at_1": 0.13808, + "map_at_3": 0.20436, + "map_at_5": 0.23331, + "map_at_10": 0.26154, + "map_at_20": 0.27145, + "map_at_100": 0.28282, + "map_at_1000": 0.28614, + "recall_at_1": 0.13808, + "recall_at_3": 0.27533, + "recall_at_5": 0.36002, + "recall_at_10": 0.48732, + "recall_at_20": 0.56032, + "recall_at_100": 0.76478, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.2625, + "cv_recall_at_3": 0.4375, + "cv_recall_at_5": 0.525, + "cv_recall_at_10": 0.66875, + "cv_recall_at_20": 0.71875, + "cv_recall_at_100": 0.89375, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.2625, + "precision_at_3": 0.17708, + "precision_at_5": 0.1475, + "precision_at_10": 0.10875, + "precision_at_20": 0.06531, + "precision_at_100": 0.02094, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.2625, + "mrr_at_3": 0.338542, + "mrr_at_5": 0.358854, + "mrr_at_10": 0.378036, + "mrr_at_20": 0.381496, + "mrr_at_100": 0.385689, + "mrr_at_1000": 0.386222, + "nauc_ndcg_at_1_max": 0.019438, + "nauc_ndcg_at_1_std": -0.064898, + "nauc_ndcg_at_1_diff1": 0.217389, + "nauc_ndcg_at_3_max": 0.040264, + "nauc_ndcg_at_3_std": -0.124926, + "nauc_ndcg_at_3_diff1": 0.253425, + "nauc_ndcg_at_5_max": 0.074608, + "nauc_ndcg_at_5_std": -0.091575, + "nauc_ndcg_at_5_diff1": 0.263438, + "nauc_ndcg_at_10_max": 0.084853, + "nauc_ndcg_at_10_std": -0.080415, + "nauc_ndcg_at_10_diff1": 0.241922, + "nauc_ndcg_at_20_max": 0.0896, + "nauc_ndcg_at_20_std": -0.081225, + "nauc_ndcg_at_20_diff1": 0.232342, + "nauc_ndcg_at_100_max": 0.086881, + "nauc_ndcg_at_100_std": -0.081808, + "nauc_ndcg_at_100_diff1": 0.216619, + "nauc_ndcg_at_1000_max": 0.076361, + "nauc_ndcg_at_1000_std": -0.079901, + "nauc_ndcg_at_1000_diff1": 0.211814, + "nauc_map_at_1_max": 0.152342, + "nauc_map_at_1_std": 0.045647, + "nauc_map_at_1_diff1": 0.333641, + "nauc_map_at_3_max": 0.099787, + "nauc_map_at_3_std": -0.064933, + "nauc_map_at_3_diff1": 0.317338, + "nauc_map_at_5_max": 0.113602, + "nauc_map_at_5_std": -0.049429, + "nauc_map_at_5_diff1": 0.28836, + "nauc_map_at_10_max": 0.108368, + "nauc_map_at_10_std": -0.058209, + "nauc_map_at_10_diff1": 0.259236, + "nauc_map_at_20_max": 0.109342, + "nauc_map_at_20_std": -0.057468, + "nauc_map_at_20_diff1": 0.248521, + "nauc_map_at_100_max": 0.106005, + "nauc_map_at_100_std": -0.060177, + "nauc_map_at_100_diff1": 0.242671, + "nauc_map_at_1000_max": 0.105353, + "nauc_map_at_1000_std": -0.059942, + "nauc_map_at_1000_diff1": 0.242512, + "nauc_recall_at_1_max": 0.152342, + "nauc_recall_at_1_std": 0.045647, + "nauc_recall_at_1_diff1": 0.333641, + "nauc_recall_at_3_max": 0.05495, + "nauc_recall_at_3_std": -0.129389, + "nauc_recall_at_3_diff1": 0.310568, + "nauc_recall_at_5_max": 0.074113, + "nauc_recall_at_5_std": -0.090888, + "nauc_recall_at_5_diff1": 0.286005, + "nauc_recall_at_10_max": 0.085924, + "nauc_recall_at_10_std": -0.07021, + "nauc_recall_at_10_diff1": 0.227237, + "nauc_recall_at_20_max": 0.091155, + "nauc_recall_at_20_std": -0.085262, + "nauc_recall_at_20_diff1": 0.212966, + "nauc_recall_at_100_max": 0.111615, + "nauc_recall_at_100_std": -0.08459, + "nauc_recall_at_100_diff1": 0.16263, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.019438, + "nauc_precision_at_1_std": -0.064898, + "nauc_precision_at_1_diff1": 0.217389, + "nauc_precision_at_3_max": -0.039308, + "nauc_precision_at_3_std": -0.183277, + "nauc_precision_at_3_diff1": 0.103547, + "nauc_precision_at_5_max": 0.013761, + "nauc_precision_at_5_std": -0.12676, + "nauc_precision_at_5_diff1": 0.031443, + "nauc_precision_at_10_max": 0.023299, + "nauc_precision_at_10_std": -0.095124, + "nauc_precision_at_10_diff1": -0.08073, + "nauc_precision_at_20_max": 0.005648, + "nauc_precision_at_20_std": -0.088854, + "nauc_precision_at_20_diff1": -0.131057, + "nauc_precision_at_100_max": -0.051857, + "nauc_precision_at_100_std": -0.081979, + "nauc_precision_at_100_diff1": -0.166189, + "nauc_precision_at_1000_max": -0.124434, + "nauc_precision_at_1000_std": -0.073329, + "nauc_precision_at_1000_diff1": -0.159615, + "nauc_cv_recall_at_1_max": 0.019438, + "nauc_cv_recall_at_1_std": -0.064898, + "nauc_cv_recall_at_1_diff1": 0.217389, + "nauc_cv_recall_at_3_max": -0.029566, + "nauc_cv_recall_at_3_std": -0.168047, + "nauc_cv_recall_at_3_diff1": 0.215638, + "nauc_cv_recall_at_5_max": -0.02563, + "nauc_cv_recall_at_5_std": -0.194091, + "nauc_cv_recall_at_5_diff1": 0.236181, + "nauc_cv_recall_at_10_max": 0.055292, + "nauc_cv_recall_at_10_std": -0.118669, + "nauc_cv_recall_at_10_diff1": 0.139123, + "nauc_cv_recall_at_20_max": 0.071981, + "nauc_cv_recall_at_20_std": -0.150698, + "nauc_cv_recall_at_20_diff1": 0.17049, + "nauc_cv_recall_at_100_max": 0.15996, + "nauc_cv_recall_at_100_std": -0.045857, + "nauc_cv_recall_at_100_diff1": 0.135689, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.019438, + "nauc_mrr_at_1_std": -0.064898, + "nauc_mrr_at_1_diff1": 0.217389, + "nauc_mrr_at_3_max": -0.001866, + "nauc_mrr_at_3_std": -0.114609, + "nauc_mrr_at_3_diff1": 0.213182, + "nauc_mrr_at_5_max": 0.001082, + "nauc_mrr_at_5_std": -0.119476, + "nauc_mrr_at_5_diff1": 0.216568, + "nauc_mrr_at_10_max": 0.011154, + "nauc_mrr_at_10_std": -0.108907, + "nauc_mrr_at_10_diff1": 0.204758, + "nauc_mrr_at_20_max": 0.011778, + "nauc_mrr_at_20_std": -0.110814, + "nauc_mrr_at_20_diff1": 0.206478, + "nauc_mrr_at_100_max": 0.01288, + "nauc_mrr_at_100_std": -0.108636, + "nauc_mrr_at_100_diff1": 0.207073, + "nauc_mrr_at_1000_max": 0.012452, + "nauc_mrr_at_1000_std": -0.108885, + "nauc_mrr_at_1000_diff1": 0.20703, + "main_score": 0.30546, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.28125, + "ndcg_at_3": 0.33255, + "ndcg_at_5": 0.36261, + "ndcg_at_10": 0.40191, + "ndcg_at_20": 0.42069, + "ndcg_at_100": 0.48443, + "ndcg_at_1000": 0.51583, + "map_at_1": 0.15005, + "map_at_3": 0.24755, + "map_at_5": 0.27946, + "map_at_10": 0.30692, + "map_at_20": 0.31634, + "map_at_100": 0.33022, + "map_at_1000": 0.33309, + "recall_at_1": 0.15005, + "recall_at_3": 0.33641, + "recall_at_5": 0.42752, + "recall_at_10": 0.54492, + "recall_at_20": 0.59581, + "recall_at_100": 0.84864, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.28125, + "cv_recall_at_3": 0.51875, + "cv_recall_at_5": 0.6125, + "cv_recall_at_10": 0.7375, + "cv_recall_at_20": 0.775, + "cv_recall_at_100": 0.9625, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.28125, + "precision_at_3": 0.21875, + "precision_at_5": 0.18, + "precision_at_10": 0.12438, + "precision_at_20": 0.075, + "precision_at_100": 0.02344, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.28125, + "mrr_at_3": 0.3875, + "mrr_at_5": 0.409375, + "mrr_at_10": 0.425439, + "mrr_at_20": 0.428031, + "mrr_at_100": 0.433349, + "mrr_at_1000": 0.433507, + "nauc_ndcg_at_1_max": 0.051631, + "nauc_ndcg_at_1_std": -0.107618, + "nauc_ndcg_at_1_diff1": 0.183161, + "nauc_ndcg_at_3_max": -0.006929, + "nauc_ndcg_at_3_std": -0.258689, + "nauc_ndcg_at_3_diff1": 0.263614, + "nauc_ndcg_at_5_max": 0.017205, + "nauc_ndcg_at_5_std": -0.23224, + "nauc_ndcg_at_5_diff1": 0.277958, + "nauc_ndcg_at_10_max": 0.008487, + "nauc_ndcg_at_10_std": -0.253655, + "nauc_ndcg_at_10_diff1": 0.298943, + "nauc_ndcg_at_20_max": 0.009887, + "nauc_ndcg_at_20_std": -0.242858, + "nauc_ndcg_at_20_diff1": 0.305649, + "nauc_ndcg_at_100_max": 0.02719, + "nauc_ndcg_at_100_std": -0.226087, + "nauc_ndcg_at_100_diff1": 0.294005, + "nauc_ndcg_at_1000_max": 0.042325, + "nauc_ndcg_at_1000_std": -0.181121, + "nauc_ndcg_at_1000_diff1": 0.272153, + "nauc_map_at_1_max": 0.020359, + "nauc_map_at_1_std": -0.125631, + "nauc_map_at_1_diff1": 0.278684, + "nauc_map_at_3_max": -0.002419, + "nauc_map_at_3_std": -0.24553, + "nauc_map_at_3_diff1": 0.304273, + "nauc_map_at_5_max": 0.022432, + "nauc_map_at_5_std": -0.226521, + "nauc_map_at_5_diff1": 0.297968, + "nauc_map_at_10_max": 0.03606, + "nauc_map_at_10_std": -0.236414, + "nauc_map_at_10_diff1": 0.301684, + "nauc_map_at_20_max": 0.035244, + "nauc_map_at_20_std": -0.232755, + "nauc_map_at_20_diff1": 0.305525, + "nauc_map_at_100_max": 0.036331, + "nauc_map_at_100_std": -0.230744, + "nauc_map_at_100_diff1": 0.301677, + "nauc_map_at_1000_max": 0.037762, + "nauc_map_at_1000_std": -0.226863, + "nauc_map_at_1000_diff1": 0.300169, + "nauc_recall_at_1_max": 0.020359, + "nauc_recall_at_1_std": -0.125631, + "nauc_recall_at_1_diff1": 0.278684, + "nauc_recall_at_3_max": -0.068703, + "nauc_recall_at_3_std": -0.305474, + "nauc_recall_at_3_diff1": 0.28125, + "nauc_recall_at_5_max": -0.027436, + "nauc_recall_at_5_std": -0.257876, + "nauc_recall_at_5_diff1": 0.275056, + "nauc_recall_at_10_max": -0.04204, + "nauc_recall_at_10_std": -0.290709, + "nauc_recall_at_10_diff1": 0.303712, + "nauc_recall_at_20_max": -0.053391, + "nauc_recall_at_20_std": -0.295085, + "nauc_recall_at_20_diff1": 0.317696, + "nauc_recall_at_100_max": -0.05331, + "nauc_recall_at_100_std": -0.381829, + "nauc_recall_at_100_diff1": 0.355736, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.051631, + "nauc_precision_at_1_std": -0.107618, + "nauc_precision_at_1_diff1": 0.183161, + "nauc_precision_at_3_max": 0.048466, + "nauc_precision_at_3_std": -0.229786, + "nauc_precision_at_3_diff1": 0.191389, + "nauc_precision_at_5_max": 0.122132, + "nauc_precision_at_5_std": -0.114976, + "nauc_precision_at_5_diff1": 0.153859, + "nauc_precision_at_10_max": 0.104895, + "nauc_precision_at_10_std": -0.117547, + "nauc_precision_at_10_diff1": 0.117156, + "nauc_precision_at_20_max": 0.061332, + "nauc_precision_at_20_std": -0.05922, + "nauc_precision_at_20_diff1": 0.086024, + "nauc_precision_at_100_max": 0.068676, + "nauc_precision_at_100_std": 0.091251, + "nauc_precision_at_100_diff1": -0.076416, + "nauc_precision_at_1000_max": 0.091853, + "nauc_precision_at_1000_std": 0.278442, + "nauc_precision_at_1000_diff1": -0.156486, + "nauc_cv_recall_at_1_max": 0.051631, + "nauc_cv_recall_at_1_std": -0.107618, + "nauc_cv_recall_at_1_diff1": 0.183161, + "nauc_cv_recall_at_3_max": -0.059111, + "nauc_cv_recall_at_3_std": -0.254027, + "nauc_cv_recall_at_3_diff1": 0.229336, + "nauc_cv_recall_at_5_max": -0.050022, + "nauc_cv_recall_at_5_std": -0.292856, + "nauc_cv_recall_at_5_diff1": 0.252186, + "nauc_cv_recall_at_10_max": -0.118855, + "nauc_cv_recall_at_10_std": -0.354616, + "nauc_cv_recall_at_10_diff1": 0.27548, + "nauc_cv_recall_at_20_max": -0.174543, + "nauc_cv_recall_at_20_std": -0.359548, + "nauc_cv_recall_at_20_diff1": 0.269245, + "nauc_cv_recall_at_100_max": 0.127607, + "nauc_cv_recall_at_100_std": -0.267507, + "nauc_cv_recall_at_100_diff1": 0.399082, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.051631, + "nauc_mrr_at_1_std": -0.107618, + "nauc_mrr_at_1_diff1": 0.183161, + "nauc_mrr_at_3_max": 0.007978, + "nauc_mrr_at_3_std": -0.171922, + "nauc_mrr_at_3_diff1": 0.209194, + "nauc_mrr_at_5_max": 0.012031, + "nauc_mrr_at_5_std": -0.178128, + "nauc_mrr_at_5_diff1": 0.213914, + "nauc_mrr_at_10_max": 0.008875, + "nauc_mrr_at_10_std": -0.179782, + "nauc_mrr_at_10_diff1": 0.21791, + "nauc_mrr_at_20_max": 0.007709, + "nauc_mrr_at_20_std": -0.178442, + "nauc_mrr_at_20_diff1": 0.217792, + "nauc_mrr_at_100_max": 0.013971, + "nauc_mrr_at_100_std": -0.173085, + "nauc_mrr_at_100_diff1": 0.217119, + "nauc_mrr_at_1000_max": 0.013871, + "nauc_mrr_at_1000_std": -0.172947, + "nauc_mrr_at_1000_diff1": 0.216916, + "main_score": 0.36261, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.23125, + "ndcg_at_3": 0.26971, + "ndcg_at_5": 0.28825, + "ndcg_at_10": 0.32652, + "ndcg_at_20": 0.34575, + "ndcg_at_100": 0.40765, + "ndcg_at_1000": 0.45341, + "map_at_1": 0.11115, + "map_at_3": 0.19203, + "map_at_5": 0.21721, + "map_at_10": 0.24319, + "map_at_20": 0.25135, + "map_at_100": 0.26471, + "map_at_1000": 0.26812, + "recall_at_1": 0.11115, + "recall_at_3": 0.26198, + "recall_at_5": 0.33616, + "recall_at_10": 0.44782, + "recall_at_20": 0.51106, + "recall_at_100": 0.75926, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.23125, + "cv_recall_at_3": 0.4125, + "cv_recall_at_5": 0.5, + "cv_recall_at_10": 0.61875, + "cv_recall_at_20": 0.6875, + "cv_recall_at_100": 0.88125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.23125, + "precision_at_3": 0.18333, + "precision_at_5": 0.1475, + "precision_at_10": 0.10875, + "precision_at_20": 0.06469, + "precision_at_100": 0.02112, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.23125, + "mrr_at_3": 0.313542, + "mrr_at_5": 0.333229, + "mrr_at_10": 0.349266, + "mrr_at_20": 0.354393, + "mrr_at_100": 0.35954, + "mrr_at_1000": 0.360226, + "nauc_ndcg_at_1_max": 0.056896, + "nauc_ndcg_at_1_std": -0.129067, + "nauc_ndcg_at_1_diff1": 0.283843, + "nauc_ndcg_at_3_max": 0.064769, + "nauc_ndcg_at_3_std": -0.141173, + "nauc_ndcg_at_3_diff1": 0.223542, + "nauc_ndcg_at_5_max": 0.012838, + "nauc_ndcg_at_5_std": -0.160319, + "nauc_ndcg_at_5_diff1": 0.186755, + "nauc_ndcg_at_10_max": 0.043616, + "nauc_ndcg_at_10_std": -0.127455, + "nauc_ndcg_at_10_diff1": 0.181936, + "nauc_ndcg_at_20_max": 0.05429, + "nauc_ndcg_at_20_std": -0.133596, + "nauc_ndcg_at_20_diff1": 0.202153, + "nauc_ndcg_at_100_max": 0.056827, + "nauc_ndcg_at_100_std": -0.128777, + "nauc_ndcg_at_100_diff1": 0.23367, + "nauc_ndcg_at_1000_max": 0.064546, + "nauc_ndcg_at_1000_std": -0.132585, + "nauc_ndcg_at_1000_diff1": 0.227448, + "nauc_map_at_1_max": 0.097279, + "nauc_map_at_1_std": -0.118654, + "nauc_map_at_1_diff1": 0.320251, + "nauc_map_at_3_max": 0.061712, + "nauc_map_at_3_std": -0.140379, + "nauc_map_at_3_diff1": 0.217222, + "nauc_map_at_5_max": 0.022931, + "nauc_map_at_5_std": -0.160205, + "nauc_map_at_5_diff1": 0.201667, + "nauc_map_at_10_max": 0.032856, + "nauc_map_at_10_std": -0.144534, + "nauc_map_at_10_diff1": 0.201345, + "nauc_map_at_20_max": 0.034769, + "nauc_map_at_20_std": -0.150163, + "nauc_map_at_20_diff1": 0.211872, + "nauc_map_at_100_max": 0.03734, + "nauc_map_at_100_std": -0.148343, + "nauc_map_at_100_diff1": 0.224031, + "nauc_map_at_1000_max": 0.038617, + "nauc_map_at_1000_std": -0.147961, + "nauc_map_at_1000_diff1": 0.224392, + "nauc_recall_at_1_max": 0.097279, + "nauc_recall_at_1_std": -0.118654, + "nauc_recall_at_1_diff1": 0.320251, + "nauc_recall_at_3_max": 0.071141, + "nauc_recall_at_3_std": -0.128442, + "nauc_recall_at_3_diff1": 0.150485, + "nauc_recall_at_5_max": -0.024195, + "nauc_recall_at_5_std": -0.152368, + "nauc_recall_at_5_diff1": 0.096191, + "nauc_recall_at_10_max": 0.034408, + "nauc_recall_at_10_std": -0.079381, + "nauc_recall_at_10_diff1": 0.078963, + "nauc_recall_at_20_max": 0.053063, + "nauc_recall_at_20_std": -0.087304, + "nauc_recall_at_20_diff1": 0.120173, + "nauc_recall_at_100_max": 0.005383, + "nauc_recall_at_100_std": -0.083931, + "nauc_recall_at_100_diff1": 0.214845, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.056896, + "nauc_precision_at_1_std": -0.129067, + "nauc_precision_at_1_diff1": 0.283843, + "nauc_precision_at_3_max": 0.055292, + "nauc_precision_at_3_std": -0.122616, + "nauc_precision_at_3_diff1": 0.177076, + "nauc_precision_at_5_max": -0.057566, + "nauc_precision_at_5_std": -0.156351, + "nauc_precision_at_5_diff1": 0.110036, + "nauc_precision_at_10_max": 0.01937, + "nauc_precision_at_10_std": -0.054971, + "nauc_precision_at_10_diff1": 0.122183, + "nauc_precision_at_20_max": 0.023571, + "nauc_precision_at_20_std": -0.064903, + "nauc_precision_at_20_diff1": 0.151275, + "nauc_precision_at_100_max": 0.023199, + "nauc_precision_at_100_std": -0.020785, + "nauc_precision_at_100_diff1": 0.118955, + "nauc_precision_at_1000_max": -0.014491, + "nauc_precision_at_1000_std": -0.032476, + "nauc_precision_at_1000_diff1": -0.022769, + "nauc_cv_recall_at_1_max": 0.056896, + "nauc_cv_recall_at_1_std": -0.129067, + "nauc_cv_recall_at_1_diff1": 0.283843, + "nauc_cv_recall_at_3_max": 0.149206, + "nauc_cv_recall_at_3_std": -0.11573, + "nauc_cv_recall_at_3_diff1": 0.218363, + "nauc_cv_recall_at_5_max": 0.033311, + "nauc_cv_recall_at_5_std": -0.140998, + "nauc_cv_recall_at_5_diff1": 0.130555, + "nauc_cv_recall_at_10_max": 0.166144, + "nauc_cv_recall_at_10_std": -0.018715, + "nauc_cv_recall_at_10_diff1": 0.135073, + "nauc_cv_recall_at_20_max": 0.200219, + "nauc_cv_recall_at_20_std": 0.004134, + "nauc_cv_recall_at_20_diff1": 0.199276, + "nauc_cv_recall_at_100_max": -0.029781, + "nauc_cv_recall_at_100_std": -0.143133, + "nauc_cv_recall_at_100_diff1": 0.254077, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.056896, + "nauc_mrr_at_1_std": -0.129067, + "nauc_mrr_at_1_diff1": 0.283843, + "nauc_mrr_at_3_max": 0.093773, + "nauc_mrr_at_3_std": -0.126436, + "nauc_mrr_at_3_diff1": 0.247938, + "nauc_mrr_at_5_max": 0.065509, + "nauc_mrr_at_5_std": -0.133703, + "nauc_mrr_at_5_diff1": 0.22665, + "nauc_mrr_at_10_max": 0.08319, + "nauc_mrr_at_10_std": -0.116884, + "nauc_mrr_at_10_diff1": 0.228357, + "nauc_mrr_at_20_max": 0.085039, + "nauc_mrr_at_20_std": -0.116336, + "nauc_mrr_at_20_diff1": 0.23323, + "nauc_mrr_at_100_max": 0.082015, + "nauc_mrr_at_100_std": -0.117496, + "nauc_mrr_at_100_diff1": 0.235862, + "nauc_mrr_at_1000_max": 0.082158, + "nauc_mrr_at_1000_std": -0.117498, + "nauc_mrr_at_1000_diff1": 0.235781, + "main_score": 0.28825, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 164.05084085464478, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/Vidore2ESGReportsHLRetrieval.json b/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..5a798627a4 --- /dev/null +++ b/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.38462, + "ndcg_at_3": 0.35778, + "ndcg_at_5": 0.36263, + "ndcg_at_10": 0.39735, + "ndcg_at_20": 0.42664, + "ndcg_at_100": 0.49243, + "ndcg_at_1000": 0.51631, + "map_at_1": 0.27853, + "map_at_3": 0.30801, + "map_at_5": 0.31807, + "map_at_10": 0.33706, + "map_at_20": 0.34889, + "map_at_100": 0.362, + "map_at_1000": 0.36354, + "recall_at_1": 0.27853, + "recall_at_3": 0.33365, + "recall_at_5": 0.37386, + "recall_at_10": 0.47644, + "recall_at_20": 0.56842, + "recall_at_100": 0.86149, + "recall_at_1000": 0.99359, + "cv_recall_at_1": 0.38462, + "cv_recall_at_3": 0.42308, + "cv_recall_at_5": 0.5, + "cv_recall_at_10": 0.63462, + "cv_recall_at_20": 0.73077, + "cv_recall_at_100": 0.94231, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.38462, + "precision_at_3": 0.17308, + "precision_at_5": 0.11923, + "precision_at_10": 0.08462, + "precision_at_20": 0.05481, + "precision_at_100": 0.01788, + "precision_at_1000": 0.00244, + "mrr_at_1": 0.384615, + "mrr_at_3": 0.397436, + "mrr_at_5": 0.416667, + "mrr_at_10": 0.435783, + "mrr_at_20": 0.441858, + "mrr_at_100": 0.446186, + "mrr_at_1000": 0.446417, + "nauc_ndcg_at_1_max": 0.309662, + "nauc_ndcg_at_1_std": -0.096438, + "nauc_ndcg_at_1_diff1": 0.35936, + "nauc_ndcg_at_3_max": 0.313101, + "nauc_ndcg_at_3_std": -0.064876, + "nauc_ndcg_at_3_diff1": 0.255787, + "nauc_ndcg_at_5_max": 0.253115, + "nauc_ndcg_at_5_std": -0.12202, + "nauc_ndcg_at_5_diff1": 0.197058, + "nauc_ndcg_at_10_max": 0.208168, + "nauc_ndcg_at_10_std": -0.147482, + "nauc_ndcg_at_10_diff1": 0.206958, + "nauc_ndcg_at_20_max": 0.233082, + "nauc_ndcg_at_20_std": -0.104261, + "nauc_ndcg_at_20_diff1": 0.195507, + "nauc_ndcg_at_100_max": 0.219743, + "nauc_ndcg_at_100_std": -0.12458, + "nauc_ndcg_at_100_diff1": 0.20217, + "nauc_ndcg_at_1000_max": 0.254013, + "nauc_ndcg_at_1000_std": -0.112864, + "nauc_ndcg_at_1000_diff1": 0.225018, + "nauc_map_at_1_max": 0.140919, + "nauc_map_at_1_std": -0.212277, + "nauc_map_at_1_diff1": 0.266424, + "nauc_map_at_3_max": 0.20075, + "nauc_map_at_3_std": -0.160567, + "nauc_map_at_3_diff1": 0.246926, + "nauc_map_at_5_max": 0.184994, + "nauc_map_at_5_std": -0.180294, + "nauc_map_at_5_diff1": 0.224505, + "nauc_map_at_10_max": 0.172892, + "nauc_map_at_10_std": -0.186794, + "nauc_map_at_10_diff1": 0.225322, + "nauc_map_at_20_max": 0.193316, + "nauc_map_at_20_std": -0.160248, + "nauc_map_at_20_diff1": 0.21673, + "nauc_map_at_100_max": 0.198208, + "nauc_map_at_100_std": -0.157033, + "nauc_map_at_100_diff1": 0.21705, + "nauc_map_at_1000_max": 0.201463, + "nauc_map_at_1000_std": -0.155422, + "nauc_map_at_1000_diff1": 0.217997, + "nauc_recall_at_1_max": 0.140919, + "nauc_recall_at_1_std": -0.212277, + "nauc_recall_at_1_diff1": 0.266424, + "nauc_recall_at_3_max": 0.239633, + "nauc_recall_at_3_std": -0.093122, + "nauc_recall_at_3_diff1": 0.180264, + "nauc_recall_at_5_max": 0.174313, + "nauc_recall_at_5_std": -0.165045, + "nauc_recall_at_5_diff1": 0.089512, + "nauc_recall_at_10_max": 0.070867, + "nauc_recall_at_10_std": -0.1847, + "nauc_recall_at_10_diff1": 0.078623, + "nauc_recall_at_20_max": 0.121199, + "nauc_recall_at_20_std": -0.072188, + "nauc_recall_at_20_diff1": 0.030754, + "nauc_recall_at_100_max": -0.263199, + "nauc_recall_at_100_std": -0.235192, + "nauc_recall_at_100_diff1": -0.126727, + "nauc_recall_at_1000_max": -0.552063, + "nauc_recall_at_1000_std": -0.16971, + "nauc_recall_at_1000_diff1": 0.121607, + "nauc_precision_at_1_max": 0.309662, + "nauc_precision_at_1_std": -0.096438, + "nauc_precision_at_1_diff1": 0.35936, + "nauc_precision_at_3_max": 0.436832, + "nauc_precision_at_3_std": 0.113805, + "nauc_precision_at_3_diff1": 0.210995, + "nauc_precision_at_5_max": 0.428035, + "nauc_precision_at_5_std": 0.074759, + "nauc_precision_at_5_diff1": 0.112265, + "nauc_precision_at_10_max": 0.415663, + "nauc_precision_at_10_std": 0.148106, + "nauc_precision_at_10_diff1": 0.052229, + "nauc_precision_at_20_max": 0.489699, + "nauc_precision_at_20_std": 0.354738, + "nauc_precision_at_20_diff1": -0.027184, + "nauc_precision_at_100_max": 0.366289, + "nauc_precision_at_100_std": 0.300636, + "nauc_precision_at_100_diff1": -0.077658, + "nauc_precision_at_1000_max": 0.42402, + "nauc_precision_at_1000_std": 0.331297, + "nauc_precision_at_1000_diff1": -0.023429, + "nauc_cv_recall_at_1_max": 0.309662, + "nauc_cv_recall_at_1_std": -0.096438, + "nauc_cv_recall_at_1_diff1": 0.35936, + "nauc_cv_recall_at_3_max": 0.374053, + "nauc_cv_recall_at_3_std": -0.007914, + "nauc_cv_recall_at_3_diff1": 0.275484, + "nauc_cv_recall_at_5_max": 0.359303, + "nauc_cv_recall_at_5_std": -0.069199, + "nauc_cv_recall_at_5_diff1": 0.130444, + "nauc_cv_recall_at_10_max": 0.369192, + "nauc_cv_recall_at_10_std": 0.025485, + "nauc_cv_recall_at_10_diff1": 0.09996, + "nauc_cv_recall_at_20_max": 0.394588, + "nauc_cv_recall_at_20_std": 0.119604, + "nauc_cv_recall_at_20_diff1": 0.108641, + "nauc_cv_recall_at_100_max": 0.268279, + "nauc_cv_recall_at_100_std": 0.102933, + "nauc_cv_recall_at_100_diff1": 0.225359, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.309662, + "nauc_mrr_at_1_std": -0.096438, + "nauc_mrr_at_1_diff1": 0.35936, + "nauc_mrr_at_3_max": 0.331163, + "nauc_mrr_at_3_std": -0.066879, + "nauc_mrr_at_3_diff1": 0.331353, + "nauc_mrr_at_5_max": 0.32726, + "nauc_mrr_at_5_std": -0.082312, + "nauc_mrr_at_5_diff1": 0.296108, + "nauc_mrr_at_10_max": 0.329832, + "nauc_mrr_at_10_std": -0.070134, + "nauc_mrr_at_10_diff1": 0.296914, + "nauc_mrr_at_20_max": 0.33004, + "nauc_mrr_at_20_std": -0.067897, + "nauc_mrr_at_20_diff1": 0.299563, + "nauc_mrr_at_100_max": 0.328277, + "nauc_mrr_at_100_std": -0.071851, + "nauc_mrr_at_100_diff1": 0.302591, + "nauc_mrr_at_1000_max": 0.328193, + "nauc_mrr_at_1000_std": -0.072013, + "nauc_mrr_at_1000_diff1": 0.302536, + "main_score": 0.36263, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 132.5850751399994, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/Vidore2ESGReportsRetrieval.json b/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..412c88e88e --- /dev/null +++ b/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.22807, + "ndcg_at_3": 0.27611, + "ndcg_at_5": 0.27802, + "ndcg_at_10": 0.31131, + "ndcg_at_20": 0.33696, + "ndcg_at_100": 0.39448, + "ndcg_at_1000": 0.44006, + "map_at_1": 0.06608, + "map_at_3": 0.15941, + "map_at_5": 0.17696, + "map_at_10": 0.20732, + "map_at_20": 0.21918, + "map_at_100": 0.23161, + "map_at_1000": 0.23512, + "recall_at_1": 0.06608, + "recall_at_3": 0.26032, + "recall_at_5": 0.31134, + "recall_at_10": 0.43517, + "recall_at_20": 0.52391, + "recall_at_100": 0.75971, + "recall_at_1000": 0.99436, + "cv_recall_at_1": 0.22807, + "cv_recall_at_3": 0.50877, + "cv_recall_at_5": 0.57895, + "cv_recall_at_10": 0.70175, + "cv_recall_at_20": 0.78947, + "cv_recall_at_100": 0.91228, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.22807, + "precision_at_3": 0.21053, + "precision_at_5": 0.15789, + "precision_at_10": 0.11579, + "precision_at_20": 0.07368, + "precision_at_100": 0.02351, + "precision_at_1000": 0.00384, + "mrr_at_1": 0.22807, + "mrr_at_3": 0.350877, + "mrr_at_5": 0.366667, + "mrr_at_10": 0.383668, + "mrr_at_20": 0.390275, + "mrr_at_100": 0.393166, + "mrr_at_1000": 0.393457, + "nauc_ndcg_at_1_max": 0.059015, + "nauc_ndcg_at_1_std": 0.096426, + "nauc_ndcg_at_1_diff1": 0.284198, + "nauc_ndcg_at_3_max": 0.223245, + "nauc_ndcg_at_3_std": 0.184659, + "nauc_ndcg_at_3_diff1": 0.112846, + "nauc_ndcg_at_5_max": 0.241146, + "nauc_ndcg_at_5_std": 0.192151, + "nauc_ndcg_at_5_diff1": 0.126216, + "nauc_ndcg_at_10_max": 0.247856, + "nauc_ndcg_at_10_std": 0.213532, + "nauc_ndcg_at_10_diff1": 0.114894, + "nauc_ndcg_at_20_max": 0.280983, + "nauc_ndcg_at_20_std": 0.229513, + "nauc_ndcg_at_20_diff1": 0.064267, + "nauc_ndcg_at_100_max": 0.294271, + "nauc_ndcg_at_100_std": 0.258401, + "nauc_ndcg_at_100_diff1": 0.097766, + "nauc_ndcg_at_1000_max": 0.259745, + "nauc_ndcg_at_1000_std": 0.254829, + "nauc_ndcg_at_1000_diff1": 0.1212, + "nauc_map_at_1_max": 0.207166, + "nauc_map_at_1_std": 0.125852, + "nauc_map_at_1_diff1": 0.351168, + "nauc_map_at_3_max": 0.304987, + "nauc_map_at_3_std": 0.184467, + "nauc_map_at_3_diff1": 0.160567, + "nauc_map_at_5_max": 0.298071, + "nauc_map_at_5_std": 0.195705, + "nauc_map_at_5_diff1": 0.161317, + "nauc_map_at_10_max": 0.270857, + "nauc_map_at_10_std": 0.19933, + "nauc_map_at_10_diff1": 0.131571, + "nauc_map_at_20_max": 0.292307, + "nauc_map_at_20_std": 0.217443, + "nauc_map_at_20_diff1": 0.103203, + "nauc_map_at_100_max": 0.305702, + "nauc_map_at_100_std": 0.234983, + "nauc_map_at_100_diff1": 0.110053, + "nauc_map_at_1000_max": 0.301547, + "nauc_map_at_1000_std": 0.234085, + "nauc_map_at_1000_diff1": 0.114454, + "nauc_recall_at_1_max": 0.207166, + "nauc_recall_at_1_std": 0.125852, + "nauc_recall_at_1_diff1": 0.351168, + "nauc_recall_at_3_max": 0.220054, + "nauc_recall_at_3_std": 0.086336, + "nauc_recall_at_3_diff1": 0.08015, + "nauc_recall_at_5_max": 0.216767, + "nauc_recall_at_5_std": 0.100742, + "nauc_recall_at_5_diff1": 0.111424, + "nauc_recall_at_10_max": 0.225247, + "nauc_recall_at_10_std": 0.158432, + "nauc_recall_at_10_diff1": 0.075877, + "nauc_recall_at_20_max": 0.270128, + "nauc_recall_at_20_std": 0.174019, + "nauc_recall_at_20_diff1": -0.033822, + "nauc_recall_at_100_max": 0.233486, + "nauc_recall_at_100_std": 0.177135, + "nauc_recall_at_100_diff1": 0.009043, + "nauc_recall_at_1000_max": 0.836117, + "nauc_recall_at_1000_std": 0.551197, + "nauc_recall_at_1000_diff1": 0.126616, + "nauc_precision_at_1_max": 0.059015, + "nauc_precision_at_1_std": 0.096426, + "nauc_precision_at_1_diff1": 0.284198, + "nauc_precision_at_3_max": 0.151501, + "nauc_precision_at_3_std": 0.239993, + "nauc_precision_at_3_diff1": -0.000834, + "nauc_precision_at_5_max": 0.121218, + "nauc_precision_at_5_std": 0.251785, + "nauc_precision_at_5_diff1": -0.037695, + "nauc_precision_at_10_max": 0.132111, + "nauc_precision_at_10_std": 0.251555, + "nauc_precision_at_10_diff1": -0.033832, + "nauc_precision_at_20_max": 0.184231, + "nauc_precision_at_20_std": 0.253364, + "nauc_precision_at_20_diff1": -0.105062, + "nauc_precision_at_100_max": 0.103769, + "nauc_precision_at_100_std": 0.170652, + "nauc_precision_at_100_diff1": 0.017897, + "nauc_precision_at_1000_max": -0.100473, + "nauc_precision_at_1000_std": 0.058979, + "nauc_precision_at_1000_diff1": 0.077728, + "nauc_cv_recall_at_1_max": 0.059015, + "nauc_cv_recall_at_1_std": 0.096426, + "nauc_cv_recall_at_1_diff1": 0.284198, + "nauc_cv_recall_at_3_max": 0.131385, + "nauc_cv_recall_at_3_std": 0.170197, + "nauc_cv_recall_at_3_diff1": 0.04984, + "nauc_cv_recall_at_5_max": 0.050868, + "nauc_cv_recall_at_5_std": 0.155179, + "nauc_cv_recall_at_5_diff1": 0.023781, + "nauc_cv_recall_at_10_max": 0.250723, + "nauc_cv_recall_at_10_std": 0.341754, + "nauc_cv_recall_at_10_diff1": 0.151363, + "nauc_cv_recall_at_20_max": 0.361587, + "nauc_cv_recall_at_20_std": 0.421926, + "nauc_cv_recall_at_20_diff1": 0.105752, + "nauc_cv_recall_at_100_max": 0.037557, + "nauc_cv_recall_at_100_std": 0.290312, + "nauc_cv_recall_at_100_diff1": 0.270546, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.059015, + "nauc_mrr_at_1_std": 0.096426, + "nauc_mrr_at_1_diff1": 0.284198, + "nauc_mrr_at_3_max": 0.112619, + "nauc_mrr_at_3_std": 0.13394, + "nauc_mrr_at_3_diff1": 0.158747, + "nauc_mrr_at_5_max": 0.092748, + "nauc_mrr_at_5_std": 0.129115, + "nauc_mrr_at_5_diff1": 0.150854, + "nauc_mrr_at_10_max": 0.121377, + "nauc_mrr_at_10_std": 0.15564, + "nauc_mrr_at_10_diff1": 0.169922, + "nauc_mrr_at_20_max": 0.125269, + "nauc_mrr_at_20_std": 0.1569, + "nauc_mrr_at_20_diff1": 0.168509, + "nauc_mrr_at_100_max": 0.121135, + "nauc_mrr_at_100_std": 0.155855, + "nauc_mrr_at_100_diff1": 0.169525, + "nauc_mrr_at_1000_max": 0.121355, + "nauc_mrr_at_1000_std": 0.155741, + "nauc_mrr_at_1000_diff1": 0.169582, + "main_score": 0.27802, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.17544, + "ndcg_at_3": 0.21924, + "ndcg_at_5": 0.24818, + "ndcg_at_10": 0.29009, + "ndcg_at_20": 0.32773, + "ndcg_at_100": 0.37471, + "ndcg_at_1000": 0.4268, + "map_at_1": 0.08553, + "map_at_3": 0.15351, + "map_at_5": 0.17609, + "map_at_10": 0.20339, + "map_at_20": 0.2192, + "map_at_100": 0.22868, + "map_at_1000": 0.23278, + "recall_at_1": 0.08553, + "recall_at_3": 0.20249, + "recall_at_5": 0.28878, + "recall_at_10": 0.40835, + "recall_at_20": 0.53028, + "recall_at_100": 0.71972, + "recall_at_1000": 0.99217, + "cv_recall_at_1": 0.17544, + "cv_recall_at_3": 0.35088, + "cv_recall_at_5": 0.49123, + "cv_recall_at_10": 0.64912, + "cv_recall_at_20": 0.77193, + "cv_recall_at_100": 0.89474, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.17544, + "precision_at_3": 0.15789, + "precision_at_5": 0.13684, + "precision_at_10": 0.10877, + "precision_at_20": 0.07368, + "precision_at_100": 0.02246, + "precision_at_1000": 0.00379, + "mrr_at_1": 0.175439, + "mrr_at_3": 0.25731, + "mrr_at_5": 0.288012, + "mrr_at_10": 0.310185, + "mrr_at_20": 0.318795, + "mrr_at_100": 0.321754, + "mrr_at_1000": 0.322245, + "nauc_ndcg_at_1_max": 0.301842, + "nauc_ndcg_at_1_std": 0.159579, + "nauc_ndcg_at_1_diff1": 0.262313, + "nauc_ndcg_at_3_max": 0.424312, + "nauc_ndcg_at_3_std": 0.310175, + "nauc_ndcg_at_3_diff1": 0.182394, + "nauc_ndcg_at_5_max": 0.329239, + "nauc_ndcg_at_5_std": 0.199602, + "nauc_ndcg_at_5_diff1": 0.125503, + "nauc_ndcg_at_10_max": 0.280063, + "nauc_ndcg_at_10_std": 0.165709, + "nauc_ndcg_at_10_diff1": 0.150372, + "nauc_ndcg_at_20_max": 0.272587, + "nauc_ndcg_at_20_std": 0.197545, + "nauc_ndcg_at_20_diff1": 0.146837, + "nauc_ndcg_at_100_max": 0.309613, + "nauc_ndcg_at_100_std": 0.248492, + "nauc_ndcg_at_100_diff1": 0.182775, + "nauc_ndcg_at_1000_max": 0.295032, + "nauc_ndcg_at_1000_std": 0.235005, + "nauc_ndcg_at_1000_diff1": 0.139625, + "nauc_map_at_1_max": 0.179199, + "nauc_map_at_1_std": -0.008381, + "nauc_map_at_1_diff1": 0.27803, + "nauc_map_at_3_max": 0.38077, + "nauc_map_at_3_std": 0.204876, + "nauc_map_at_3_diff1": 0.25466, + "nauc_map_at_5_max": 0.318114, + "nauc_map_at_5_std": 0.14957, + "nauc_map_at_5_diff1": 0.201154, + "nauc_map_at_10_max": 0.297142, + "nauc_map_at_10_std": 0.149373, + "nauc_map_at_10_diff1": 0.200705, + "nauc_map_at_20_max": 0.299223, + "nauc_map_at_20_std": 0.176556, + "nauc_map_at_20_diff1": 0.18876, + "nauc_map_at_100_max": 0.310502, + "nauc_map_at_100_std": 0.190875, + "nauc_map_at_100_diff1": 0.183937, + "nauc_map_at_1000_max": 0.308882, + "nauc_map_at_1000_std": 0.190486, + "nauc_map_at_1000_diff1": 0.180391, + "nauc_recall_at_1_max": 0.179199, + "nauc_recall_at_1_std": -0.008381, + "nauc_recall_at_1_diff1": 0.27803, + "nauc_recall_at_3_max": 0.43371, + "nauc_recall_at_3_std": 0.263243, + "nauc_recall_at_3_diff1": 0.191652, + "nauc_recall_at_5_max": 0.224021, + "nauc_recall_at_5_std": 0.083027, + "nauc_recall_at_5_diff1": 0.056914, + "nauc_recall_at_10_max": 0.130849, + "nauc_recall_at_10_std": 0.047248, + "nauc_recall_at_10_diff1": 0.114144, + "nauc_recall_at_20_max": 0.115447, + "nauc_recall_at_20_std": 0.095736, + "nauc_recall_at_20_diff1": 0.108647, + "nauc_recall_at_100_max": 0.238423, + "nauc_recall_at_100_std": 0.262209, + "nauc_recall_at_100_diff1": 0.350667, + "nauc_recall_at_1000_max": 0.534074, + "nauc_recall_at_1000_std": 0.268354, + "nauc_recall_at_1000_diff1": 0.461937, + "nauc_precision_at_1_max": 0.301842, + "nauc_precision_at_1_std": 0.159579, + "nauc_precision_at_1_diff1": 0.262313, + "nauc_precision_at_3_max": 0.444576, + "nauc_precision_at_3_std": 0.392671, + "nauc_precision_at_3_diff1": 0.151973, + "nauc_precision_at_5_max": 0.292823, + "nauc_precision_at_5_std": 0.278087, + "nauc_precision_at_5_diff1": 0.022781, + "nauc_precision_at_10_max": 0.215297, + "nauc_precision_at_10_std": 0.248461, + "nauc_precision_at_10_diff1": -0.033212, + "nauc_precision_at_20_max": 0.215121, + "nauc_precision_at_20_std": 0.341576, + "nauc_precision_at_20_diff1": -0.067502, + "nauc_precision_at_100_max": 0.139317, + "nauc_precision_at_100_std": 0.28615, + "nauc_precision_at_100_diff1": -0.067244, + "nauc_precision_at_1000_max": -0.049191, + "nauc_precision_at_1000_std": 0.066805, + "nauc_precision_at_1000_diff1": -0.152856, + "nauc_cv_recall_at_1_max": 0.301842, + "nauc_cv_recall_at_1_std": 0.159579, + "nauc_cv_recall_at_1_diff1": 0.262313, + "nauc_cv_recall_at_3_max": 0.413867, + "nauc_cv_recall_at_3_std": 0.347801, + "nauc_cv_recall_at_3_diff1": 0.121794, + "nauc_cv_recall_at_5_max": 0.197616, + "nauc_cv_recall_at_5_std": 0.200398, + "nauc_cv_recall_at_5_diff1": -0.008147, + "nauc_cv_recall_at_10_max": 0.109535, + "nauc_cv_recall_at_10_std": 0.171108, + "nauc_cv_recall_at_10_diff1": -0.028605, + "nauc_cv_recall_at_20_max": 0.191187, + "nauc_cv_recall_at_20_std": 0.38067, + "nauc_cv_recall_at_20_diff1": 0.050783, + "nauc_cv_recall_at_100_max": 0.310337, + "nauc_cv_recall_at_100_std": 0.510328, + "nauc_cv_recall_at_100_diff1": 0.417002, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.301842, + "nauc_mrr_at_1_std": 0.159579, + "nauc_mrr_at_1_diff1": 0.262313, + "nauc_mrr_at_3_max": 0.383422, + "nauc_mrr_at_3_std": 0.278718, + "nauc_mrr_at_3_diff1": 0.192056, + "nauc_mrr_at_5_max": 0.332098, + "nauc_mrr_at_5_std": 0.246734, + "nauc_mrr_at_5_diff1": 0.156728, + "nauc_mrr_at_10_max": 0.320777, + "nauc_mrr_at_10_std": 0.242549, + "nauc_mrr_at_10_diff1": 0.15755, + "nauc_mrr_at_20_max": 0.324677, + "nauc_mrr_at_20_std": 0.252288, + "nauc_mrr_at_20_diff1": 0.161244, + "nauc_mrr_at_100_max": 0.324754, + "nauc_mrr_at_100_std": 0.251302, + "nauc_mrr_at_100_diff1": 0.162471, + "nauc_mrr_at_1000_max": 0.324536, + "nauc_mrr_at_1000_std": 0.250719, + "nauc_mrr_at_1000_diff1": 0.161769, + "main_score": 0.24818, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.33333, + "ndcg_at_3": 0.32691, + "ndcg_at_5": 0.33676, + "ndcg_at_10": 0.39001, + "ndcg_at_20": 0.42436, + "ndcg_at_100": 0.47142, + "ndcg_at_1000": 0.50977, + "map_at_1": 0.1443, + "map_at_3": 0.23207, + "map_at_5": 0.24827, + "map_at_10": 0.28455, + "map_at_20": 0.29832, + "map_at_100": 0.31071, + "map_at_1000": 0.31407, + "recall_at_1": 0.1443, + "recall_at_3": 0.30863, + "recall_at_5": 0.35121, + "recall_at_10": 0.51046, + "recall_at_20": 0.62878, + "recall_at_100": 0.80209, + "recall_at_1000": 0.99363, + "cv_recall_at_1": 0.33333, + "cv_recall_at_3": 0.50877, + "cv_recall_at_5": 0.57895, + "cv_recall_at_10": 0.77193, + "cv_recall_at_20": 0.87719, + "cv_recall_at_100": 0.92982, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.33333, + "precision_at_3": 0.20468, + "precision_at_5": 0.1614, + "precision_at_10": 0.12982, + "precision_at_20": 0.08333, + "precision_at_100": 0.02491, + "precision_at_1000": 0.00382, + "mrr_at_1": 0.333333, + "mrr_at_3": 0.418129, + "mrr_at_5": 0.433041, + "mrr_at_10": 0.461689, + "mrr_at_20": 0.468391, + "mrr_at_100": 0.469978, + "mrr_at_1000": 0.470187, + "nauc_ndcg_at_1_max": 0.19095, + "nauc_ndcg_at_1_std": -0.019503, + "nauc_ndcg_at_1_diff1": 0.151565, + "nauc_ndcg_at_3_max": 0.342759, + "nauc_ndcg_at_3_std": 0.102143, + "nauc_ndcg_at_3_diff1": 0.181734, + "nauc_ndcg_at_5_max": 0.352796, + "nauc_ndcg_at_5_std": 0.080781, + "nauc_ndcg_at_5_diff1": 0.205678, + "nauc_ndcg_at_10_max": 0.253964, + "nauc_ndcg_at_10_std": 0.019409, + "nauc_ndcg_at_10_diff1": 0.182502, + "nauc_ndcg_at_20_max": 0.230579, + "nauc_ndcg_at_20_std": 0.047748, + "nauc_ndcg_at_20_diff1": 0.156241, + "nauc_ndcg_at_100_max": 0.247055, + "nauc_ndcg_at_100_std": 0.08979, + "nauc_ndcg_at_100_diff1": 0.160689, + "nauc_ndcg_at_1000_max": 0.271873, + "nauc_ndcg_at_1000_std": 0.106738, + "nauc_ndcg_at_1000_diff1": 0.136935, + "nauc_map_at_1_max": 0.295237, + "nauc_map_at_1_std": 0.010538, + "nauc_map_at_1_diff1": 0.220834, + "nauc_map_at_3_max": 0.34076, + "nauc_map_at_3_std": 0.05101, + "nauc_map_at_3_diff1": 0.222379, + "nauc_map_at_5_max": 0.358652, + "nauc_map_at_5_std": 0.064548, + "nauc_map_at_5_diff1": 0.223114, + "nauc_map_at_10_max": 0.297536, + "nauc_map_at_10_std": 0.033867, + "nauc_map_at_10_diff1": 0.199803, + "nauc_map_at_20_max": 0.30062, + "nauc_map_at_20_std": 0.050718, + "nauc_map_at_20_diff1": 0.183125, + "nauc_map_at_100_max": 0.310732, + "nauc_map_at_100_std": 0.070916, + "nauc_map_at_100_diff1": 0.178917, + "nauc_map_at_1000_max": 0.311987, + "nauc_map_at_1000_std": 0.072626, + "nauc_map_at_1000_diff1": 0.178297, + "nauc_recall_at_1_max": 0.295237, + "nauc_recall_at_1_std": 0.010538, + "nauc_recall_at_1_diff1": 0.220834, + "nauc_recall_at_3_max": 0.340004, + "nauc_recall_at_3_std": 0.042769, + "nauc_recall_at_3_diff1": 0.228445, + "nauc_recall_at_5_max": 0.310855, + "nauc_recall_at_5_std": 0.013358, + "nauc_recall_at_5_diff1": 0.27991, + "nauc_recall_at_10_max": 0.13117, + "nauc_recall_at_10_std": -0.090391, + "nauc_recall_at_10_diff1": 0.247629, + "nauc_recall_at_20_max": 0.006485, + "nauc_recall_at_20_std": -0.054373, + "nauc_recall_at_20_diff1": 0.189087, + "nauc_recall_at_100_max": -0.029156, + "nauc_recall_at_100_std": 0.018022, + "nauc_recall_at_100_diff1": 0.296293, + "nauc_recall_at_1000_max": 0.624771, + "nauc_recall_at_1000_std": 0.488749, + "nauc_recall_at_1000_diff1": 0.464496, + "nauc_precision_at_1_max": 0.19095, + "nauc_precision_at_1_std": -0.019503, + "nauc_precision_at_1_diff1": 0.151565, + "nauc_precision_at_3_max": 0.321983, + "nauc_precision_at_3_std": 0.175936, + "nauc_precision_at_3_diff1": 0.100715, + "nauc_precision_at_5_max": 0.334192, + "nauc_precision_at_5_std": 0.202119, + "nauc_precision_at_5_diff1": 0.024314, + "nauc_precision_at_10_max": 0.102557, + "nauc_precision_at_10_std": 0.131975, + "nauc_precision_at_10_diff1": -0.13383, + "nauc_precision_at_20_max": 0.059974, + "nauc_precision_at_20_std": 0.213389, + "nauc_precision_at_20_diff1": -0.209716, + "nauc_precision_at_100_max": 0.03981, + "nauc_precision_at_100_std": 0.216891, + "nauc_precision_at_100_diff1": -0.162627, + "nauc_precision_at_1000_max": 0.004314, + "nauc_precision_at_1000_std": 0.124501, + "nauc_precision_at_1000_diff1": -0.141243, + "nauc_cv_recall_at_1_max": 0.19095, + "nauc_cv_recall_at_1_std": -0.019503, + "nauc_cv_recall_at_1_diff1": 0.151565, + "nauc_cv_recall_at_3_max": 0.301708, + "nauc_cv_recall_at_3_std": 0.111289, + "nauc_cv_recall_at_3_diff1": 0.134299, + "nauc_cv_recall_at_5_max": 0.260496, + "nauc_cv_recall_at_5_std": 0.082922, + "nauc_cv_recall_at_5_diff1": 0.138929, + "nauc_cv_recall_at_10_max": 0.066023, + "nauc_cv_recall_at_10_std": 0.125922, + "nauc_cv_recall_at_10_diff1": 0.021879, + "nauc_cv_recall_at_20_max": -0.277596, + "nauc_cv_recall_at_20_std": 0.365477, + "nauc_cv_recall_at_20_diff1": -0.143571, + "nauc_cv_recall_at_100_max": -0.030922, + "nauc_cv_recall_at_100_std": 0.260564, + "nauc_cv_recall_at_100_diff1": 0.388229, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.19095, + "nauc_mrr_at_1_std": -0.019503, + "nauc_mrr_at_1_diff1": 0.151565, + "nauc_mrr_at_3_max": 0.236122, + "nauc_mrr_at_3_std": 0.051742, + "nauc_mrr_at_3_diff1": 0.132996, + "nauc_mrr_at_5_max": 0.226727, + "nauc_mrr_at_5_std": 0.045269, + "nauc_mrr_at_5_diff1": 0.134389, + "nauc_mrr_at_10_max": 0.205236, + "nauc_mrr_at_10_std": 0.043768, + "nauc_mrr_at_10_diff1": 0.124557, + "nauc_mrr_at_20_max": 0.199418, + "nauc_mrr_at_20_std": 0.048817, + "nauc_mrr_at_20_diff1": 0.121651, + "nauc_mrr_at_100_max": 0.20221, + "nauc_mrr_at_100_std": 0.046478, + "nauc_mrr_at_100_diff1": 0.12507, + "nauc_mrr_at_1000_max": 0.202464, + "nauc_mrr_at_1000_std": 0.046413, + "nauc_mrr_at_1000_diff1": 0.124817, + "main_score": 0.33676, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.26316, + "ndcg_at_3": 0.27494, + "ndcg_at_5": 0.29227, + "ndcg_at_10": 0.31877, + "ndcg_at_20": 0.35074, + "ndcg_at_100": 0.40958, + "ndcg_at_1000": 0.45545, + "map_at_1": 0.11276, + "map_at_3": 0.18539, + "map_at_5": 0.20992, + "map_at_10": 0.23218, + "map_at_20": 0.24656, + "map_at_100": 0.25962, + "map_at_1000": 0.26314, + "recall_at_1": 0.11276, + "recall_at_3": 0.24716, + "recall_at_5": 0.31587, + "recall_at_10": 0.40856, + "recall_at_20": 0.52279, + "recall_at_100": 0.75257, + "recall_at_1000": 0.99353, + "cv_recall_at_1": 0.26316, + "cv_recall_at_3": 0.42105, + "cv_recall_at_5": 0.50877, + "cv_recall_at_10": 0.66667, + "cv_recall_at_20": 0.77193, + "cv_recall_at_100": 0.89474, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.26316, + "precision_at_3": 0.19883, + "precision_at_5": 0.1614, + "precision_at_10": 0.11754, + "precision_at_20": 0.07456, + "precision_at_100": 0.02368, + "precision_at_1000": 0.00382, + "mrr_at_1": 0.263158, + "mrr_at_3": 0.327485, + "mrr_at_5": 0.347661, + "mrr_at_10": 0.36856, + "mrr_at_20": 0.376121, + "mrr_at_100": 0.379873, + "mrr_at_1000": 0.380358, + "nauc_ndcg_at_1_max": -0.021618, + "nauc_ndcg_at_1_std": -0.157185, + "nauc_ndcg_at_1_diff1": -0.011463, + "nauc_ndcg_at_3_max": 0.23055, + "nauc_ndcg_at_3_std": 0.045855, + "nauc_ndcg_at_3_diff1": 0.153298, + "nauc_ndcg_at_5_max": 0.279733, + "nauc_ndcg_at_5_std": 0.075783, + "nauc_ndcg_at_5_diff1": 0.184493, + "nauc_ndcg_at_10_max": 0.245448, + "nauc_ndcg_at_10_std": 0.075499, + "nauc_ndcg_at_10_diff1": 0.190137, + "nauc_ndcg_at_20_max": 0.228448, + "nauc_ndcg_at_20_std": 0.109898, + "nauc_ndcg_at_20_diff1": 0.191452, + "nauc_ndcg_at_100_max": 0.200417, + "nauc_ndcg_at_100_std": 0.120761, + "nauc_ndcg_at_100_diff1": 0.153262, + "nauc_ndcg_at_1000_max": 0.192135, + "nauc_ndcg_at_1000_std": 0.084913, + "nauc_ndcg_at_1000_diff1": 0.126958, + "nauc_map_at_1_max": 0.030544, + "nauc_map_at_1_std": -0.173447, + "nauc_map_at_1_diff1": 0.133672, + "nauc_map_at_3_max": 0.214292, + "nauc_map_at_3_std": -0.009957, + "nauc_map_at_3_diff1": 0.246882, + "nauc_map_at_5_max": 0.252775, + "nauc_map_at_5_std": 0.018725, + "nauc_map_at_5_diff1": 0.228826, + "nauc_map_at_10_max": 0.261381, + "nauc_map_at_10_std": 0.050451, + "nauc_map_at_10_diff1": 0.221531, + "nauc_map_at_20_max": 0.256416, + "nauc_map_at_20_std": 0.065936, + "nauc_map_at_20_diff1": 0.210273, + "nauc_map_at_100_max": 0.260229, + "nauc_map_at_100_std": 0.077064, + "nauc_map_at_100_diff1": 0.20284, + "nauc_map_at_1000_max": 0.25867, + "nauc_map_at_1000_std": 0.075042, + "nauc_map_at_1000_diff1": 0.201033, + "nauc_recall_at_1_max": 0.030544, + "nauc_recall_at_1_std": -0.173447, + "nauc_recall_at_1_diff1": 0.133672, + "nauc_recall_at_3_max": 0.250226, + "nauc_recall_at_3_std": 0.02009, + "nauc_recall_at_3_diff1": 0.324773, + "nauc_recall_at_5_max": 0.325464, + "nauc_recall_at_5_std": 0.078133, + "nauc_recall_at_5_diff1": 0.307533, + "nauc_recall_at_10_max": 0.283271, + "nauc_recall_at_10_std": 0.119554, + "nauc_recall_at_10_diff1": 0.263052, + "nauc_recall_at_20_max": 0.244279, + "nauc_recall_at_20_std": 0.249308, + "nauc_recall_at_20_diff1": 0.285768, + "nauc_recall_at_100_max": 0.064868, + "nauc_recall_at_100_std": 0.309167, + "nauc_recall_at_100_diff1": 0.162068, + "nauc_recall_at_1000_max": 0.670509, + "nauc_recall_at_1000_std": 0.304414, + "nauc_recall_at_1000_diff1": 0.680647, + "nauc_precision_at_1_max": -0.021618, + "nauc_precision_at_1_std": -0.157185, + "nauc_precision_at_1_diff1": -0.011463, + "nauc_precision_at_3_max": 0.249677, + "nauc_precision_at_3_std": 0.125343, + "nauc_precision_at_3_diff1": 0.053768, + "nauc_precision_at_5_max": 0.284712, + "nauc_precision_at_5_std": 0.19347, + "nauc_precision_at_5_diff1": -0.010483, + "nauc_precision_at_10_max": 0.281532, + "nauc_precision_at_10_std": 0.2751, + "nauc_precision_at_10_diff1": -0.030899, + "nauc_precision_at_20_max": 0.225694, + "nauc_precision_at_20_std": 0.298902, + "nauc_precision_at_20_diff1": -0.091774, + "nauc_precision_at_100_max": 0.063252, + "nauc_precision_at_100_std": 0.221088, + "nauc_precision_at_100_diff1": -0.207646, + "nauc_precision_at_1000_max": -0.118162, + "nauc_precision_at_1000_std": 0.029449, + "nauc_precision_at_1000_diff1": -0.291492, + "nauc_cv_recall_at_1_max": -0.021618, + "nauc_cv_recall_at_1_std": -0.157185, + "nauc_cv_recall_at_1_diff1": -0.011463, + "nauc_cv_recall_at_3_max": 0.157742, + "nauc_cv_recall_at_3_std": 0.027554, + "nauc_cv_recall_at_3_diff1": 0.179035, + "nauc_cv_recall_at_5_max": 0.24225, + "nauc_cv_recall_at_5_std": 0.148652, + "nauc_cv_recall_at_5_diff1": 0.191878, + "nauc_cv_recall_at_10_max": 0.14285, + "nauc_cv_recall_at_10_std": 0.204433, + "nauc_cv_recall_at_10_diff1": 0.011657, + "nauc_cv_recall_at_20_max": 0.154627, + "nauc_cv_recall_at_20_std": 0.46583, + "nauc_cv_recall_at_20_diff1": 0.112519, + "nauc_cv_recall_at_100_max": -0.255582, + "nauc_cv_recall_at_100_std": 0.350163, + "nauc_cv_recall_at_100_diff1": -0.175761, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.021618, + "nauc_mrr_at_1_std": -0.157185, + "nauc_mrr_at_1_diff1": -0.011463, + "nauc_mrr_at_3_max": 0.080514, + "nauc_mrr_at_3_std": -0.050811, + "nauc_mrr_at_3_diff1": 0.072101, + "nauc_mrr_at_5_max": 0.102105, + "nauc_mrr_at_5_std": -0.020315, + "nauc_mrr_at_5_diff1": 0.077808, + "nauc_mrr_at_10_max": 0.0857, + "nauc_mrr_at_10_std": -0.021377, + "nauc_mrr_at_10_diff1": 0.053491, + "nauc_mrr_at_20_max": 0.083048, + "nauc_mrr_at_20_std": -0.016068, + "nauc_mrr_at_20_diff1": 0.057641, + "nauc_mrr_at_100_max": 0.0786, + "nauc_mrr_at_100_std": -0.021004, + "nauc_mrr_at_100_diff1": 0.054448, + "nauc_mrr_at_1000_max": 0.079382, + "nauc_mrr_at_1000_std": -0.021613, + "nauc_mrr_at_1000_diff1": 0.055317, + "main_score": 0.29227, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 577.6250951290131, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/Vidore2EconomicsReportsRetrieval.json b/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..31516aada2 --- /dev/null +++ b/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.25862, + "ndcg_at_3": 0.22236, + "ndcg_at_5": 0.23274, + "ndcg_at_10": 0.24455, + "ndcg_at_20": 0.29575, + "ndcg_at_100": 0.4251, + "ndcg_at_1000": 0.51512, + "map_at_1": 0.02363, + "map_at_3": 0.04751, + "map_at_5": 0.06393, + "map_at_10": 0.08924, + "map_at_20": 0.1262, + "map_at_100": 0.18349, + "map_at_1000": 0.21, + "recall_at_1": 0.02363, + "recall_at_3": 0.07149, + "recall_at_5": 0.10959, + "recall_at_10": 0.2016, + "recall_at_20": 0.37194, + "recall_at_100": 0.73726, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.25862, + "cv_recall_at_3": 0.46552, + "cv_recall_at_5": 0.55172, + "cv_recall_at_10": 0.7931, + "cv_recall_at_20": 0.94828, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.25862, + "precision_at_3": 0.21264, + "precision_at_5": 0.22414, + "precision_at_10": 0.20517, + "precision_at_20": 0.1819, + "precision_at_100": 0.09638, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.258621, + "mrr_at_3": 0.336207, + "mrr_at_5": 0.356897, + "mrr_at_10": 0.392063, + "mrr_at_20": 0.403675, + "mrr_at_100": 0.405768, + "mrr_at_1000": 0.405768, + "nauc_ndcg_at_1_max": 0.361733, + "nauc_ndcg_at_1_std": 0.285224, + "nauc_ndcg_at_1_diff1": 0.441794, + "nauc_ndcg_at_3_max": 0.390598, + "nauc_ndcg_at_3_std": 0.286688, + "nauc_ndcg_at_3_diff1": 0.39963, + "nauc_ndcg_at_5_max": 0.366694, + "nauc_ndcg_at_5_std": 0.306354, + "nauc_ndcg_at_5_diff1": 0.325965, + "nauc_ndcg_at_10_max": 0.370895, + "nauc_ndcg_at_10_std": 0.361892, + "nauc_ndcg_at_10_diff1": 0.24167, + "nauc_ndcg_at_20_max": 0.434755, + "nauc_ndcg_at_20_std": 0.402871, + "nauc_ndcg_at_20_diff1": 0.238491, + "nauc_ndcg_at_100_max": 0.533998, + "nauc_ndcg_at_100_std": 0.491582, + "nauc_ndcg_at_100_diff1": 0.358749, + "nauc_ndcg_at_1000_max": 0.347827, + "nauc_ndcg_at_1000_std": 0.392186, + "nauc_ndcg_at_1000_diff1": 0.251214, + "nauc_map_at_1_max": 0.405952, + "nauc_map_at_1_std": 0.382218, + "nauc_map_at_1_diff1": 0.451313, + "nauc_map_at_3_max": 0.393916, + "nauc_map_at_3_std": 0.267724, + "nauc_map_at_3_diff1": 0.340111, + "nauc_map_at_5_max": 0.488437, + "nauc_map_at_5_std": 0.315381, + "nauc_map_at_5_diff1": 0.406157, + "nauc_map_at_10_max": 0.553095, + "nauc_map_at_10_std": 0.352167, + "nauc_map_at_10_diff1": 0.39192, + "nauc_map_at_20_max": 0.574149, + "nauc_map_at_20_std": 0.397164, + "nauc_map_at_20_diff1": 0.369418, + "nauc_map_at_100_max": 0.52927, + "nauc_map_at_100_std": 0.480809, + "nauc_map_at_100_diff1": 0.341642, + "nauc_map_at_1000_max": 0.409994, + "nauc_map_at_1000_std": 0.432668, + "nauc_map_at_1000_diff1": 0.247134, + "nauc_recall_at_1_max": 0.405952, + "nauc_recall_at_1_std": 0.382218, + "nauc_recall_at_1_diff1": 0.451313, + "nauc_recall_at_3_max": 0.236919, + "nauc_recall_at_3_std": 0.129231, + "nauc_recall_at_3_diff1": 0.138291, + "nauc_recall_at_5_max": 0.401286, + "nauc_recall_at_5_std": 0.252231, + "nauc_recall_at_5_diff1": 0.258336, + "nauc_recall_at_10_max": 0.362914, + "nauc_recall_at_10_std": 0.182545, + "nauc_recall_at_10_diff1": 0.184184, + "nauc_recall_at_20_max": 0.331339, + "nauc_recall_at_20_std": 0.17159, + "nauc_recall_at_20_diff1": 0.130972, + "nauc_recall_at_100_max": 0.411361, + "nauc_recall_at_100_std": 0.285043, + "nauc_recall_at_100_diff1": 0.188063, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.361733, + "nauc_precision_at_1_std": 0.285224, + "nauc_precision_at_1_diff1": 0.441794, + "nauc_precision_at_3_max": 0.415402, + "nauc_precision_at_3_std": 0.305236, + "nauc_precision_at_3_diff1": 0.393559, + "nauc_precision_at_5_max": 0.334894, + "nauc_precision_at_5_std": 0.30485, + "nauc_precision_at_5_diff1": 0.259093, + "nauc_precision_at_10_max": 0.25385, + "nauc_precision_at_10_std": 0.32064, + "nauc_precision_at_10_diff1": 0.109625, + "nauc_precision_at_20_max": 0.201157, + "nauc_precision_at_20_std": 0.325356, + "nauc_precision_at_20_diff1": 0.053773, + "nauc_precision_at_100_max": -0.07788, + "nauc_precision_at_100_std": 0.219752, + "nauc_precision_at_100_diff1": -0.121588, + "nauc_precision_at_1000_max": -0.217304, + "nauc_precision_at_1000_std": 0.103118, + "nauc_precision_at_1000_diff1": -0.225668, + "nauc_cv_recall_at_1_max": 0.361733, + "nauc_cv_recall_at_1_std": 0.285224, + "nauc_cv_recall_at_1_diff1": 0.441794, + "nauc_cv_recall_at_3_max": 0.298675, + "nauc_cv_recall_at_3_std": 0.293386, + "nauc_cv_recall_at_3_diff1": 0.330183, + "nauc_cv_recall_at_5_max": 0.388959, + "nauc_cv_recall_at_5_std": 0.421412, + "nauc_cv_recall_at_5_diff1": 0.371938, + "nauc_cv_recall_at_10_max": 0.523638, + "nauc_cv_recall_at_10_std": 0.476732, + "nauc_cv_recall_at_10_diff1": 0.394012, + "nauc_cv_recall_at_20_max": 0.35164, + "nauc_cv_recall_at_20_std": 0.472102, + "nauc_cv_recall_at_20_diff1": -0.108925, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.361733, + "nauc_mrr_at_1_std": 0.285224, + "nauc_mrr_at_1_diff1": 0.441794, + "nauc_mrr_at_3_max": 0.339395, + "nauc_mrr_at_3_std": 0.294752, + "nauc_mrr_at_3_diff1": 0.413658, + "nauc_mrr_at_5_max": 0.357797, + "nauc_mrr_at_5_std": 0.321906, + "nauc_mrr_at_5_diff1": 0.422129, + "nauc_mrr_at_10_max": 0.369291, + "nauc_mrr_at_10_std": 0.319907, + "nauc_mrr_at_10_diff1": 0.426435, + "nauc_mrr_at_20_max": 0.363057, + "nauc_mrr_at_20_std": 0.316382, + "nauc_mrr_at_20_diff1": 0.421152, + "nauc_mrr_at_100_max": 0.363266, + "nauc_mrr_at_100_std": 0.315639, + "nauc_mrr_at_100_diff1": 0.425125, + "nauc_mrr_at_1000_max": 0.363266, + "nauc_mrr_at_1000_std": 0.315639, + "nauc_mrr_at_1000_diff1": 0.425125, + "main_score": 0.23274, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.25862, + "ndcg_at_3": 0.28258, + "ndcg_at_5": 0.27655, + "ndcg_at_10": 0.2912, + "ndcg_at_20": 0.33321, + "ndcg_at_100": 0.46492, + "ndcg_at_1000": 0.54083, + "map_at_1": 0.02557, + "map_at_3": 0.06052, + "map_at_5": 0.07846, + "map_at_10": 0.10951, + "map_at_20": 0.14566, + "map_at_100": 0.20864, + "map_at_1000": 0.23261, + "recall_at_1": 0.02557, + "recall_at_3": 0.08485, + "recall_at_5": 0.13324, + "recall_at_10": 0.24802, + "recall_at_20": 0.41017, + "recall_at_100": 0.78096, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.25862, + "cv_recall_at_3": 0.56897, + "cv_recall_at_5": 0.68966, + "cv_recall_at_10": 0.87931, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.25862, + "precision_at_3": 0.28161, + "precision_at_5": 0.25862, + "precision_at_10": 0.23448, + "precision_at_20": 0.19655, + "precision_at_100": 0.10397, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.258621, + "mrr_at_3": 0.402299, + "mrr_at_5": 0.429885, + "mrr_at_10": 0.456938, + "mrr_at_20": 0.463791, + "mrr_at_100": 0.46443, + "mrr_at_1000": 0.46443, + "nauc_ndcg_at_1_max": 0.24885, + "nauc_ndcg_at_1_std": 0.163594, + "nauc_ndcg_at_1_diff1": 0.266734, + "nauc_ndcg_at_3_max": 0.244579, + "nauc_ndcg_at_3_std": 0.133611, + "nauc_ndcg_at_3_diff1": 0.175731, + "nauc_ndcg_at_5_max": 0.241932, + "nauc_ndcg_at_5_std": 0.148462, + "nauc_ndcg_at_5_diff1": 0.181238, + "nauc_ndcg_at_10_max": 0.224639, + "nauc_ndcg_at_10_std": 0.190417, + "nauc_ndcg_at_10_diff1": 0.021926, + "nauc_ndcg_at_20_max": 0.292875, + "nauc_ndcg_at_20_std": 0.201831, + "nauc_ndcg_at_20_diff1": 0.065275, + "nauc_ndcg_at_100_max": 0.364171, + "nauc_ndcg_at_100_std": 0.299192, + "nauc_ndcg_at_100_diff1": 0.113389, + "nauc_ndcg_at_1000_max": 0.246145, + "nauc_ndcg_at_1000_std": 0.249844, + "nauc_ndcg_at_1000_diff1": 0.090601, + "nauc_map_at_1_max": 0.206322, + "nauc_map_at_1_std": 0.10607, + "nauc_map_at_1_diff1": 0.159424, + "nauc_map_at_3_max": 0.319499, + "nauc_map_at_3_std": 0.152798, + "nauc_map_at_3_diff1": 0.111586, + "nauc_map_at_5_max": 0.347027, + "nauc_map_at_5_std": 0.126255, + "nauc_map_at_5_diff1": 0.200287, + "nauc_map_at_10_max": 0.363782, + "nauc_map_at_10_std": 0.149896, + "nauc_map_at_10_diff1": 0.096471, + "nauc_map_at_20_max": 0.392269, + "nauc_map_at_20_std": 0.206378, + "nauc_map_at_20_diff1": 0.098881, + "nauc_map_at_100_max": 0.315745, + "nauc_map_at_100_std": 0.233464, + "nauc_map_at_100_diff1": 0.083715, + "nauc_map_at_1000_max": 0.237263, + "nauc_map_at_1000_std": 0.208471, + "nauc_map_at_1000_diff1": 0.056415, + "nauc_recall_at_1_max": 0.206322, + "nauc_recall_at_1_std": 0.10607, + "nauc_recall_at_1_diff1": 0.159424, + "nauc_recall_at_3_max": 0.324603, + "nauc_recall_at_3_std": 0.215381, + "nauc_recall_at_3_diff1": 0.041782, + "nauc_recall_at_5_max": 0.298566, + "nauc_recall_at_5_std": 0.114006, + "nauc_recall_at_5_diff1": 0.153707, + "nauc_recall_at_10_max": 0.214489, + "nauc_recall_at_10_std": 0.071414, + "nauc_recall_at_10_diff1": -0.109138, + "nauc_recall_at_20_max": 0.188356, + "nauc_recall_at_20_std": 0.034034, + "nauc_recall_at_20_diff1": -0.018067, + "nauc_recall_at_100_max": 0.286854, + "nauc_recall_at_100_std": 0.171705, + "nauc_recall_at_100_diff1": 0.033477, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.24885, + "nauc_precision_at_1_std": 0.163594, + "nauc_precision_at_1_diff1": 0.266734, + "nauc_precision_at_3_max": 0.223467, + "nauc_precision_at_3_std": 0.112276, + "nauc_precision_at_3_diff1": 0.146753, + "nauc_precision_at_5_max": 0.226932, + "nauc_precision_at_5_std": 0.15566, + "nauc_precision_at_5_diff1": 0.15161, + "nauc_precision_at_10_max": 0.195821, + "nauc_precision_at_10_std": 0.24135, + "nauc_precision_at_10_diff1": -0.007115, + "nauc_precision_at_20_max": 0.126681, + "nauc_precision_at_20_std": 0.23494, + "nauc_precision_at_20_diff1": 0.012036, + "nauc_precision_at_100_max": -0.129374, + "nauc_precision_at_100_std": 0.08278, + "nauc_precision_at_100_diff1": -0.033732, + "nauc_precision_at_1000_max": -0.207321, + "nauc_precision_at_1000_std": 0.018166, + "nauc_precision_at_1000_diff1": -0.066372, + "nauc_cv_recall_at_1_max": 0.24885, + "nauc_cv_recall_at_1_std": 0.163594, + "nauc_cv_recall_at_1_diff1": 0.266734, + "nauc_cv_recall_at_3_max": 0.230547, + "nauc_cv_recall_at_3_std": 0.21375, + "nauc_cv_recall_at_3_diff1": 0.114583, + "nauc_cv_recall_at_5_max": 0.392862, + "nauc_cv_recall_at_5_std": 0.409446, + "nauc_cv_recall_at_5_diff1": 0.018318, + "nauc_cv_recall_at_10_max": 0.496906, + "nauc_cv_recall_at_10_std": 0.717024, + "nauc_cv_recall_at_10_diff1": -0.375751, + "nauc_cv_recall_at_20_max": 0.720441, + "nauc_cv_recall_at_20_std": 0.554013, + "nauc_cv_recall_at_20_diff1": 0.123761, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.24885, + "nauc_mrr_at_1_std": 0.163594, + "nauc_mrr_at_1_diff1": 0.266734, + "nauc_mrr_at_3_max": 0.215988, + "nauc_mrr_at_3_std": 0.168518, + "nauc_mrr_at_3_diff1": 0.195044, + "nauc_mrr_at_5_max": 0.240549, + "nauc_mrr_at_5_std": 0.202806, + "nauc_mrr_at_5_diff1": 0.179165, + "nauc_mrr_at_10_max": 0.235864, + "nauc_mrr_at_10_std": 0.202219, + "nauc_mrr_at_10_diff1": 0.164898, + "nauc_mrr_at_20_max": 0.231715, + "nauc_mrr_at_20_std": 0.189156, + "nauc_mrr_at_20_diff1": 0.180713, + "nauc_mrr_at_100_max": 0.230631, + "nauc_mrr_at_100_std": 0.188347, + "nauc_mrr_at_100_diff1": 0.18084, + "nauc_mrr_at_1000_max": 0.230631, + "nauc_mrr_at_1000_std": 0.188347, + "nauc_mrr_at_1000_diff1": 0.18084, + "main_score": 0.27655, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.44828, + "ndcg_at_3": 0.42899, + "ndcg_at_5": 0.39747, + "ndcg_at_10": 0.37169, + "ndcg_at_20": 0.40084, + "ndcg_at_100": 0.53416, + "ndcg_at_1000": 0.59938, + "map_at_1": 0.04017, + "map_at_3": 0.10253, + "map_at_5": 0.12597, + "map_at_10": 0.16198, + "map_at_20": 0.1995, + "map_at_100": 0.27571, + "map_at_1000": 0.29748, + "recall_at_1": 0.04017, + "recall_at_3": 0.14206, + "recall_at_5": 0.19583, + "recall_at_10": 0.27487, + "recall_at_20": 0.43622, + "recall_at_100": 0.80679, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.44828, + "cv_recall_at_3": 0.67241, + "cv_recall_at_5": 0.81034, + "cv_recall_at_10": 0.86207, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.44828, + "precision_at_3": 0.4023, + "precision_at_5": 0.34828, + "precision_at_10": 0.27931, + "precision_at_20": 0.21638, + "precision_at_100": 0.11241, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.448276, + "mrr_at_3": 0.551724, + "mrr_at_5": 0.585345, + "mrr_at_10": 0.592289, + "mrr_at_20": 0.601636, + "mrr_at_100": 0.602211, + "mrr_at_1000": 0.602211, + "nauc_ndcg_at_1_max": 0.181774, + "nauc_ndcg_at_1_std": 0.222131, + "nauc_ndcg_at_1_diff1": 0.33036, + "nauc_ndcg_at_3_max": 0.104473, + "nauc_ndcg_at_3_std": 0.13506, + "nauc_ndcg_at_3_diff1": 0.215842, + "nauc_ndcg_at_5_max": 0.126886, + "nauc_ndcg_at_5_std": 0.164081, + "nauc_ndcg_at_5_diff1": 0.180682, + "nauc_ndcg_at_10_max": 0.161542, + "nauc_ndcg_at_10_std": 0.187033, + "nauc_ndcg_at_10_diff1": 0.134547, + "nauc_ndcg_at_20_max": 0.20884, + "nauc_ndcg_at_20_std": 0.208663, + "nauc_ndcg_at_20_diff1": 0.096818, + "nauc_ndcg_at_100_max": 0.313146, + "nauc_ndcg_at_100_std": 0.277282, + "nauc_ndcg_at_100_diff1": 0.257598, + "nauc_ndcg_at_1000_max": 0.223489, + "nauc_ndcg_at_1000_std": 0.278149, + "nauc_ndcg_at_1000_diff1": 0.234691, + "nauc_map_at_1_max": 0.172222, + "nauc_map_at_1_std": 0.050948, + "nauc_map_at_1_diff1": 0.284753, + "nauc_map_at_3_max": -0.000492, + "nauc_map_at_3_std": -0.172454, + "nauc_map_at_3_diff1": 0.152481, + "nauc_map_at_5_max": 0.103942, + "nauc_map_at_5_std": -0.117132, + "nauc_map_at_5_diff1": 0.193193, + "nauc_map_at_10_max": 0.193134, + "nauc_map_at_10_std": -0.066025, + "nauc_map_at_10_diff1": 0.194879, + "nauc_map_at_20_max": 0.246399, + "nauc_map_at_20_std": 0.009936, + "nauc_map_at_20_diff1": 0.182073, + "nauc_map_at_100_max": 0.244562, + "nauc_map_at_100_std": 0.186724, + "nauc_map_at_100_diff1": 0.165386, + "nauc_map_at_1000_max": 0.202582, + "nauc_map_at_1000_std": 0.225845, + "nauc_map_at_1000_diff1": 0.130686, + "nauc_recall_at_1_max": 0.172222, + "nauc_recall_at_1_std": 0.050948, + "nauc_recall_at_1_diff1": 0.284753, + "nauc_recall_at_3_max": -0.125121, + "nauc_recall_at_3_std": -0.239655, + "nauc_recall_at_3_diff1": 0.04284, + "nauc_recall_at_5_max": 0.011144, + "nauc_recall_at_5_std": -0.139642, + "nauc_recall_at_5_diff1": 0.073601, + "nauc_recall_at_10_max": 0.124567, + "nauc_recall_at_10_std": -0.09936, + "nauc_recall_at_10_diff1": 0.070521, + "nauc_recall_at_20_max": 0.154152, + "nauc_recall_at_20_std": -0.019387, + "nauc_recall_at_20_diff1": -0.036995, + "nauc_recall_at_100_max": 0.319872, + "nauc_recall_at_100_std": 0.149655, + "nauc_recall_at_100_diff1": 0.083621, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.181774, + "nauc_precision_at_1_std": 0.222131, + "nauc_precision_at_1_diff1": 0.33036, + "nauc_precision_at_3_max": 0.141082, + "nauc_precision_at_3_std": 0.164815, + "nauc_precision_at_3_diff1": 0.18703, + "nauc_precision_at_5_max": 0.204884, + "nauc_precision_at_5_std": 0.236094, + "nauc_precision_at_5_diff1": 0.144823, + "nauc_precision_at_10_max": 0.197261, + "nauc_precision_at_10_std": 0.258883, + "nauc_precision_at_10_diff1": 0.097128, + "nauc_precision_at_20_max": 0.165993, + "nauc_precision_at_20_std": 0.369027, + "nauc_precision_at_20_diff1": 0.042724, + "nauc_precision_at_100_max": 0.006861, + "nauc_precision_at_100_std": 0.410102, + "nauc_precision_at_100_diff1": -0.055583, + "nauc_precision_at_1000_max": -0.071082, + "nauc_precision_at_1000_std": 0.3803, + "nauc_precision_at_1000_diff1": -0.123951, + "nauc_cv_recall_at_1_max": 0.181774, + "nauc_cv_recall_at_1_std": 0.222131, + "nauc_cv_recall_at_1_diff1": 0.33036, + "nauc_cv_recall_at_3_max": 0.139401, + "nauc_cv_recall_at_3_std": 0.237429, + "nauc_cv_recall_at_3_diff1": 0.328735, + "nauc_cv_recall_at_5_max": 0.385906, + "nauc_cv_recall_at_5_std": 0.461798, + "nauc_cv_recall_at_5_diff1": 0.319709, + "nauc_cv_recall_at_10_max": 0.477388, + "nauc_cv_recall_at_10_std": 0.395966, + "nauc_cv_recall_at_10_diff1": 0.487517, + "nauc_cv_recall_at_20_max": 1.0, + "nauc_cv_recall_at_20_std": 0.868778, + "nauc_cv_recall_at_20_diff1": 0.554013, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.181774, + "nauc_mrr_at_1_std": 0.222131, + "nauc_mrr_at_1_diff1": 0.33036, + "nauc_mrr_at_3_max": 0.155553, + "nauc_mrr_at_3_std": 0.207555, + "nauc_mrr_at_3_diff1": 0.326146, + "nauc_mrr_at_5_max": 0.195097, + "nauc_mrr_at_5_std": 0.239349, + "nauc_mrr_at_5_diff1": 0.325367, + "nauc_mrr_at_10_max": 0.200353, + "nauc_mrr_at_10_std": 0.232031, + "nauc_mrr_at_10_diff1": 0.334317, + "nauc_mrr_at_20_max": 0.195626, + "nauc_mrr_at_20_std": 0.23112, + "nauc_mrr_at_20_diff1": 0.330118, + "nauc_mrr_at_100_max": 0.193982, + "nauc_mrr_at_100_std": 0.229818, + "nauc_mrr_at_100_diff1": 0.32966, + "nauc_mrr_at_1000_max": 0.193982, + "nauc_mrr_at_1000_std": 0.229818, + "nauc_mrr_at_1000_diff1": 0.32966, + "main_score": 0.39747, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.36207, + "ndcg_at_3": 0.29746, + "ndcg_at_5": 0.29071, + "ndcg_at_10": 0.28996, + "ndcg_at_20": 0.33672, + "ndcg_at_100": 0.48011, + "ndcg_at_1000": 0.55015, + "map_at_1": 0.03451, + "map_at_3": 0.06443, + "map_at_5": 0.08136, + "map_at_10": 0.11527, + "map_at_20": 0.15136, + "map_at_100": 0.22253, + "map_at_1000": 0.24449, + "recall_at_1": 0.03451, + "recall_at_3": 0.08022, + "recall_at_5": 0.1242, + "recall_at_10": 0.22182, + "recall_at_20": 0.39854, + "recall_at_100": 0.79536, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.36207, + "cv_recall_at_3": 0.55172, + "cv_recall_at_5": 0.72414, + "cv_recall_at_10": 0.81034, + "cv_recall_at_20": 0.94828, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.36207, + "precision_at_3": 0.28161, + "precision_at_5": 0.27241, + "precision_at_10": 0.23276, + "precision_at_20": 0.19741, + "precision_at_100": 0.10793, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.362069, + "mrr_at_3": 0.442529, + "mrr_at_5": 0.48046, + "mrr_at_10": 0.490873, + "mrr_at_20": 0.501599, + "mrr_at_100": 0.503466, + "mrr_at_1000": 0.503466, + "nauc_ndcg_at_1_max": 0.234748, + "nauc_ndcg_at_1_std": 0.386373, + "nauc_ndcg_at_1_diff1": 0.213938, + "nauc_ndcg_at_3_max": 0.311929, + "nauc_ndcg_at_3_std": 0.327936, + "nauc_ndcg_at_3_diff1": 0.250624, + "nauc_ndcg_at_5_max": 0.359669, + "nauc_ndcg_at_5_std": 0.358029, + "nauc_ndcg_at_5_diff1": 0.208163, + "nauc_ndcg_at_10_max": 0.36206, + "nauc_ndcg_at_10_std": 0.322323, + "nauc_ndcg_at_10_diff1": 0.170149, + "nauc_ndcg_at_20_max": 0.330201, + "nauc_ndcg_at_20_std": 0.21977, + "nauc_ndcg_at_20_diff1": 0.114346, + "nauc_ndcg_at_100_max": 0.35172, + "nauc_ndcg_at_100_std": 0.317777, + "nauc_ndcg_at_100_diff1": 0.196291, + "nauc_ndcg_at_1000_max": 0.3347, + "nauc_ndcg_at_1000_std": 0.366751, + "nauc_ndcg_at_1000_diff1": 0.151425, + "nauc_map_at_1_max": 0.126704, + "nauc_map_at_1_std": 0.095147, + "nauc_map_at_1_diff1": 0.287069, + "nauc_map_at_3_max": 0.332952, + "nauc_map_at_3_std": 0.083504, + "nauc_map_at_3_diff1": 0.355972, + "nauc_map_at_5_max": 0.373098, + "nauc_map_at_5_std": 0.084885, + "nauc_map_at_5_diff1": 0.357887, + "nauc_map_at_10_max": 0.330915, + "nauc_map_at_10_std": 0.082972, + "nauc_map_at_10_diff1": 0.321947, + "nauc_map_at_20_max": 0.316647, + "nauc_map_at_20_std": 0.074791, + "nauc_map_at_20_diff1": 0.267326, + "nauc_map_at_100_max": 0.333859, + "nauc_map_at_100_std": 0.257706, + "nauc_map_at_100_diff1": 0.163434, + "nauc_map_at_1000_max": 0.334496, + "nauc_map_at_1000_std": 0.312916, + "nauc_map_at_1000_diff1": 0.097621, + "nauc_recall_at_1_max": 0.126704, + "nauc_recall_at_1_std": 0.095147, + "nauc_recall_at_1_diff1": 0.287069, + "nauc_recall_at_3_max": 0.371435, + "nauc_recall_at_3_std": 0.103456, + "nauc_recall_at_3_diff1": 0.352738, + "nauc_recall_at_5_max": 0.364732, + "nauc_recall_at_5_std": 0.022222, + "nauc_recall_at_5_diff1": 0.327019, + "nauc_recall_at_10_max": 0.312448, + "nauc_recall_at_10_std": 0.016996, + "nauc_recall_at_10_diff1": 0.245238, + "nauc_recall_at_20_max": 0.075369, + "nauc_recall_at_20_std": -0.191957, + "nauc_recall_at_20_diff1": 0.017761, + "nauc_recall_at_100_max": 0.152209, + "nauc_recall_at_100_std": 0.044503, + "nauc_recall_at_100_diff1": 0.05163, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.234748, + "nauc_precision_at_1_std": 0.386373, + "nauc_precision_at_1_diff1": 0.213938, + "nauc_precision_at_3_max": 0.30999, + "nauc_precision_at_3_std": 0.285643, + "nauc_precision_at_3_diff1": 0.258115, + "nauc_precision_at_5_max": 0.372, + "nauc_precision_at_5_std": 0.352178, + "nauc_precision_at_5_diff1": 0.176224, + "nauc_precision_at_10_max": 0.30588, + "nauc_precision_at_10_std": 0.322952, + "nauc_precision_at_10_diff1": 0.075835, + "nauc_precision_at_20_max": 0.236128, + "nauc_precision_at_20_std": 0.327738, + "nauc_precision_at_20_diff1": -0.074044, + "nauc_precision_at_100_max": 0.150694, + "nauc_precision_at_100_std": 0.453979, + "nauc_precision_at_100_diff1": -0.214201, + "nauc_precision_at_1000_max": 0.119511, + "nauc_precision_at_1000_std": 0.420409, + "nauc_precision_at_1000_diff1": -0.273479, + "nauc_cv_recall_at_1_max": 0.234748, + "nauc_cv_recall_at_1_std": 0.386373, + "nauc_cv_recall_at_1_diff1": 0.213938, + "nauc_cv_recall_at_3_max": 0.303659, + "nauc_cv_recall_at_3_std": 0.426884, + "nauc_cv_recall_at_3_diff1": 0.313175, + "nauc_cv_recall_at_5_max": 0.303661, + "nauc_cv_recall_at_5_std": 0.215107, + "nauc_cv_recall_at_5_diff1": 0.346211, + "nauc_cv_recall_at_10_max": 0.393255, + "nauc_cv_recall_at_10_std": 0.2242, + "nauc_cv_recall_at_10_diff1": 0.201778, + "nauc_cv_recall_at_20_max": 0.401086, + "nauc_cv_recall_at_20_std": -0.021143, + "nauc_cv_recall_at_20_diff1": -0.074022, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.234748, + "nauc_mrr_at_1_std": 0.386373, + "nauc_mrr_at_1_diff1": 0.213938, + "nauc_mrr_at_3_max": 0.272615, + "nauc_mrr_at_3_std": 0.411577, + "nauc_mrr_at_3_diff1": 0.266547, + "nauc_mrr_at_5_max": 0.272319, + "nauc_mrr_at_5_std": 0.375983, + "nauc_mrr_at_5_diff1": 0.27116, + "nauc_mrr_at_10_max": 0.279471, + "nauc_mrr_at_10_std": 0.380573, + "nauc_mrr_at_10_diff1": 0.260462, + "nauc_mrr_at_20_max": 0.27849, + "nauc_mrr_at_20_std": 0.384833, + "nauc_mrr_at_20_diff1": 0.260354, + "nauc_mrr_at_100_max": 0.277408, + "nauc_mrr_at_100_std": 0.386115, + "nauc_mrr_at_100_diff1": 0.262218, + "nauc_mrr_at_1000_max": 0.277408, + "nauc_mrr_at_1000_std": 0.386115, + "nauc_mrr_at_1000_diff1": 0.262218, + "main_score": 0.29071, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 76.92450594902039, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/model_meta.json b/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/model_meta.json index c621116e7b..80d2d03c44 100644 --- a/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/model_meta.json +++ b/results/TIGER-Lab__VLM2Vec-Full/e9afa98002097ac2471827ba23ea1f2ddd229480/model_meta.json @@ -1 +1 @@ -{"name": "TIGER-Lab/VLM2Vec-Full", "revision": "e9afa98002097ac2471827ba23ea1f2ddd229480", "release_date": "2024-10-08", "languages": ["eng_Latn"], "n_parameters": 4150000000, "memory_usage_mb": 7909.0, "max_tokens": 131072.0, "embed_dim": 3072, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/TIGER-AI-Lab/VLM2Vec", "public_training_data": "https://huggingface.co/TIGER-Lab/VLM2Vec-Full", "framework": ["PyTorch"], "reference": "https://huggingface.co/TIGER-Lab/VLM2Vec-Full", "similarity_fn_name": null, "use_instructions": true, "training_datasets": {}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "VLM2VecWrapper"} \ No newline at end of file +{"name": "TIGER-Lab/VLM2Vec-Full", "revision": "e9afa98002097ac2471827ba23ea1f2ddd229480", "release_date": "2024-10-08", "languages": ["eng-Latn"], "n_parameters": 4150000000, "memory_usage_mb": 7909.0, "max_tokens": 131072.0, "embed_dim": 3072, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/TIGER-AI-Lab/VLM2Vec", "public_training_data": "https://huggingface.co/TIGER-Lab/VLM2Vec-Full", "framework": ["PyTorch"], "reference": "https://huggingface.co/TIGER-Lab/VLM2Vec-Full", "similarity_fn_name": null, "use_instructions": true, "training_datasets": {}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "VLM2VecWrapper"} \ No newline at end of file diff --git a/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/Vidore2BioMedicalLecturesRetrieval.json b/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..8803e15e20 --- /dev/null +++ b/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.2, + "ndcg_at_3": 0.27338, + "ndcg_at_5": 0.27696, + "ndcg_at_10": 0.3069, + "ndcg_at_20": 0.33071, + "ndcg_at_100": 0.39172, + "ndcg_at_1000": 0.44277, + "map_at_1": 0.1211, + "map_at_3": 0.20742, + "map_at_5": 0.2207, + "map_at_10": 0.24208, + "map_at_20": 0.25115, + "map_at_100": 0.26425, + "map_at_1000": 0.26755, + "recall_at_1": 0.1211, + "recall_at_3": 0.28259, + "recall_at_5": 0.31031, + "recall_at_10": 0.4024, + "recall_at_20": 0.47666, + "recall_at_100": 0.71133, + "recall_at_1000": 0.99844, + "cv_recall_at_1": 0.2, + "cv_recall_at_3": 0.41875, + "cv_recall_at_5": 0.4625, + "cv_recall_at_10": 0.55, + "cv_recall_at_20": 0.64375, + "cv_recall_at_100": 0.83125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.2, + "precision_at_3": 0.18333, + "precision_at_5": 0.13625, + "precision_at_10": 0.09375, + "precision_at_20": 0.05875, + "precision_at_100": 0.02006, + "precision_at_1000": 0.00321, + "mrr_at_1": 0.2, + "mrr_at_3": 0.298958, + "mrr_at_5": 0.309583, + "mrr_at_10": 0.321071, + "mrr_at_20": 0.327178, + "mrr_at_100": 0.332352, + "mrr_at_1000": 0.333148, + "nauc_ndcg_at_1_max": 0.249475, + "nauc_ndcg_at_1_std": 0.049887, + "nauc_ndcg_at_1_diff1": 0.342297, + "nauc_ndcg_at_3_max": 0.165398, + "nauc_ndcg_at_3_std": -0.034694, + "nauc_ndcg_at_3_diff1": 0.243599, + "nauc_ndcg_at_5_max": 0.190911, + "nauc_ndcg_at_5_std": -0.006148, + "nauc_ndcg_at_5_diff1": 0.257712, + "nauc_ndcg_at_10_max": 0.22192, + "nauc_ndcg_at_10_std": 0.030957, + "nauc_ndcg_at_10_diff1": 0.260945, + "nauc_ndcg_at_20_max": 0.236151, + "nauc_ndcg_at_20_std": 0.059503, + "nauc_ndcg_at_20_diff1": 0.2481, + "nauc_ndcg_at_100_max": 0.260657, + "nauc_ndcg_at_100_std": 0.102315, + "nauc_ndcg_at_100_diff1": 0.247821, + "nauc_ndcg_at_1000_max": 0.225313, + "nauc_ndcg_at_1000_std": 0.045827, + "nauc_ndcg_at_1000_diff1": 0.235966, + "nauc_map_at_1_max": 0.203741, + "nauc_map_at_1_std": 0.010912, + "nauc_map_at_1_diff1": 0.39391, + "nauc_map_at_3_max": 0.140578, + "nauc_map_at_3_std": -0.01462, + "nauc_map_at_3_diff1": 0.27528, + "nauc_map_at_5_max": 0.176331, + "nauc_map_at_5_std": 0.005143, + "nauc_map_at_5_diff1": 0.291553, + "nauc_map_at_10_max": 0.207996, + "nauc_map_at_10_std": 0.022956, + "nauc_map_at_10_diff1": 0.291514, + "nauc_map_at_20_max": 0.212332, + "nauc_map_at_20_std": 0.02824, + "nauc_map_at_20_diff1": 0.284041, + "nauc_map_at_100_max": 0.22398, + "nauc_map_at_100_std": 0.041836, + "nauc_map_at_100_diff1": 0.280596, + "nauc_map_at_1000_max": 0.221046, + "nauc_map_at_1000_std": 0.038092, + "nauc_map_at_1000_diff1": 0.280096, + "nauc_recall_at_1_max": 0.203741, + "nauc_recall_at_1_std": 0.010912, + "nauc_recall_at_1_diff1": 0.39391, + "nauc_recall_at_3_max": 0.075416, + "nauc_recall_at_3_std": -0.057088, + "nauc_recall_at_3_diff1": 0.181474, + "nauc_recall_at_5_max": 0.141346, + "nauc_recall_at_5_std": -0.015148, + "nauc_recall_at_5_diff1": 0.200954, + "nauc_recall_at_10_max": 0.212158, + "nauc_recall_at_10_std": 0.062505, + "nauc_recall_at_10_diff1": 0.217639, + "nauc_recall_at_20_max": 0.239197, + "nauc_recall_at_20_std": 0.129412, + "nauc_recall_at_20_diff1": 0.188098, + "nauc_recall_at_100_max": 0.3076, + "nauc_recall_at_100_std": 0.319012, + "nauc_recall_at_100_diff1": 0.209293, + "nauc_recall_at_1000_max": -0.563492, + "nauc_recall_at_1000_std": 0.358077, + "nauc_recall_at_1000_diff1": -1.739963, + "nauc_precision_at_1_max": 0.249475, + "nauc_precision_at_1_std": 0.049887, + "nauc_precision_at_1_diff1": 0.342297, + "nauc_precision_at_3_max": 0.158322, + "nauc_precision_at_3_std": -0.022035, + "nauc_precision_at_3_diff1": 0.158386, + "nauc_precision_at_5_max": 0.232556, + "nauc_precision_at_5_std": 0.019402, + "nauc_precision_at_5_diff1": 0.1675, + "nauc_precision_at_10_max": 0.286658, + "nauc_precision_at_10_std": 0.064969, + "nauc_precision_at_10_diff1": 0.10837, + "nauc_precision_at_20_max": 0.288204, + "nauc_precision_at_20_std": 0.101644, + "nauc_precision_at_20_diff1": 0.046354, + "nauc_precision_at_100_max": 0.226013, + "nauc_precision_at_100_std": 0.128941, + "nauc_precision_at_100_diff1": -0.052325, + "nauc_precision_at_1000_max": -0.083848, + "nauc_precision_at_1000_std": -0.144619, + "nauc_precision_at_1000_diff1": -0.19539, + "nauc_cv_recall_at_1_max": 0.249475, + "nauc_cv_recall_at_1_std": 0.049887, + "nauc_cv_recall_at_1_diff1": 0.342297, + "nauc_cv_recall_at_3_max": 0.136252, + "nauc_cv_recall_at_3_std": -0.055798, + "nauc_cv_recall_at_3_diff1": 0.150026, + "nauc_cv_recall_at_5_max": 0.183835, + "nauc_cv_recall_at_5_std": -0.019062, + "nauc_cv_recall_at_5_diff1": 0.1303, + "nauc_cv_recall_at_10_max": 0.222987, + "nauc_cv_recall_at_10_std": 0.080294, + "nauc_cv_recall_at_10_diff1": 0.186995, + "nauc_cv_recall_at_20_max": 0.278948, + "nauc_cv_recall_at_20_std": 0.207942, + "nauc_cv_recall_at_20_diff1": 0.162072, + "nauc_cv_recall_at_100_max": 0.237736, + "nauc_cv_recall_at_100_std": 0.227015, + "nauc_cv_recall_at_100_diff1": 0.254812, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.249475, + "nauc_mrr_at_1_std": 0.049887, + "nauc_mrr_at_1_diff1": 0.342297, + "nauc_mrr_at_3_max": 0.193248, + "nauc_mrr_at_3_std": -0.002495, + "nauc_mrr_at_3_diff1": 0.237817, + "nauc_mrr_at_5_max": 0.204867, + "nauc_mrr_at_5_std": 0.006147, + "nauc_mrr_at_5_diff1": 0.233696, + "nauc_mrr_at_10_max": 0.208478, + "nauc_mrr_at_10_std": 0.018869, + "nauc_mrr_at_10_diff1": 0.240618, + "nauc_mrr_at_20_max": 0.211792, + "nauc_mrr_at_20_std": 0.026879, + "nauc_mrr_at_20_diff1": 0.237972, + "nauc_mrr_at_100_max": 0.210817, + "nauc_mrr_at_100_std": 0.026309, + "nauc_mrr_at_100_diff1": 0.239733, + "nauc_mrr_at_1000_max": 0.210483, + "nauc_mrr_at_1000_std": 0.025684, + "nauc_mrr_at_1000_diff1": 0.239906, + "main_score": 0.27696, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.175, + "ndcg_at_3": 0.1954, + "ndcg_at_5": 0.21323, + "ndcg_at_10": 0.2421, + "ndcg_at_20": 0.27126, + "ndcg_at_100": 0.33425, + "ndcg_at_1000": 0.39277, + "map_at_1": 0.11491, + "map_at_3": 0.14988, + "map_at_5": 0.16677, + "map_at_10": 0.18555, + "map_at_20": 0.19586, + "map_at_100": 0.20811, + "map_at_1000": 0.2116, + "recall_at_1": 0.11491, + "recall_at_3": 0.18525, + "recall_at_5": 0.23599, + "recall_at_10": 0.31961, + "recall_at_20": 0.40539, + "recall_at_100": 0.66348, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.175, + "cv_recall_at_3": 0.29375, + "cv_recall_at_5": 0.38125, + "cv_recall_at_10": 0.49375, + "cv_recall_at_20": 0.6, + "cv_recall_at_100": 0.8125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.175, + "precision_at_3": 0.11875, + "precision_at_5": 0.105, + "precision_at_10": 0.07687, + "precision_at_20": 0.05219, + "precision_at_100": 0.0185, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.175, + "mrr_at_3": 0.227083, + "mrr_at_5": 0.247083, + "mrr_at_10": 0.262096, + "mrr_at_20": 0.269767, + "mrr_at_100": 0.275122, + "mrr_at_1000": 0.275949, + "nauc_ndcg_at_1_max": 0.132469, + "nauc_ndcg_at_1_std": -0.062363, + "nauc_ndcg_at_1_diff1": 0.127516, + "nauc_ndcg_at_3_max": 0.267864, + "nauc_ndcg_at_3_std": 0.027747, + "nauc_ndcg_at_3_diff1": 0.179678, + "nauc_ndcg_at_5_max": 0.282066, + "nauc_ndcg_at_5_std": 0.037355, + "nauc_ndcg_at_5_diff1": 0.1761, + "nauc_ndcg_at_10_max": 0.258144, + "nauc_ndcg_at_10_std": 0.046036, + "nauc_ndcg_at_10_diff1": 0.189105, + "nauc_ndcg_at_20_max": 0.277395, + "nauc_ndcg_at_20_std": 0.081751, + "nauc_ndcg_at_20_diff1": 0.189045, + "nauc_ndcg_at_100_max": 0.278879, + "nauc_ndcg_at_100_std": 0.093236, + "nauc_ndcg_at_100_diff1": 0.178058, + "nauc_ndcg_at_1000_max": 0.256613, + "nauc_ndcg_at_1000_std": 0.060775, + "nauc_ndcg_at_1000_diff1": 0.178943, + "nauc_map_at_1_max": 0.16362, + "nauc_map_at_1_std": -0.065885, + "nauc_map_at_1_diff1": 0.125653, + "nauc_map_at_3_max": 0.231515, + "nauc_map_at_3_std": -0.007982, + "nauc_map_at_3_diff1": 0.162466, + "nauc_map_at_5_max": 0.235376, + "nauc_map_at_5_std": 0.000811, + "nauc_map_at_5_diff1": 0.155828, + "nauc_map_at_10_max": 0.228267, + "nauc_map_at_10_std": 0.012342, + "nauc_map_at_10_diff1": 0.162768, + "nauc_map_at_20_max": 0.236734, + "nauc_map_at_20_std": 0.028155, + "nauc_map_at_20_diff1": 0.162629, + "nauc_map_at_100_max": 0.242782, + "nauc_map_at_100_std": 0.035883, + "nauc_map_at_100_diff1": 0.162988, + "nauc_map_at_1000_max": 0.241681, + "nauc_map_at_1000_std": 0.034383, + "nauc_map_at_1000_diff1": 0.163298, + "nauc_recall_at_1_max": 0.16362, + "nauc_recall_at_1_std": -0.065885, + "nauc_recall_at_1_diff1": 0.125653, + "nauc_recall_at_3_max": 0.298832, + "nauc_recall_at_3_std": 0.02884, + "nauc_recall_at_3_diff1": 0.21789, + "nauc_recall_at_5_max": 0.297771, + "nauc_recall_at_5_std": 0.038984, + "nauc_recall_at_5_diff1": 0.195849, + "nauc_recall_at_10_max": 0.248596, + "nauc_recall_at_10_std": 0.057455, + "nauc_recall_at_10_diff1": 0.200664, + "nauc_recall_at_20_max": 0.288139, + "nauc_recall_at_20_std": 0.150523, + "nauc_recall_at_20_diff1": 0.202339, + "nauc_recall_at_100_max": 0.277654, + "nauc_recall_at_100_std": 0.209858, + "nauc_recall_at_100_diff1": 0.165298, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.132469, + "nauc_precision_at_1_std": -0.062363, + "nauc_precision_at_1_diff1": 0.127516, + "nauc_precision_at_3_max": 0.314787, + "nauc_precision_at_3_std": 0.106202, + "nauc_precision_at_3_diff1": 0.194577, + "nauc_precision_at_5_max": 0.326287, + "nauc_precision_at_5_std": 0.150674, + "nauc_precision_at_5_diff1": 0.145442, + "nauc_precision_at_10_max": 0.233119, + "nauc_precision_at_10_std": 0.155226, + "nauc_precision_at_10_diff1": 0.145764, + "nauc_precision_at_20_max": 0.229244, + "nauc_precision_at_20_std": 0.188094, + "nauc_precision_at_20_diff1": 0.101638, + "nauc_precision_at_100_max": 0.115433, + "nauc_precision_at_100_std": 0.113721, + "nauc_precision_at_100_diff1": 0.023705, + "nauc_precision_at_1000_max": -0.126698, + "nauc_precision_at_1000_std": -0.089562, + "nauc_precision_at_1000_diff1": 0.041261, + "nauc_cv_recall_at_1_max": 0.132469, + "nauc_cv_recall_at_1_std": -0.062363, + "nauc_cv_recall_at_1_diff1": 0.127516, + "nauc_cv_recall_at_3_max": 0.362336, + "nauc_cv_recall_at_3_std": 0.095761, + "nauc_cv_recall_at_3_diff1": 0.268981, + "nauc_cv_recall_at_5_max": 0.43188, + "nauc_cv_recall_at_5_std": 0.168297, + "nauc_cv_recall_at_5_diff1": 0.234124, + "nauc_cv_recall_at_10_max": 0.340427, + "nauc_cv_recall_at_10_std": 0.158987, + "nauc_cv_recall_at_10_diff1": 0.238618, + "nauc_cv_recall_at_20_max": 0.356059, + "nauc_cv_recall_at_20_std": 0.232673, + "nauc_cv_recall_at_20_diff1": 0.237125, + "nauc_cv_recall_at_100_max": 0.265413, + "nauc_cv_recall_at_100_std": 0.248757, + "nauc_cv_recall_at_100_diff1": 0.180494, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.132469, + "nauc_mrr_at_1_std": -0.062363, + "nauc_mrr_at_1_diff1": 0.127516, + "nauc_mrr_at_3_max": 0.254815, + "nauc_mrr_at_3_std": 0.021606, + "nauc_mrr_at_3_diff1": 0.195197, + "nauc_mrr_at_5_max": 0.27839, + "nauc_mrr_at_5_std": 0.04502, + "nauc_mrr_at_5_diff1": 0.188172, + "nauc_mrr_at_10_max": 0.265186, + "nauc_mrr_at_10_std": 0.045624, + "nauc_mrr_at_10_diff1": 0.1881, + "nauc_mrr_at_20_max": 0.266187, + "nauc_mrr_at_20_std": 0.04925, + "nauc_mrr_at_20_diff1": 0.187688, + "nauc_mrr_at_100_max": 0.26409, + "nauc_mrr_at_100_std": 0.047149, + "nauc_mrr_at_100_diff1": 0.18647, + "nauc_mrr_at_1000_max": 0.264075, + "nauc_mrr_at_1000_std": 0.046707, + "nauc_mrr_at_1000_diff1": 0.186246, + "main_score": 0.21323, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.44375, + "ndcg_at_3": 0.4552, + "ndcg_at_5": 0.47518, + "ndcg_at_10": 0.51121, + "ndcg_at_20": 0.55617, + "ndcg_at_100": 0.59767, + "ndcg_at_1000": 0.62171, + "map_at_1": 0.27507, + "map_at_3": 0.36464, + "map_at_5": 0.39128, + "map_at_10": 0.41846, + "map_at_20": 0.44004, + "map_at_100": 0.45322, + "map_at_1000": 0.45535, + "recall_at_1": 0.27507, + "recall_at_3": 0.43331, + "recall_at_5": 0.5051, + "recall_at_10": 0.61081, + "recall_at_20": 0.74249, + "recall_at_100": 0.88398, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.44375, + "cv_recall_at_3": 0.65625, + "cv_recall_at_5": 0.75, + "cv_recall_at_10": 0.825, + "cv_recall_at_20": 0.90625, + "cv_recall_at_100": 0.9625, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.44375, + "precision_at_3": 0.275, + "precision_at_5": 0.2125, + "precision_at_10": 0.14125, + "precision_at_20": 0.09375, + "precision_at_100": 0.02556, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.44375, + "mrr_at_3": 0.5375, + "mrr_at_5": 0.558437, + "mrr_at_10": 0.568269, + "mrr_at_20": 0.573738, + "mrr_at_100": 0.574974, + "mrr_at_1000": 0.57526, + "nauc_ndcg_at_1_max": 0.162511, + "nauc_ndcg_at_1_std": -0.081219, + "nauc_ndcg_at_1_diff1": 0.344214, + "nauc_ndcg_at_3_max": 0.177593, + "nauc_ndcg_at_3_std": -0.058124, + "nauc_ndcg_at_3_diff1": 0.272481, + "nauc_ndcg_at_5_max": 0.20595, + "nauc_ndcg_at_5_std": -0.031268, + "nauc_ndcg_at_5_diff1": 0.286658, + "nauc_ndcg_at_10_max": 0.209412, + "nauc_ndcg_at_10_std": -0.014941, + "nauc_ndcg_at_10_diff1": 0.303755, + "nauc_ndcg_at_20_max": 0.240841, + "nauc_ndcg_at_20_std": 0.011343, + "nauc_ndcg_at_20_diff1": 0.3053, + "nauc_ndcg_at_100_max": 0.248603, + "nauc_ndcg_at_100_std": 0.016563, + "nauc_ndcg_at_100_diff1": 0.285152, + "nauc_ndcg_at_1000_max": 0.228005, + "nauc_ndcg_at_1000_std": -0.007194, + "nauc_ndcg_at_1000_diff1": 0.278248, + "nauc_map_at_1_max": 0.09605, + "nauc_map_at_1_std": -0.08604, + "nauc_map_at_1_diff1": 0.384397, + "nauc_map_at_3_max": 0.145376, + "nauc_map_at_3_std": -0.074285, + "nauc_map_at_3_diff1": 0.31959, + "nauc_map_at_5_max": 0.172321, + "nauc_map_at_5_std": -0.045765, + "nauc_map_at_5_diff1": 0.319347, + "nauc_map_at_10_max": 0.185304, + "nauc_map_at_10_std": -0.028933, + "nauc_map_at_10_diff1": 0.316962, + "nauc_map_at_20_max": 0.201561, + "nauc_map_at_20_std": -0.019408, + "nauc_map_at_20_diff1": 0.312503, + "nauc_map_at_100_max": 0.20266, + "nauc_map_at_100_std": -0.02019, + "nauc_map_at_100_diff1": 0.304405, + "nauc_map_at_1000_max": 0.201367, + "nauc_map_at_1000_std": -0.021609, + "nauc_map_at_1000_diff1": 0.303422, + "nauc_recall_at_1_max": 0.09605, + "nauc_recall_at_1_std": -0.08604, + "nauc_recall_at_1_diff1": 0.384397, + "nauc_recall_at_3_max": 0.146547, + "nauc_recall_at_3_std": -0.065752, + "nauc_recall_at_3_diff1": 0.257807, + "nauc_recall_at_5_max": 0.210233, + "nauc_recall_at_5_std": -0.007988, + "nauc_recall_at_5_diff1": 0.251197, + "nauc_recall_at_10_max": 0.18932, + "nauc_recall_at_10_std": 0.016499, + "nauc_recall_at_10_diff1": 0.253931, + "nauc_recall_at_20_max": 0.268227, + "nauc_recall_at_20_std": 0.106784, + "nauc_recall_at_20_diff1": 0.250337, + "nauc_recall_at_100_max": 0.40112, + "nauc_recall_at_100_std": 0.301916, + "nauc_recall_at_100_diff1": 0.186218, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.162511, + "nauc_precision_at_1_std": -0.081219, + "nauc_precision_at_1_diff1": 0.344214, + "nauc_precision_at_3_max": 0.235144, + "nauc_precision_at_3_std": 0.037167, + "nauc_precision_at_3_diff1": 0.069745, + "nauc_precision_at_5_max": 0.286219, + "nauc_precision_at_5_std": 0.1348, + "nauc_precision_at_5_diff1": 0.036334, + "nauc_precision_at_10_max": 0.229127, + "nauc_precision_at_10_std": 0.17476, + "nauc_precision_at_10_diff1": -0.037721, + "nauc_precision_at_20_max": 0.209031, + "nauc_precision_at_20_std": 0.156341, + "nauc_precision_at_20_diff1": -0.104231, + "nauc_precision_at_100_max": 0.108595, + "nauc_precision_at_100_std": 0.081198, + "nauc_precision_at_100_diff1": -0.198101, + "nauc_precision_at_1000_max": -0.026095, + "nauc_precision_at_1000_std": 0.00461, + "nauc_precision_at_1000_diff1": -0.275759, + "nauc_cv_recall_at_1_max": 0.162511, + "nauc_cv_recall_at_1_std": -0.081219, + "nauc_cv_recall_at_1_diff1": 0.344214, + "nauc_cv_recall_at_3_max": 0.276098, + "nauc_cv_recall_at_3_std": -0.006754, + "nauc_cv_recall_at_3_diff1": 0.217997, + "nauc_cv_recall_at_5_max": 0.385505, + "nauc_cv_recall_at_5_std": 0.034746, + "nauc_cv_recall_at_5_diff1": 0.266117, + "nauc_cv_recall_at_10_max": 0.359704, + "nauc_cv_recall_at_10_std": 0.044175, + "nauc_cv_recall_at_10_diff1": 0.309683, + "nauc_cv_recall_at_20_max": 0.544911, + "nauc_cv_recall_at_20_std": 0.188049, + "nauc_cv_recall_at_20_diff1": 0.321289, + "nauc_cv_recall_at_100_max": 0.5554, + "nauc_cv_recall_at_100_std": 0.516962, + "nauc_cv_recall_at_100_diff1": 0.112356, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.162511, + "nauc_mrr_at_1_std": -0.081219, + "nauc_mrr_at_1_diff1": 0.344214, + "nauc_mrr_at_3_max": 0.208093, + "nauc_mrr_at_3_std": -0.051425, + "nauc_mrr_at_3_diff1": 0.290121, + "nauc_mrr_at_5_max": 0.225247, + "nauc_mrr_at_5_std": -0.044797, + "nauc_mrr_at_5_diff1": 0.299462, + "nauc_mrr_at_10_max": 0.219685, + "nauc_mrr_at_10_std": -0.046564, + "nauc_mrr_at_10_diff1": 0.30222, + "nauc_mrr_at_20_max": 0.223168, + "nauc_mrr_at_20_std": -0.043378, + "nauc_mrr_at_20_diff1": 0.30385, + "nauc_mrr_at_100_max": 0.221604, + "nauc_mrr_at_100_std": -0.043742, + "nauc_mrr_at_100_diff1": 0.303076, + "nauc_mrr_at_1000_max": 0.221261, + "nauc_mrr_at_1000_std": -0.044335, + "nauc_mrr_at_1000_diff1": 0.303367, + "main_score": 0.47518, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.11875, + "ndcg_at_3": 0.12845, + "ndcg_at_5": 0.15035, + "ndcg_at_10": 0.1764, + "ndcg_at_20": 0.20117, + "ndcg_at_100": 0.26467, + "ndcg_at_1000": 0.33577, + "map_at_1": 0.0649, + "map_at_3": 0.09793, + "map_at_5": 0.11267, + "map_at_10": 0.12586, + "map_at_20": 0.13449, + "map_at_100": 0.1458, + "map_at_1000": 0.15023, + "recall_at_1": 0.0649, + "recall_at_3": 0.12492, + "recall_at_5": 0.18091, + "recall_at_10": 0.25503, + "recall_at_20": 0.32952, + "recall_at_100": 0.58669, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.11875, + "cv_recall_at_3": 0.2125, + "cv_recall_at_5": 0.30625, + "cv_recall_at_10": 0.4, + "cv_recall_at_20": 0.48125, + "cv_recall_at_100": 0.7375, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.11875, + "precision_at_3": 0.08958, + "precision_at_5": 0.08, + "precision_at_10": 0.05813, + "precision_at_20": 0.04063, + "precision_at_100": 0.01706, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.11875, + "mrr_at_3": 0.157292, + "mrr_at_5": 0.178542, + "mrr_at_10": 0.191419, + "mrr_at_20": 0.197441, + "mrr_at_100": 0.203574, + "mrr_at_1000": 0.204793, + "nauc_ndcg_at_1_max": 0.338475, + "nauc_ndcg_at_1_std": 0.281205, + "nauc_ndcg_at_1_diff1": -0.004863, + "nauc_ndcg_at_3_max": 0.260803, + "nauc_ndcg_at_3_std": 0.20807, + "nauc_ndcg_at_3_diff1": -0.042128, + "nauc_ndcg_at_5_max": 0.243195, + "nauc_ndcg_at_5_std": 0.194994, + "nauc_ndcg_at_5_diff1": -0.015882, + "nauc_ndcg_at_10_max": 0.250668, + "nauc_ndcg_at_10_std": 0.210981, + "nauc_ndcg_at_10_diff1": 0.023775, + "nauc_ndcg_at_20_max": 0.283039, + "nauc_ndcg_at_20_std": 0.226054, + "nauc_ndcg_at_20_diff1": 0.042196, + "nauc_ndcg_at_100_max": 0.269995, + "nauc_ndcg_at_100_std": 0.228998, + "nauc_ndcg_at_100_diff1": 0.032313, + "nauc_ndcg_at_1000_max": 0.285011, + "nauc_ndcg_at_1000_std": 0.230077, + "nauc_ndcg_at_1000_diff1": 0.013673, + "nauc_map_at_1_max": 0.228696, + "nauc_map_at_1_std": 0.168302, + "nauc_map_at_1_diff1": 0.024616, + "nauc_map_at_3_max": 0.212561, + "nauc_map_at_3_std": 0.184918, + "nauc_map_at_3_diff1": -0.038364, + "nauc_map_at_5_max": 0.219441, + "nauc_map_at_5_std": 0.187376, + "nauc_map_at_5_diff1": -0.020188, + "nauc_map_at_10_max": 0.237705, + "nauc_map_at_10_std": 0.207657, + "nauc_map_at_10_diff1": -0.00052, + "nauc_map_at_20_max": 0.252506, + "nauc_map_at_20_std": 0.213163, + "nauc_map_at_20_diff1": 0.008331, + "nauc_map_at_100_max": 0.254695, + "nauc_map_at_100_std": 0.21722, + "nauc_map_at_100_diff1": 0.00407, + "nauc_map_at_1000_max": 0.257482, + "nauc_map_at_1000_std": 0.219327, + "nauc_map_at_1000_diff1": 0.003264, + "nauc_recall_at_1_max": 0.228696, + "nauc_recall_at_1_std": 0.168302, + "nauc_recall_at_1_diff1": 0.024616, + "nauc_recall_at_3_max": 0.158333, + "nauc_recall_at_3_std": 0.172882, + "nauc_recall_at_3_diff1": -0.070812, + "nauc_recall_at_5_max": 0.140966, + "nauc_recall_at_5_std": 0.138781, + "nauc_recall_at_5_diff1": -0.003543, + "nauc_recall_at_10_max": 0.161541, + "nauc_recall_at_10_std": 0.154817, + "nauc_recall_at_10_diff1": 0.053942, + "nauc_recall_at_20_max": 0.224367, + "nauc_recall_at_20_std": 0.177511, + "nauc_recall_at_20_diff1": 0.0965, + "nauc_recall_at_100_max": 0.137759, + "nauc_recall_at_100_std": 0.140497, + "nauc_recall_at_100_diff1": 0.08031, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.338475, + "nauc_precision_at_1_std": 0.281205, + "nauc_precision_at_1_diff1": -0.004863, + "nauc_precision_at_3_max": 0.328229, + "nauc_precision_at_3_std": 0.251242, + "nauc_precision_at_3_diff1": -0.077061, + "nauc_precision_at_5_max": 0.359813, + "nauc_precision_at_5_std": 0.247184, + "nauc_precision_at_5_diff1": -0.029733, + "nauc_precision_at_10_max": 0.397828, + "nauc_precision_at_10_std": 0.289203, + "nauc_precision_at_10_diff1": 0.025942, + "nauc_precision_at_20_max": 0.418798, + "nauc_precision_at_20_std": 0.280914, + "nauc_precision_at_20_diff1": 0.041259, + "nauc_precision_at_100_max": 0.244324, + "nauc_precision_at_100_std": 0.190412, + "nauc_precision_at_100_diff1": -0.003575, + "nauc_precision_at_1000_max": 0.075809, + "nauc_precision_at_1000_std": 0.000868, + "nauc_precision_at_1000_diff1": -0.045976, + "nauc_cv_recall_at_1_max": 0.338475, + "nauc_cv_recall_at_1_std": 0.281205, + "nauc_cv_recall_at_1_diff1": -0.004863, + "nauc_cv_recall_at_3_max": 0.314673, + "nauc_cv_recall_at_3_std": 0.262032, + "nauc_cv_recall_at_3_diff1": -0.068973, + "nauc_cv_recall_at_5_max": 0.279055, + "nauc_cv_recall_at_5_std": 0.183191, + "nauc_cv_recall_at_5_diff1": 0.004352, + "nauc_cv_recall_at_10_max": 0.272729, + "nauc_cv_recall_at_10_std": 0.176244, + "nauc_cv_recall_at_10_diff1": 0.073088, + "nauc_cv_recall_at_20_max": 0.350081, + "nauc_cv_recall_at_20_std": 0.205777, + "nauc_cv_recall_at_20_diff1": 0.060504, + "nauc_cv_recall_at_100_max": 0.247806, + "nauc_cv_recall_at_100_std": 0.166445, + "nauc_cv_recall_at_100_diff1": 0.114605, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.338475, + "nauc_mrr_at_1_std": 0.281205, + "nauc_mrr_at_1_diff1": -0.004863, + "nauc_mrr_at_3_max": 0.33582, + "nauc_mrr_at_3_std": 0.276451, + "nauc_mrr_at_3_diff1": -0.028716, + "nauc_mrr_at_5_max": 0.321823, + "nauc_mrr_at_5_std": 0.250603, + "nauc_mrr_at_5_diff1": -0.007738, + "nauc_mrr_at_10_max": 0.319704, + "nauc_mrr_at_10_std": 0.249297, + "nauc_mrr_at_10_diff1": 0.010173, + "nauc_mrr_at_20_max": 0.327789, + "nauc_mrr_at_20_std": 0.253076, + "nauc_mrr_at_20_diff1": 0.008738, + "nauc_mrr_at_100_max": 0.323501, + "nauc_mrr_at_100_std": 0.252066, + "nauc_mrr_at_100_diff1": 0.007245, + "nauc_mrr_at_1000_max": 0.323978, + "nauc_mrr_at_1000_std": 0.252917, + "nauc_mrr_at_1000_diff1": 0.005837, + "main_score": 0.15035, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 120.56661701202393, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/Vidore2ESGReportsHLRetrieval.json b/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..9172b4d919 --- /dev/null +++ b/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.41026, + "ndcg_at_3": 0.43293, + "ndcg_at_5": 0.44545, + "ndcg_at_10": 0.50082, + "ndcg_at_20": 0.53246, + "ndcg_at_100": 0.57678, + "ndcg_at_1000": 0.59128, + "map_at_1": 0.26859, + "map_at_3": 0.35093, + "map_at_5": 0.38239, + "map_at_10": 0.41499, + "map_at_20": 0.42902, + "map_at_100": 0.44184, + "map_at_1000": 0.443, + "recall_at_1": 0.26859, + "recall_at_3": 0.41425, + "recall_at_5": 0.47305, + "recall_at_10": 0.63509, + "recall_at_20": 0.7258, + "recall_at_100": 0.90487, + "recall_at_1000": 0.99038, + "cv_recall_at_1": 0.42308, + "cv_recall_at_3": 0.61538, + "cv_recall_at_5": 0.65385, + "cv_recall_at_10": 0.80769, + "cv_recall_at_20": 0.84615, + "cv_recall_at_100": 0.94231, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.42308, + "precision_at_3": 0.25, + "precision_at_5": 0.19231, + "precision_at_10": 0.12885, + "precision_at_20": 0.07885, + "precision_at_100": 0.02192, + "precision_at_1000": 0.00244, + "mrr_at_1": 0.423077, + "mrr_at_3": 0.512821, + "mrr_at_5": 0.521474, + "mrr_at_10": 0.541369, + "mrr_at_20": 0.543933, + "mrr_at_100": 0.547084, + "mrr_at_1000": 0.547321, + "nauc_ndcg_at_1_max": 0.225305, + "nauc_ndcg_at_1_std": 0.359139, + "nauc_ndcg_at_1_diff1": 0.546132, + "nauc_ndcg_at_3_max": 0.289095, + "nauc_ndcg_at_3_std": 0.349712, + "nauc_ndcg_at_3_diff1": 0.547639, + "nauc_ndcg_at_5_max": 0.302866, + "nauc_ndcg_at_5_std": 0.331044, + "nauc_ndcg_at_5_diff1": 0.551801, + "nauc_ndcg_at_10_max": 0.255107, + "nauc_ndcg_at_10_std": 0.306525, + "nauc_ndcg_at_10_diff1": 0.545323, + "nauc_ndcg_at_20_max": 0.260574, + "nauc_ndcg_at_20_std": 0.330531, + "nauc_ndcg_at_20_diff1": 0.495174, + "nauc_ndcg_at_100_max": 0.326773, + "nauc_ndcg_at_100_std": 0.380446, + "nauc_ndcg_at_100_diff1": 0.502549, + "nauc_ndcg_at_1000_max": 0.31302, + "nauc_ndcg_at_1000_std": 0.357495, + "nauc_ndcg_at_1000_diff1": 0.507661, + "nauc_map_at_1_max": 0.214142, + "nauc_map_at_1_std": 0.106457, + "nauc_map_at_1_diff1": 0.716575, + "nauc_map_at_3_max": 0.219124, + "nauc_map_at_3_std": 0.194373, + "nauc_map_at_3_diff1": 0.642673, + "nauc_map_at_5_max": 0.248666, + "nauc_map_at_5_std": 0.257705, + "nauc_map_at_5_diff1": 0.608856, + "nauc_map_at_10_max": 0.238461, + "nauc_map_at_10_std": 0.249283, + "nauc_map_at_10_diff1": 0.592654, + "nauc_map_at_20_max": 0.247841, + "nauc_map_at_20_std": 0.266474, + "nauc_map_at_20_diff1": 0.572234, + "nauc_map_at_100_max": 0.275343, + "nauc_map_at_100_std": 0.284824, + "nauc_map_at_100_diff1": 0.568783, + "nauc_map_at_1000_max": 0.27511, + "nauc_map_at_1000_std": 0.283685, + "nauc_map_at_1000_diff1": 0.568328, + "nauc_recall_at_1_max": 0.214142, + "nauc_recall_at_1_std": 0.106457, + "nauc_recall_at_1_diff1": 0.716575, + "nauc_recall_at_3_max": 0.230019, + "nauc_recall_at_3_std": 0.245414, + "nauc_recall_at_3_diff1": 0.562994, + "nauc_recall_at_5_max": 0.283714, + "nauc_recall_at_5_std": 0.321559, + "nauc_recall_at_5_diff1": 0.516538, + "nauc_recall_at_10_max": 0.205485, + "nauc_recall_at_10_std": 0.285954, + "nauc_recall_at_10_diff1": 0.465835, + "nauc_recall_at_20_max": 0.165686, + "nauc_recall_at_20_std": 0.342236, + "nauc_recall_at_20_diff1": 0.282135, + "nauc_recall_at_100_max": 0.592285, + "nauc_recall_at_100_std": 0.758763, + "nauc_recall_at_100_diff1": 0.21538, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 0.554244, + "nauc_precision_at_1_max": 0.213581, + "nauc_precision_at_1_std": 0.356366, + "nauc_precision_at_1_diff1": 0.512913, + "nauc_precision_at_3_max": 0.263735, + "nauc_precision_at_3_std": 0.457658, + "nauc_precision_at_3_diff1": 0.120712, + "nauc_precision_at_5_max": 0.272135, + "nauc_precision_at_5_std": 0.491354, + "nauc_precision_at_5_diff1": -0.016878, + "nauc_precision_at_10_max": 0.212748, + "nauc_precision_at_10_std": 0.442238, + "nauc_precision_at_10_diff1": -0.148213, + "nauc_precision_at_20_max": 0.246996, + "nauc_precision_at_20_std": 0.429607, + "nauc_precision_at_20_diff1": -0.267093, + "nauc_precision_at_100_max": 0.337086, + "nauc_precision_at_100_std": 0.450698, + "nauc_precision_at_100_diff1": -0.354942, + "nauc_precision_at_1000_max": 0.268107, + "nauc_precision_at_1000_std": 0.338352, + "nauc_precision_at_1000_diff1": -0.388036, + "nauc_cv_recall_at_1_max": 0.213581, + "nauc_cv_recall_at_1_std": 0.356366, + "nauc_cv_recall_at_1_diff1": 0.512913, + "nauc_cv_recall_at_3_max": 0.343402, + "nauc_cv_recall_at_3_std": 0.476923, + "nauc_cv_recall_at_3_diff1": 0.358636, + "nauc_cv_recall_at_5_max": 0.410298, + "nauc_cv_recall_at_5_std": 0.491169, + "nauc_cv_recall_at_5_diff1": 0.339834, + "nauc_cv_recall_at_10_max": 0.418941, + "nauc_cv_recall_at_10_std": 0.580702, + "nauc_cv_recall_at_10_diff1": 0.309972, + "nauc_cv_recall_at_20_max": 0.353508, + "nauc_cv_recall_at_20_std": 0.640596, + "nauc_cv_recall_at_20_diff1": 0.135114, + "nauc_cv_recall_at_100_max": 0.693693, + "nauc_cv_recall_at_100_std": 0.864672, + "nauc_cv_recall_at_100_diff1": 0.185475, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.213581, + "nauc_mrr_at_1_std": 0.356366, + "nauc_mrr_at_1_diff1": 0.512913, + "nauc_mrr_at_3_max": 0.275097, + "nauc_mrr_at_3_std": 0.416217, + "nauc_mrr_at_3_diff1": 0.456056, + "nauc_mrr_at_5_max": 0.289535, + "nauc_mrr_at_5_std": 0.420157, + "nauc_mrr_at_5_diff1": 0.454459, + "nauc_mrr_at_10_max": 0.28493, + "nauc_mrr_at_10_std": 0.422871, + "nauc_mrr_at_10_diff1": 0.455714, + "nauc_mrr_at_20_max": 0.281961, + "nauc_mrr_at_20_std": 0.423607, + "nauc_mrr_at_20_diff1": 0.451339, + "nauc_mrr_at_100_max": 0.285957, + "nauc_mrr_at_100_std": 0.424027, + "nauc_mrr_at_100_diff1": 0.454713, + "nauc_mrr_at_1000_max": 0.285653, + "nauc_mrr_at_1000_std": 0.423697, + "nauc_mrr_at_1000_diff1": 0.455071, + "main_score": 0.44545, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 103.0151150226593, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/Vidore2ESGReportsRetrieval.json b/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..63a38d5ec4 --- /dev/null +++ b/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.36842, + "ndcg_at_3": 0.31957, + "ndcg_at_5": 0.34964, + "ndcg_at_10": 0.39406, + "ndcg_at_20": 0.42776, + "ndcg_at_100": 0.49891, + "ndcg_at_1000": 0.52127, + "map_at_1": 0.15, + "map_at_3": 0.22332, + "map_at_5": 0.25749, + "map_at_10": 0.28856, + "map_at_20": 0.3076, + "map_at_100": 0.3305, + "map_at_1000": 0.33246, + "recall_at_1": 0.15, + "recall_at_3": 0.26316, + "recall_at_5": 0.37135, + "recall_at_10": 0.51671, + "recall_at_20": 0.61393, + "recall_at_100": 0.86247, + "recall_at_1000": 0.97744, + "cv_recall_at_1": 0.36842, + "cv_recall_at_3": 0.49123, + "cv_recall_at_5": 0.63158, + "cv_recall_at_10": 0.75439, + "cv_recall_at_20": 0.78947, + "cv_recall_at_100": 0.92982, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.36842, + "precision_at_3": 0.22807, + "precision_at_5": 0.18947, + "precision_at_10": 0.13333, + "precision_at_20": 0.08947, + "precision_at_100": 0.03211, + "precision_at_1000": 0.00384, + "mrr_at_1": 0.368421, + "mrr_at_3": 0.426901, + "mrr_at_5": 0.457602, + "mrr_at_10": 0.473385, + "mrr_at_20": 0.475724, + "mrr_at_100": 0.479558, + "mrr_at_1000": 0.479824, + "nauc_ndcg_at_1_max": 0.403392, + "nauc_ndcg_at_1_std": 0.237199, + "nauc_ndcg_at_1_diff1": 0.454195, + "nauc_ndcg_at_3_max": 0.396447, + "nauc_ndcg_at_3_std": 0.170939, + "nauc_ndcg_at_3_diff1": 0.469811, + "nauc_ndcg_at_5_max": 0.407751, + "nauc_ndcg_at_5_std": 0.183176, + "nauc_ndcg_at_5_diff1": 0.4097, + "nauc_ndcg_at_10_max": 0.425117, + "nauc_ndcg_at_10_std": 0.313052, + "nauc_ndcg_at_10_diff1": 0.35155, + "nauc_ndcg_at_20_max": 0.454118, + "nauc_ndcg_at_20_std": 0.339688, + "nauc_ndcg_at_20_diff1": 0.346414, + "nauc_ndcg_at_100_max": 0.468017, + "nauc_ndcg_at_100_std": 0.305891, + "nauc_ndcg_at_100_diff1": 0.415706, + "nauc_ndcg_at_1000_max": 0.445238, + "nauc_ndcg_at_1000_std": 0.27097, + "nauc_ndcg_at_1000_diff1": 0.418411, + "nauc_map_at_1_max": 0.527197, + "nauc_map_at_1_std": 0.145901, + "nauc_map_at_1_diff1": 0.499402, + "nauc_map_at_3_max": 0.423434, + "nauc_map_at_3_std": 0.167565, + "nauc_map_at_3_diff1": 0.417809, + "nauc_map_at_5_max": 0.404758, + "nauc_map_at_5_std": 0.162804, + "nauc_map_at_5_diff1": 0.387736, + "nauc_map_at_10_max": 0.42273, + "nauc_map_at_10_std": 0.221789, + "nauc_map_at_10_diff1": 0.359591, + "nauc_map_at_20_max": 0.443568, + "nauc_map_at_20_std": 0.243741, + "nauc_map_at_20_diff1": 0.362205, + "nauc_map_at_100_max": 0.45545, + "nauc_map_at_100_std": 0.241842, + "nauc_map_at_100_diff1": 0.386884, + "nauc_map_at_1000_max": 0.453979, + "nauc_map_at_1000_std": 0.239358, + "nauc_map_at_1000_diff1": 0.388173, + "nauc_recall_at_1_max": 0.527197, + "nauc_recall_at_1_std": 0.145901, + "nauc_recall_at_1_diff1": 0.499402, + "nauc_recall_at_3_max": 0.388295, + "nauc_recall_at_3_std": 0.172903, + "nauc_recall_at_3_diff1": 0.373892, + "nauc_recall_at_5_max": 0.308911, + "nauc_recall_at_5_std": 0.140682, + "nauc_recall_at_5_diff1": 0.245389, + "nauc_recall_at_10_max": 0.341082, + "nauc_recall_at_10_std": 0.470621, + "nauc_recall_at_10_diff1": 0.141969, + "nauc_recall_at_20_max": 0.410059, + "nauc_recall_at_20_std": 0.537318, + "nauc_recall_at_20_diff1": 0.116476, + "nauc_recall_at_100_max": 0.647206, + "nauc_recall_at_100_std": 0.710501, + "nauc_recall_at_100_diff1": 0.401006, + "nauc_recall_at_1000_max": 0.760511, + "nauc_recall_at_1000_std": 0.992298, + "nauc_recall_at_1000_diff1": 0.658856, + "nauc_precision_at_1_max": 0.403392, + "nauc_precision_at_1_std": 0.237199, + "nauc_precision_at_1_diff1": 0.454195, + "nauc_precision_at_3_max": 0.234935, + "nauc_precision_at_3_std": 0.155153, + "nauc_precision_at_3_diff1": 0.381294, + "nauc_precision_at_5_max": 0.210712, + "nauc_precision_at_5_std": 0.125007, + "nauc_precision_at_5_diff1": 0.32017, + "nauc_precision_at_10_max": 0.140439, + "nauc_precision_at_10_std": 0.197288, + "nauc_precision_at_10_diff1": 0.181856, + "nauc_precision_at_20_max": 0.210856, + "nauc_precision_at_20_std": 0.236677, + "nauc_precision_at_20_diff1": 0.138819, + "nauc_precision_at_100_max": 0.064527, + "nauc_precision_at_100_std": 0.014781, + "nauc_precision_at_100_diff1": 0.129528, + "nauc_precision_at_1000_max": -0.044518, + "nauc_precision_at_1000_std": -0.099045, + "nauc_precision_at_1000_diff1": 0.042795, + "nauc_cv_recall_at_1_max": 0.403392, + "nauc_cv_recall_at_1_std": 0.237199, + "nauc_cv_recall_at_1_diff1": 0.454195, + "nauc_cv_recall_at_3_max": 0.382175, + "nauc_cv_recall_at_3_std": 0.224381, + "nauc_cv_recall_at_3_diff1": 0.48164, + "nauc_cv_recall_at_5_max": 0.357216, + "nauc_cv_recall_at_5_std": 0.211004, + "nauc_cv_recall_at_5_diff1": 0.355887, + "nauc_cv_recall_at_10_max": 0.410377, + "nauc_cv_recall_at_10_std": 0.703586, + "nauc_cv_recall_at_10_diff1": 0.399232, + "nauc_cv_recall_at_20_max": 0.424167, + "nauc_cv_recall_at_20_std": 0.639807, + "nauc_cv_recall_at_20_diff1": 0.389777, + "nauc_cv_recall_at_100_max": 0.748782, + "nauc_cv_recall_at_100_std": 0.888771, + "nauc_cv_recall_at_100_diff1": 0.717265, + "nauc_cv_recall_at_1000_max": 0.868512, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_1000_diff1": 0.868512, + "nauc_mrr_at_1_max": 0.403392, + "nauc_mrr_at_1_std": 0.237199, + "nauc_mrr_at_1_diff1": 0.454195, + "nauc_mrr_at_3_max": 0.391084, + "nauc_mrr_at_3_std": 0.225957, + "nauc_mrr_at_3_diff1": 0.468373, + "nauc_mrr_at_5_max": 0.388306, + "nauc_mrr_at_5_std": 0.225931, + "nauc_mrr_at_5_diff1": 0.44305, + "nauc_mrr_at_10_max": 0.393628, + "nauc_mrr_at_10_std": 0.267576, + "nauc_mrr_at_10_diff1": 0.447127, + "nauc_mrr_at_20_max": 0.394083, + "nauc_mrr_at_20_std": 0.262838, + "nauc_mrr_at_20_diff1": 0.447002, + "nauc_mrr_at_100_max": 0.398632, + "nauc_mrr_at_100_std": 0.260312, + "nauc_mrr_at_100_diff1": 0.449948, + "nauc_mrr_at_1000_max": 0.398335, + "nauc_mrr_at_1000_std": 0.259808, + "nauc_mrr_at_1000_diff1": 0.449808, + "main_score": 0.34964, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.31579, + "ndcg_at_3": 0.29768, + "ndcg_at_5": 0.33398, + "ndcg_at_10": 0.37537, + "ndcg_at_20": 0.42094, + "ndcg_at_100": 0.47982, + "ndcg_at_1000": 0.51152, + "map_at_1": 0.14327, + "map_at_3": 0.20789, + "map_at_5": 0.24314, + "map_at_10": 0.27775, + "map_at_20": 0.29672, + "map_at_100": 0.31474, + "map_at_1000": 0.31789, + "recall_at_1": 0.14327, + "recall_at_3": 0.26594, + "recall_at_5": 0.36462, + "recall_at_10": 0.48645, + "recall_at_20": 0.62782, + "recall_at_100": 0.83306, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.31579, + "cv_recall_at_3": 0.49123, + "cv_recall_at_5": 0.63158, + "cv_recall_at_10": 0.7193, + "cv_recall_at_20": 0.82456, + "cv_recall_at_100": 0.89474, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.31579, + "precision_at_3": 0.21053, + "precision_at_5": 0.18246, + "precision_at_10": 0.13333, + "precision_at_20": 0.09035, + "precision_at_100": 0.02895, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.315789, + "mrr_at_3": 0.394737, + "mrr_at_5": 0.42807, + "mrr_at_10": 0.439376, + "mrr_at_20": 0.446935, + "mrr_at_100": 0.449073, + "mrr_at_1000": 0.449448, + "nauc_ndcg_at_1_max": 0.271086, + "nauc_ndcg_at_1_std": 0.041177, + "nauc_ndcg_at_1_diff1": 0.421682, + "nauc_ndcg_at_3_max": 0.208963, + "nauc_ndcg_at_3_std": 0.040929, + "nauc_ndcg_at_3_diff1": 0.354708, + "nauc_ndcg_at_5_max": 0.274917, + "nauc_ndcg_at_5_std": 0.098547, + "nauc_ndcg_at_5_diff1": 0.301121, + "nauc_ndcg_at_10_max": 0.234243, + "nauc_ndcg_at_10_std": 0.069251, + "nauc_ndcg_at_10_diff1": 0.250978, + "nauc_ndcg_at_20_max": 0.26117, + "nauc_ndcg_at_20_std": 0.136556, + "nauc_ndcg_at_20_diff1": 0.301042, + "nauc_ndcg_at_100_max": 0.2843, + "nauc_ndcg_at_100_std": 0.138366, + "nauc_ndcg_at_100_diff1": 0.276971, + "nauc_ndcg_at_1000_max": 0.235286, + "nauc_ndcg_at_1000_std": 0.074248, + "nauc_ndcg_at_1000_diff1": 0.284991, + "nauc_map_at_1_max": 0.238236, + "nauc_map_at_1_std": 0.147652, + "nauc_map_at_1_diff1": 0.363383, + "nauc_map_at_3_max": 0.217576, + "nauc_map_at_3_std": 0.111406, + "nauc_map_at_3_diff1": 0.331128, + "nauc_map_at_5_max": 0.275193, + "nauc_map_at_5_std": 0.131943, + "nauc_map_at_5_diff1": 0.32801, + "nauc_map_at_10_max": 0.284546, + "nauc_map_at_10_std": 0.120026, + "nauc_map_at_10_diff1": 0.332764, + "nauc_map_at_20_max": 0.290771, + "nauc_map_at_20_std": 0.139518, + "nauc_map_at_20_diff1": 0.349713, + "nauc_map_at_100_max": 0.283258, + "nauc_map_at_100_std": 0.136028, + "nauc_map_at_100_diff1": 0.336479, + "nauc_map_at_1000_max": 0.277607, + "nauc_map_at_1000_std": 0.128588, + "nauc_map_at_1000_diff1": 0.335792, + "nauc_recall_at_1_max": 0.238236, + "nauc_recall_at_1_std": 0.147652, + "nauc_recall_at_1_diff1": 0.363383, + "nauc_recall_at_3_max": 0.120918, + "nauc_recall_at_3_std": 0.016004, + "nauc_recall_at_3_diff1": 0.216097, + "nauc_recall_at_5_max": 0.208196, + "nauc_recall_at_5_std": 0.08493, + "nauc_recall_at_5_diff1": 0.121706, + "nauc_recall_at_10_max": 0.101167, + "nauc_recall_at_10_std": 0.015707, + "nauc_recall_at_10_diff1": 0.030345, + "nauc_recall_at_20_max": 0.162291, + "nauc_recall_at_20_std": 0.234554, + "nauc_recall_at_20_diff1": 0.141571, + "nauc_recall_at_100_max": 0.513757, + "nauc_recall_at_100_std": 0.48246, + "nauc_recall_at_100_diff1": 0.020693, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.271086, + "nauc_precision_at_1_std": 0.041177, + "nauc_precision_at_1_diff1": 0.421682, + "nauc_precision_at_3_max": 0.194151, + "nauc_precision_at_3_std": -0.024803, + "nauc_precision_at_3_diff1": 0.294492, + "nauc_precision_at_5_max": 0.299247, + "nauc_precision_at_5_std": 0.045377, + "nauc_precision_at_5_diff1": 0.249228, + "nauc_precision_at_10_max": 0.161393, + "nauc_precision_at_10_std": -0.037407, + "nauc_precision_at_10_diff1": 0.174916, + "nauc_precision_at_20_max": 0.052695, + "nauc_precision_at_20_std": -0.036414, + "nauc_precision_at_20_diff1": 0.143735, + "nauc_precision_at_100_max": -0.128526, + "nauc_precision_at_100_std": -0.165509, + "nauc_precision_at_100_diff1": -0.00898, + "nauc_precision_at_1000_max": -0.275997, + "nauc_precision_at_1000_std": -0.323795, + "nauc_precision_at_1000_diff1": -0.110155, + "nauc_cv_recall_at_1_max": 0.271086, + "nauc_cv_recall_at_1_std": 0.041177, + "nauc_cv_recall_at_1_diff1": 0.421682, + "nauc_cv_recall_at_3_max": 0.204362, + "nauc_cv_recall_at_3_std": -0.016019, + "nauc_cv_recall_at_3_diff1": 0.265269, + "nauc_cv_recall_at_5_max": 0.197253, + "nauc_cv_recall_at_5_std": 0.062909, + "nauc_cv_recall_at_5_diff1": 0.024446, + "nauc_cv_recall_at_10_max": 0.035036, + "nauc_cv_recall_at_10_std": -0.031197, + "nauc_cv_recall_at_10_diff1": -0.127566, + "nauc_cv_recall_at_20_max": 0.067015, + "nauc_cv_recall_at_20_std": 0.205199, + "nauc_cv_recall_at_20_diff1": -0.068769, + "nauc_cv_recall_at_100_max": 0.463344, + "nauc_cv_recall_at_100_std": 0.363991, + "nauc_cv_recall_at_100_diff1": -0.052112, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.271086, + "nauc_mrr_at_1_std": 0.041177, + "nauc_mrr_at_1_diff1": 0.421682, + "nauc_mrr_at_3_max": 0.254789, + "nauc_mrr_at_3_std": 0.030384, + "nauc_mrr_at_3_diff1": 0.354443, + "nauc_mrr_at_5_max": 0.258963, + "nauc_mrr_at_5_std": 0.050953, + "nauc_mrr_at_5_diff1": 0.305181, + "nauc_mrr_at_10_max": 0.244934, + "nauc_mrr_at_10_std": 0.040213, + "nauc_mrr_at_10_diff1": 0.298488, + "nauc_mrr_at_20_max": 0.252244, + "nauc_mrr_at_20_std": 0.051586, + "nauc_mrr_at_20_diff1": 0.309095, + "nauc_mrr_at_100_max": 0.257544, + "nauc_mrr_at_100_std": 0.052406, + "nauc_mrr_at_100_diff1": 0.311046, + "nauc_mrr_at_1000_max": 0.257449, + "nauc_mrr_at_1000_std": 0.052274, + "nauc_mrr_at_1000_diff1": 0.311348, + "main_score": 0.33398, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.50877, + "ndcg_at_3": 0.48444, + "ndcg_at_5": 0.49116, + "ndcg_at_10": 0.52706, + "ndcg_at_20": 0.57848, + "ndcg_at_100": 0.62951, + "ndcg_at_1000": 0.63459, + "map_at_1": 0.21364, + "map_at_3": 0.33029, + "map_at_5": 0.37077, + "map_at_10": 0.4044, + "map_at_20": 0.43648, + "map_at_100": 0.45981, + "map_at_1000": 0.46036, + "recall_at_1": 0.21364, + "recall_at_3": 0.39735, + "recall_at_5": 0.47717, + "recall_at_10": 0.63285, + "recall_at_20": 0.79624, + "recall_at_100": 0.95792, + "recall_at_1000": 0.99561, + "cv_recall_at_1": 0.50877, + "cv_recall_at_3": 0.68421, + "cv_recall_at_5": 0.7193, + "cv_recall_at_10": 0.85965, + "cv_recall_at_20": 0.91228, + "cv_recall_at_100": 0.96491, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.50877, + "precision_at_3": 0.35673, + "precision_at_5": 0.27368, + "precision_at_10": 0.17719, + "precision_at_20": 0.12368, + "precision_at_100": 0.03772, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.508772, + "mrr_at_3": 0.584795, + "mrr_at_5": 0.59269, + "mrr_at_10": 0.612288, + "mrr_at_20": 0.616731, + "mrr_at_100": 0.618721, + "mrr_at_1000": 0.618793, + "nauc_ndcg_at_1_max": 0.260565, + "nauc_ndcg_at_1_std": 0.082815, + "nauc_ndcg_at_1_diff1": 0.343867, + "nauc_ndcg_at_3_max": 0.187461, + "nauc_ndcg_at_3_std": -0.028199, + "nauc_ndcg_at_3_diff1": 0.354998, + "nauc_ndcg_at_5_max": 0.218263, + "nauc_ndcg_at_5_std": 0.006352, + "nauc_ndcg_at_5_diff1": 0.352624, + "nauc_ndcg_at_10_max": 0.199358, + "nauc_ndcg_at_10_std": -0.02167, + "nauc_ndcg_at_10_diff1": 0.378752, + "nauc_ndcg_at_20_max": 0.209988, + "nauc_ndcg_at_20_std": -0.002784, + "nauc_ndcg_at_20_diff1": 0.380647, + "nauc_ndcg_at_100_max": 0.259132, + "nauc_ndcg_at_100_std": 0.039148, + "nauc_ndcg_at_100_diff1": 0.381707, + "nauc_ndcg_at_1000_max": 0.249943, + "nauc_ndcg_at_1000_std": 0.019964, + "nauc_ndcg_at_1000_diff1": 0.382183, + "nauc_map_at_1_max": 0.261125, + "nauc_map_at_1_std": 0.104593, + "nauc_map_at_1_diff1": 0.362463, + "nauc_map_at_3_max": 0.265528, + "nauc_map_at_3_std": 0.037754, + "nauc_map_at_3_diff1": 0.40679, + "nauc_map_at_5_max": 0.249479, + "nauc_map_at_5_std": 0.013071, + "nauc_map_at_5_diff1": 0.401607, + "nauc_map_at_10_max": 0.229074, + "nauc_map_at_10_std": -0.010939, + "nauc_map_at_10_diff1": 0.401434, + "nauc_map_at_20_max": 0.237751, + "nauc_map_at_20_std": -0.003432, + "nauc_map_at_20_diff1": 0.397991, + "nauc_map_at_100_max": 0.255505, + "nauc_map_at_100_std": 0.001521, + "nauc_map_at_100_diff1": 0.407538, + "nauc_map_at_1000_max": 0.254324, + "nauc_map_at_1000_std": -0.000167, + "nauc_map_at_1000_diff1": 0.408371, + "nauc_recall_at_1_max": 0.261125, + "nauc_recall_at_1_std": 0.104593, + "nauc_recall_at_1_diff1": 0.362463, + "nauc_recall_at_3_max": 0.151587, + "nauc_recall_at_3_std": -0.002679, + "nauc_recall_at_3_diff1": 0.333711, + "nauc_recall_at_5_max": 0.109795, + "nauc_recall_at_5_std": -0.033675, + "nauc_recall_at_5_diff1": 0.290203, + "nauc_recall_at_10_max": -0.001152, + "nauc_recall_at_10_std": -0.116127, + "nauc_recall_at_10_diff1": 0.258912, + "nauc_recall_at_20_max": 0.006976, + "nauc_recall_at_20_std": -0.039776, + "nauc_recall_at_20_diff1": 0.306866, + "nauc_recall_at_100_max": 0.569669, + "nauc_recall_at_100_std": 0.87203, + "nauc_recall_at_100_diff1": 0.462011, + "nauc_recall_at_1000_max": 0.555085, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 1.0, + "nauc_precision_at_1_max": 0.260565, + "nauc_precision_at_1_std": 0.082815, + "nauc_precision_at_1_diff1": 0.343867, + "nauc_precision_at_3_max": 0.094345, + "nauc_precision_at_3_std": -0.096555, + "nauc_precision_at_3_diff1": 0.207837, + "nauc_precision_at_5_max": 0.105888, + "nauc_precision_at_5_std": -0.055444, + "nauc_precision_at_5_diff1": 0.14087, + "nauc_precision_at_10_max": 0.032662, + "nauc_precision_at_10_std": -0.13283, + "nauc_precision_at_10_diff1": 0.122434, + "nauc_precision_at_20_max": 0.060994, + "nauc_precision_at_20_std": -0.064981, + "nauc_precision_at_20_diff1": 0.013832, + "nauc_precision_at_100_max": 0.04323, + "nauc_precision_at_100_std": -0.029747, + "nauc_precision_at_100_diff1": -0.036033, + "nauc_precision_at_1000_max": 0.023938, + "nauc_precision_at_1000_std": -0.052608, + "nauc_precision_at_1000_diff1": -0.032103, + "nauc_cv_recall_at_1_max": 0.260565, + "nauc_cv_recall_at_1_std": 0.082815, + "nauc_cv_recall_at_1_diff1": 0.343867, + "nauc_cv_recall_at_3_max": 0.126648, + "nauc_cv_recall_at_3_std": 0.017445, + "nauc_cv_recall_at_3_diff1": 0.231599, + "nauc_cv_recall_at_5_max": 0.106674, + "nauc_cv_recall_at_5_std": 0.007339, + "nauc_cv_recall_at_5_diff1": 0.155511, + "nauc_cv_recall_at_10_max": 0.207594, + "nauc_cv_recall_at_10_std": -0.159017, + "nauc_cv_recall_at_10_diff1": 0.435254, + "nauc_cv_recall_at_20_max": 0.207633, + "nauc_cv_recall_at_20_std": -0.123815, + "nauc_cv_recall_at_20_diff1": 0.645383, + "nauc_cv_recall_at_100_max": 0.563308, + "nauc_cv_recall_at_100_std": 0.861366, + "nauc_cv_recall_at_100_diff1": 0.456469, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.260565, + "nauc_mrr_at_1_std": 0.082815, + "nauc_mrr_at_1_diff1": 0.343867, + "nauc_mrr_at_3_max": 0.229025, + "nauc_mrr_at_3_std": 0.070324, + "nauc_mrr_at_3_diff1": 0.303546, + "nauc_mrr_at_5_max": 0.224864, + "nauc_mrr_at_5_std": 0.067548, + "nauc_mrr_at_5_diff1": 0.291988, + "nauc_mrr_at_10_max": 0.238134, + "nauc_mrr_at_10_std": 0.062171, + "nauc_mrr_at_10_diff1": 0.318934, + "nauc_mrr_at_20_max": 0.239339, + "nauc_mrr_at_20_std": 0.065021, + "nauc_mrr_at_20_diff1": 0.323614, + "nauc_mrr_at_100_max": 0.241127, + "nauc_mrr_at_100_std": 0.071383, + "nauc_mrr_at_100_diff1": 0.320449, + "nauc_mrr_at_1000_max": 0.241047, + "nauc_mrr_at_1000_std": 0.071184, + "nauc_mrr_at_1000_diff1": 0.320415, + "main_score": 0.49116, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.24561, + "ndcg_at_3": 0.2615, + "ndcg_at_5": 0.27558, + "ndcg_at_10": 0.31175, + "ndcg_at_20": 0.35291, + "ndcg_at_100": 0.44986, + "ndcg_at_1000": 0.46681, + "map_at_1": 0.07607, + "map_at_3": 0.1686, + "map_at_5": 0.19126, + "map_at_10": 0.22561, + "map_at_20": 0.24297, + "map_at_100": 0.2708, + "map_at_1000": 0.27259, + "recall_at_1": 0.07607, + "recall_at_3": 0.22231, + "recall_at_5": 0.30272, + "recall_at_10": 0.4089, + "recall_at_20": 0.54238, + "recall_at_100": 0.90602, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.24561, + "cv_recall_at_3": 0.40351, + "cv_recall_at_5": 0.50877, + "cv_recall_at_10": 0.57895, + "cv_recall_at_20": 0.7193, + "cv_recall_at_100": 0.94737, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.24561, + "precision_at_3": 0.21053, + "precision_at_5": 0.16491, + "precision_at_10": 0.12632, + "precision_at_20": 0.08684, + "precision_at_100": 0.03368, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.245614, + "mrr_at_3": 0.312865, + "mrr_at_5": 0.335673, + "mrr_at_10": 0.345071, + "mrr_at_20": 0.355213, + "mrr_at_100": 0.362355, + "mrr_at_1000": 0.362476, + "nauc_ndcg_at_1_max": 0.146012, + "nauc_ndcg_at_1_std": 0.147152, + "nauc_ndcg_at_1_diff1": 0.297569, + "nauc_ndcg_at_3_max": 0.252687, + "nauc_ndcg_at_3_std": 0.152352, + "nauc_ndcg_at_3_diff1": 0.316266, + "nauc_ndcg_at_5_max": 0.193288, + "nauc_ndcg_at_5_std": 0.14243, + "nauc_ndcg_at_5_diff1": 0.290765, + "nauc_ndcg_at_10_max": 0.202136, + "nauc_ndcg_at_10_std": 0.132041, + "nauc_ndcg_at_10_diff1": 0.267908, + "nauc_ndcg_at_20_max": 0.237821, + "nauc_ndcg_at_20_std": 0.154622, + "nauc_ndcg_at_20_diff1": 0.226625, + "nauc_ndcg_at_100_max": 0.273735, + "nauc_ndcg_at_100_std": 0.185905, + "nauc_ndcg_at_100_diff1": 0.284865, + "nauc_ndcg_at_1000_max": 0.230191, + "nauc_ndcg_at_1000_std": 0.149425, + "nauc_ndcg_at_1000_diff1": 0.286119, + "nauc_map_at_1_max": 0.129605, + "nauc_map_at_1_std": 0.226887, + "nauc_map_at_1_diff1": 0.293264, + "nauc_map_at_3_max": 0.250853, + "nauc_map_at_3_std": 0.182447, + "nauc_map_at_3_diff1": 0.266823, + "nauc_map_at_5_max": 0.204497, + "nauc_map_at_5_std": 0.172315, + "nauc_map_at_5_diff1": 0.245673, + "nauc_map_at_10_max": 0.223086, + "nauc_map_at_10_std": 0.163683, + "nauc_map_at_10_diff1": 0.263733, + "nauc_map_at_20_max": 0.245311, + "nauc_map_at_20_std": 0.17545, + "nauc_map_at_20_diff1": 0.255007, + "nauc_map_at_100_max": 0.259415, + "nauc_map_at_100_std": 0.181687, + "nauc_map_at_100_diff1": 0.272042, + "nauc_map_at_1000_max": 0.253528, + "nauc_map_at_1000_std": 0.177264, + "nauc_map_at_1000_diff1": 0.274219, + "nauc_recall_at_1_max": 0.129605, + "nauc_recall_at_1_std": 0.226887, + "nauc_recall_at_1_diff1": 0.293264, + "nauc_recall_at_3_max": 0.251182, + "nauc_recall_at_3_std": 0.132969, + "nauc_recall_at_3_diff1": 0.21924, + "nauc_recall_at_5_max": 0.114798, + "nauc_recall_at_5_std": 0.085, + "nauc_recall_at_5_diff1": 0.17963, + "nauc_recall_at_10_max": 0.12857, + "nauc_recall_at_10_std": 0.052571, + "nauc_recall_at_10_diff1": 0.178461, + "nauc_recall_at_20_max": 0.195959, + "nauc_recall_at_20_std": 0.081385, + "nauc_recall_at_20_diff1": 0.069651, + "nauc_recall_at_100_max": 0.63671, + "nauc_recall_at_100_std": 0.456428, + "nauc_recall_at_100_diff1": 0.38114, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.146012, + "nauc_precision_at_1_std": 0.147152, + "nauc_precision_at_1_diff1": 0.297569, + "nauc_precision_at_3_max": 0.26317, + "nauc_precision_at_3_std": 0.115289, + "nauc_precision_at_3_diff1": 0.299014, + "nauc_precision_at_5_max": 0.169383, + "nauc_precision_at_5_std": 0.068625, + "nauc_precision_at_5_diff1": 0.280413, + "nauc_precision_at_10_max": 0.181455, + "nauc_precision_at_10_std": 0.059082, + "nauc_precision_at_10_diff1": 0.249841, + "nauc_precision_at_20_max": 0.233606, + "nauc_precision_at_20_std": 0.092642, + "nauc_precision_at_20_diff1": 0.16873, + "nauc_precision_at_100_max": 0.024947, + "nauc_precision_at_100_std": -0.047125, + "nauc_precision_at_100_diff1": 0.137874, + "nauc_precision_at_1000_max": -0.109055, + "nauc_precision_at_1000_std": -0.146756, + "nauc_precision_at_1000_diff1": 0.172165, + "nauc_cv_recall_at_1_max": 0.146012, + "nauc_cv_recall_at_1_std": 0.147152, + "nauc_cv_recall_at_1_diff1": 0.297569, + "nauc_cv_recall_at_3_max": 0.279892, + "nauc_cv_recall_at_3_std": 0.163987, + "nauc_cv_recall_at_3_diff1": 0.273535, + "nauc_cv_recall_at_5_max": 0.230844, + "nauc_cv_recall_at_5_std": 0.120746, + "nauc_cv_recall_at_5_diff1": 0.364239, + "nauc_cv_recall_at_10_max": 0.062438, + "nauc_cv_recall_at_10_std": -0.001943, + "nauc_cv_recall_at_10_diff1": 0.219356, + "nauc_cv_recall_at_20_max": 0.16677, + "nauc_cv_recall_at_20_std": -0.032066, + "nauc_cv_recall_at_20_diff1": 0.046223, + "nauc_cv_recall_at_100_max": 0.436368, + "nauc_cv_recall_at_100_std": 0.091883, + "nauc_cv_recall_at_100_diff1": 0.567856, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.146012, + "nauc_mrr_at_1_std": 0.147152, + "nauc_mrr_at_1_diff1": 0.297569, + "nauc_mrr_at_3_max": 0.235552, + "nauc_mrr_at_3_std": 0.187094, + "nauc_mrr_at_3_diff1": 0.297164, + "nauc_mrr_at_5_max": 0.222479, + "nauc_mrr_at_5_std": 0.17709, + "nauc_mrr_at_5_diff1": 0.315689, + "nauc_mrr_at_10_max": 0.200458, + "nauc_mrr_at_10_std": 0.161217, + "nauc_mrr_at_10_diff1": 0.296194, + "nauc_mrr_at_20_max": 0.205963, + "nauc_mrr_at_20_std": 0.160592, + "nauc_mrr_at_20_diff1": 0.290762, + "nauc_mrr_at_100_max": 0.210365, + "nauc_mrr_at_100_std": 0.166786, + "nauc_mrr_at_100_diff1": 0.297711, + "nauc_mrr_at_1000_max": 0.210423, + "nauc_mrr_at_1000_std": 0.166897, + "nauc_mrr_at_1000_diff1": 0.297712, + "main_score": 0.27558, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 402.9583468437195, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/Vidore2EconomicsReportsRetrieval.json b/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..937cf17e59 --- /dev/null +++ b/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.2069, + "ndcg_at_3": 0.1937, + "ndcg_at_5": 0.18579, + "ndcg_at_10": 0.18198, + "ndcg_at_20": 0.20745, + "ndcg_at_100": 0.32688, + "ndcg_at_1000": 0.46438, + "map_at_1": 0.02318, + "map_at_3": 0.04475, + "map_at_5": 0.05753, + "map_at_10": 0.08067, + "map_at_20": 0.09933, + "map_at_100": 0.1331, + "map_at_1000": 0.1605, + "recall_at_1": 0.02318, + "recall_at_3": 0.05317, + "recall_at_5": 0.07871, + "recall_at_10": 0.14065, + "recall_at_20": 0.23922, + "recall_at_100": 0.55405, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.2069, + "cv_recall_at_3": 0.36207, + "cv_recall_at_5": 0.46552, + "cv_recall_at_10": 0.56897, + "cv_recall_at_20": 0.68966, + "cv_recall_at_100": 0.91379, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.2069, + "precision_at_3": 0.18966, + "precision_at_5": 0.17931, + "precision_at_10": 0.15, + "precision_at_20": 0.1181, + "precision_at_100": 0.0719, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.206897, + "mrr_at_3": 0.275862, + "mrr_at_5": 0.300862, + "mrr_at_10": 0.315846, + "mrr_at_20": 0.324723, + "mrr_at_100": 0.332376, + "mrr_at_1000": 0.333068, + "nauc_ndcg_at_1_max": 0.452681, + "nauc_ndcg_at_1_std": 0.210225, + "nauc_ndcg_at_1_diff1": 0.582569, + "nauc_ndcg_at_3_max": 0.43052, + "nauc_ndcg_at_3_std": 0.221808, + "nauc_ndcg_at_3_diff1": 0.422184, + "nauc_ndcg_at_5_max": 0.434886, + "nauc_ndcg_at_5_std": 0.295478, + "nauc_ndcg_at_5_diff1": 0.342229, + "nauc_ndcg_at_10_max": 0.501811, + "nauc_ndcg_at_10_std": 0.350988, + "nauc_ndcg_at_10_diff1": 0.347591, + "nauc_ndcg_at_20_max": 0.494579, + "nauc_ndcg_at_20_std": 0.368487, + "nauc_ndcg_at_20_diff1": 0.314898, + "nauc_ndcg_at_100_max": 0.52479, + "nauc_ndcg_at_100_std": 0.373521, + "nauc_ndcg_at_100_diff1": 0.296172, + "nauc_ndcg_at_1000_max": 0.498897, + "nauc_ndcg_at_1000_std": 0.429551, + "nauc_ndcg_at_1000_diff1": 0.216141, + "nauc_map_at_1_max": 0.427877, + "nauc_map_at_1_std": 0.276935, + "nauc_map_at_1_diff1": 0.450837, + "nauc_map_at_3_max": 0.475138, + "nauc_map_at_3_std": 0.291276, + "nauc_map_at_3_diff1": 0.359381, + "nauc_map_at_5_max": 0.465726, + "nauc_map_at_5_std": 0.302777, + "nauc_map_at_5_diff1": 0.306039, + "nauc_map_at_10_max": 0.540398, + "nauc_map_at_10_std": 0.307679, + "nauc_map_at_10_diff1": 0.296089, + "nauc_map_at_20_max": 0.516738, + "nauc_map_at_20_std": 0.300761, + "nauc_map_at_20_diff1": 0.305776, + "nauc_map_at_100_max": 0.554814, + "nauc_map_at_100_std": 0.349937, + "nauc_map_at_100_diff1": 0.283324, + "nauc_map_at_1000_max": 0.578008, + "nauc_map_at_1000_std": 0.426111, + "nauc_map_at_1000_diff1": 0.216906, + "nauc_recall_at_1_max": 0.427877, + "nauc_recall_at_1_std": 0.276935, + "nauc_recall_at_1_diff1": 0.450837, + "nauc_recall_at_3_max": 0.425105, + "nauc_recall_at_3_std": 0.269631, + "nauc_recall_at_3_diff1": 0.341844, + "nauc_recall_at_5_max": 0.382332, + "nauc_recall_at_5_std": 0.295725, + "nauc_recall_at_5_diff1": 0.233917, + "nauc_recall_at_10_max": 0.515643, + "nauc_recall_at_10_std": 0.333371, + "nauc_recall_at_10_diff1": 0.332411, + "nauc_recall_at_20_max": 0.363745, + "nauc_recall_at_20_std": 0.289069, + "nauc_recall_at_20_diff1": 0.218022, + "nauc_recall_at_100_max": 0.35015, + "nauc_recall_at_100_std": 0.209864, + "nauc_recall_at_100_diff1": 0.187416, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.452681, + "nauc_precision_at_1_std": 0.210225, + "nauc_precision_at_1_diff1": 0.582569, + "nauc_precision_at_3_max": 0.421533, + "nauc_precision_at_3_std": 0.215192, + "nauc_precision_at_3_diff1": 0.383014, + "nauc_precision_at_5_max": 0.409452, + "nauc_precision_at_5_std": 0.320325, + "nauc_precision_at_5_diff1": 0.252211, + "nauc_precision_at_10_max": 0.492587, + "nauc_precision_at_10_std": 0.366596, + "nauc_precision_at_10_diff1": 0.232198, + "nauc_precision_at_20_max": 0.432166, + "nauc_precision_at_20_std": 0.377048, + "nauc_precision_at_20_diff1": 0.209837, + "nauc_precision_at_100_max": 0.384212, + "nauc_precision_at_100_std": 0.459873, + "nauc_precision_at_100_diff1": -0.121991, + "nauc_precision_at_1000_max": 0.26732, + "nauc_precision_at_1000_std": 0.439072, + "nauc_precision_at_1000_diff1": -0.22685, + "nauc_cv_recall_at_1_max": 0.452681, + "nauc_cv_recall_at_1_std": 0.210225, + "nauc_cv_recall_at_1_diff1": 0.582569, + "nauc_cv_recall_at_3_max": 0.282738, + "nauc_cv_recall_at_3_std": 0.190198, + "nauc_cv_recall_at_3_diff1": 0.355092, + "nauc_cv_recall_at_5_max": 0.310146, + "nauc_cv_recall_at_5_std": 0.34807, + "nauc_cv_recall_at_5_diff1": 0.26306, + "nauc_cv_recall_at_10_max": 0.463726, + "nauc_cv_recall_at_10_std": 0.534206, + "nauc_cv_recall_at_10_diff1": 0.317703, + "nauc_cv_recall_at_20_max": 0.439924, + "nauc_cv_recall_at_20_std": 0.630713, + "nauc_cv_recall_at_20_diff1": 0.1466, + "nauc_cv_recall_at_100_max": 0.305158, + "nauc_cv_recall_at_100_std": 0.266262, + "nauc_cv_recall_at_100_diff1": 0.152894, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.452681, + "nauc_mrr_at_1_std": 0.210225, + "nauc_mrr_at_1_diff1": 0.582569, + "nauc_mrr_at_3_max": 0.371505, + "nauc_mrr_at_3_std": 0.208859, + "nauc_mrr_at_3_diff1": 0.458902, + "nauc_mrr_at_5_max": 0.378589, + "nauc_mrr_at_5_std": 0.247416, + "nauc_mrr_at_5_diff1": 0.435281, + "nauc_mrr_at_10_max": 0.404034, + "nauc_mrr_at_10_std": 0.273748, + "nauc_mrr_at_10_diff1": 0.44812, + "nauc_mrr_at_20_max": 0.404218, + "nauc_mrr_at_20_std": 0.278039, + "nauc_mrr_at_20_diff1": 0.439917, + "nauc_mrr_at_100_max": 0.40396, + "nauc_mrr_at_100_std": 0.268724, + "nauc_mrr_at_100_diff1": 0.444907, + "nauc_mrr_at_1000_max": 0.404296, + "nauc_mrr_at_1000_std": 0.268842, + "nauc_mrr_at_1000_diff1": 0.445734, + "main_score": 0.18579, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.18966, + "ndcg_at_3": 0.20603, + "ndcg_at_5": 0.1996, + "ndcg_at_10": 0.18851, + "ndcg_at_20": 0.20748, + "ndcg_at_100": 0.32076, + "ndcg_at_1000": 0.46095, + "map_at_1": 0.02027, + "map_at_3": 0.04324, + "map_at_5": 0.06103, + "map_at_10": 0.07974, + "map_at_20": 0.09799, + "map_at_100": 0.13061, + "map_at_1000": 0.15813, + "recall_at_1": 0.02027, + "recall_at_3": 0.0551, + "recall_at_5": 0.09142, + "recall_at_10": 0.1426, + "recall_at_20": 0.21518, + "recall_at_100": 0.56517, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.18966, + "cv_recall_at_3": 0.43103, + "cv_recall_at_5": 0.5, + "cv_recall_at_10": 0.56897, + "cv_recall_at_20": 0.65517, + "cv_recall_at_100": 0.96552, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.18966, + "precision_at_3": 0.2069, + "precision_at_5": 0.1931, + "precision_at_10": 0.15517, + "precision_at_20": 0.12672, + "precision_at_100": 0.0669, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.189655, + "mrr_at_3": 0.298851, + "mrr_at_5": 0.314368, + "mrr_at_10": 0.325041, + "mrr_at_20": 0.330239, + "mrr_at_100": 0.337658, + "mrr_at_1000": 0.337915, + "nauc_ndcg_at_1_max": 0.596393, + "nauc_ndcg_at_1_std": 0.504927, + "nauc_ndcg_at_1_diff1": 0.414057, + "nauc_ndcg_at_3_max": 0.482832, + "nauc_ndcg_at_3_std": 0.387866, + "nauc_ndcg_at_3_diff1": 0.26614, + "nauc_ndcg_at_5_max": 0.541846, + "nauc_ndcg_at_5_std": 0.437157, + "nauc_ndcg_at_5_diff1": 0.235359, + "nauc_ndcg_at_10_max": 0.520007, + "nauc_ndcg_at_10_std": 0.474073, + "nauc_ndcg_at_10_diff1": 0.224539, + "nauc_ndcg_at_20_max": 0.495927, + "nauc_ndcg_at_20_std": 0.492634, + "nauc_ndcg_at_20_diff1": 0.18787, + "nauc_ndcg_at_100_max": 0.477709, + "nauc_ndcg_at_100_std": 0.508516, + "nauc_ndcg_at_100_diff1": 0.199052, + "nauc_ndcg_at_1000_max": 0.460945, + "nauc_ndcg_at_1000_std": 0.426181, + "nauc_ndcg_at_1000_diff1": 0.138307, + "nauc_map_at_1_max": 0.472878, + "nauc_map_at_1_std": 0.474173, + "nauc_map_at_1_diff1": 0.207629, + "nauc_map_at_3_max": 0.380969, + "nauc_map_at_3_std": 0.352335, + "nauc_map_at_3_diff1": 0.105275, + "nauc_map_at_5_max": 0.437336, + "nauc_map_at_5_std": 0.395602, + "nauc_map_at_5_diff1": 0.07774, + "nauc_map_at_10_max": 0.442647, + "nauc_map_at_10_std": 0.433325, + "nauc_map_at_10_diff1": 0.149455, + "nauc_map_at_20_max": 0.451976, + "nauc_map_at_20_std": 0.447972, + "nauc_map_at_20_diff1": 0.169858, + "nauc_map_at_100_max": 0.470623, + "nauc_map_at_100_std": 0.487983, + "nauc_map_at_100_diff1": 0.162702, + "nauc_map_at_1000_max": 0.502602, + "nauc_map_at_1000_std": 0.486313, + "nauc_map_at_1000_diff1": 0.169676, + "nauc_recall_at_1_max": 0.472878, + "nauc_recall_at_1_std": 0.474173, + "nauc_recall_at_1_diff1": 0.207629, + "nauc_recall_at_3_max": 0.319285, + "nauc_recall_at_3_std": 0.260739, + "nauc_recall_at_3_diff1": 0.067165, + "nauc_recall_at_5_max": 0.425452, + "nauc_recall_at_5_std": 0.375266, + "nauc_recall_at_5_diff1": -0.023409, + "nauc_recall_at_10_max": 0.373784, + "nauc_recall_at_10_std": 0.416799, + "nauc_recall_at_10_diff1": 0.083991, + "nauc_recall_at_20_max": 0.368444, + "nauc_recall_at_20_std": 0.467193, + "nauc_recall_at_20_diff1": 0.096081, + "nauc_recall_at_100_max": 0.242759, + "nauc_recall_at_100_std": 0.329516, + "nauc_recall_at_100_diff1": 0.207776, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.596393, + "nauc_precision_at_1_std": 0.504927, + "nauc_precision_at_1_diff1": 0.414057, + "nauc_precision_at_3_max": 0.445119, + "nauc_precision_at_3_std": 0.336925, + "nauc_precision_at_3_diff1": 0.222288, + "nauc_precision_at_5_max": 0.548027, + "nauc_precision_at_5_std": 0.42906, + "nauc_precision_at_5_diff1": 0.198391, + "nauc_precision_at_10_max": 0.518863, + "nauc_precision_at_10_std": 0.472361, + "nauc_precision_at_10_diff1": 0.274672, + "nauc_precision_at_20_max": 0.422349, + "nauc_precision_at_20_std": 0.395638, + "nauc_precision_at_20_diff1": 0.218902, + "nauc_precision_at_100_max": 0.323376, + "nauc_precision_at_100_std": 0.281597, + "nauc_precision_at_100_diff1": 0.124699, + "nauc_precision_at_1000_max": 0.215022, + "nauc_precision_at_1000_std": 0.083314, + "nauc_precision_at_1000_diff1": 0.094304, + "nauc_cv_recall_at_1_max": 0.596393, + "nauc_cv_recall_at_1_std": 0.504927, + "nauc_cv_recall_at_1_diff1": 0.414057, + "nauc_cv_recall_at_3_max": 0.480382, + "nauc_cv_recall_at_3_std": 0.375148, + "nauc_cv_recall_at_3_diff1": 0.146474, + "nauc_cv_recall_at_5_max": 0.595569, + "nauc_cv_recall_at_5_std": 0.512763, + "nauc_cv_recall_at_5_diff1": 0.065002, + "nauc_cv_recall_at_10_max": 0.493407, + "nauc_cv_recall_at_10_std": 0.46662, + "nauc_cv_recall_at_10_diff1": 0.138537, + "nauc_cv_recall_at_20_max": 0.413208, + "nauc_cv_recall_at_20_std": 0.529582, + "nauc_cv_recall_at_20_diff1": 0.057955, + "nauc_cv_recall_at_100_max": 0.711396, + "nauc_cv_recall_at_100_std": 0.934389, + "nauc_cv_recall_at_100_diff1": 0.359055, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.596393, + "nauc_mrr_at_1_std": 0.504927, + "nauc_mrr_at_1_diff1": 0.414057, + "nauc_mrr_at_3_max": 0.498605, + "nauc_mrr_at_3_std": 0.417519, + "nauc_mrr_at_3_diff1": 0.266103, + "nauc_mrr_at_5_max": 0.524137, + "nauc_mrr_at_5_std": 0.447944, + "nauc_mrr_at_5_diff1": 0.249573, + "nauc_mrr_at_10_max": 0.506768, + "nauc_mrr_at_10_std": 0.439256, + "nauc_mrr_at_10_diff1": 0.261582, + "nauc_mrr_at_20_max": 0.502565, + "nauc_mrr_at_20_std": 0.442987, + "nauc_mrr_at_20_diff1": 0.258223, + "nauc_mrr_at_100_max": 0.500711, + "nauc_mrr_at_100_std": 0.440637, + "nauc_mrr_at_100_diff1": 0.261274, + "nauc_mrr_at_1000_max": 0.500572, + "nauc_mrr_at_1000_std": 0.440234, + "nauc_mrr_at_1000_diff1": 0.261191, + "main_score": 0.1996, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.43103, + "ndcg_at_3": 0.44457, + "ndcg_at_5": 0.41537, + "ndcg_at_10": 0.43479, + "ndcg_at_20": 0.44322, + "ndcg_at_100": 0.56613, + "ndcg_at_1000": 0.62734, + "map_at_1": 0.07556, + "map_at_3": 0.12902, + "map_at_5": 0.15533, + "map_at_10": 0.21242, + "map_at_20": 0.24615, + "map_at_100": 0.31859, + "map_at_1000": 0.33847, + "recall_at_1": 0.07556, + "recall_at_3": 0.1629, + "recall_at_5": 0.20643, + "recall_at_10": 0.36504, + "recall_at_20": 0.46785, + "recall_at_100": 0.82141, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.43103, + "cv_recall_at_3": 0.75862, + "cv_recall_at_5": 0.7931, + "cv_recall_at_10": 0.91379, + "cv_recall_at_20": 0.93103, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.43103, + "precision_at_3": 0.41379, + "precision_at_5": 0.35862, + "precision_at_10": 0.32241, + "precision_at_20": 0.23362, + "precision_at_100": 0.11224, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.431034, + "mrr_at_3": 0.566092, + "mrr_at_5": 0.573851, + "mrr_at_10": 0.590681, + "mrr_at_20": 0.591639, + "mrr_at_100": 0.593723, + "mrr_at_1000": 0.593723, + "nauc_ndcg_at_1_max": -0.055356, + "nauc_ndcg_at_1_std": -0.263134, + "nauc_ndcg_at_1_diff1": 0.344669, + "nauc_ndcg_at_3_max": -0.01823, + "nauc_ndcg_at_3_std": -0.234061, + "nauc_ndcg_at_3_diff1": 0.299563, + "nauc_ndcg_at_5_max": -0.041739, + "nauc_ndcg_at_5_std": -0.260219, + "nauc_ndcg_at_5_diff1": 0.254757, + "nauc_ndcg_at_10_max": -0.014379, + "nauc_ndcg_at_10_std": -0.231489, + "nauc_ndcg_at_10_diff1": 0.217193, + "nauc_ndcg_at_20_max": 0.011958, + "nauc_ndcg_at_20_std": -0.171496, + "nauc_ndcg_at_20_diff1": 0.063953, + "nauc_ndcg_at_100_max": 0.11311, + "nauc_ndcg_at_100_std": -0.131308, + "nauc_ndcg_at_100_diff1": 0.17475, + "nauc_ndcg_at_1000_max": 0.066463, + "nauc_ndcg_at_1000_std": -0.121695, + "nauc_ndcg_at_1000_diff1": 0.216068, + "nauc_map_at_1_max": -0.329496, + "nauc_map_at_1_std": -0.383818, + "nauc_map_at_1_diff1": 0.314221, + "nauc_map_at_3_max": -0.200052, + "nauc_map_at_3_std": -0.32232, + "nauc_map_at_3_diff1": 0.177576, + "nauc_map_at_5_max": -0.15632, + "nauc_map_at_5_std": -0.314679, + "nauc_map_at_5_diff1": 0.112244, + "nauc_map_at_10_max": -0.08373, + "nauc_map_at_10_std": -0.27743, + "nauc_map_at_10_diff1": 0.043278, + "nauc_map_at_20_max": -0.023824, + "nauc_map_at_20_std": -0.214514, + "nauc_map_at_20_diff1": 0.028604, + "nauc_map_at_100_max": 0.038386, + "nauc_map_at_100_std": -0.148324, + "nauc_map_at_100_diff1": 0.119183, + "nauc_map_at_1000_max": 0.025941, + "nauc_map_at_1000_std": -0.143791, + "nauc_map_at_1000_diff1": 0.138768, + "nauc_recall_at_1_max": -0.329496, + "nauc_recall_at_1_std": -0.383818, + "nauc_recall_at_1_diff1": 0.314221, + "nauc_recall_at_3_max": -0.206463, + "nauc_recall_at_3_std": -0.320763, + "nauc_recall_at_3_diff1": 0.090857, + "nauc_recall_at_5_max": -0.167788, + "nauc_recall_at_5_std": -0.312434, + "nauc_recall_at_5_diff1": -0.008006, + "nauc_recall_at_10_max": -0.12367, + "nauc_recall_at_10_std": -0.289496, + "nauc_recall_at_10_diff1": -0.0544, + "nauc_recall_at_20_max": -0.053436, + "nauc_recall_at_20_std": -0.198236, + "nauc_recall_at_20_diff1": -0.109078, + "nauc_recall_at_100_max": 0.177936, + "nauc_recall_at_100_std": -0.076182, + "nauc_recall_at_100_diff1": 0.006618, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.055356, + "nauc_precision_at_1_std": -0.263134, + "nauc_precision_at_1_diff1": 0.344669, + "nauc_precision_at_3_max": 0.097673, + "nauc_precision_at_3_std": -0.120629, + "nauc_precision_at_3_diff1": 0.222187, + "nauc_precision_at_5_max": 0.102678, + "nauc_precision_at_5_std": -0.109837, + "nauc_precision_at_5_diff1": 0.132275, + "nauc_precision_at_10_max": 0.208851, + "nauc_precision_at_10_std": 0.002865, + "nauc_precision_at_10_diff1": 0.102963, + "nauc_precision_at_20_max": 0.215765, + "nauc_precision_at_20_std": 0.147256, + "nauc_precision_at_20_diff1": 0.073679, + "nauc_precision_at_100_max": 0.097174, + "nauc_precision_at_100_std": 0.126681, + "nauc_precision_at_100_diff1": 0.208353, + "nauc_precision_at_1000_max": 0.018343, + "nauc_precision_at_1000_std": 0.073069, + "nauc_precision_at_1000_diff1": 0.191562, + "nauc_cv_recall_at_1_max": -0.055356, + "nauc_cv_recall_at_1_std": -0.263134, + "nauc_cv_recall_at_1_diff1": 0.344669, + "nauc_cv_recall_at_3_max": 0.058024, + "nauc_cv_recall_at_3_std": -0.186098, + "nauc_cv_recall_at_3_diff1": 0.05829, + "nauc_cv_recall_at_5_max": -0.030794, + "nauc_cv_recall_at_5_std": -0.270459, + "nauc_cv_recall_at_5_diff1": -0.007983, + "nauc_cv_recall_at_10_max": -0.202055, + "nauc_cv_recall_at_10_std": -0.691601, + "nauc_cv_recall_at_10_diff1": 0.479177, + "nauc_cv_recall_at_20_max": -0.502569, + "nauc_cv_recall_at_20_std": -1.003005, + "nauc_cv_recall_at_20_diff1": 0.460468, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.055356, + "nauc_mrr_at_1_std": -0.263134, + "nauc_mrr_at_1_diff1": 0.344669, + "nauc_mrr_at_3_max": -0.0202, + "nauc_mrr_at_3_std": -0.225073, + "nauc_mrr_at_3_diff1": 0.241589, + "nauc_mrr_at_5_max": -0.03404, + "nauc_mrr_at_5_std": -0.237434, + "nauc_mrr_at_5_diff1": 0.237073, + "nauc_mrr_at_10_max": -0.03789, + "nauc_mrr_at_10_std": -0.250419, + "nauc_mrr_at_10_diff1": 0.273228, + "nauc_mrr_at_20_max": -0.041395, + "nauc_mrr_at_20_std": -0.253135, + "nauc_mrr_at_20_diff1": 0.27228, + "nauc_mrr_at_100_max": -0.039384, + "nauc_mrr_at_100_std": -0.248191, + "nauc_mrr_at_100_diff1": 0.270926, + "nauc_mrr_at_1000_max": -0.039384, + "nauc_mrr_at_1000_std": -0.248191, + "nauc_mrr_at_1000_diff1": 0.270926, + "main_score": 0.41537, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.15517, + "ndcg_at_3": 0.12474, + "ndcg_at_5": 0.13117, + "ndcg_at_10": 0.13904, + "ndcg_at_20": 0.17907, + "ndcg_at_100": 0.28104, + "ndcg_at_1000": 0.43423, + "map_at_1": 0.01564, + "map_at_3": 0.02638, + "map_at_5": 0.03568, + "map_at_10": 0.05315, + "map_at_20": 0.07285, + "map_at_100": 0.10101, + "map_at_1000": 0.13009, + "recall_at_1": 0.01564, + "recall_at_3": 0.03387, + "recall_at_5": 0.05636, + "recall_at_10": 0.12232, + "recall_at_20": 0.22096, + "recall_at_100": 0.49595, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.15517, + "cv_recall_at_3": 0.27586, + "cv_recall_at_5": 0.43103, + "cv_recall_at_10": 0.55172, + "cv_recall_at_20": 0.7069, + "cv_recall_at_100": 0.89655, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.15517, + "precision_at_3": 0.11494, + "precision_at_5": 0.12759, + "precision_at_10": 0.11552, + "precision_at_20": 0.11121, + "precision_at_100": 0.06431, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.155172, + "mrr_at_3": 0.20977, + "mrr_at_5": 0.244253, + "mrr_at_10": 0.260961, + "mrr_at_20": 0.272881, + "mrr_at_100": 0.278319, + "mrr_at_1000": 0.278967, + "nauc_ndcg_at_1_max": 0.339883, + "nauc_ndcg_at_1_std": 0.273853, + "nauc_ndcg_at_1_diff1": 0.163696, + "nauc_ndcg_at_3_max": 0.2758, + "nauc_ndcg_at_3_std": 0.171472, + "nauc_ndcg_at_3_diff1": 0.062936, + "nauc_ndcg_at_5_max": 0.214275, + "nauc_ndcg_at_5_std": 0.122269, + "nauc_ndcg_at_5_diff1": 0.009505, + "nauc_ndcg_at_10_max": 0.203746, + "nauc_ndcg_at_10_std": 0.154257, + "nauc_ndcg_at_10_diff1": 0.149684, + "nauc_ndcg_at_20_max": 0.263762, + "nauc_ndcg_at_20_std": 0.167564, + "nauc_ndcg_at_20_diff1": 0.21588, + "nauc_ndcg_at_100_max": 0.327088, + "nauc_ndcg_at_100_std": 0.277576, + "nauc_ndcg_at_100_diff1": 0.154927, + "nauc_ndcg_at_1000_max": 0.192421, + "nauc_ndcg_at_1000_std": 0.103719, + "nauc_ndcg_at_1000_diff1": 0.063556, + "nauc_map_at_1_max": 0.223943, + "nauc_map_at_1_std": 0.116487, + "nauc_map_at_1_diff1": 0.133757, + "nauc_map_at_3_max": 0.228874, + "nauc_map_at_3_std": 0.074433, + "nauc_map_at_3_diff1": 0.135271, + "nauc_map_at_5_max": 0.210508, + "nauc_map_at_5_std": 0.057236, + "nauc_map_at_5_diff1": 0.132352, + "nauc_map_at_10_max": 0.194856, + "nauc_map_at_10_std": 0.093673, + "nauc_map_at_10_diff1": 0.239682, + "nauc_map_at_20_max": 0.242628, + "nauc_map_at_20_std": 0.116619, + "nauc_map_at_20_diff1": 0.213802, + "nauc_map_at_100_max": 0.263376, + "nauc_map_at_100_std": 0.158048, + "nauc_map_at_100_diff1": 0.176939, + "nauc_map_at_1000_max": 0.216867, + "nauc_map_at_1000_std": 0.102065, + "nauc_map_at_1000_diff1": 0.131908, + "nauc_recall_at_1_max": 0.223943, + "nauc_recall_at_1_std": 0.116487, + "nauc_recall_at_1_diff1": 0.133757, + "nauc_recall_at_3_max": 0.21195, + "nauc_recall_at_3_std": 0.029998, + "nauc_recall_at_3_diff1": 0.088915, + "nauc_recall_at_5_max": 0.194777, + "nauc_recall_at_5_std": 0.043292, + "nauc_recall_at_5_diff1": 0.090166, + "nauc_recall_at_10_max": 0.184783, + "nauc_recall_at_10_std": 0.183853, + "nauc_recall_at_10_diff1": 0.334859, + "nauc_recall_at_20_max": 0.247775, + "nauc_recall_at_20_std": 0.202037, + "nauc_recall_at_20_diff1": 0.379129, + "nauc_recall_at_100_max": 0.341023, + "nauc_recall_at_100_std": 0.408215, + "nauc_recall_at_100_diff1": 0.211428, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.339883, + "nauc_precision_at_1_std": 0.273853, + "nauc_precision_at_1_diff1": 0.163696, + "nauc_precision_at_3_max": 0.264511, + "nauc_precision_at_3_std": 0.141097, + "nauc_precision_at_3_diff1": 0.032334, + "nauc_precision_at_5_max": 0.158731, + "nauc_precision_at_5_std": 0.06451, + "nauc_precision_at_5_diff1": -0.049443, + "nauc_precision_at_10_max": 0.130796, + "nauc_precision_at_10_std": 0.121358, + "nauc_precision_at_10_diff1": 0.117877, + "nauc_precision_at_20_max": 0.219317, + "nauc_precision_at_20_std": 0.124738, + "nauc_precision_at_20_diff1": 0.04642, + "nauc_precision_at_100_max": 0.078102, + "nauc_precision_at_100_std": 0.060306, + "nauc_precision_at_100_diff1": -0.120225, + "nauc_precision_at_1000_max": -0.086997, + "nauc_precision_at_1000_std": -0.119497, + "nauc_precision_at_1000_diff1": -0.132004, + "nauc_cv_recall_at_1_max": 0.339883, + "nauc_cv_recall_at_1_std": 0.273853, + "nauc_cv_recall_at_1_diff1": 0.163696, + "nauc_cv_recall_at_3_max": 0.18759, + "nauc_cv_recall_at_3_std": 0.034218, + "nauc_cv_recall_at_3_diff1": 0.007867, + "nauc_cv_recall_at_5_max": 0.070896, + "nauc_cv_recall_at_5_std": -0.005741, + "nauc_cv_recall_at_5_diff1": -0.03318, + "nauc_cv_recall_at_10_max": 0.117899, + "nauc_cv_recall_at_10_std": 0.102595, + "nauc_cv_recall_at_10_diff1": 0.276743, + "nauc_cv_recall_at_20_max": 0.272215, + "nauc_cv_recall_at_20_std": 0.212035, + "nauc_cv_recall_at_20_diff1": 0.39471, + "nauc_cv_recall_at_100_max": 0.482464, + "nauc_cv_recall_at_100_std": 0.611881, + "nauc_cv_recall_at_100_diff1": 0.206779, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.339883, + "nauc_mrr_at_1_std": 0.273853, + "nauc_mrr_at_1_diff1": 0.163696, + "nauc_mrr_at_3_max": 0.253297, + "nauc_mrr_at_3_std": 0.144987, + "nauc_mrr_at_3_diff1": 0.068571, + "nauc_mrr_at_5_max": 0.228744, + "nauc_mrr_at_5_std": 0.138716, + "nauc_mrr_at_5_diff1": 0.061718, + "nauc_mrr_at_10_max": 0.243776, + "nauc_mrr_at_10_std": 0.161705, + "nauc_mrr_at_10_diff1": 0.111934, + "nauc_mrr_at_20_max": 0.260678, + "nauc_mrr_at_20_std": 0.173937, + "nauc_mrr_at_20_diff1": 0.117176, + "nauc_mrr_at_100_max": 0.263857, + "nauc_mrr_at_100_std": 0.177565, + "nauc_mrr_at_100_diff1": 0.109772, + "nauc_mrr_at_1000_max": 0.263651, + "nauc_mrr_at_1000_std": 0.176671, + "nauc_mrr_at_1000_diff1": 0.109718, + "main_score": 0.13117, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 54.49580216407776, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/model_meta.json b/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/model_meta.json index 5d672a49bf..6df7903d19 100644 --- a/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/model_meta.json +++ b/results/google__siglip-so400m-patch14-384/9fdffc58afc957d1a03a25b10dba0329ab15c2a3/model_meta.json @@ -1 +1 @@ -{"name": "google/siglip-so400m-patch14-384", "revision": "9fdffc58afc957d1a03a25b10dba0329ab15c2a3", "release_date": "2024-01-08", "languages": ["eng_Latn"], "n_parameters": 878000000, "memory_usage_mb": 3349.0, "max_tokens": 64.0, "embed_dim": 1152, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py", "public_training_data": null, "framework": ["PyTorch"], "reference": "https://huggingface.co/google/siglip-so400m-patch14-384", "similarity_fn_name": null, "use_instructions": false, "training_datasets": {}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "SiglipModelWrapper"} \ No newline at end of file +{"name": "google/siglip-so400m-patch14-384", "revision": "9fdffc58afc957d1a03a25b10dba0329ab15c2a3", "release_date": "2024-01-08", "languages": ["eng-Latn"], "n_parameters": 878000000, "memory_usage_mb": 3349.0, "max_tokens": 64.0, "embed_dim": 1152, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py", "public_training_data": null, "framework": ["PyTorch"], "reference": "https://huggingface.co/google/siglip-so400m-patch14-384", "similarity_fn_name": null, "use_instructions": false, "training_datasets": {}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "SiglipModelWrapper"} \ No newline at end of file diff --git a/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Vidore2BioMedicalLecturesRetrieval.json b/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..75b59ca318 --- /dev/null +++ b/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.0875, + "ndcg_at_3": 0.10767, + "ndcg_at_5": 0.12621, + "ndcg_at_10": 0.1462, + "ndcg_at_20": 0.17638, + "ndcg_at_100": 0.23752, + "ndcg_at_1000": 0.31286, + "map_at_1": 0.04974, + "map_at_3": 0.07775, + "map_at_5": 0.0903, + "map_at_10": 0.10155, + "map_at_20": 0.11103, + "map_at_100": 0.12095, + "map_at_1000": 0.12542, + "recall_at_1": 0.04974, + "recall_at_3": 0.10933, + "recall_at_5": 0.1582, + "recall_at_10": 0.22498, + "recall_at_20": 0.31964, + "recall_at_100": 0.56969, + "recall_at_1000": 0.99978, + "cv_recall_at_1": 0.0875, + "cv_recall_at_3": 0.175, + "cv_recall_at_5": 0.2625, + "cv_recall_at_10": 0.34375, + "cv_recall_at_20": 0.48125, + "cv_recall_at_100": 0.76875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.0875, + "precision_at_3": 0.06667, + "precision_at_5": 0.0625, + "precision_at_10": 0.04438, + "precision_at_20": 0.03563, + "precision_at_100": 0.01519, + "precision_at_1000": 0.00321, + "mrr_at_1": 0.0875, + "mrr_at_3": 0.126042, + "mrr_at_5": 0.146042, + "mrr_at_10": 0.157262, + "mrr_at_20": 0.166462, + "mrr_at_100": 0.173167, + "mrr_at_1000": 0.174231, + "nauc_ndcg_at_1_max": 0.085342, + "nauc_ndcg_at_1_std": 0.059872, + "nauc_ndcg_at_1_diff1": 0.230768, + "nauc_ndcg_at_3_max": 0.242262, + "nauc_ndcg_at_3_std": 0.144322, + "nauc_ndcg_at_3_diff1": 0.172912, + "nauc_ndcg_at_5_max": 0.268728, + "nauc_ndcg_at_5_std": 0.210794, + "nauc_ndcg_at_5_diff1": 0.148957, + "nauc_ndcg_at_10_max": 0.288743, + "nauc_ndcg_at_10_std": 0.238986, + "nauc_ndcg_at_10_diff1": 0.137039, + "nauc_ndcg_at_20_max": 0.255332, + "nauc_ndcg_at_20_std": 0.222166, + "nauc_ndcg_at_20_diff1": 0.115298, + "nauc_ndcg_at_100_max": 0.252227, + "nauc_ndcg_at_100_std": 0.18709, + "nauc_ndcg_at_100_diff1": 0.113168, + "nauc_ndcg_at_1000_max": 0.202945, + "nauc_ndcg_at_1000_std": 0.175171, + "nauc_ndcg_at_1000_diff1": 0.091333, + "nauc_map_at_1_max": 0.040098, + "nauc_map_at_1_std": 0.076558, + "nauc_map_at_1_diff1": 0.29731, + "nauc_map_at_3_max": 0.196405, + "nauc_map_at_3_std": 0.118645, + "nauc_map_at_3_diff1": 0.225848, + "nauc_map_at_5_max": 0.229566, + "nauc_map_at_5_std": 0.181438, + "nauc_map_at_5_diff1": 0.193739, + "nauc_map_at_10_max": 0.245222, + "nauc_map_at_10_std": 0.205684, + "nauc_map_at_10_diff1": 0.179229, + "nauc_map_at_20_max": 0.230554, + "nauc_map_at_20_std": 0.198928, + "nauc_map_at_20_diff1": 0.16967, + "nauc_map_at_100_max": 0.224674, + "nauc_map_at_100_std": 0.186658, + "nauc_map_at_100_diff1": 0.167927, + "nauc_map_at_1000_max": 0.219951, + "nauc_map_at_1000_std": 0.186706, + "nauc_map_at_1000_diff1": 0.163847, + "nauc_recall_at_1_max": 0.040098, + "nauc_recall_at_1_std": 0.076558, + "nauc_recall_at_1_diff1": 0.29731, + "nauc_recall_at_3_max": 0.278838, + "nauc_recall_at_3_std": 0.1641, + "nauc_recall_at_3_diff1": 0.159744, + "nauc_recall_at_5_max": 0.337058, + "nauc_recall_at_5_std": 0.307836, + "nauc_recall_at_5_diff1": 0.11459, + "nauc_recall_at_10_max": 0.352294, + "nauc_recall_at_10_std": 0.324881, + "nauc_recall_at_10_diff1": 0.063773, + "nauc_recall_at_20_max": 0.280866, + "nauc_recall_at_20_std": 0.282544, + "nauc_recall_at_20_diff1": 0.027113, + "nauc_recall_at_100_max": 0.28245, + "nauc_recall_at_100_std": 0.198068, + "nauc_recall_at_100_diff1": 0.026228, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.122782, + "nauc_recall_at_1000_diff1": 0.722222, + "nauc_precision_at_1_max": 0.085342, + "nauc_precision_at_1_std": 0.059872, + "nauc_precision_at_1_diff1": 0.230768, + "nauc_precision_at_3_max": 0.324944, + "nauc_precision_at_3_std": 0.168814, + "nauc_precision_at_3_diff1": 0.117741, + "nauc_precision_at_5_max": 0.298135, + "nauc_precision_at_5_std": 0.217093, + "nauc_precision_at_5_diff1": 0.04401, + "nauc_precision_at_10_max": 0.254428, + "nauc_precision_at_10_std": 0.245337, + "nauc_precision_at_10_diff1": 0.033579, + "nauc_precision_at_20_max": 0.093259, + "nauc_precision_at_20_std": 0.135479, + "nauc_precision_at_20_diff1": -0.029525, + "nauc_precision_at_100_max": -0.032575, + "nauc_precision_at_100_std": -0.029385, + "nauc_precision_at_100_diff1": -0.082112, + "nauc_precision_at_1000_max": -0.17937, + "nauc_precision_at_1000_std": -0.047893, + "nauc_precision_at_1000_diff1": -0.202077, + "nauc_cv_recall_at_1_max": 0.085342, + "nauc_cv_recall_at_1_std": 0.059872, + "nauc_cv_recall_at_1_diff1": 0.230768, + "nauc_cv_recall_at_3_max": 0.3063, + "nauc_cv_recall_at_3_std": 0.191862, + "nauc_cv_recall_at_3_diff1": 0.117195, + "nauc_cv_recall_at_5_max": 0.312987, + "nauc_cv_recall_at_5_std": 0.216418, + "nauc_cv_recall_at_5_diff1": 0.067237, + "nauc_cv_recall_at_10_max": 0.344067, + "nauc_cv_recall_at_10_std": 0.243634, + "nauc_cv_recall_at_10_diff1": 0.060016, + "nauc_cv_recall_at_20_max": 0.245456, + "nauc_cv_recall_at_20_std": 0.236885, + "nauc_cv_recall_at_20_diff1": -0.014321, + "nauc_cv_recall_at_100_max": 0.332721, + "nauc_cv_recall_at_100_std": 0.249349, + "nauc_cv_recall_at_100_diff1": -0.129138, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.085342, + "nauc_mrr_at_1_std": 0.059872, + "nauc_mrr_at_1_diff1": 0.230768, + "nauc_mrr_at_3_max": 0.216401, + "nauc_mrr_at_3_std": 0.128922, + "nauc_mrr_at_3_diff1": 0.172449, + "nauc_mrr_at_5_max": 0.220777, + "nauc_mrr_at_5_std": 0.139253, + "nauc_mrr_at_5_diff1": 0.148378, + "nauc_mrr_at_10_max": 0.234651, + "nauc_mrr_at_10_std": 0.14973, + "nauc_mrr_at_10_diff1": 0.145473, + "nauc_mrr_at_20_max": 0.222803, + "nauc_mrr_at_20_std": 0.147759, + "nauc_mrr_at_20_diff1": 0.13612, + "nauc_mrr_at_100_max": 0.225472, + "nauc_mrr_at_100_std": 0.145836, + "nauc_mrr_at_100_diff1": 0.134827, + "nauc_mrr_at_1000_max": 0.224826, + "nauc_mrr_at_1000_std": 0.14534, + "nauc_mrr_at_1000_diff1": 0.135503, + "main_score": 0.12621, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.10625, + "ndcg_at_3": 0.12771, + "ndcg_at_5": 0.13519, + "ndcg_at_10": 0.15018, + "ndcg_at_20": 0.17304, + "ndcg_at_100": 0.23965, + "ndcg_at_1000": 0.31717, + "map_at_1": 0.06328, + "map_at_3": 0.09769, + "map_at_5": 0.10469, + "map_at_10": 0.11218, + "map_at_20": 0.11846, + "map_at_100": 0.12883, + "map_at_1000": 0.13336, + "recall_at_1": 0.06328, + "recall_at_3": 0.13125, + "recall_at_5": 0.15568, + "recall_at_10": 0.19849, + "recall_at_20": 0.27996, + "recall_at_100": 0.54875, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.10625, + "cv_recall_at_3": 0.21875, + "cv_recall_at_5": 0.26875, + "cv_recall_at_10": 0.3375, + "cv_recall_at_20": 0.45625, + "cv_recall_at_100": 0.75625, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.10625, + "precision_at_3": 0.07917, + "precision_at_5": 0.06125, + "precision_at_10": 0.04188, + "precision_at_20": 0.03, + "precision_at_100": 0.01531, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.10625, + "mrr_at_3": 0.155208, + "mrr_at_5": 0.166458, + "mrr_at_10": 0.176186, + "mrr_at_20": 0.184354, + "mrr_at_100": 0.190922, + "mrr_at_1000": 0.192104, + "nauc_ndcg_at_1_max": 0.131484, + "nauc_ndcg_at_1_std": -0.005829, + "nauc_ndcg_at_1_diff1": 0.161935, + "nauc_ndcg_at_3_max": 0.148239, + "nauc_ndcg_at_3_std": 0.063053, + "nauc_ndcg_at_3_diff1": 0.03203, + "nauc_ndcg_at_5_max": 0.147854, + "nauc_ndcg_at_5_std": 0.049167, + "nauc_ndcg_at_5_diff1": 0.047361, + "nauc_ndcg_at_10_max": 0.13977, + "nauc_ndcg_at_10_std": 0.053232, + "nauc_ndcg_at_10_diff1": 0.067054, + "nauc_ndcg_at_20_max": 0.138966, + "nauc_ndcg_at_20_std": 0.030691, + "nauc_ndcg_at_20_diff1": 0.035435, + "nauc_ndcg_at_100_max": 0.120724, + "nauc_ndcg_at_100_std": 0.042771, + "nauc_ndcg_at_100_diff1": 0.065613, + "nauc_ndcg_at_1000_max": 0.09865, + "nauc_ndcg_at_1000_std": 0.027287, + "nauc_ndcg_at_1000_diff1": 0.045332, + "nauc_map_at_1_max": 0.15793, + "nauc_map_at_1_std": 0.021186, + "nauc_map_at_1_diff1": 0.140062, + "nauc_map_at_3_max": 0.178661, + "nauc_map_at_3_std": 0.082797, + "nauc_map_at_3_diff1": 0.0222, + "nauc_map_at_5_max": 0.164069, + "nauc_map_at_5_std": 0.077561, + "nauc_map_at_5_diff1": 0.027086, + "nauc_map_at_10_max": 0.16362, + "nauc_map_at_10_std": 0.08132, + "nauc_map_at_10_diff1": 0.042171, + "nauc_map_at_20_max": 0.163508, + "nauc_map_at_20_std": 0.074297, + "nauc_map_at_20_diff1": 0.031257, + "nauc_map_at_100_max": 0.156606, + "nauc_map_at_100_std": 0.075033, + "nauc_map_at_100_diff1": 0.036784, + "nauc_map_at_1000_max": 0.153282, + "nauc_map_at_1000_std": 0.07295, + "nauc_map_at_1000_diff1": 0.036179, + "nauc_recall_at_1_max": 0.15793, + "nauc_recall_at_1_std": 0.021186, + "nauc_recall_at_1_diff1": 0.140062, + "nauc_recall_at_3_max": 0.172606, + "nauc_recall_at_3_std": 0.086325, + "nauc_recall_at_3_diff1": -0.022352, + "nauc_recall_at_5_max": 0.149185, + "nauc_recall_at_5_std": 0.042, + "nauc_recall_at_5_diff1": 0.020783, + "nauc_recall_at_10_max": 0.142028, + "nauc_recall_at_10_std": 0.05599, + "nauc_recall_at_10_diff1": 0.092512, + "nauc_recall_at_20_max": 0.116681, + "nauc_recall_at_20_std": -0.017636, + "nauc_recall_at_20_diff1": 0.014179, + "nauc_recall_at_100_max": 0.108978, + "nauc_recall_at_100_std": 0.036481, + "nauc_recall_at_100_diff1": 0.111113, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.131484, + "nauc_precision_at_1_std": -0.005829, + "nauc_precision_at_1_diff1": 0.161935, + "nauc_precision_at_3_max": 0.108028, + "nauc_precision_at_3_std": 0.088318, + "nauc_precision_at_3_diff1": -0.052166, + "nauc_precision_at_5_max": 0.123212, + "nauc_precision_at_5_std": 0.044121, + "nauc_precision_at_5_diff1": 0.0213, + "nauc_precision_at_10_max": 0.067197, + "nauc_precision_at_10_std": 0.016124, + "nauc_precision_at_10_diff1": 0.052955, + "nauc_precision_at_20_max": 0.052186, + "nauc_precision_at_20_std": -0.035499, + "nauc_precision_at_20_diff1": 0.00601, + "nauc_precision_at_100_max": -0.076657, + "nauc_precision_at_100_std": -0.014163, + "nauc_precision_at_100_diff1": 0.051768, + "nauc_precision_at_1000_max": -0.151359, + "nauc_precision_at_1000_std": -0.092337, + "nauc_precision_at_1000_diff1": -0.044923, + "nauc_cv_recall_at_1_max": 0.131484, + "nauc_cv_recall_at_1_std": -0.005829, + "nauc_cv_recall_at_1_diff1": 0.161935, + "nauc_cv_recall_at_3_max": 0.08331, + "nauc_cv_recall_at_3_std": 0.040195, + "nauc_cv_recall_at_3_diff1": -0.004585, + "nauc_cv_recall_at_5_max": 0.132247, + "nauc_cv_recall_at_5_std": -0.005908, + "nauc_cv_recall_at_5_diff1": 0.049377, + "nauc_cv_recall_at_10_max": 0.106883, + "nauc_cv_recall_at_10_std": 0.001227, + "nauc_cv_recall_at_10_diff1": 0.103044, + "nauc_cv_recall_at_20_max": 0.046776, + "nauc_cv_recall_at_20_std": -0.095669, + "nauc_cv_recall_at_20_diff1": 0.044111, + "nauc_cv_recall_at_100_max": 0.027893, + "nauc_cv_recall_at_100_std": -0.068872, + "nauc_cv_recall_at_100_diff1": 0.117621, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.131484, + "nauc_mrr_at_1_std": -0.005829, + "nauc_mrr_at_1_diff1": 0.161935, + "nauc_mrr_at_3_max": 0.110006, + "nauc_mrr_at_3_std": 0.029431, + "nauc_mrr_at_3_diff1": 0.07224, + "nauc_mrr_at_5_max": 0.122216, + "nauc_mrr_at_5_std": 0.014374, + "nauc_mrr_at_5_diff1": 0.082365, + "nauc_mrr_at_10_max": 0.114933, + "nauc_mrr_at_10_std": 0.016621, + "nauc_mrr_at_10_diff1": 0.093751, + "nauc_mrr_at_20_max": 0.108554, + "nauc_mrr_at_20_std": 0.007355, + "nauc_mrr_at_20_diff1": 0.087661, + "nauc_mrr_at_100_max": 0.108425, + "nauc_mrr_at_100_std": 0.008671, + "nauc_mrr_at_100_diff1": 0.089209, + "nauc_mrr_at_1000_max": 0.108534, + "nauc_mrr_at_1000_std": 0.00892, + "nauc_mrr_at_1000_diff1": 0.088993, + "main_score": 0.13519, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.21875, + "ndcg_at_3": 0.22679, + "ndcg_at_5": 0.23464, + "ndcg_at_10": 0.2605, + "ndcg_at_20": 0.2948, + "ndcg_at_100": 0.3554, + "ndcg_at_1000": 0.41389, + "map_at_1": 0.13693, + "map_at_3": 0.17762, + "map_at_5": 0.19041, + "map_at_10": 0.2071, + "map_at_20": 0.21859, + "map_at_100": 0.23138, + "map_at_1000": 0.23516, + "recall_at_1": 0.13693, + "recall_at_3": 0.21559, + "recall_at_5": 0.25127, + "recall_at_10": 0.32389, + "recall_at_20": 0.43167, + "recall_at_100": 0.66727, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.21875, + "cv_recall_at_3": 0.325, + "cv_recall_at_5": 0.3875, + "cv_recall_at_10": 0.49375, + "cv_recall_at_20": 0.6125, + "cv_recall_at_100": 0.80625, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.21875, + "precision_at_3": 0.13542, + "precision_at_5": 0.105, + "precision_at_10": 0.07812, + "precision_at_20": 0.055, + "precision_at_100": 0.01906, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.21875, + "mrr_at_3": 0.263542, + "mrr_at_5": 0.277917, + "mrr_at_10": 0.291677, + "mrr_at_20": 0.300492, + "mrr_at_100": 0.306149, + "mrr_at_1000": 0.306944, + "nauc_ndcg_at_1_max": 0.192611, + "nauc_ndcg_at_1_std": -0.057445, + "nauc_ndcg_at_1_diff1": 0.284994, + "nauc_ndcg_at_3_max": 0.200198, + "nauc_ndcg_at_3_std": -0.051499, + "nauc_ndcg_at_3_diff1": 0.279312, + "nauc_ndcg_at_5_max": 0.17571, + "nauc_ndcg_at_5_std": -0.061206, + "nauc_ndcg_at_5_diff1": 0.261356, + "nauc_ndcg_at_10_max": 0.200843, + "nauc_ndcg_at_10_std": -0.033992, + "nauc_ndcg_at_10_diff1": 0.258111, + "nauc_ndcg_at_20_max": 0.196664, + "nauc_ndcg_at_20_std": -0.002231, + "nauc_ndcg_at_20_diff1": 0.230037, + "nauc_ndcg_at_100_max": 0.180777, + "nauc_ndcg_at_100_std": 0.004287, + "nauc_ndcg_at_100_diff1": 0.239234, + "nauc_ndcg_at_1000_max": 0.201337, + "nauc_ndcg_at_1000_std": -0.012623, + "nauc_ndcg_at_1000_diff1": 0.222677, + "nauc_map_at_1_max": 0.156431, + "nauc_map_at_1_std": -0.125347, + "nauc_map_at_1_diff1": 0.373939, + "nauc_map_at_3_max": 0.180928, + "nauc_map_at_3_std": -0.072561, + "nauc_map_at_3_diff1": 0.325406, + "nauc_map_at_5_max": 0.173299, + "nauc_map_at_5_std": -0.059805, + "nauc_map_at_5_diff1": 0.302702, + "nauc_map_at_10_max": 0.179003, + "nauc_map_at_10_std": -0.043473, + "nauc_map_at_10_diff1": 0.289618, + "nauc_map_at_20_max": 0.178971, + "nauc_map_at_20_std": -0.032228, + "nauc_map_at_20_diff1": 0.276723, + "nauc_map_at_100_max": 0.176005, + "nauc_map_at_100_std": -0.028247, + "nauc_map_at_100_diff1": 0.273818, + "nauc_map_at_1000_max": 0.178557, + "nauc_map_at_1000_std": -0.028847, + "nauc_map_at_1000_diff1": 0.271296, + "nauc_recall_at_1_max": 0.156431, + "nauc_recall_at_1_std": -0.125347, + "nauc_recall_at_1_diff1": 0.373939, + "nauc_recall_at_3_max": 0.189521, + "nauc_recall_at_3_std": -0.040627, + "nauc_recall_at_3_diff1": 0.299795, + "nauc_recall_at_5_max": 0.146723, + "nauc_recall_at_5_std": -0.045948, + "nauc_recall_at_5_diff1": 0.243874, + "nauc_recall_at_10_max": 0.219284, + "nauc_recall_at_10_std": 0.02171, + "nauc_recall_at_10_diff1": 0.224692, + "nauc_recall_at_20_max": 0.221259, + "nauc_recall_at_20_std": 0.108845, + "nauc_recall_at_20_diff1": 0.136031, + "nauc_recall_at_100_max": 0.113588, + "nauc_recall_at_100_std": 0.132983, + "nauc_recall_at_100_diff1": 0.217151, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.192611, + "nauc_precision_at_1_std": -0.057445, + "nauc_precision_at_1_diff1": 0.284994, + "nauc_precision_at_3_max": 0.207087, + "nauc_precision_at_3_std": 0.007461, + "nauc_precision_at_3_diff1": 0.143535, + "nauc_precision_at_5_max": 0.153964, + "nauc_precision_at_5_std": 0.01065, + "nauc_precision_at_5_diff1": 0.057802, + "nauc_precision_at_10_max": 0.174725, + "nauc_precision_at_10_std": 0.046327, + "nauc_precision_at_10_diff1": -0.011376, + "nauc_precision_at_20_max": 0.124038, + "nauc_precision_at_20_std": 0.096869, + "nauc_precision_at_20_diff1": -0.124277, + "nauc_precision_at_100_max": 0.12765, + "nauc_precision_at_100_std": 0.094118, + "nauc_precision_at_100_diff1": -0.167923, + "nauc_precision_at_1000_max": 0.181221, + "nauc_precision_at_1000_std": 0.073343, + "nauc_precision_at_1000_diff1": -0.217693, + "nauc_cv_recall_at_1_max": 0.192611, + "nauc_cv_recall_at_1_std": -0.057445, + "nauc_cv_recall_at_1_diff1": 0.284994, + "nauc_cv_recall_at_3_max": 0.229013, + "nauc_cv_recall_at_3_std": -0.051783, + "nauc_cv_recall_at_3_diff1": 0.25644, + "nauc_cv_recall_at_5_max": 0.15768, + "nauc_cv_recall_at_5_std": -0.092971, + "nauc_cv_recall_at_5_diff1": 0.173196, + "nauc_cv_recall_at_10_max": 0.257248, + "nauc_cv_recall_at_10_std": 0.053314, + "nauc_cv_recall_at_10_diff1": 0.153943, + "nauc_cv_recall_at_20_max": 0.2534, + "nauc_cv_recall_at_20_std": 0.115257, + "nauc_cv_recall_at_20_diff1": 0.095652, + "nauc_cv_recall_at_100_max": 0.316324, + "nauc_cv_recall_at_100_std": 0.20091, + "nauc_cv_recall_at_100_diff1": 0.165461, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.192611, + "nauc_mrr_at_1_std": -0.057445, + "nauc_mrr_at_1_diff1": 0.284994, + "nauc_mrr_at_3_max": 0.208293, + "nauc_mrr_at_3_std": -0.058549, + "nauc_mrr_at_3_diff1": 0.272726, + "nauc_mrr_at_5_max": 0.191324, + "nauc_mrr_at_5_std": -0.069151, + "nauc_mrr_at_5_diff1": 0.252083, + "nauc_mrr_at_10_max": 0.202992, + "nauc_mrr_at_10_std": -0.049825, + "nauc_mrr_at_10_diff1": 0.247475, + "nauc_mrr_at_20_max": 0.202316, + "nauc_mrr_at_20_std": -0.045776, + "nauc_mrr_at_20_diff1": 0.245692, + "nauc_mrr_at_100_max": 0.200722, + "nauc_mrr_at_100_std": -0.045204, + "nauc_mrr_at_100_diff1": 0.249097, + "nauc_mrr_at_1000_max": 0.200627, + "nauc_mrr_at_1000_std": -0.045792, + "nauc_mrr_at_1000_diff1": 0.249117, + "main_score": 0.23464, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.0625, + "ndcg_at_3": 0.08524, + "ndcg_at_5": 0.08699, + "ndcg_at_10": 0.10073, + "ndcg_at_20": 0.11859, + "ndcg_at_100": 0.16084, + "ndcg_at_1000": 0.26358, + "map_at_1": 0.03349, + "map_at_3": 0.05757, + "map_at_5": 0.06121, + "map_at_10": 0.06912, + "map_at_20": 0.0744, + "map_at_100": 0.08072, + "map_at_1000": 0.08573, + "recall_at_1": 0.03349, + "recall_at_3": 0.08271, + "recall_at_5": 0.09537, + "recall_at_10": 0.14428, + "recall_at_20": 0.20418, + "recall_at_100": 0.37304, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.0625, + "cv_recall_at_3": 0.1625, + "cv_recall_at_5": 0.19375, + "cv_recall_at_10": 0.25625, + "cv_recall_at_20": 0.34375, + "cv_recall_at_100": 0.55, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.0625, + "precision_at_3": 0.0625, + "precision_at_5": 0.0475, + "precision_at_10": 0.0325, + "precision_at_20": 0.02406, + "precision_at_100": 0.01087, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.0625, + "mrr_at_3": 0.104167, + "mrr_at_5": 0.111354, + "mrr_at_10": 0.119841, + "mrr_at_20": 0.126255, + "mrr_at_100": 0.130972, + "mrr_at_1000": 0.132806, + "nauc_ndcg_at_1_max": 0.071747, + "nauc_ndcg_at_1_std": 0.225087, + "nauc_ndcg_at_1_diff1": -0.060504, + "nauc_ndcg_at_3_max": 0.108823, + "nauc_ndcg_at_3_std": 0.110847, + "nauc_ndcg_at_3_diff1": -0.005824, + "nauc_ndcg_at_5_max": 0.15244, + "nauc_ndcg_at_5_std": 0.131839, + "nauc_ndcg_at_5_diff1": 0.00494, + "nauc_ndcg_at_10_max": 0.141391, + "nauc_ndcg_at_10_std": 0.108548, + "nauc_ndcg_at_10_diff1": 0.002882, + "nauc_ndcg_at_20_max": 0.133806, + "nauc_ndcg_at_20_std": 0.105219, + "nauc_ndcg_at_20_diff1": 0.001957, + "nauc_ndcg_at_100_max": 0.152195, + "nauc_ndcg_at_100_std": 0.111043, + "nauc_ndcg_at_100_diff1": -0.009277, + "nauc_ndcg_at_1000_max": 0.097374, + "nauc_ndcg_at_1000_std": 0.092397, + "nauc_ndcg_at_1000_diff1": -0.018187, + "nauc_map_at_1_max": 0.166999, + "nauc_map_at_1_std": 0.125388, + "nauc_map_at_1_diff1": 0.040024, + "nauc_map_at_3_max": 0.215918, + "nauc_map_at_3_std": 0.159614, + "nauc_map_at_3_diff1": 0.019718, + "nauc_map_at_5_max": 0.227756, + "nauc_map_at_5_std": 0.161237, + "nauc_map_at_5_diff1": 0.025113, + "nauc_map_at_10_max": 0.208099, + "nauc_map_at_10_std": 0.142984, + "nauc_map_at_10_diff1": 0.01504, + "nauc_map_at_20_max": 0.201005, + "nauc_map_at_20_std": 0.13823, + "nauc_map_at_20_diff1": 0.013022, + "nauc_map_at_100_max": 0.196275, + "nauc_map_at_100_std": 0.13487, + "nauc_map_at_100_diff1": 0.00532, + "nauc_map_at_1000_max": 0.189996, + "nauc_map_at_1000_std": 0.133736, + "nauc_map_at_1000_diff1": 0.002265, + "nauc_recall_at_1_max": 0.166999, + "nauc_recall_at_1_std": 0.125388, + "nauc_recall_at_1_diff1": 0.040024, + "nauc_recall_at_3_max": 0.167029, + "nauc_recall_at_3_std": 0.135886, + "nauc_recall_at_3_diff1": 0.029246, + "nauc_recall_at_5_max": 0.202394, + "nauc_recall_at_5_std": 0.144744, + "nauc_recall_at_5_diff1": 0.037618, + "nauc_recall_at_10_max": 0.152694, + "nauc_recall_at_10_std": 0.091553, + "nauc_recall_at_10_diff1": 0.014098, + "nauc_recall_at_20_max": 0.123301, + "nauc_recall_at_20_std": 0.082441, + "nauc_recall_at_20_diff1": 0.018692, + "nauc_recall_at_100_max": 0.189619, + "nauc_recall_at_100_std": 0.112195, + "nauc_recall_at_100_diff1": 0.011297, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.071747, + "nauc_precision_at_1_std": 0.225087, + "nauc_precision_at_1_diff1": -0.060504, + "nauc_precision_at_3_max": 0.06201, + "nauc_precision_at_3_std": 0.101139, + "nauc_precision_at_3_diff1": -0.063693, + "nauc_precision_at_5_max": 0.096931, + "nauc_precision_at_5_std": 0.111134, + "nauc_precision_at_5_diff1": -0.046388, + "nauc_precision_at_10_max": 0.0239, + "nauc_precision_at_10_std": 0.063552, + "nauc_precision_at_10_diff1": -0.052072, + "nauc_precision_at_20_max": -0.024525, + "nauc_precision_at_20_std": 0.033319, + "nauc_precision_at_20_diff1": -0.055355, + "nauc_precision_at_100_max": -0.015664, + "nauc_precision_at_100_std": 0.03582, + "nauc_precision_at_100_diff1": -0.05651, + "nauc_precision_at_1000_max": -0.163697, + "nauc_precision_at_1000_std": -0.012751, + "nauc_precision_at_1000_diff1": 0.031881, + "nauc_cv_recall_at_1_max": 0.071747, + "nauc_cv_recall_at_1_std": 0.225087, + "nauc_cv_recall_at_1_diff1": -0.060504, + "nauc_cv_recall_at_3_max": -0.001447, + "nauc_cv_recall_at_3_std": 0.042822, + "nauc_cv_recall_at_3_diff1": -0.026037, + "nauc_cv_recall_at_5_max": 0.061157, + "nauc_cv_recall_at_5_std": 0.086737, + "nauc_cv_recall_at_5_diff1": -0.021545, + "nauc_cv_recall_at_10_max": 0.021606, + "nauc_cv_recall_at_10_std": 0.050533, + "nauc_cv_recall_at_10_diff1": -0.016196, + "nauc_cv_recall_at_20_max": -0.012225, + "nauc_cv_recall_at_20_std": 0.043369, + "nauc_cv_recall_at_20_diff1": -0.024185, + "nauc_cv_recall_at_100_max": 0.146905, + "nauc_cv_recall_at_100_std": 0.071167, + "nauc_cv_recall_at_100_diff1": 0.033938, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.071747, + "nauc_mrr_at_1_std": 0.225087, + "nauc_mrr_at_1_diff1": -0.060504, + "nauc_mrr_at_3_max": 0.038365, + "nauc_mrr_at_3_std": 0.10416, + "nauc_mrr_at_3_diff1": -0.031335, + "nauc_mrr_at_5_max": 0.060146, + "nauc_mrr_at_5_std": 0.11749, + "nauc_mrr_at_5_diff1": -0.030802, + "nauc_mrr_at_10_max": 0.051588, + "nauc_mrr_at_10_std": 0.10617, + "nauc_mrr_at_10_diff1": -0.032332, + "nauc_mrr_at_20_max": 0.048647, + "nauc_mrr_at_20_std": 0.104859, + "nauc_mrr_at_20_diff1": -0.035611, + "nauc_mrr_at_100_max": 0.05382, + "nauc_mrr_at_100_std": 0.104898, + "nauc_mrr_at_100_diff1": -0.035232, + "nauc_mrr_at_1000_max": 0.052118, + "nauc_mrr_at_1000_std": 0.104634, + "nauc_mrr_at_1000_diff1": -0.036808, + "main_score": 0.08699, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 53.58569622039795, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Vidore2ESGReportsHLRetrieval.json b/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..b5ffeb929a --- /dev/null +++ b/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.09615, + "ndcg_at_3": 0.09341, + "ndcg_at_5": 0.09313, + "ndcg_at_10": 0.10771, + "ndcg_at_20": 0.12764, + "ndcg_at_100": 0.16199, + "ndcg_at_1000": 0.24794, + "map_at_1": 0.05944, + "map_at_3": 0.07241, + "map_at_5": 0.07433, + "map_at_10": 0.08056, + "map_at_20": 0.08555, + "map_at_100": 0.09018, + "map_at_1000": 0.09441, + "recall_at_1": 0.05944, + "recall_at_3": 0.08523, + "recall_at_5": 0.09484, + "recall_at_10": 0.14612, + "recall_at_20": 0.21407, + "recall_at_100": 0.36854, + "recall_at_1000": 0.92674, + "cv_recall_at_1": 0.09615, + "cv_recall_at_3": 0.13462, + "cv_recall_at_5": 0.15385, + "cv_recall_at_10": 0.23077, + "cv_recall_at_20": 0.32692, + "cv_recall_at_100": 0.53846, + "cv_recall_at_1000": 0.98077, + "precision_at_1": 0.09615, + "precision_at_3": 0.05128, + "precision_at_5": 0.03462, + "precision_at_10": 0.02692, + "precision_at_20": 0.01923, + "precision_at_100": 0.00788, + "precision_at_1000": 0.00227, + "mrr_at_1": 0.096154, + "mrr_at_3": 0.112179, + "mrr_at_5": 0.116026, + "mrr_at_10": 0.125694, + "mrr_at_20": 0.131951, + "mrr_at_100": 0.136792, + "mrr_at_1000": 0.138688, + "nauc_ndcg_at_1_max": 0.068391, + "nauc_ndcg_at_1_std": 0.270469, + "nauc_ndcg_at_1_diff1": 0.053567, + "nauc_ndcg_at_3_max": 0.169196, + "nauc_ndcg_at_3_std": 0.354532, + "nauc_ndcg_at_3_diff1": 0.194809, + "nauc_ndcg_at_5_max": 0.158772, + "nauc_ndcg_at_5_std": 0.388668, + "nauc_ndcg_at_5_diff1": 0.234227, + "nauc_ndcg_at_10_max": 0.14183, + "nauc_ndcg_at_10_std": 0.300663, + "nauc_ndcg_at_10_diff1": 0.164682, + "nauc_ndcg_at_20_max": 0.06594, + "nauc_ndcg_at_20_std": 0.233374, + "nauc_ndcg_at_20_diff1": 0.183383, + "nauc_ndcg_at_100_max": 0.141163, + "nauc_ndcg_at_100_std": 0.263057, + "nauc_ndcg_at_100_diff1": 0.187743, + "nauc_ndcg_at_1000_max": 0.132954, + "nauc_ndcg_at_1000_std": 0.253404, + "nauc_ndcg_at_1000_diff1": 0.207992, + "nauc_map_at_1_max": -0.08195, + "nauc_map_at_1_std": 0.216827, + "nauc_map_at_1_diff1": 0.148218, + "nauc_map_at_3_max": 0.078811, + "nauc_map_at_3_std": 0.302009, + "nauc_map_at_3_diff1": 0.255731, + "nauc_map_at_5_max": 0.085745, + "nauc_map_at_5_std": 0.323458, + "nauc_map_at_5_diff1": 0.259563, + "nauc_map_at_10_max": 0.092306, + "nauc_map_at_10_std": 0.289752, + "nauc_map_at_10_diff1": 0.216107, + "nauc_map_at_20_max": 0.070849, + "nauc_map_at_20_std": 0.271306, + "nauc_map_at_20_diff1": 0.220992, + "nauc_map_at_100_max": 0.089403, + "nauc_map_at_100_std": 0.280284, + "nauc_map_at_100_diff1": 0.222012, + "nauc_map_at_1000_max": 0.091614, + "nauc_map_at_1000_std": 0.279442, + "nauc_map_at_1000_diff1": 0.221861, + "nauc_recall_at_1_max": -0.08195, + "nauc_recall_at_1_std": 0.216827, + "nauc_recall_at_1_diff1": 0.148218, + "nauc_recall_at_3_max": 0.176519, + "nauc_recall_at_3_std": 0.374481, + "nauc_recall_at_3_diff1": 0.348193, + "nauc_recall_at_5_max": 0.197688, + "nauc_recall_at_5_std": 0.456305, + "nauc_recall_at_5_diff1": 0.35716, + "nauc_recall_at_10_max": 0.190621, + "nauc_recall_at_10_std": 0.221479, + "nauc_recall_at_10_diff1": 0.082791, + "nauc_recall_at_20_max": 0.017615, + "nauc_recall_at_20_std": 0.094516, + "nauc_recall_at_20_diff1": 0.144892, + "nauc_recall_at_100_max": 0.177705, + "nauc_recall_at_100_std": 0.150132, + "nauc_recall_at_100_diff1": 0.109319, + "nauc_recall_at_1000_max": 0.270379, + "nauc_recall_at_1000_std": -0.467315, + "nauc_recall_at_1000_diff1": 0.163961, + "nauc_precision_at_1_max": 0.068391, + "nauc_precision_at_1_std": 0.270469, + "nauc_precision_at_1_diff1": 0.053567, + "nauc_precision_at_3_max": 0.245716, + "nauc_precision_at_3_std": 0.45394, + "nauc_precision_at_3_diff1": 0.148937, + "nauc_precision_at_5_max": 0.264528, + "nauc_precision_at_5_std": 0.542118, + "nauc_precision_at_5_diff1": 0.175712, + "nauc_precision_at_10_max": 0.16857, + "nauc_precision_at_10_std": 0.258641, + "nauc_precision_at_10_diff1": 0.070422, + "nauc_precision_at_20_max": -0.015508, + "nauc_precision_at_20_std": 0.07827, + "nauc_precision_at_20_diff1": 0.113057, + "nauc_precision_at_100_max": 0.271354, + "nauc_precision_at_100_std": 0.136791, + "nauc_precision_at_100_diff1": 0.152404, + "nauc_precision_at_1000_max": 0.10554, + "nauc_precision_at_1000_std": 0.007127, + "nauc_precision_at_1000_diff1": 0.113334, + "nauc_cv_recall_at_1_max": 0.068391, + "nauc_cv_recall_at_1_std": 0.270469, + "nauc_cv_recall_at_1_diff1": 0.053567, + "nauc_cv_recall_at_3_max": 0.18079, + "nauc_cv_recall_at_3_std": 0.476603, + "nauc_cv_recall_at_3_diff1": 0.250162, + "nauc_cv_recall_at_5_max": 0.208411, + "nauc_cv_recall_at_5_std": 0.575307, + "nauc_cv_recall_at_5_diff1": 0.271256, + "nauc_cv_recall_at_10_max": 0.155388, + "nauc_cv_recall_at_10_std": 0.316906, + "nauc_cv_recall_at_10_diff1": 0.106359, + "nauc_cv_recall_at_20_max": -0.044807, + "nauc_cv_recall_at_20_std": 0.131812, + "nauc_cv_recall_at_20_diff1": 0.133297, + "nauc_cv_recall_at_100_max": 0.229346, + "nauc_cv_recall_at_100_std": 0.22487, + "nauc_cv_recall_at_100_diff1": 0.253681, + "nauc_cv_recall_at_1000_max": 0.356901, + "nauc_cv_recall_at_1000_std": -1.163827, + "nauc_cv_recall_at_1000_diff1": 0.724179, + "nauc_mrr_at_1_max": 0.068391, + "nauc_mrr_at_1_std": 0.270469, + "nauc_mrr_at_1_diff1": 0.053567, + "nauc_mrr_at_3_max": 0.139727, + "nauc_mrr_at_3_std": 0.363023, + "nauc_mrr_at_3_diff1": 0.15713, + "nauc_mrr_at_5_max": 0.147454, + "nauc_mrr_at_5_std": 0.389869, + "nauc_mrr_at_5_diff1": 0.164454, + "nauc_mrr_at_10_max": 0.139542, + "nauc_mrr_at_10_std": 0.349115, + "nauc_mrr_at_10_diff1": 0.141646, + "nauc_mrr_at_20_max": 0.118426, + "nauc_mrr_at_20_std": 0.329213, + "nauc_mrr_at_20_diff1": 0.143083, + "nauc_mrr_at_100_max": 0.133248, + "nauc_mrr_at_100_std": 0.337362, + "nauc_mrr_at_100_diff1": 0.148433, + "nauc_mrr_at_1000_max": 0.133019, + "nauc_mrr_at_1000_std": 0.337413, + "nauc_mrr_at_1000_diff1": 0.147827, + "main_score": 0.09313, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 96.24464321136475, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Vidore2ESGReportsRetrieval.json b/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..3d3d7a93e1 --- /dev/null +++ b/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.05263, + "ndcg_at_3": 0.03577, + "ndcg_at_5": 0.03876, + "ndcg_at_10": 0.05298, + "ndcg_at_20": 0.06632, + "ndcg_at_100": 0.11427, + "ndcg_at_1000": 0.20787, + "map_at_1": 0.01023, + "map_at_3": 0.01901, + "map_at_5": 0.02339, + "map_at_10": 0.02918, + "map_at_20": 0.0327, + "map_at_100": 0.03856, + "map_at_1000": 0.04264, + "recall_at_1": 0.01023, + "recall_at_3": 0.02778, + "recall_at_5": 0.04532, + "recall_at_10": 0.09357, + "recall_at_20": 0.13377, + "recall_at_100": 0.32669, + "recall_at_1000": 0.9052, + "cv_recall_at_1": 0.05263, + "cv_recall_at_3": 0.07018, + "cv_recall_at_5": 0.08772, + "cv_recall_at_10": 0.17544, + "cv_recall_at_20": 0.24561, + "cv_recall_at_100": 0.59649, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.05263, + "precision_at_3": 0.02339, + "precision_at_5": 0.01754, + "precision_at_10": 0.01754, + "precision_at_20": 0.01404, + "precision_at_100": 0.00965, + "precision_at_1000": 0.00323, + "mrr_at_1": 0.052632, + "mrr_at_3": 0.061404, + "mrr_at_5": 0.065789, + "mrr_at_10": 0.07658, + "mrr_at_20": 0.081768, + "mrr_at_100": 0.089671, + "mrr_at_1000": 0.090984, + "nauc_ndcg_at_1_max": 0.666779, + "nauc_ndcg_at_1_std": -0.081452, + "nauc_ndcg_at_1_diff1": 0.247659, + "nauc_ndcg_at_3_max": 0.489976, + "nauc_ndcg_at_3_std": -0.026703, + "nauc_ndcg_at_3_diff1": 0.041573, + "nauc_ndcg_at_5_max": 0.359922, + "nauc_ndcg_at_5_std": -0.147712, + "nauc_ndcg_at_5_diff1": 0.032284, + "nauc_ndcg_at_10_max": 0.251176, + "nauc_ndcg_at_10_std": -0.001684, + "nauc_ndcg_at_10_diff1": -0.027844, + "nauc_ndcg_at_20_max": 0.25995, + "nauc_ndcg_at_20_std": 0.015673, + "nauc_ndcg_at_20_diff1": 0.039984, + "nauc_ndcg_at_100_max": 0.335591, + "nauc_ndcg_at_100_std": 0.081551, + "nauc_ndcg_at_100_diff1": 0.120625, + "nauc_ndcg_at_1000_max": 0.233234, + "nauc_ndcg_at_1000_std": -0.009797, + "nauc_ndcg_at_1000_diff1": 0.03099, + "nauc_map_at_1_max": 0.67028, + "nauc_map_at_1_std": -0.201748, + "nauc_map_at_1_diff1": 0.274838, + "nauc_map_at_3_max": 0.404979, + "nauc_map_at_3_std": -0.064574, + "nauc_map_at_3_diff1": -0.045064, + "nauc_map_at_5_max": 0.315306, + "nauc_map_at_5_std": -0.160983, + "nauc_map_at_5_diff1": -0.018715, + "nauc_map_at_10_max": 0.262782, + "nauc_map_at_10_std": -0.060871, + "nauc_map_at_10_diff1": -0.047264, + "nauc_map_at_20_max": 0.26041, + "nauc_map_at_20_std": -0.049256, + "nauc_map_at_20_diff1": -0.010714, + "nauc_map_at_100_max": 0.287224, + "nauc_map_at_100_std": -0.017387, + "nauc_map_at_100_diff1": 0.019486, + "nauc_map_at_1000_max": 0.281319, + "nauc_map_at_1000_std": -0.024902, + "nauc_map_at_1000_diff1": 0.012208, + "nauc_recall_at_1_max": 0.67028, + "nauc_recall_at_1_std": -0.201748, + "nauc_recall_at_1_diff1": 0.274838, + "nauc_recall_at_3_max": 0.307237, + "nauc_recall_at_3_std": -0.014037, + "nauc_recall_at_3_diff1": -0.162922, + "nauc_recall_at_5_max": 0.15994, + "nauc_recall_at_5_std": -0.232636, + "nauc_recall_at_5_diff1": -0.062903, + "nauc_recall_at_10_max": 0.07699, + "nauc_recall_at_10_std": 0.059929, + "nauc_recall_at_10_diff1": -0.114605, + "nauc_recall_at_20_max": 0.103729, + "nauc_recall_at_20_std": 0.059653, + "nauc_recall_at_20_diff1": 0.011533, + "nauc_recall_at_100_max": 0.271901, + "nauc_recall_at_100_std": 0.205016, + "nauc_recall_at_100_diff1": 0.162087, + "nauc_recall_at_1000_max": -0.325047, + "nauc_recall_at_1000_std": 0.076404, + "nauc_recall_at_1000_diff1": 0.0077, + "nauc_precision_at_1_max": 0.666779, + "nauc_precision_at_1_std": -0.081452, + "nauc_precision_at_1_diff1": 0.247659, + "nauc_precision_at_3_max": 0.52395, + "nauc_precision_at_3_std": -0.037223, + "nauc_precision_at_3_diff1": 0.081174, + "nauc_precision_at_5_max": 0.404504, + "nauc_precision_at_5_std": -0.145529, + "nauc_precision_at_5_diff1": 0.084032, + "nauc_precision_at_10_max": 0.284151, + "nauc_precision_at_10_std": 0.100864, + "nauc_precision_at_10_diff1": 0.027539, + "nauc_precision_at_20_max": 0.336363, + "nauc_precision_at_20_std": 0.154849, + "nauc_precision_at_20_diff1": 0.10217, + "nauc_precision_at_100_max": 0.315616, + "nauc_precision_at_100_std": 0.044661, + "nauc_precision_at_100_diff1": 0.105299, + "nauc_precision_at_1000_max": -0.103203, + "nauc_precision_at_1000_std": -0.185868, + "nauc_precision_at_1000_diff1": -0.184321, + "nauc_cv_recall_at_1_max": 0.666779, + "nauc_cv_recall_at_1_std": -0.081452, + "nauc_cv_recall_at_1_diff1": 0.247659, + "nauc_cv_recall_at_3_max": 0.52395, + "nauc_cv_recall_at_3_std": -0.037223, + "nauc_cv_recall_at_3_diff1": 0.081174, + "nauc_cv_recall_at_5_max": 0.404504, + "nauc_cv_recall_at_5_std": -0.145529, + "nauc_cv_recall_at_5_diff1": 0.084032, + "nauc_cv_recall_at_10_max": 0.284151, + "nauc_cv_recall_at_10_std": 0.100864, + "nauc_cv_recall_at_10_diff1": 0.027539, + "nauc_cv_recall_at_20_max": 0.295536, + "nauc_cv_recall_at_20_std": 0.022836, + "nauc_cv_recall_at_20_diff1": 0.110639, + "nauc_cv_recall_at_100_max": 0.381376, + "nauc_cv_recall_at_100_std": 0.10857, + "nauc_cv_recall_at_100_diff1": 0.204026, + "nauc_cv_recall_at_1000_max": -0.164944, + "nauc_cv_recall_at_1000_std": 0.722732, + "nauc_cv_recall_at_1000_diff1": 0.357853, + "nauc_mrr_at_1_max": 0.666779, + "nauc_mrr_at_1_std": -0.081452, + "nauc_mrr_at_1_diff1": 0.247659, + "nauc_mrr_at_3_max": 0.585163, + "nauc_mrr_at_3_std": -0.056178, + "nauc_mrr_at_3_diff1": 0.152525, + "nauc_mrr_at_5_max": 0.541266, + "nauc_mrr_at_5_std": -0.091017, + "nauc_mrr_at_5_diff1": 0.148721, + "nauc_mrr_at_10_max": 0.493738, + "nauc_mrr_at_10_std": -0.029962, + "nauc_mrr_at_10_diff1": 0.124777, + "nauc_mrr_at_20_max": 0.490794, + "nauc_mrr_at_20_std": -0.039541, + "nauc_mrr_at_20_diff1": 0.141384, + "nauc_mrr_at_100_max": 0.493853, + "nauc_mrr_at_100_std": -0.029895, + "nauc_mrr_at_100_diff1": 0.153343, + "nauc_mrr_at_1000_max": 0.493546, + "nauc_mrr_at_1000_std": -0.029303, + "nauc_mrr_at_1000_diff1": 0.152628, + "main_score": 0.03876, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.03509, + "ndcg_at_3": 0.04793, + "ndcg_at_5": 0.04479, + "ndcg_at_10": 0.04188, + "ndcg_at_20": 0.07073, + "ndcg_at_100": 0.11606, + "ndcg_at_1000": 0.2173, + "map_at_1": 0.00439, + "map_at_3": 0.02388, + "map_at_5": 0.02406, + "map_at_10": 0.02448, + "map_at_20": 0.03196, + "map_at_100": 0.03834, + "map_at_1000": 0.04305, + "recall_at_1": 0.00439, + "recall_at_3": 0.04751, + "recall_at_5": 0.04825, + "recall_at_10": 0.05117, + "recall_at_20": 0.14898, + "recall_at_100": 0.33722, + "recall_at_1000": 0.94112, + "cv_recall_at_1": 0.03509, + "cv_recall_at_3": 0.10526, + "cv_recall_at_5": 0.12281, + "cv_recall_at_10": 0.14035, + "cv_recall_at_20": 0.29825, + "cv_recall_at_100": 0.57895, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.03509, + "precision_at_3": 0.04094, + "precision_at_5": 0.02807, + "precision_at_10": 0.01579, + "precision_at_20": 0.0193, + "precision_at_100": 0.00947, + "precision_at_1000": 0.00342, + "mrr_at_1": 0.035088, + "mrr_at_3": 0.064327, + "mrr_at_5": 0.068713, + "mrr_at_10": 0.07122, + "mrr_at_20": 0.08171, + "mrr_at_100": 0.088739, + "mrr_at_1000": 0.090441, + "nauc_ndcg_at_1_max": 0.661879, + "nauc_ndcg_at_1_std": 0.12524, + "nauc_ndcg_at_1_diff1": 0.46336, + "nauc_ndcg_at_3_max": 0.641804, + "nauc_ndcg_at_3_std": 0.091862, + "nauc_ndcg_at_3_diff1": 0.288558, + "nauc_ndcg_at_5_max": 0.623978, + "nauc_ndcg_at_5_std": 0.060024, + "nauc_ndcg_at_5_diff1": 0.279991, + "nauc_ndcg_at_10_max": 0.60218, + "nauc_ndcg_at_10_std": 0.04257, + "nauc_ndcg_at_10_diff1": 0.263422, + "nauc_ndcg_at_20_max": 0.558735, + "nauc_ndcg_at_20_std": 0.049736, + "nauc_ndcg_at_20_diff1": 0.241574, + "nauc_ndcg_at_100_max": 0.451611, + "nauc_ndcg_at_100_std": 0.105193, + "nauc_ndcg_at_100_diff1": 0.24317, + "nauc_ndcg_at_1000_max": 0.377533, + "nauc_ndcg_at_1000_std": -0.000344, + "nauc_ndcg_at_1000_diff1": 0.282951, + "nauc_map_at_1_max": 0.661879, + "nauc_map_at_1_std": 0.12524, + "nauc_map_at_1_diff1": 0.46336, + "nauc_map_at_3_max": 0.658314, + "nauc_map_at_3_std": 0.11864, + "nauc_map_at_3_diff1": 0.176873, + "nauc_map_at_5_max": 0.657388, + "nauc_map_at_5_std": 0.115583, + "nauc_map_at_5_diff1": 0.181114, + "nauc_map_at_10_max": 0.65138, + "nauc_map_at_10_std": 0.108713, + "nauc_map_at_10_diff1": 0.190643, + "nauc_map_at_20_max": 0.635595, + "nauc_map_at_20_std": 0.111094, + "nauc_map_at_20_diff1": 0.203802, + "nauc_map_at_100_max": 0.595364, + "nauc_map_at_100_std": 0.11923, + "nauc_map_at_100_diff1": 0.212206, + "nauc_map_at_1000_max": 0.579474, + "nauc_map_at_1000_std": 0.099006, + "nauc_map_at_1000_diff1": 0.221105, + "nauc_recall_at_1_max": 0.661879, + "nauc_recall_at_1_std": 0.12524, + "nauc_recall_at_1_diff1": 0.46336, + "nauc_recall_at_3_max": 0.630761, + "nauc_recall_at_3_std": 0.051153, + "nauc_recall_at_3_diff1": 0.178156, + "nauc_recall_at_5_max": 0.629193, + "nauc_recall_at_5_std": 0.045735, + "nauc_recall_at_5_diff1": 0.186623, + "nauc_recall_at_10_max": 0.607005, + "nauc_recall_at_10_std": 0.025015, + "nauc_recall_at_10_diff1": 0.217783, + "nauc_recall_at_20_max": 0.470068, + "nauc_recall_at_20_std": 0.054872, + "nauc_recall_at_20_diff1": 0.141663, + "nauc_recall_at_100_max": 0.284727, + "nauc_recall_at_100_std": 0.157757, + "nauc_recall_at_100_diff1": 0.148746, + "nauc_recall_at_1000_max": 0.074076, + "nauc_recall_at_1000_std": -0.100579, + "nauc_recall_at_1000_diff1": 0.228164, + "nauc_precision_at_1_max": 0.661879, + "nauc_precision_at_1_std": 0.12524, + "nauc_precision_at_1_diff1": 0.46336, + "nauc_precision_at_3_max": 0.598752, + "nauc_precision_at_3_std": 0.030921, + "nauc_precision_at_3_diff1": 0.351272, + "nauc_precision_at_5_max": 0.588155, + "nauc_precision_at_5_std": -0.013653, + "nauc_precision_at_5_diff1": 0.408448, + "nauc_precision_at_10_max": 0.551945, + "nauc_precision_at_10_std": -0.051006, + "nauc_precision_at_10_diff1": 0.456362, + "nauc_precision_at_20_max": 0.456471, + "nauc_precision_at_20_std": -0.051171, + "nauc_precision_at_20_diff1": 0.391354, + "nauc_precision_at_100_max": 0.208514, + "nauc_precision_at_100_std": -0.006545, + "nauc_precision_at_100_diff1": 0.317711, + "nauc_precision_at_1000_max": -0.124939, + "nauc_precision_at_1000_std": -0.114028, + "nauc_precision_at_1000_diff1": 0.121759, + "nauc_cv_recall_at_1_max": 0.661879, + "nauc_cv_recall_at_1_std": 0.12524, + "nauc_cv_recall_at_1_diff1": 0.46336, + "nauc_cv_recall_at_3_max": 0.531877, + "nauc_cv_recall_at_3_std": -0.017885, + "nauc_cv_recall_at_3_diff1": 0.243151, + "nauc_cv_recall_at_5_max": 0.52647, + "nauc_cv_recall_at_5_std": -0.064189, + "nauc_cv_recall_at_5_diff1": 0.319847, + "nauc_cv_recall_at_10_max": 0.491011, + "nauc_cv_recall_at_10_std": -0.101973, + "nauc_cv_recall_at_10_diff1": 0.38243, + "nauc_cv_recall_at_20_max": 0.408901, + "nauc_cv_recall_at_20_std": -0.108669, + "nauc_cv_recall_at_20_diff1": 0.350093, + "nauc_cv_recall_at_100_max": 0.204501, + "nauc_cv_recall_at_100_std": 0.084655, + "nauc_cv_recall_at_100_diff1": 0.207302, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.661879, + "nauc_mrr_at_1_std": 0.12524, + "nauc_mrr_at_1_diff1": 0.46336, + "nauc_mrr_at_3_max": 0.580592, + "nauc_mrr_at_3_std": 0.065219, + "nauc_mrr_at_3_diff1": 0.283189, + "nauc_mrr_at_5_max": 0.576091, + "nauc_mrr_at_5_std": 0.041405, + "nauc_mrr_at_5_diff1": 0.314798, + "nauc_mrr_at_10_max": 0.565399, + "nauc_mrr_at_10_std": 0.028717, + "nauc_mrr_at_10_diff1": 0.331639, + "nauc_mrr_at_20_max": 0.542239, + "nauc_mrr_at_20_std": 0.016795, + "nauc_mrr_at_20_diff1": 0.327185, + "nauc_mrr_at_100_max": 0.528755, + "nauc_mrr_at_100_std": 0.030099, + "nauc_mrr_at_100_diff1": 0.311631, + "nauc_mrr_at_1000_max": 0.529848, + "nauc_mrr_at_1000_std": 0.027694, + "nauc_mrr_at_1000_diff1": 0.313453, + "main_score": 0.04479, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.14035, + "ndcg_at_3": 0.1148, + "ndcg_at_5": 0.11808, + "ndcg_at_10": 0.12577, + "ndcg_at_20": 0.15573, + "ndcg_at_100": 0.19553, + "ndcg_at_1000": 0.28677, + "map_at_1": 0.06798, + "map_at_3": 0.0787, + "map_at_5": 0.08367, + "map_at_10": 0.08895, + "map_at_20": 0.09678, + "map_at_100": 0.10286, + "map_at_1000": 0.10791, + "recall_at_1": 0.06798, + "recall_at_3": 0.0943, + "recall_at_5": 0.11769, + "recall_at_10": 0.15058, + "recall_at_20": 0.2534, + "recall_at_100": 0.41587, + "recall_at_1000": 0.92914, + "cv_recall_at_1": 0.14035, + "cv_recall_at_3": 0.21053, + "cv_recall_at_5": 0.26316, + "cv_recall_at_10": 0.31579, + "cv_recall_at_20": 0.47368, + "cv_recall_at_100": 0.66667, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.14035, + "precision_at_3": 0.07602, + "precision_at_5": 0.05614, + "precision_at_10": 0.03509, + "precision_at_20": 0.02895, + "precision_at_100": 0.01123, + "precision_at_1000": 0.00347, + "mrr_at_1": 0.140351, + "mrr_at_3": 0.169591, + "mrr_at_5": 0.180994, + "mrr_at_10": 0.187329, + "mrr_at_20": 0.197901, + "mrr_at_100": 0.202066, + "mrr_at_1000": 0.203464, + "nauc_ndcg_at_1_max": 0.212149, + "nauc_ndcg_at_1_std": -0.281631, + "nauc_ndcg_at_1_diff1": 0.222409, + "nauc_ndcg_at_3_max": 0.241058, + "nauc_ndcg_at_3_std": -0.117827, + "nauc_ndcg_at_3_diff1": 0.349965, + "nauc_ndcg_at_5_max": 0.279748, + "nauc_ndcg_at_5_std": -0.056944, + "nauc_ndcg_at_5_diff1": 0.375978, + "nauc_ndcg_at_10_max": 0.291506, + "nauc_ndcg_at_10_std": -0.08868, + "nauc_ndcg_at_10_diff1": 0.352127, + "nauc_ndcg_at_20_max": 0.223987, + "nauc_ndcg_at_20_std": -0.050146, + "nauc_ndcg_at_20_diff1": 0.384964, + "nauc_ndcg_at_100_max": 0.25528, + "nauc_ndcg_at_100_std": 0.043226, + "nauc_ndcg_at_100_diff1": 0.382142, + "nauc_ndcg_at_1000_max": 0.310727, + "nauc_ndcg_at_1000_std": -0.002646, + "nauc_ndcg_at_1000_diff1": 0.374778, + "nauc_map_at_1_max": 0.113271, + "nauc_map_at_1_std": -0.291801, + "nauc_map_at_1_diff1": 0.258191, + "nauc_map_at_3_max": 0.172443, + "nauc_map_at_3_std": -0.198112, + "nauc_map_at_3_diff1": 0.333936, + "nauc_map_at_5_max": 0.200348, + "nauc_map_at_5_std": -0.148226, + "nauc_map_at_5_diff1": 0.345635, + "nauc_map_at_10_max": 0.209625, + "nauc_map_at_10_std": -0.166085, + "nauc_map_at_10_diff1": 0.349207, + "nauc_map_at_20_max": 0.186867, + "nauc_map_at_20_std": -0.151347, + "nauc_map_at_20_diff1": 0.360664, + "nauc_map_at_100_max": 0.199632, + "nauc_map_at_100_std": -0.125203, + "nauc_map_at_100_diff1": 0.358735, + "nauc_map_at_1000_max": 0.210344, + "nauc_map_at_1000_std": -0.123629, + "nauc_map_at_1000_diff1": 0.359444, + "nauc_recall_at_1_max": 0.113271, + "nauc_recall_at_1_std": -0.291801, + "nauc_recall_at_1_diff1": 0.258191, + "nauc_recall_at_3_max": 0.248322, + "nauc_recall_at_3_std": -0.111431, + "nauc_recall_at_3_diff1": 0.430975, + "nauc_recall_at_5_max": 0.335451, + "nauc_recall_at_5_std": 0.07167, + "nauc_recall_at_5_diff1": 0.457288, + "nauc_recall_at_10_max": 0.321941, + "nauc_recall_at_10_std": -0.037495, + "nauc_recall_at_10_diff1": 0.344957, + "nauc_recall_at_20_max": 0.087145, + "nauc_recall_at_20_std": 0.017642, + "nauc_recall_at_20_diff1": 0.369907, + "nauc_recall_at_100_max": 0.161134, + "nauc_recall_at_100_std": 0.248827, + "nauc_recall_at_100_diff1": 0.359359, + "nauc_recall_at_1000_max": 0.399659, + "nauc_recall_at_1000_std": 0.269226, + "nauc_recall_at_1000_diff1": 0.358718, + "nauc_precision_at_1_max": 0.212149, + "nauc_precision_at_1_std": -0.281631, + "nauc_precision_at_1_diff1": 0.222409, + "nauc_precision_at_3_max": 0.338733, + "nauc_precision_at_3_std": 0.056806, + "nauc_precision_at_3_diff1": 0.411876, + "nauc_precision_at_5_max": 0.43209, + "nauc_precision_at_5_std": 0.223374, + "nauc_precision_at_5_diff1": 0.443916, + "nauc_precision_at_10_max": 0.476262, + "nauc_precision_at_10_std": 0.15945, + "nauc_precision_at_10_diff1": 0.378283, + "nauc_precision_at_20_max": 0.339142, + "nauc_precision_at_20_std": 0.249861, + "nauc_precision_at_20_diff1": 0.375236, + "nauc_precision_at_100_max": 0.31324, + "nauc_precision_at_100_std": 0.340354, + "nauc_precision_at_100_diff1": 0.284111, + "nauc_precision_at_1000_max": 0.183276, + "nauc_precision_at_1000_std": -0.005378, + "nauc_precision_at_1000_diff1": 0.088094, + "nauc_cv_recall_at_1_max": 0.212149, + "nauc_cv_recall_at_1_std": -0.281631, + "nauc_cv_recall_at_1_diff1": 0.222409, + "nauc_cv_recall_at_3_max": 0.340276, + "nauc_cv_recall_at_3_std": -0.040363, + "nauc_cv_recall_at_3_diff1": 0.384606, + "nauc_cv_recall_at_5_max": 0.442236, + "nauc_cv_recall_at_5_std": 0.150642, + "nauc_cv_recall_at_5_diff1": 0.422127, + "nauc_cv_recall_at_10_max": 0.502968, + "nauc_cv_recall_at_10_std": 0.127942, + "nauc_cv_recall_at_10_diff1": 0.297728, + "nauc_cv_recall_at_20_max": 0.39025, + "nauc_cv_recall_at_20_std": 0.245601, + "nauc_cv_recall_at_20_diff1": 0.421968, + "nauc_cv_recall_at_100_max": 0.389386, + "nauc_cv_recall_at_100_std": 0.316682, + "nauc_cv_recall_at_100_diff1": 0.47303, + "nauc_cv_recall_at_1000_max": 1.0, + "nauc_cv_recall_at_1000_std": 0.722732, + "nauc_cv_recall_at_1000_diff1": 0.868512, + "nauc_mrr_at_1_max": 0.212149, + "nauc_mrr_at_1_std": -0.281631, + "nauc_mrr_at_1_diff1": 0.222409, + "nauc_mrr_at_3_max": 0.264352, + "nauc_mrr_at_3_std": -0.154314, + "nauc_mrr_at_3_diff1": 0.281112, + "nauc_mrr_at_5_max": 0.292859, + "nauc_mrr_at_5_std": -0.101975, + "nauc_mrr_at_5_diff1": 0.290398, + "nauc_mrr_at_10_max": 0.304628, + "nauc_mrr_at_10_std": -0.102218, + "nauc_mrr_at_10_diff1": 0.272287, + "nauc_mrr_at_20_max": 0.293493, + "nauc_mrr_at_20_std": -0.091105, + "nauc_mrr_at_20_diff1": 0.283092, + "nauc_mrr_at_100_max": 0.293708, + "nauc_mrr_at_100_std": -0.090589, + "nauc_mrr_at_100_diff1": 0.281149, + "nauc_mrr_at_1000_max": 0.294684, + "nauc_mrr_at_1000_std": -0.092014, + "nauc_mrr_at_1000_diff1": 0.281052, + "main_score": 0.11808, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.05263, + "ndcg_at_3": 0.04957, + "ndcg_at_5": 0.05215, + "ndcg_at_10": 0.05431, + "ndcg_at_20": 0.0667, + "ndcg_at_100": 0.10957, + "ndcg_at_1000": 0.20981, + "map_at_1": 0.02485, + "map_at_3": 0.03509, + "map_at_5": 0.03576, + "map_at_10": 0.03708, + "map_at_20": 0.03998, + "map_at_100": 0.04512, + "map_at_1000": 0.04971, + "recall_at_1": 0.02485, + "recall_at_3": 0.05117, + "recall_at_5": 0.05399, + "recall_at_10": 0.06558, + "recall_at_20": 0.11412, + "recall_at_100": 0.30109, + "recall_at_1000": 0.90236, + "cv_recall_at_1": 0.05263, + "cv_recall_at_3": 0.08772, + "cv_recall_at_5": 0.12281, + "cv_recall_at_10": 0.15789, + "cv_recall_at_20": 0.24561, + "cv_recall_at_100": 0.52632, + "cv_recall_at_1000": 0.96491, + "precision_at_1": 0.05263, + "precision_at_3": 0.02924, + "precision_at_5": 0.02456, + "precision_at_10": 0.01754, + "precision_at_20": 0.01404, + "precision_at_100": 0.0086, + "precision_at_1000": 0.00326, + "mrr_at_1": 0.052632, + "mrr_at_3": 0.067251, + "mrr_at_5": 0.075146, + "mrr_at_10": 0.079407, + "mrr_at_20": 0.084459, + "mrr_at_100": 0.090862, + "mrr_at_1000": 0.092447, + "nauc_ndcg_at_1_max": 0.365267, + "nauc_ndcg_at_1_std": -0.275158, + "nauc_ndcg_at_1_diff1": 0.146803, + "nauc_ndcg_at_3_max": 0.318563, + "nauc_ndcg_at_3_std": -0.260298, + "nauc_ndcg_at_3_diff1": 0.195185, + "nauc_ndcg_at_5_max": 0.248079, + "nauc_ndcg_at_5_std": -0.234723, + "nauc_ndcg_at_5_diff1": 0.122859, + "nauc_ndcg_at_10_max": 0.185765, + "nauc_ndcg_at_10_std": -0.213651, + "nauc_ndcg_at_10_diff1": 0.082389, + "nauc_ndcg_at_20_max": 0.214412, + "nauc_ndcg_at_20_std": -0.226632, + "nauc_ndcg_at_20_diff1": 0.105414, + "nauc_ndcg_at_100_max": 0.226581, + "nauc_ndcg_at_100_std": -0.048127, + "nauc_ndcg_at_100_diff1": 0.036323, + "nauc_ndcg_at_1000_max": 0.072537, + "nauc_ndcg_at_1000_std": -0.116752, + "nauc_ndcg_at_1000_diff1": -0.011277, + "nauc_map_at_1_max": 0.290796, + "nauc_map_at_1_std": -0.302364, + "nauc_map_at_1_diff1": 0.198848, + "nauc_map_at_3_max": 0.278587, + "nauc_map_at_3_std": -0.25748, + "nauc_map_at_3_diff1": 0.197231, + "nauc_map_at_5_max": 0.269003, + "nauc_map_at_5_std": -0.255761, + "nauc_map_at_5_diff1": 0.183991, + "nauc_map_at_10_max": 0.24903, + "nauc_map_at_10_std": -0.250869, + "nauc_map_at_10_diff1": 0.166605, + "nauc_map_at_20_max": 0.256602, + "nauc_map_at_20_std": -0.255134, + "nauc_map_at_20_diff1": 0.166327, + "nauc_map_at_100_max": 0.250689, + "nauc_map_at_100_std": -0.218324, + "nauc_map_at_100_diff1": 0.146715, + "nauc_map_at_1000_max": 0.228904, + "nauc_map_at_1000_std": -0.215574, + "nauc_map_at_1000_diff1": 0.134436, + "nauc_recall_at_1_max": 0.290796, + "nauc_recall_at_1_std": -0.302364, + "nauc_recall_at_1_diff1": 0.198848, + "nauc_recall_at_3_max": 0.318141, + "nauc_recall_at_3_std": -0.263699, + "nauc_recall_at_3_diff1": 0.184815, + "nauc_recall_at_5_max": 0.289346, + "nauc_recall_at_5_std": -0.258202, + "nauc_recall_at_5_diff1": 0.148587, + "nauc_recall_at_10_max": 0.18073, + "nauc_recall_at_10_std": -0.22864, + "nauc_recall_at_10_diff1": 0.064175, + "nauc_recall_at_20_max": 0.228106, + "nauc_recall_at_20_std": -0.240445, + "nauc_recall_at_20_diff1": 0.087183, + "nauc_recall_at_100_max": 0.273615, + "nauc_recall_at_100_std": 0.187925, + "nauc_recall_at_100_diff1": -0.023279, + "nauc_recall_at_1000_max": -0.330772, + "nauc_recall_at_1000_std": 0.231006, + "nauc_recall_at_1000_diff1": -0.249839, + "nauc_precision_at_1_max": 0.365267, + "nauc_precision_at_1_std": -0.275158, + "nauc_precision_at_1_diff1": 0.146803, + "nauc_precision_at_3_max": 0.290225, + "nauc_precision_at_3_std": -0.200874, + "nauc_precision_at_3_diff1": 0.171926, + "nauc_precision_at_5_max": 0.123676, + "nauc_precision_at_5_std": -0.167154, + "nauc_precision_at_5_diff1": -0.033217, + "nauc_precision_at_10_max": -0.036777, + "nauc_precision_at_10_std": -0.129252, + "nauc_precision_at_10_diff1": -0.15981, + "nauc_precision_at_20_max": 0.046642, + "nauc_precision_at_20_std": -0.164543, + "nauc_precision_at_20_diff1": -0.050628, + "nauc_precision_at_100_max": 0.004502, + "nauc_precision_at_100_std": 0.022906, + "nauc_precision_at_100_diff1": -0.162357, + "nauc_precision_at_1000_max": -0.210259, + "nauc_precision_at_1000_std": -0.085944, + "nauc_precision_at_1000_diff1": -0.230579, + "nauc_cv_recall_at_1_max": 0.365267, + "nauc_cv_recall_at_1_std": -0.275158, + "nauc_cv_recall_at_1_diff1": 0.146803, + "nauc_cv_recall_at_3_max": 0.290225, + "nauc_cv_recall_at_3_std": -0.200874, + "nauc_cv_recall_at_3_diff1": 0.171926, + "nauc_cv_recall_at_5_max": 0.123676, + "nauc_cv_recall_at_5_std": -0.167154, + "nauc_cv_recall_at_5_diff1": -0.033217, + "nauc_cv_recall_at_10_max": 0.010734, + "nauc_cv_recall_at_10_std": -0.157237, + "nauc_cv_recall_at_10_diff1": -0.107636, + "nauc_cv_recall_at_20_max": 0.030706, + "nauc_cv_recall_at_20_std": -0.156999, + "nauc_cv_recall_at_20_diff1": -0.03578, + "nauc_cv_recall_at_100_max": 0.044869, + "nauc_cv_recall_at_100_std": 0.17154, + "nauc_cv_recall_at_100_diff1": -0.199095, + "nauc_cv_recall_at_1000_max": -0.800621, + "nauc_cv_recall_at_1000_std": 0.34085, + "nauc_cv_recall_at_1000_diff1": -1.143633, + "nauc_mrr_at_1_max": 0.365267, + "nauc_mrr_at_1_std": -0.275158, + "nauc_mrr_at_1_diff1": 0.146803, + "nauc_mrr_at_3_max": 0.302791, + "nauc_mrr_at_3_std": -0.21682, + "nauc_mrr_at_3_diff1": 0.165419, + "nauc_mrr_at_5_max": 0.243032, + "nauc_mrr_at_5_std": -0.204765, + "nauc_mrr_at_5_diff1": 0.093085, + "nauc_mrr_at_10_max": 0.212958, + "nauc_mrr_at_10_std": -0.201148, + "nauc_mrr_at_10_diff1": 0.071795, + "nauc_mrr_at_20_max": 0.209325, + "nauc_mrr_at_20_std": -0.198054, + "nauc_mrr_at_20_diff1": 0.075616, + "nauc_mrr_at_100_max": 0.202539, + "nauc_mrr_at_100_std": -0.182535, + "nauc_mrr_at_100_diff1": 0.063867, + "nauc_mrr_at_1000_max": 0.203693, + "nauc_mrr_at_1000_std": -0.184387, + "nauc_mrr_at_1000_diff1": 0.065871, + "main_score": 0.05215, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 399.75314831733704, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Vidore2EconomicsReportsRetrieval.json b/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..7adc94039f --- /dev/null +++ b/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.15517, + "ndcg_at_3": 0.11857, + "ndcg_at_5": 0.10255, + "ndcg_at_10": 0.09908, + "ndcg_at_20": 0.10471, + "ndcg_at_100": 0.18886, + "ndcg_at_1000": 0.37655, + "map_at_1": 0.00856, + "map_at_3": 0.01746, + "map_at_5": 0.02006, + "map_at_10": 0.02493, + "map_at_20": 0.02964, + "map_at_100": 0.05142, + "map_at_1000": 0.08184, + "recall_at_1": 0.00856, + "recall_at_3": 0.02399, + "recall_at_5": 0.03174, + "recall_at_10": 0.05285, + "recall_at_20": 0.08125, + "recall_at_100": 0.3701, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.15517, + "cv_recall_at_3": 0.24138, + "cv_recall_at_5": 0.31034, + "cv_recall_at_10": 0.39655, + "cv_recall_at_20": 0.55172, + "cv_recall_at_100": 0.87931, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.15517, + "precision_at_3": 0.1092, + "precision_at_5": 0.08966, + "precision_at_10": 0.07931, + "precision_at_20": 0.07069, + "precision_at_100": 0.05466, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.155172, + "mrr_at_3": 0.195402, + "mrr_at_5": 0.21092, + "mrr_at_10": 0.223851, + "mrr_at_20": 0.235586, + "mrr_at_100": 0.244228, + "mrr_at_1000": 0.245037, + "nauc_ndcg_at_1_max": 0.495075, + "nauc_ndcg_at_1_std": 0.245399, + "nauc_ndcg_at_1_diff1": 0.042836, + "nauc_ndcg_at_3_max": 0.492038, + "nauc_ndcg_at_3_std": 0.259178, + "nauc_ndcg_at_3_diff1": -0.018747, + "nauc_ndcg_at_5_max": 0.478173, + "nauc_ndcg_at_5_std": 0.254108, + "nauc_ndcg_at_5_diff1": 0.00883, + "nauc_ndcg_at_10_max": 0.432951, + "nauc_ndcg_at_10_std": 0.195888, + "nauc_ndcg_at_10_diff1": -0.055324, + "nauc_ndcg_at_20_max": 0.449366, + "nauc_ndcg_at_20_std": 0.206492, + "nauc_ndcg_at_20_diff1": -0.09786, + "nauc_ndcg_at_100_max": 0.485368, + "nauc_ndcg_at_100_std": 0.308373, + "nauc_ndcg_at_100_diff1": -0.00697, + "nauc_ndcg_at_1000_max": 0.368344, + "nauc_ndcg_at_1000_std": 0.088871, + "nauc_ndcg_at_1000_diff1": 0.043351, + "nauc_map_at_1_max": 0.523033, + "nauc_map_at_1_std": 0.519589, + "nauc_map_at_1_diff1": 0.218814, + "nauc_map_at_3_max": 0.475237, + "nauc_map_at_3_std": 0.467005, + "nauc_map_at_3_diff1": 0.098534, + "nauc_map_at_5_max": 0.448829, + "nauc_map_at_5_std": 0.499891, + "nauc_map_at_5_diff1": 0.078601, + "nauc_map_at_10_max": 0.415178, + "nauc_map_at_10_std": 0.451992, + "nauc_map_at_10_diff1": 0.025846, + "nauc_map_at_20_max": 0.453224, + "nauc_map_at_20_std": 0.424103, + "nauc_map_at_20_diff1": -0.00542, + "nauc_map_at_100_max": 0.469598, + "nauc_map_at_100_std": 0.307538, + "nauc_map_at_100_diff1": -0.040397, + "nauc_map_at_1000_max": 0.445267, + "nauc_map_at_1000_std": 0.162467, + "nauc_map_at_1000_diff1": -0.012278, + "nauc_recall_at_1_max": 0.523033, + "nauc_recall_at_1_std": 0.519589, + "nauc_recall_at_1_diff1": 0.218814, + "nauc_recall_at_3_max": 0.349607, + "nauc_recall_at_3_std": 0.349981, + "nauc_recall_at_3_diff1": 0.010581, + "nauc_recall_at_5_max": 0.32924, + "nauc_recall_at_5_std": 0.411499, + "nauc_recall_at_5_diff1": -0.011042, + "nauc_recall_at_10_max": 0.229151, + "nauc_recall_at_10_std": 0.335043, + "nauc_recall_at_10_diff1": -0.139947, + "nauc_recall_at_20_max": 0.421768, + "nauc_recall_at_20_std": 0.354204, + "nauc_recall_at_20_diff1": -0.110509, + "nauc_recall_at_100_max": 0.388649, + "nauc_recall_at_100_std": 0.358995, + "nauc_recall_at_100_diff1": 0.046924, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.495075, + "nauc_precision_at_1_std": 0.245399, + "nauc_precision_at_1_diff1": 0.042836, + "nauc_precision_at_3_max": 0.506088, + "nauc_precision_at_3_std": 0.260906, + "nauc_precision_at_3_diff1": -0.050307, + "nauc_precision_at_5_max": 0.475428, + "nauc_precision_at_5_std": 0.245163, + "nauc_precision_at_5_diff1": -0.002939, + "nauc_precision_at_10_max": 0.38748, + "nauc_precision_at_10_std": 0.099867, + "nauc_precision_at_10_diff1": -0.090534, + "nauc_precision_at_20_max": 0.360576, + "nauc_precision_at_20_std": 0.00366, + "nauc_precision_at_20_diff1": -0.154413, + "nauc_precision_at_100_max": 0.366756, + "nauc_precision_at_100_std": -0.042916, + "nauc_precision_at_100_diff1": -0.077426, + "nauc_precision_at_1000_max": 0.306497, + "nauc_precision_at_1000_std": -0.156344, + "nauc_precision_at_1000_diff1": -0.027917, + "nauc_cv_recall_at_1_max": 0.495075, + "nauc_cv_recall_at_1_std": 0.245399, + "nauc_cv_recall_at_1_diff1": 0.042836, + "nauc_cv_recall_at_3_max": 0.337034, + "nauc_cv_recall_at_3_std": 0.194308, + "nauc_cv_recall_at_3_diff1": -0.043918, + "nauc_cv_recall_at_5_max": 0.329588, + "nauc_cv_recall_at_5_std": 0.190643, + "nauc_cv_recall_at_5_diff1": 0.022935, + "nauc_cv_recall_at_10_max": 0.304017, + "nauc_cv_recall_at_10_std": 0.243555, + "nauc_cv_recall_at_10_diff1": -0.11403, + "nauc_cv_recall_at_20_max": 0.417377, + "nauc_cv_recall_at_20_std": 0.269024, + "nauc_cv_recall_at_20_diff1": -0.099691, + "nauc_cv_recall_at_100_max": 0.632789, + "nauc_cv_recall_at_100_std": 0.185623, + "nauc_cv_recall_at_100_diff1": 0.199324, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.495075, + "nauc_mrr_at_1_std": 0.245399, + "nauc_mrr_at_1_diff1": 0.042836, + "nauc_mrr_at_3_max": 0.408478, + "nauc_mrr_at_3_std": 0.218797, + "nauc_mrr_at_3_diff1": 0.000764, + "nauc_mrr_at_5_max": 0.404601, + "nauc_mrr_at_5_std": 0.216615, + "nauc_mrr_at_5_diff1": 0.024983, + "nauc_mrr_at_10_max": 0.39736, + "nauc_mrr_at_10_std": 0.224425, + "nauc_mrr_at_10_diff1": -0.000167, + "nauc_mrr_at_20_max": 0.409353, + "nauc_mrr_at_20_std": 0.22744, + "nauc_mrr_at_20_diff1": 0.001579, + "nauc_mrr_at_100_max": 0.404364, + "nauc_mrr_at_100_std": 0.218031, + "nauc_mrr_at_100_diff1": 0.006633, + "nauc_mrr_at_1000_max": 0.403714, + "nauc_mrr_at_1000_std": 0.218064, + "nauc_mrr_at_1000_diff1": 0.005971, + "main_score": 0.10255, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.06897, + "ndcg_at_3": 0.08938, + "ndcg_at_5": 0.09657, + "ndcg_at_10": 0.0876, + "ndcg_at_20": 0.09694, + "ndcg_at_100": 0.1873, + "ndcg_at_1000": 0.37121, + "map_at_1": 0.0058, + "map_at_3": 0.01409, + "map_at_5": 0.01943, + "map_at_10": 0.02275, + "map_at_20": 0.02736, + "map_at_100": 0.04637, + "map_at_1000": 0.0754, + "recall_at_1": 0.0058, + "recall_at_3": 0.02059, + "recall_at_5": 0.03776, + "recall_at_10": 0.05366, + "recall_at_20": 0.09589, + "recall_at_100": 0.38309, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.06897, + "cv_recall_at_3": 0.22414, + "cv_recall_at_5": 0.37931, + "cv_recall_at_10": 0.46552, + "cv_recall_at_20": 0.58621, + "cv_recall_at_100": 0.91379, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.06897, + "precision_at_3": 0.09195, + "precision_at_5": 0.1, + "precision_at_10": 0.07586, + "precision_at_20": 0.06379, + "precision_at_100": 0.05259, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.068966, + "mrr_at_3": 0.137931, + "mrr_at_5": 0.174138, + "mrr_at_10": 0.183689, + "mrr_at_20": 0.191386, + "mrr_at_100": 0.198221, + "mrr_at_1000": 0.198763, + "nauc_ndcg_at_1_max": 0.004934, + "nauc_ndcg_at_1_std": 0.043126, + "nauc_ndcg_at_1_diff1": 0.317943, + "nauc_ndcg_at_3_max": 0.241379, + "nauc_ndcg_at_3_std": 0.053322, + "nauc_ndcg_at_3_diff1": 0.373974, + "nauc_ndcg_at_5_max": 0.204973, + "nauc_ndcg_at_5_std": 0.002239, + "nauc_ndcg_at_5_diff1": 0.310179, + "nauc_ndcg_at_10_max": 0.249422, + "nauc_ndcg_at_10_std": -0.024932, + "nauc_ndcg_at_10_diff1": 0.293936, + "nauc_ndcg_at_20_max": 0.318668, + "nauc_ndcg_at_20_std": -0.038274, + "nauc_ndcg_at_20_diff1": 0.288062, + "nauc_ndcg_at_100_max": 0.367712, + "nauc_ndcg_at_100_std": 0.081618, + "nauc_ndcg_at_100_diff1": 0.313251, + "nauc_ndcg_at_1000_max": 0.196068, + "nauc_ndcg_at_1000_std": -0.111048, + "nauc_ndcg_at_1000_diff1": 0.236772, + "nauc_map_at_1_max": 0.313233, + "nauc_map_at_1_std": 0.393821, + "nauc_map_at_1_diff1": 0.666584, + "nauc_map_at_3_max": 0.43945, + "nauc_map_at_3_std": 0.31183, + "nauc_map_at_3_diff1": 0.483742, + "nauc_map_at_5_max": 0.368979, + "nauc_map_at_5_std": 0.235694, + "nauc_map_at_5_diff1": 0.382699, + "nauc_map_at_10_max": 0.381753, + "nauc_map_at_10_std": 0.205435, + "nauc_map_at_10_diff1": 0.385726, + "nauc_map_at_20_max": 0.409953, + "nauc_map_at_20_std": 0.165382, + "nauc_map_at_20_diff1": 0.386894, + "nauc_map_at_100_max": 0.391297, + "nauc_map_at_100_std": 0.087727, + "nauc_map_at_100_diff1": 0.357495, + "nauc_map_at_1000_max": 0.304287, + "nauc_map_at_1000_std": -0.035663, + "nauc_map_at_1000_diff1": 0.253344, + "nauc_recall_at_1_max": 0.313233, + "nauc_recall_at_1_std": 0.393821, + "nauc_recall_at_1_diff1": 0.666584, + "nauc_recall_at_3_max": 0.421193, + "nauc_recall_at_3_std": 0.222653, + "nauc_recall_at_3_diff1": 0.408489, + "nauc_recall_at_5_max": 0.241414, + "nauc_recall_at_5_std": 0.105876, + "nauc_recall_at_5_diff1": 0.256615, + "nauc_recall_at_10_max": 0.268998, + "nauc_recall_at_10_std": 0.079208, + "nauc_recall_at_10_diff1": 0.225794, + "nauc_recall_at_20_max": 0.309142, + "nauc_recall_at_20_std": -0.043029, + "nauc_recall_at_20_diff1": 0.229619, + "nauc_recall_at_100_max": 0.289073, + "nauc_recall_at_100_std": 0.314562, + "nauc_recall_at_100_diff1": 0.049346, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.004934, + "nauc_precision_at_1_std": 0.043126, + "nauc_precision_at_1_diff1": 0.317943, + "nauc_precision_at_3_max": 0.290126, + "nauc_precision_at_3_std": 0.068692, + "nauc_precision_at_3_diff1": 0.389816, + "nauc_precision_at_5_max": 0.224508, + "nauc_precision_at_5_std": -0.010997, + "nauc_precision_at_5_diff1": 0.288127, + "nauc_precision_at_10_max": 0.261917, + "nauc_precision_at_10_std": -0.069925, + "nauc_precision_at_10_diff1": 0.250357, + "nauc_precision_at_20_max": 0.294002, + "nauc_precision_at_20_std": -0.136881, + "nauc_precision_at_20_diff1": 0.193413, + "nauc_precision_at_100_max": 0.174173, + "nauc_precision_at_100_std": -0.172463, + "nauc_precision_at_100_diff1": 0.098214, + "nauc_precision_at_1000_max": 0.069652, + "nauc_precision_at_1000_std": -0.235259, + "nauc_precision_at_1000_diff1": -0.000202, + "nauc_cv_recall_at_1_max": 0.004934, + "nauc_cv_recall_at_1_std": 0.043126, + "nauc_cv_recall_at_1_diff1": 0.317943, + "nauc_cv_recall_at_3_max": 0.340189, + "nauc_cv_recall_at_3_std": 0.073327, + "nauc_cv_recall_at_3_diff1": 0.452308, + "nauc_cv_recall_at_5_max": 0.231452, + "nauc_cv_recall_at_5_std": -0.014359, + "nauc_cv_recall_at_5_diff1": 0.387674, + "nauc_cv_recall_at_10_max": 0.235637, + "nauc_cv_recall_at_10_std": -0.014029, + "nauc_cv_recall_at_10_diff1": 0.356624, + "nauc_cv_recall_at_20_max": 0.154663, + "nauc_cv_recall_at_20_std": -0.206678, + "nauc_cv_recall_at_20_diff1": 0.351524, + "nauc_cv_recall_at_100_max": 0.287204, + "nauc_cv_recall_at_100_std": 0.107194, + "nauc_cv_recall_at_100_diff1": 0.125326, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.004934, + "nauc_mrr_at_1_std": 0.043126, + "nauc_mrr_at_1_diff1": 0.317943, + "nauc_mrr_at_3_max": 0.221918, + "nauc_mrr_at_3_std": 0.032789, + "nauc_mrr_at_3_diff1": 0.396341, + "nauc_mrr_at_5_max": 0.177288, + "nauc_mrr_at_5_std": -0.000477, + "nauc_mrr_at_5_diff1": 0.372105, + "nauc_mrr_at_10_max": 0.176868, + "nauc_mrr_at_10_std": -0.001333, + "nauc_mrr_at_10_diff1": 0.363717, + "nauc_mrr_at_20_max": 0.169853, + "nauc_mrr_at_20_std": -0.018291, + "nauc_mrr_at_20_diff1": 0.363695, + "nauc_mrr_at_100_max": 0.174458, + "nauc_mrr_at_100_std": -0.013004, + "nauc_mrr_at_100_diff1": 0.365874, + "nauc_mrr_at_1000_max": 0.173984, + "nauc_mrr_at_1000_std": -0.013636, + "nauc_mrr_at_1000_diff1": 0.366457, + "main_score": 0.09657, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.17241, + "ndcg_at_3": 0.12666, + "ndcg_at_5": 0.11276, + "ndcg_at_10": 0.1007, + "ndcg_at_20": 0.12369, + "ndcg_at_100": 0.21092, + "ndcg_at_1000": 0.39044, + "map_at_1": 0.01271, + "map_at_3": 0.02204, + "map_at_5": 0.0246, + "map_at_10": 0.02858, + "map_at_20": 0.03752, + "map_at_100": 0.05794, + "map_at_1000": 0.0883, + "recall_at_1": 0.01271, + "recall_at_3": 0.02813, + "recall_at_5": 0.03933, + "recall_at_10": 0.05555, + "recall_at_20": 0.12532, + "recall_at_100": 0.40359, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.17241, + "cv_recall_at_3": 0.27586, + "cv_recall_at_5": 0.37931, + "cv_recall_at_10": 0.44828, + "cv_recall_at_20": 0.63793, + "cv_recall_at_100": 0.94828, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.17241, + "precision_at_3": 0.11494, + "precision_at_5": 0.1, + "precision_at_10": 0.07759, + "precision_at_20": 0.0819, + "precision_at_100": 0.05655, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.172414, + "mrr_at_3": 0.215517, + "mrr_at_5": 0.236207, + "mrr_at_10": 0.246983, + "mrr_at_20": 0.26112, + "mrr_at_100": 0.26854, + "mrr_at_1000": 0.26886, + "nauc_ndcg_at_1_max": 0.087678, + "nauc_ndcg_at_1_std": -0.000488, + "nauc_ndcg_at_1_diff1": -0.173013, + "nauc_ndcg_at_3_max": 0.012146, + "nauc_ndcg_at_3_std": -0.079406, + "nauc_ndcg_at_3_diff1": -0.109292, + "nauc_ndcg_at_5_max": 0.145529, + "nauc_ndcg_at_5_std": -0.071895, + "nauc_ndcg_at_5_diff1": -0.081878, + "nauc_ndcg_at_10_max": 0.23026, + "nauc_ndcg_at_10_std": -0.061242, + "nauc_ndcg_at_10_diff1": -0.026902, + "nauc_ndcg_at_20_max": 0.365761, + "nauc_ndcg_at_20_std": -0.015063, + "nauc_ndcg_at_20_diff1": -0.017683, + "nauc_ndcg_at_100_max": 0.309143, + "nauc_ndcg_at_100_std": 0.044555, + "nauc_ndcg_at_100_diff1": -0.08957, + "nauc_ndcg_at_1000_max": 0.292392, + "nauc_ndcg_at_1000_std": -0.007092, + "nauc_ndcg_at_1000_diff1": -0.051858, + "nauc_map_at_1_max": -0.132866, + "nauc_map_at_1_std": -0.011989, + "nauc_map_at_1_diff1": -0.224902, + "nauc_map_at_3_max": -0.108611, + "nauc_map_at_3_std": 0.010159, + "nauc_map_at_3_diff1": -0.171553, + "nauc_map_at_5_max": -0.047051, + "nauc_map_at_5_std": -0.014151, + "nauc_map_at_5_diff1": -0.159982, + "nauc_map_at_10_max": 0.011849, + "nauc_map_at_10_std": -0.036802, + "nauc_map_at_10_diff1": -0.092222, + "nauc_map_at_20_max": 0.088488, + "nauc_map_at_20_std": -0.047031, + "nauc_map_at_20_diff1": -0.058048, + "nauc_map_at_100_max": 0.221905, + "nauc_map_at_100_std": 0.007716, + "nauc_map_at_100_diff1": -0.099295, + "nauc_map_at_1000_max": 0.313697, + "nauc_map_at_1000_std": 0.046442, + "nauc_map_at_1000_diff1": -0.102705, + "nauc_recall_at_1_max": -0.132866, + "nauc_recall_at_1_std": -0.011989, + "nauc_recall_at_1_diff1": -0.224902, + "nauc_recall_at_3_max": -0.147708, + "nauc_recall_at_3_std": -0.055714, + "nauc_recall_at_3_diff1": -0.105774, + "nauc_recall_at_5_max": 0.052581, + "nauc_recall_at_5_std": -0.121045, + "nauc_recall_at_5_diff1": -0.100753, + "nauc_recall_at_10_max": 0.17375, + "nauc_recall_at_10_std": -0.132246, + "nauc_recall_at_10_diff1": 0.075078, + "nauc_recall_at_20_max": 0.403304, + "nauc_recall_at_20_std": -0.029376, + "nauc_recall_at_20_diff1": 0.143427, + "nauc_recall_at_100_max": 0.196095, + "nauc_recall_at_100_std": 0.26268, + "nauc_recall_at_100_diff1": -0.092887, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.087678, + "nauc_precision_at_1_std": -0.000488, + "nauc_precision_at_1_diff1": -0.173013, + "nauc_precision_at_3_max": -0.036764, + "nauc_precision_at_3_std": -0.12064, + "nauc_precision_at_3_diff1": -0.075394, + "nauc_precision_at_5_max": 0.2143, + "nauc_precision_at_5_std": -0.103202, + "nauc_precision_at_5_diff1": -0.024422, + "nauc_precision_at_10_max": 0.33673, + "nauc_precision_at_10_std": -0.081875, + "nauc_precision_at_10_diff1": 0.046531, + "nauc_precision_at_20_max": 0.482054, + "nauc_precision_at_20_std": -0.028976, + "nauc_precision_at_20_diff1": 0.048886, + "nauc_precision_at_100_max": 0.49251, + "nauc_precision_at_100_std": 0.122467, + "nauc_precision_at_100_diff1": -0.089025, + "nauc_precision_at_1000_max": 0.446998, + "nauc_precision_at_1000_std": 0.120735, + "nauc_precision_at_1000_diff1": -0.08111, + "nauc_cv_recall_at_1_max": 0.087678, + "nauc_cv_recall_at_1_std": -0.000488, + "nauc_cv_recall_at_1_diff1": -0.173013, + "nauc_cv_recall_at_3_max": 0.017981, + "nauc_cv_recall_at_3_std": -0.152787, + "nauc_cv_recall_at_3_diff1": -0.012379, + "nauc_cv_recall_at_5_max": 0.257466, + "nauc_cv_recall_at_5_std": -0.190664, + "nauc_cv_recall_at_5_diff1": -0.003923, + "nauc_cv_recall_at_10_max": 0.281364, + "nauc_cv_recall_at_10_std": -0.186461, + "nauc_cv_recall_at_10_diff1": 0.047239, + "nauc_cv_recall_at_20_max": 0.730958, + "nauc_cv_recall_at_20_std": 0.006657, + "nauc_cv_recall_at_20_diff1": 0.031366, + "nauc_cv_recall_at_100_max": 0.565288, + "nauc_cv_recall_at_100_std": 0.956259, + "nauc_cv_recall_at_100_diff1": 0.072043, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.087678, + "nauc_mrr_at_1_std": -0.000488, + "nauc_mrr_at_1_diff1": -0.173013, + "nauc_mrr_at_3_max": 0.061177, + "nauc_mrr_at_3_std": -0.078726, + "nauc_mrr_at_3_diff1": -0.102051, + "nauc_mrr_at_5_max": 0.118775, + "nauc_mrr_at_5_std": -0.089194, + "nauc_mrr_at_5_diff1": -0.098486, + "nauc_mrr_at_10_max": 0.125038, + "nauc_mrr_at_10_std": -0.086009, + "nauc_mrr_at_10_diff1": -0.087463, + "nauc_mrr_at_20_max": 0.152324, + "nauc_mrr_at_20_std": -0.070781, + "nauc_mrr_at_20_diff1": -0.091826, + "nauc_mrr_at_100_max": 0.140127, + "nauc_mrr_at_100_std": -0.065465, + "nauc_mrr_at_100_diff1": -0.093793, + "nauc_mrr_at_1000_max": 0.139684, + "nauc_mrr_at_1000_std": -0.066609, + "nauc_mrr_at_1000_diff1": -0.093921, + "main_score": 0.11276, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.03448, + "ndcg_at_3": 0.03853, + "ndcg_at_5": 0.03741, + "ndcg_at_10": 0.0424, + "ndcg_at_20": 0.05681, + "ndcg_at_100": 0.13551, + "ndcg_at_1000": 0.34105, + "map_at_1": 0.00217, + "map_at_3": 0.00342, + "map_at_5": 0.0041, + "map_at_10": 0.00624, + "map_at_20": 0.0107, + "map_at_100": 0.02698, + "map_at_1000": 0.05828, + "recall_at_1": 0.00217, + "recall_at_3": 0.00526, + "recall_at_5": 0.00811, + "recall_at_10": 0.01854, + "recall_at_20": 0.05697, + "recall_at_100": 0.27649, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.03448, + "cv_recall_at_3": 0.12069, + "cv_recall_at_5": 0.17241, + "cv_recall_at_10": 0.27586, + "cv_recall_at_20": 0.44828, + "cv_recall_at_100": 0.77586, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.03448, + "precision_at_3": 0.04023, + "precision_at_5": 0.03793, + "precision_at_10": 0.0431, + "precision_at_20": 0.05086, + "precision_at_100": 0.04845, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.034483, + "mrr_at_3": 0.068966, + "mrr_at_5": 0.081034, + "mrr_at_10": 0.096086, + "mrr_at_20": 0.106889, + "mrr_at_100": 0.114602, + "mrr_at_1000": 0.11626, + "nauc_ndcg_at_1_max": -0.042693, + "nauc_ndcg_at_1_std": 0.338155, + "nauc_ndcg_at_1_diff1": -0.054378, + "nauc_ndcg_at_3_max": -0.036895, + "nauc_ndcg_at_3_std": 0.227276, + "nauc_ndcg_at_3_diff1": -0.194905, + "nauc_ndcg_at_5_max": 0.022269, + "nauc_ndcg_at_5_std": 0.24285, + "nauc_ndcg_at_5_diff1": -0.145915, + "nauc_ndcg_at_10_max": -0.000843, + "nauc_ndcg_at_10_std": 0.142247, + "nauc_ndcg_at_10_diff1": -0.177688, + "nauc_ndcg_at_20_max": 0.023388, + "nauc_ndcg_at_20_std": 0.081217, + "nauc_ndcg_at_20_diff1": -0.128296, + "nauc_ndcg_at_100_max": 0.042225, + "nauc_ndcg_at_100_std": 0.051634, + "nauc_ndcg_at_100_diff1": -0.129943, + "nauc_ndcg_at_1000_max": 0.120369, + "nauc_ndcg_at_1000_std": 0.010647, + "nauc_ndcg_at_1000_diff1": 0.039659, + "nauc_map_at_1_max": -0.333643, + "nauc_map_at_1_std": 0.847267, + "nauc_map_at_1_diff1": -0.17392, + "nauc_map_at_3_max": -0.230842, + "nauc_map_at_3_std": 0.578235, + "nauc_map_at_3_diff1": -0.228188, + "nauc_map_at_5_max": -0.155272, + "nauc_map_at_5_std": 0.605792, + "nauc_map_at_5_diff1": -0.162662, + "nauc_map_at_10_max": -0.130938, + "nauc_map_at_10_std": 0.435138, + "nauc_map_at_10_diff1": -0.167098, + "nauc_map_at_20_max": -0.090206, + "nauc_map_at_20_std": 0.254737, + "nauc_map_at_20_diff1": -0.13506, + "nauc_map_at_100_max": 0.005087, + "nauc_map_at_100_std": 0.102827, + "nauc_map_at_100_diff1": -0.125663, + "nauc_map_at_1000_max": 0.112355, + "nauc_map_at_1000_std": -0.038422, + "nauc_map_at_1000_diff1": -0.019212, + "nauc_recall_at_1_max": -0.333643, + "nauc_recall_at_1_std": 0.847267, + "nauc_recall_at_1_diff1": -0.17392, + "nauc_recall_at_3_max": -0.196308, + "nauc_recall_at_3_std": 0.376423, + "nauc_recall_at_3_diff1": -0.234921, + "nauc_recall_at_5_max": -0.05165, + "nauc_recall_at_5_std": 0.51737, + "nauc_recall_at_5_diff1": -0.093334, + "nauc_recall_at_10_max": -0.071493, + "nauc_recall_at_10_std": 0.227562, + "nauc_recall_at_10_diff1": -0.179297, + "nauc_recall_at_20_max": 0.009427, + "nauc_recall_at_20_std": 0.021691, + "nauc_recall_at_20_diff1": -0.047283, + "nauc_recall_at_100_max": 0.063325, + "nauc_recall_at_100_std": -0.09984, + "nauc_recall_at_100_diff1": -0.179249, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.042693, + "nauc_precision_at_1_std": 0.338155, + "nauc_precision_at_1_diff1": -0.054378, + "nauc_precision_at_3_max": -0.057391, + "nauc_precision_at_3_std": 0.180836, + "nauc_precision_at_3_diff1": -0.212888, + "nauc_precision_at_5_max": 0.035006, + "nauc_precision_at_5_std": 0.220133, + "nauc_precision_at_5_diff1": -0.135172, + "nauc_precision_at_10_max": 0.00857, + "nauc_precision_at_10_std": 0.088922, + "nauc_precision_at_10_diff1": -0.17955, + "nauc_precision_at_20_max": 0.058605, + "nauc_precision_at_20_std": 0.013436, + "nauc_precision_at_20_diff1": -0.123245, + "nauc_precision_at_100_max": 0.139024, + "nauc_precision_at_100_std": -0.088002, + "nauc_precision_at_100_diff1": -0.039697, + "nauc_precision_at_1000_max": 0.216881, + "nauc_precision_at_1000_std": -0.180577, + "nauc_precision_at_1000_diff1": 0.088815, + "nauc_cv_recall_at_1_max": -0.042693, + "nauc_cv_recall_at_1_std": 0.338155, + "nauc_cv_recall_at_1_diff1": -0.054378, + "nauc_cv_recall_at_3_max": -0.057391, + "nauc_cv_recall_at_3_std": 0.180836, + "nauc_cv_recall_at_3_diff1": -0.212888, + "nauc_cv_recall_at_5_max": 0.000347, + "nauc_cv_recall_at_5_std": 0.28285, + "nauc_cv_recall_at_5_diff1": -0.162412, + "nauc_cv_recall_at_10_max": 0.05837, + "nauc_cv_recall_at_10_std": 0.204528, + "nauc_cv_recall_at_10_diff1": -0.210459, + "nauc_cv_recall_at_20_max": 0.17176, + "nauc_cv_recall_at_20_std": 0.145558, + "nauc_cv_recall_at_20_diff1": -0.069486, + "nauc_cv_recall_at_100_max": 0.061284, + "nauc_cv_recall_at_100_std": -0.399799, + "nauc_cv_recall_at_100_diff1": -0.028581, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.042693, + "nauc_mrr_at_1_std": 0.338155, + "nauc_mrr_at_1_diff1": -0.054378, + "nauc_mrr_at_3_max": -0.026332, + "nauc_mrr_at_3_std": 0.253133, + "nauc_mrr_at_3_diff1": -0.183377, + "nauc_mrr_at_5_max": -0.000546, + "nauc_mrr_at_5_std": 0.286045, + "nauc_mrr_at_5_diff1": -0.162229, + "nauc_mrr_at_10_max": 0.011376, + "nauc_mrr_at_10_std": 0.248362, + "nauc_mrr_at_10_diff1": -0.178668, + "nauc_mrr_at_20_max": 0.025846, + "nauc_mrr_at_20_std": 0.237719, + "nauc_mrr_at_20_diff1": -0.156441, + "nauc_mrr_at_100_max": 0.012639, + "nauc_mrr_at_100_std": 0.223168, + "nauc_mrr_at_100_diff1": -0.153857, + "nauc_mrr_at_1000_max": 0.013729, + "nauc_mrr_at_1000_std": 0.230385, + "nauc_mrr_at_1000_diff1": -0.154062, + "main_score": 0.03741, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 22.84238362312317, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/model_meta.json b/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/model_meta.json index e4ccb721d2..a3fd50809f 100644 --- a/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/model_meta.json +++ b/results/jinaai__jina-clip-v1/06150c7c382d7a4faedc7d5a0d8cdb59308968f4/model_meta.json @@ -1 +1 @@ -{"name": "jinaai/jina-clip-v1", "revision": "06150c7c382d7a4faedc7d5a0d8cdb59308968f4", "release_date": "2024-05-30", "languages": ["eng_Latn"], "n_parameters": 223000000, "memory_usage_mb": 849.0, "max_tokens": 8192.0, "embed_dim": 768, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["PyTorch"], "reference": "https://huggingface.co/jinaai/jina-clip-v1", "similarity_fn_name": null, "use_instructions": true, "training_datasets": {"MSMARCO": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "JinaCLIPModelWrapper"} \ No newline at end of file +{"name": "jinaai/jina-clip-v1", "revision": "06150c7c382d7a4faedc7d5a0d8cdb59308968f4", "release_date": "2024-05-30", "languages": ["eng-Latn"], "n_parameters": 223000000, "memory_usage_mb": 849.0, "max_tokens": 8192.0, "embed_dim": 768, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["PyTorch"], "reference": "https://huggingface.co/jinaai/jina-clip-v1", "similarity_fn_name": null, "use_instructions": true, "training_datasets": {"MSMARCO": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "JinaCLIPModelWrapper"} \ No newline at end of file diff --git a/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/Vidore2BioMedicalLecturesRetrieval.json b/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..66d975b374 --- /dev/null +++ b/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.275, + "ndcg_at_3": 0.28268, + "ndcg_at_5": 0.31573, + "ndcg_at_10": 0.34823, + "ndcg_at_20": 0.37049, + "ndcg_at_100": 0.42965, + "ndcg_at_1000": 0.47791, + "map_at_1": 0.17931, + "map_at_3": 0.22454, + "map_at_5": 0.25074, + "map_at_10": 0.27041, + "map_at_20": 0.27986, + "map_at_100": 0.2919, + "map_at_1000": 0.29561, + "recall_at_1": 0.17931, + "recall_at_3": 0.26619, + "recall_at_5": 0.35098, + "recall_at_10": 0.45296, + "recall_at_20": 0.52304, + "recall_at_100": 0.7449, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.275, + "cv_recall_at_3": 0.41875, + "cv_recall_at_5": 0.51875, + "cv_recall_at_10": 0.64375, + "cv_recall_at_20": 0.73125, + "cv_recall_at_100": 0.9, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.275, + "precision_at_3": 0.15833, + "precision_at_5": 0.135, + "precision_at_10": 0.09188, + "precision_at_20": 0.05969, + "precision_at_100": 0.02075, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.275, + "mrr_at_3": 0.341667, + "mrr_at_5": 0.363542, + "mrr_at_10": 0.380871, + "mrr_at_20": 0.386745, + "mrr_at_100": 0.390653, + "mrr_at_1000": 0.391258, + "nauc_ndcg_at_1_max": 0.326346, + "nauc_ndcg_at_1_std": -0.054856, + "nauc_ndcg_at_1_diff1": 0.358409, + "nauc_ndcg_at_3_max": 0.252257, + "nauc_ndcg_at_3_std": -0.029285, + "nauc_ndcg_at_3_diff1": 0.282046, + "nauc_ndcg_at_5_max": 0.238808, + "nauc_ndcg_at_5_std": -0.014458, + "nauc_ndcg_at_5_diff1": 0.268528, + "nauc_ndcg_at_10_max": 0.220017, + "nauc_ndcg_at_10_std": -0.028065, + "nauc_ndcg_at_10_diff1": 0.284214, + "nauc_ndcg_at_20_max": 0.225422, + "nauc_ndcg_at_20_std": -0.025696, + "nauc_ndcg_at_20_diff1": 0.262873, + "nauc_ndcg_at_100_max": 0.200394, + "nauc_ndcg_at_100_std": -0.042359, + "nauc_ndcg_at_100_diff1": 0.263216, + "nauc_ndcg_at_1000_max": 0.234926, + "nauc_ndcg_at_1000_std": -0.040862, + "nauc_ndcg_at_1000_diff1": 0.280011, + "nauc_map_at_1_max": 0.304192, + "nauc_map_at_1_std": 0.040957, + "nauc_map_at_1_diff1": 0.320916, + "nauc_map_at_3_max": 0.256542, + "nauc_map_at_3_std": 0.020072, + "nauc_map_at_3_diff1": 0.289236, + "nauc_map_at_5_max": 0.241228, + "nauc_map_at_5_std": 0.006822, + "nauc_map_at_5_diff1": 0.277993, + "nauc_map_at_10_max": 0.234807, + "nauc_map_at_10_std": -0.003592, + "nauc_map_at_10_diff1": 0.284356, + "nauc_map_at_20_max": 0.233789, + "nauc_map_at_20_std": -0.008087, + "nauc_map_at_20_diff1": 0.274481, + "nauc_map_at_100_max": 0.23035, + "nauc_map_at_100_std": -0.012229, + "nauc_map_at_100_diff1": 0.277714, + "nauc_map_at_1000_max": 0.232322, + "nauc_map_at_1000_std": -0.012031, + "nauc_map_at_1000_diff1": 0.278072, + "nauc_recall_at_1_max": 0.304192, + "nauc_recall_at_1_std": 0.040957, + "nauc_recall_at_1_diff1": 0.320916, + "nauc_recall_at_3_max": 0.205584, + "nauc_recall_at_3_std": 0.004017, + "nauc_recall_at_3_diff1": 0.245758, + "nauc_recall_at_5_max": 0.160725, + "nauc_recall_at_5_std": 0.019429, + "nauc_recall_at_5_diff1": 0.185682, + "nauc_recall_at_10_max": 0.126128, + "nauc_recall_at_10_std": -0.016224, + "nauc_recall_at_10_diff1": 0.198035, + "nauc_recall_at_20_max": 0.156187, + "nauc_recall_at_20_std": -0.000995, + "nauc_recall_at_20_diff1": 0.113719, + "nauc_recall_at_100_max": -0.027926, + "nauc_recall_at_100_std": -0.040919, + "nauc_recall_at_100_diff1": 0.043457, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.326346, + "nauc_precision_at_1_std": -0.054856, + "nauc_precision_at_1_diff1": 0.358409, + "nauc_precision_at_3_max": 0.180544, + "nauc_precision_at_3_std": -0.114906, + "nauc_precision_at_3_diff1": 0.266668, + "nauc_precision_at_5_max": 0.135872, + "nauc_precision_at_5_std": -0.095938, + "nauc_precision_at_5_diff1": 0.229241, + "nauc_precision_at_10_max": 0.07396, + "nauc_precision_at_10_std": -0.155257, + "nauc_precision_at_10_diff1": 0.191698, + "nauc_precision_at_20_max": 0.051039, + "nauc_precision_at_20_std": -0.137533, + "nauc_precision_at_20_diff1": 0.079181, + "nauc_precision_at_100_max": -0.020585, + "nauc_precision_at_100_std": -0.12626, + "nauc_precision_at_100_diff1": -0.037418, + "nauc_precision_at_1000_max": 0.049373, + "nauc_precision_at_1000_std": -0.100195, + "nauc_precision_at_1000_diff1": -0.081609, + "nauc_cv_recall_at_1_max": 0.288235, + "nauc_cv_recall_at_1_std": -0.060799, + "nauc_cv_recall_at_1_diff1": 0.358409, + "nauc_cv_recall_at_3_max": 0.246755, + "nauc_cv_recall_at_3_std": -0.100713, + "nauc_cv_recall_at_3_diff1": 0.347088, + "nauc_cv_recall_at_5_max": 0.19596, + "nauc_cv_recall_at_5_std": 0.006232, + "nauc_cv_recall_at_5_diff1": 0.286163, + "nauc_cv_recall_at_10_max": 0.170434, + "nauc_cv_recall_at_10_std": -0.020704, + "nauc_cv_recall_at_10_diff1": 0.288847, + "nauc_cv_recall_at_20_max": 0.17279, + "nauc_cv_recall_at_20_std": -0.020843, + "nauc_cv_recall_at_20_diff1": 0.240981, + "nauc_cv_recall_at_100_max": 0.024481, + "nauc_cv_recall_at_100_std": -0.042338, + "nauc_cv_recall_at_100_diff1": 0.043447, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.288235, + "nauc_mrr_at_1_std": -0.060799, + "nauc_mrr_at_1_diff1": 0.358409, + "nauc_mrr_at_3_max": 0.263327, + "nauc_mrr_at_3_std": -0.078861, + "nauc_mrr_at_3_diff1": 0.35605, + "nauc_mrr_at_5_max": 0.252013, + "nauc_mrr_at_5_std": -0.055984, + "nauc_mrr_at_5_diff1": 0.341806, + "nauc_mrr_at_10_max": 0.25158, + "nauc_mrr_at_10_std": -0.057939, + "nauc_mrr_at_10_diff1": 0.342485, + "nauc_mrr_at_20_max": 0.252328, + "nauc_mrr_at_20_std": -0.058076, + "nauc_mrr_at_20_diff1": 0.340401, + "nauc_mrr_at_100_max": 0.251982, + "nauc_mrr_at_100_std": -0.060911, + "nauc_mrr_at_100_diff1": 0.339945, + "nauc_mrr_at_1000_max": 0.252329, + "nauc_mrr_at_1000_std": -0.060663, + "nauc_mrr_at_1000_diff1": 0.340076, + "main_score": 0.31573, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.2125, + "ndcg_at_3": 0.25943, + "ndcg_at_5": 0.29073, + "ndcg_at_10": 0.31799, + "ndcg_at_20": 0.34852, + "ndcg_at_100": 0.39869, + "ndcg_at_1000": 0.45103, + "map_at_1": 0.1263, + "map_at_3": 0.19561, + "map_at_5": 0.22012, + "map_at_10": 0.23817, + "map_at_20": 0.24978, + "map_at_100": 0.25899, + "map_at_1000": 0.26313, + "recall_at_1": 0.1263, + "recall_at_3": 0.27186, + "recall_at_5": 0.352, + "recall_at_10": 0.43473, + "recall_at_20": 0.53491, + "recall_at_100": 0.72351, + "recall_at_1000": 0.99978, + "cv_recall_at_1": 0.225, + "cv_recall_at_3": 0.43125, + "cv_recall_at_5": 0.5375, + "cv_recall_at_10": 0.60625, + "cv_recall_at_20": 0.7125, + "cv_recall_at_100": 0.88125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.2125, + "precision_at_3": 0.1625, + "precision_at_5": 0.135, + "precision_at_10": 0.09062, + "precision_at_20": 0.05937, + "precision_at_100": 0.01988, + "precision_at_1000": 0.00321, + "mrr_at_1": 0.225, + "mrr_at_3": 0.315625, + "mrr_at_5": 0.34125, + "mrr_at_10": 0.350456, + "mrr_at_20": 0.358023, + "mrr_at_100": 0.3618, + "mrr_at_1000": 0.362533, + "nauc_ndcg_at_1_max": 0.213076, + "nauc_ndcg_at_1_std": -0.111192, + "nauc_ndcg_at_1_diff1": 0.333754, + "nauc_ndcg_at_3_max": 0.114958, + "nauc_ndcg_at_3_std": -0.108474, + "nauc_ndcg_at_3_diff1": 0.204818, + "nauc_ndcg_at_5_max": 0.121931, + "nauc_ndcg_at_5_std": -0.086942, + "nauc_ndcg_at_5_diff1": 0.209707, + "nauc_ndcg_at_10_max": 0.115584, + "nauc_ndcg_at_10_std": -0.041957, + "nauc_ndcg_at_10_diff1": 0.21091, + "nauc_ndcg_at_20_max": 0.130613, + "nauc_ndcg_at_20_std": -0.043331, + "nauc_ndcg_at_20_diff1": 0.209773, + "nauc_ndcg_at_100_max": 0.129447, + "nauc_ndcg_at_100_std": -0.059195, + "nauc_ndcg_at_100_diff1": 0.211688, + "nauc_ndcg_at_1000_max": 0.128929, + "nauc_ndcg_at_1000_std": -0.084893, + "nauc_ndcg_at_1000_diff1": 0.208731, + "nauc_map_at_1_max": 0.173199, + "nauc_map_at_1_std": -0.020482, + "nauc_map_at_1_diff1": 0.273429, + "nauc_map_at_3_max": 0.120643, + "nauc_map_at_3_std": -0.052451, + "nauc_map_at_3_diff1": 0.202018, + "nauc_map_at_5_max": 0.125165, + "nauc_map_at_5_std": -0.055513, + "nauc_map_at_5_diff1": 0.21147, + "nauc_map_at_10_max": 0.117402, + "nauc_map_at_10_std": -0.049349, + "nauc_map_at_10_diff1": 0.217994, + "nauc_map_at_20_max": 0.120265, + "nauc_map_at_20_std": -0.055023, + "nauc_map_at_20_diff1": 0.219534, + "nauc_map_at_100_max": 0.12019, + "nauc_map_at_100_std": -0.057861, + "nauc_map_at_100_diff1": 0.21777, + "nauc_map_at_1000_max": 0.119569, + "nauc_map_at_1000_std": -0.060013, + "nauc_map_at_1000_diff1": 0.216739, + "nauc_recall_at_1_max": 0.173199, + "nauc_recall_at_1_std": -0.020482, + "nauc_recall_at_1_diff1": 0.273429, + "nauc_recall_at_3_max": 0.067653, + "nauc_recall_at_3_std": -0.062154, + "nauc_recall_at_3_diff1": 0.10464, + "nauc_recall_at_5_max": 0.079599, + "nauc_recall_at_5_std": -0.039067, + "nauc_recall_at_5_diff1": 0.110439, + "nauc_recall_at_10_max": 0.074004, + "nauc_recall_at_10_std": 0.050841, + "nauc_recall_at_10_diff1": 0.101194, + "nauc_recall_at_20_max": 0.124902, + "nauc_recall_at_20_std": 0.070472, + "nauc_recall_at_20_diff1": 0.111451, + "nauc_recall_at_100_max": 0.095248, + "nauc_recall_at_100_std": 0.05525, + "nauc_recall_at_100_diff1": 0.110845, + "nauc_recall_at_1000_max": 0.869281, + "nauc_recall_at_1000_std": 0.869281, + "nauc_recall_at_1000_diff1": 0.554155, + "nauc_precision_at_1_max": 0.213076, + "nauc_precision_at_1_std": -0.111192, + "nauc_precision_at_1_diff1": 0.333754, + "nauc_precision_at_3_max": 0.071254, + "nauc_precision_at_3_std": -0.183494, + "nauc_precision_at_3_diff1": 0.200473, + "nauc_precision_at_5_max": 0.053551, + "nauc_precision_at_5_std": -0.187385, + "nauc_precision_at_5_diff1": 0.191653, + "nauc_precision_at_10_max": -0.008149, + "nauc_precision_at_10_std": -0.127334, + "nauc_precision_at_10_diff1": 0.172005, + "nauc_precision_at_20_max": -0.005506, + "nauc_precision_at_20_std": -0.148176, + "nauc_precision_at_20_diff1": 0.101224, + "nauc_precision_at_100_max": -0.016921, + "nauc_precision_at_100_std": -0.15078, + "nauc_precision_at_100_diff1": -0.005054, + "nauc_precision_at_1000_max": -0.041512, + "nauc_precision_at_1000_std": -0.19101, + "nauc_precision_at_1000_diff1": -0.081724, + "nauc_cv_recall_at_1_max": 0.194175, + "nauc_cv_recall_at_1_std": -0.101547, + "nauc_cv_recall_at_1_diff1": 0.284782, + "nauc_cv_recall_at_3_max": 0.124369, + "nauc_cv_recall_at_3_std": -0.127662, + "nauc_cv_recall_at_3_diff1": 0.22348, + "nauc_cv_recall_at_5_max": 0.141023, + "nauc_cv_recall_at_5_std": -0.072724, + "nauc_cv_recall_at_5_diff1": 0.153977, + "nauc_cv_recall_at_10_max": 0.118194, + "nauc_cv_recall_at_10_std": 0.059941, + "nauc_cv_recall_at_10_diff1": 0.134111, + "nauc_cv_recall_at_20_max": 0.177317, + "nauc_cv_recall_at_20_std": 0.12957, + "nauc_cv_recall_at_20_diff1": 0.119374, + "nauc_cv_recall_at_100_max": 0.30832, + "nauc_cv_recall_at_100_std": 0.11265, + "nauc_cv_recall_at_100_diff1": 0.168975, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.194175, + "nauc_mrr_at_1_std": -0.101547, + "nauc_mrr_at_1_diff1": 0.284782, + "nauc_mrr_at_3_max": 0.164108, + "nauc_mrr_at_3_std": -0.122368, + "nauc_mrr_at_3_diff1": 0.265958, + "nauc_mrr_at_5_max": 0.168044, + "nauc_mrr_at_5_std": -0.10857, + "nauc_mrr_at_5_diff1": 0.250319, + "nauc_mrr_at_10_max": 0.165955, + "nauc_mrr_at_10_std": -0.091133, + "nauc_mrr_at_10_diff1": 0.249137, + "nauc_mrr_at_20_max": 0.167189, + "nauc_mrr_at_20_std": -0.091657, + "nauc_mrr_at_20_diff1": 0.248693, + "nauc_mrr_at_100_max": 0.168757, + "nauc_mrr_at_100_std": -0.094173, + "nauc_mrr_at_100_diff1": 0.250386, + "nauc_mrr_at_1000_max": 0.16821, + "nauc_mrr_at_1000_std": -0.094261, + "nauc_mrr_at_1000_diff1": 0.250093, + "main_score": 0.29073, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.3, + "ndcg_at_3": 0.30914, + "ndcg_at_5": 0.33976, + "ndcg_at_10": 0.37473, + "ndcg_at_20": 0.3988, + "ndcg_at_100": 0.45473, + "ndcg_at_1000": 0.49662, + "map_at_1": 0.17735, + "map_at_3": 0.23514, + "map_at_5": 0.26625, + "map_at_10": 0.29005, + "map_at_20": 0.29927, + "map_at_100": 0.31065, + "map_at_1000": 0.31413, + "recall_at_1": 0.17735, + "recall_at_3": 0.2983, + "recall_at_5": 0.38435, + "recall_at_10": 0.49171, + "recall_at_20": 0.57105, + "recall_at_100": 0.7855, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.30625, + "cv_recall_at_3": 0.48125, + "cv_recall_at_5": 0.58125, + "cv_recall_at_10": 0.6625, + "cv_recall_at_20": 0.7625, + "cv_recall_at_100": 0.9375, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.3, + "precision_at_3": 0.18333, + "precision_at_5": 0.15625, + "precision_at_10": 0.1075, + "precision_at_20": 0.06594, + "precision_at_100": 0.02144, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.30625, + "mrr_at_3": 0.384375, + "mrr_at_5": 0.405938, + "mrr_at_10": 0.416731, + "mrr_at_20": 0.423504, + "mrr_at_100": 0.427435, + "mrr_at_1000": 0.427768, + "nauc_ndcg_at_1_max": 0.068285, + "nauc_ndcg_at_1_std": -0.199348, + "nauc_ndcg_at_1_diff1": 0.29708, + "nauc_ndcg_at_3_max": -0.011197, + "nauc_ndcg_at_3_std": -0.115241, + "nauc_ndcg_at_3_diff1": 0.263671, + "nauc_ndcg_at_5_max": 0.011403, + "nauc_ndcg_at_5_std": -0.136647, + "nauc_ndcg_at_5_diff1": 0.263115, + "nauc_ndcg_at_10_max": -0.00901, + "nauc_ndcg_at_10_std": -0.11264, + "nauc_ndcg_at_10_diff1": 0.236615, + "nauc_ndcg_at_20_max": 0.005317, + "nauc_ndcg_at_20_std": -0.11825, + "nauc_ndcg_at_20_diff1": 0.250771, + "nauc_ndcg_at_100_max": -0.00953, + "nauc_ndcg_at_100_std": -0.133614, + "nauc_ndcg_at_100_diff1": 0.24475, + "nauc_ndcg_at_1000_max": 0.017934, + "nauc_ndcg_at_1000_std": -0.150596, + "nauc_ndcg_at_1000_diff1": 0.261694, + "nauc_map_at_1_max": 0.019234, + "nauc_map_at_1_std": -0.141208, + "nauc_map_at_1_diff1": 0.394401, + "nauc_map_at_3_max": -0.027518, + "nauc_map_at_3_std": -0.090329, + "nauc_map_at_3_diff1": 0.32737, + "nauc_map_at_5_max": -0.026348, + "nauc_map_at_5_std": -0.12402, + "nauc_map_at_5_diff1": 0.306907, + "nauc_map_at_10_max": -0.040905, + "nauc_map_at_10_std": -0.12226, + "nauc_map_at_10_diff1": 0.277677, + "nauc_map_at_20_max": -0.033866, + "nauc_map_at_20_std": -0.124731, + "nauc_map_at_20_diff1": 0.281085, + "nauc_map_at_100_max": -0.034047, + "nauc_map_at_100_std": -0.130007, + "nauc_map_at_100_diff1": 0.277558, + "nauc_map_at_1000_max": -0.032145, + "nauc_map_at_1000_std": -0.132097, + "nauc_map_at_1000_diff1": 0.278342, + "nauc_recall_at_1_max": 0.019234, + "nauc_recall_at_1_std": -0.141208, + "nauc_recall_at_1_diff1": 0.394401, + "nauc_recall_at_3_max": -0.038276, + "nauc_recall_at_3_std": -0.005445, + "nauc_recall_at_3_diff1": 0.243474, + "nauc_recall_at_5_max": -0.004059, + "nauc_recall_at_5_std": -0.074459, + "nauc_recall_at_5_diff1": 0.18771, + "nauc_recall_at_10_max": -0.035073, + "nauc_recall_at_10_std": -0.011152, + "nauc_recall_at_10_diff1": 0.087043, + "nauc_recall_at_20_max": 0.025221, + "nauc_recall_at_20_std": -0.035539, + "nauc_recall_at_20_diff1": 0.12072, + "nauc_recall_at_100_max": -0.127432, + "nauc_recall_at_100_std": -0.008745, + "nauc_recall_at_100_diff1": 0.029244, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.068285, + "nauc_precision_at_1_std": -0.199348, + "nauc_precision_at_1_diff1": 0.29708, + "nauc_precision_at_3_max": 0.007012, + "nauc_precision_at_3_std": -0.151722, + "nauc_precision_at_3_diff1": 0.161315, + "nauc_precision_at_5_max": 0.02274, + "nauc_precision_at_5_std": -0.177116, + "nauc_precision_at_5_diff1": 0.094527, + "nauc_precision_at_10_max": 0.004819, + "nauc_precision_at_10_std": -0.161246, + "nauc_precision_at_10_diff1": -0.005353, + "nauc_precision_at_20_max": 0.05403, + "nauc_precision_at_20_std": -0.157124, + "nauc_precision_at_20_diff1": -0.009366, + "nauc_precision_at_100_max": 0.070926, + "nauc_precision_at_100_std": -0.15244, + "nauc_precision_at_100_diff1": -0.08828, + "nauc_precision_at_1000_max": 0.157294, + "nauc_precision_at_1000_std": -0.158348, + "nauc_precision_at_1000_diff1": -0.10256, + "nauc_cv_recall_at_1_max": 0.046715, + "nauc_cv_recall_at_1_std": -0.170184, + "nauc_cv_recall_at_1_diff1": 0.278129, + "nauc_cv_recall_at_3_max": 0.063945, + "nauc_cv_recall_at_3_std": -0.065782, + "nauc_cv_recall_at_3_diff1": 0.262069, + "nauc_cv_recall_at_5_max": 0.087877, + "nauc_cv_recall_at_5_std": -0.095178, + "nauc_cv_recall_at_5_diff1": 0.187887, + "nauc_cv_recall_at_10_max": 0.058861, + "nauc_cv_recall_at_10_std": -0.015896, + "nauc_cv_recall_at_10_diff1": 0.1002, + "nauc_cv_recall_at_20_max": 0.162395, + "nauc_cv_recall_at_20_std": -0.139258, + "nauc_cv_recall_at_20_diff1": 0.127383, + "nauc_cv_recall_at_100_max": 0.088235, + "nauc_cv_recall_at_100_std": -0.167834, + "nauc_cv_recall_at_100_diff1": 0.249486, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.046715, + "nauc_mrr_at_1_std": -0.170184, + "nauc_mrr_at_1_diff1": 0.278129, + "nauc_mrr_at_3_max": 0.043133, + "nauc_mrr_at_3_std": -0.134124, + "nauc_mrr_at_3_diff1": 0.270429, + "nauc_mrr_at_5_max": 0.050712, + "nauc_mrr_at_5_std": -0.140546, + "nauc_mrr_at_5_diff1": 0.255222, + "nauc_mrr_at_10_max": 0.045752, + "nauc_mrr_at_10_std": -0.132923, + "nauc_mrr_at_10_diff1": 0.245974, + "nauc_mrr_at_20_max": 0.049752, + "nauc_mrr_at_20_std": -0.139361, + "nauc_mrr_at_20_diff1": 0.248721, + "nauc_mrr_at_100_max": 0.047368, + "nauc_mrr_at_100_std": -0.140396, + "nauc_mrr_at_100_diff1": 0.249172, + "nauc_mrr_at_1000_max": 0.047302, + "nauc_mrr_at_1000_std": -0.140207, + "nauc_mrr_at_1000_diff1": 0.248983, + "main_score": 0.33976, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.25, + "ndcg_at_3": 0.26134, + "ndcg_at_5": 0.29687, + "ndcg_at_10": 0.32714, + "ndcg_at_20": 0.34893, + "ndcg_at_100": 0.4107, + "ndcg_at_1000": 0.46004, + "map_at_1": 0.16416, + "map_at_3": 0.20991, + "map_at_5": 0.23662, + "map_at_10": 0.25759, + "map_at_20": 0.26496, + "map_at_100": 0.27674, + "map_at_1000": 0.28038, + "recall_at_1": 0.16416, + "recall_at_3": 0.25172, + "recall_at_5": 0.3459, + "recall_at_10": 0.43045, + "recall_at_20": 0.50277, + "recall_at_100": 0.73546, + "recall_at_1000": 0.99978, + "cv_recall_at_1": 0.25625, + "cv_recall_at_3": 0.3875, + "cv_recall_at_5": 0.4875, + "cv_recall_at_10": 0.575, + "cv_recall_at_20": 0.675, + "cv_recall_at_100": 0.89375, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.25, + "precision_at_3": 0.15208, + "precision_at_5": 0.1275, + "precision_at_10": 0.09, + "precision_at_20": 0.05531, + "precision_at_100": 0.02031, + "precision_at_1000": 0.00321, + "mrr_at_1": 0.25625, + "mrr_at_3": 0.316667, + "mrr_at_5": 0.340417, + "mrr_at_10": 0.352636, + "mrr_at_20": 0.359398, + "mrr_at_100": 0.36527, + "mrr_at_1000": 0.365844, + "nauc_ndcg_at_1_max": 0.203843, + "nauc_ndcg_at_1_std": -0.012643, + "nauc_ndcg_at_1_diff1": 0.303981, + "nauc_ndcg_at_3_max": 0.167123, + "nauc_ndcg_at_3_std": 0.020744, + "nauc_ndcg_at_3_diff1": 0.223313, + "nauc_ndcg_at_5_max": 0.151675, + "nauc_ndcg_at_5_std": -0.011203, + "nauc_ndcg_at_5_diff1": 0.241265, + "nauc_ndcg_at_10_max": 0.179298, + "nauc_ndcg_at_10_std": 0.020742, + "nauc_ndcg_at_10_diff1": 0.263463, + "nauc_ndcg_at_20_max": 0.170228, + "nauc_ndcg_at_20_std": 0.022746, + "nauc_ndcg_at_20_diff1": 0.27932, + "nauc_ndcg_at_100_max": 0.154231, + "nauc_ndcg_at_100_std": -0.011576, + "nauc_ndcg_at_100_diff1": 0.275138, + "nauc_ndcg_at_1000_max": 0.16156, + "nauc_ndcg_at_1000_std": -0.023326, + "nauc_ndcg_at_1000_diff1": 0.261193, + "nauc_map_at_1_max": 0.151, + "nauc_map_at_1_std": 0.09159, + "nauc_map_at_1_diff1": 0.259206, + "nauc_map_at_3_max": 0.152597, + "nauc_map_at_3_std": 0.074636, + "nauc_map_at_3_diff1": 0.24169, + "nauc_map_at_5_max": 0.137425, + "nauc_map_at_5_std": 0.034497, + "nauc_map_at_5_diff1": 0.248777, + "nauc_map_at_10_max": 0.139792, + "nauc_map_at_10_std": 0.033852, + "nauc_map_at_10_diff1": 0.247545, + "nauc_map_at_20_max": 0.135897, + "nauc_map_at_20_std": 0.032935, + "nauc_map_at_20_diff1": 0.251182, + "nauc_map_at_100_max": 0.133487, + "nauc_map_at_100_std": 0.023776, + "nauc_map_at_100_diff1": 0.252466, + "nauc_map_at_1000_max": 0.133681, + "nauc_map_at_1000_std": 0.021844, + "nauc_map_at_1000_diff1": 0.250607, + "nauc_recall_at_1_max": 0.151, + "nauc_recall_at_1_std": 0.09159, + "nauc_recall_at_1_diff1": 0.259206, + "nauc_recall_at_3_max": 0.165846, + "nauc_recall_at_3_std": 0.087341, + "nauc_recall_at_3_diff1": 0.223439, + "nauc_recall_at_5_max": 0.128582, + "nauc_recall_at_5_std": -0.003202, + "nauc_recall_at_5_diff1": 0.22969, + "nauc_recall_at_10_max": 0.202569, + "nauc_recall_at_10_std": 0.06798, + "nauc_recall_at_10_diff1": 0.244909, + "nauc_recall_at_20_max": 0.179137, + "nauc_recall_at_20_std": 0.06708, + "nauc_recall_at_20_diff1": 0.282943, + "nauc_recall_at_100_max": 0.095079, + "nauc_recall_at_100_std": -0.004931, + "nauc_recall_at_100_diff1": 0.265987, + "nauc_recall_at_1000_max": 0.869281, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 0.722222, + "nauc_precision_at_1_max": 0.203843, + "nauc_precision_at_1_std": -0.012643, + "nauc_precision_at_1_diff1": 0.303981, + "nauc_precision_at_3_max": 0.143639, + "nauc_precision_at_3_std": -0.097818, + "nauc_precision_at_3_diff1": 0.175538, + "nauc_precision_at_5_max": 0.072855, + "nauc_precision_at_5_std": -0.194726, + "nauc_precision_at_5_diff1": 0.14742, + "nauc_precision_at_10_max": 0.094366, + "nauc_precision_at_10_std": -0.157748, + "nauc_precision_at_10_diff1": 0.11967, + "nauc_precision_at_20_max": 0.069415, + "nauc_precision_at_20_std": -0.166858, + "nauc_precision_at_20_diff1": 0.104408, + "nauc_precision_at_100_max": -0.013752, + "nauc_precision_at_100_std": -0.252209, + "nauc_precision_at_100_diff1": -0.066247, + "nauc_precision_at_1000_max": -0.028799, + "nauc_precision_at_1000_std": -0.261217, + "nauc_precision_at_1000_diff1": -0.158363, + "nauc_cv_recall_at_1_max": 0.182127, + "nauc_cv_recall_at_1_std": -0.029436, + "nauc_cv_recall_at_1_diff1": 0.281166, + "nauc_cv_recall_at_3_max": 0.238293, + "nauc_cv_recall_at_3_std": -0.016212, + "nauc_cv_recall_at_3_diff1": 0.244241, + "nauc_cv_recall_at_5_max": 0.193788, + "nauc_cv_recall_at_5_std": -0.071066, + "nauc_cv_recall_at_5_diff1": 0.275592, + "nauc_cv_recall_at_10_max": 0.297864, + "nauc_cv_recall_at_10_std": 0.012924, + "nauc_cv_recall_at_10_diff1": 0.291109, + "nauc_cv_recall_at_20_max": 0.246772, + "nauc_cv_recall_at_20_std": -0.016278, + "nauc_cv_recall_at_20_diff1": 0.321876, + "nauc_cv_recall_at_100_max": 0.30041, + "nauc_cv_recall_at_100_std": -0.175458, + "nauc_cv_recall_at_100_diff1": 0.450047, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.182127, + "nauc_mrr_at_1_std": -0.029436, + "nauc_mrr_at_1_diff1": 0.281166, + "nauc_mrr_at_3_max": 0.212397, + "nauc_mrr_at_3_std": -0.023317, + "nauc_mrr_at_3_diff1": 0.266225, + "nauc_mrr_at_5_max": 0.203056, + "nauc_mrr_at_5_std": -0.037152, + "nauc_mrr_at_5_diff1": 0.276745, + "nauc_mrr_at_10_max": 0.216965, + "nauc_mrr_at_10_std": -0.026388, + "nauc_mrr_at_10_diff1": 0.278754, + "nauc_mrr_at_20_max": 0.212708, + "nauc_mrr_at_20_std": -0.028408, + "nauc_mrr_at_20_diff1": 0.28233, + "nauc_mrr_at_100_max": 0.211022, + "nauc_mrr_at_100_std": -0.031394, + "nauc_mrr_at_100_diff1": 0.283238, + "nauc_mrr_at_1000_max": 0.210744, + "nauc_mrr_at_1000_std": -0.031134, + "nauc_mrr_at_1000_diff1": 0.282754, + "main_score": 0.29687, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 143.7470419406891, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/Vidore2ESGReportsHLRetrieval.json b/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..d9885e3bc9 --- /dev/null +++ b/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.21154, + "ndcg_at_3": 0.2229, + "ndcg_at_5": 0.25332, + "ndcg_at_10": 0.29016, + "ndcg_at_20": 0.31971, + "ndcg_at_100": 0.35654, + "ndcg_at_1000": 0.40748, + "map_at_1": 0.14167, + "map_at_3": 0.17885, + "map_at_5": 0.20149, + "map_at_10": 0.22042, + "map_at_20": 0.23136, + "map_at_100": 0.23763, + "map_at_1000": 0.24077, + "recall_at_1": 0.14167, + "recall_at_3": 0.23205, + "recall_at_5": 0.31717, + "recall_at_10": 0.41924, + "recall_at_20": 0.51127, + "recall_at_100": 0.66332, + "recall_at_1000": 0.98077, + "cv_recall_at_1": 0.19231, + "cv_recall_at_3": 0.34615, + "cv_recall_at_5": 0.44231, + "cv_recall_at_10": 0.53846, + "cv_recall_at_20": 0.65385, + "cv_recall_at_100": 0.80769, + "cv_recall_at_1000": 0.98077, + "precision_at_1": 0.21154, + "precision_at_3": 0.12179, + "precision_at_5": 0.10385, + "precision_at_10": 0.075, + "precision_at_20": 0.05, + "precision_at_100": 0.01462, + "precision_at_1000": 0.00244, + "mrr_at_1": 0.192308, + "mrr_at_3": 0.25641, + "mrr_at_5": 0.276603, + "mrr_at_10": 0.289019, + "mrr_at_20": 0.296744, + "mrr_at_100": 0.301183, + "mrr_at_1000": 0.302052, + "nauc_ndcg_at_1_max": -0.035295, + "nauc_ndcg_at_1_std": -0.111527, + "nauc_ndcg_at_1_diff1": 0.261929, + "nauc_ndcg_at_3_max": 0.086625, + "nauc_ndcg_at_3_std": -0.07529, + "nauc_ndcg_at_3_diff1": 0.367759, + "nauc_ndcg_at_5_max": 0.038822, + "nauc_ndcg_at_5_std": -0.094191, + "nauc_ndcg_at_5_diff1": 0.399152, + "nauc_ndcg_at_10_max": 0.045792, + "nauc_ndcg_at_10_std": -0.012491, + "nauc_ndcg_at_10_diff1": 0.338527, + "nauc_ndcg_at_20_max": 0.040451, + "nauc_ndcg_at_20_std": 0.028329, + "nauc_ndcg_at_20_diff1": 0.28825, + "nauc_ndcg_at_100_max": 0.049907, + "nauc_ndcg_at_100_std": 0.052423, + "nauc_ndcg_at_100_diff1": 0.293234, + "nauc_ndcg_at_1000_max": 0.057791, + "nauc_ndcg_at_1000_std": 0.024382, + "nauc_ndcg_at_1000_diff1": 0.289358, + "nauc_map_at_1_max": -0.043081, + "nauc_map_at_1_std": -0.124478, + "nauc_map_at_1_diff1": 0.461699, + "nauc_map_at_3_max": 0.040807, + "nauc_map_at_3_std": -0.112059, + "nauc_map_at_3_diff1": 0.444489, + "nauc_map_at_5_max": 0.015615, + "nauc_map_at_5_std": -0.120243, + "nauc_map_at_5_diff1": 0.455923, + "nauc_map_at_10_max": 0.030505, + "nauc_map_at_10_std": -0.073073, + "nauc_map_at_10_diff1": 0.414432, + "nauc_map_at_20_max": 0.028544, + "nauc_map_at_20_std": -0.054727, + "nauc_map_at_20_diff1": 0.393466, + "nauc_map_at_100_max": 0.033178, + "nauc_map_at_100_std": -0.051606, + "nauc_map_at_100_diff1": 0.392571, + "nauc_map_at_1000_max": 0.03386, + "nauc_map_at_1000_std": -0.053018, + "nauc_map_at_1000_diff1": 0.391521, + "nauc_recall_at_1_max": -0.043081, + "nauc_recall_at_1_std": -0.124478, + "nauc_recall_at_1_diff1": 0.461699, + "nauc_recall_at_3_max": 0.129912, + "nauc_recall_at_3_std": -0.076907, + "nauc_recall_at_3_diff1": 0.386334, + "nauc_recall_at_5_max": 0.031544, + "nauc_recall_at_5_std": -0.078092, + "nauc_recall_at_5_diff1": 0.395687, + "nauc_recall_at_10_max": 0.04854, + "nauc_recall_at_10_std": 0.066138, + "nauc_recall_at_10_diff1": 0.269858, + "nauc_recall_at_20_max": 0.003545, + "nauc_recall_at_20_std": 0.130071, + "nauc_recall_at_20_diff1": 0.144563, + "nauc_recall_at_100_max": 0.019149, + "nauc_recall_at_100_std": 0.264938, + "nauc_recall_at_100_diff1": 0.185605, + "nauc_recall_at_1000_max": -0.552063, + "nauc_recall_at_1000_std": 0.869837, + "nauc_recall_at_1000_diff1": -0.552063, + "nauc_precision_at_1_max": -0.035295, + "nauc_precision_at_1_std": -0.111527, + "nauc_precision_at_1_diff1": 0.261929, + "nauc_precision_at_3_max": 0.172708, + "nauc_precision_at_3_std": 0.012071, + "nauc_precision_at_3_diff1": 0.216618, + "nauc_precision_at_5_max": 0.184547, + "nauc_precision_at_5_std": 0.057542, + "nauc_precision_at_5_diff1": 0.166831, + "nauc_precision_at_10_max": 0.210089, + "nauc_precision_at_10_std": 0.231235, + "nauc_precision_at_10_diff1": -0.026211, + "nauc_precision_at_20_max": 0.212896, + "nauc_precision_at_20_std": 0.326718, + "nauc_precision_at_20_diff1": -0.165902, + "nauc_precision_at_100_max": 0.229752, + "nauc_precision_at_100_std": 0.287216, + "nauc_precision_at_100_diff1": -0.206336, + "nauc_precision_at_1000_max": 0.259881, + "nauc_precision_at_1000_std": 0.142331, + "nauc_precision_at_1000_diff1": -0.194626, + "nauc_cv_recall_at_1_max": 0.012653, + "nauc_cv_recall_at_1_std": -0.078005, + "nauc_cv_recall_at_1_diff1": 0.339808, + "nauc_cv_recall_at_3_max": 0.159334, + "nauc_cv_recall_at_3_std": 0.043445, + "nauc_cv_recall_at_3_diff1": 0.228331, + "nauc_cv_recall_at_5_max": 0.139162, + "nauc_cv_recall_at_5_std": 0.132381, + "nauc_cv_recall_at_5_diff1": 0.204283, + "nauc_cv_recall_at_10_max": 0.152873, + "nauc_cv_recall_at_10_std": 0.182951, + "nauc_cv_recall_at_10_diff1": 0.162966, + "nauc_cv_recall_at_20_max": 0.105444, + "nauc_cv_recall_at_20_std": 0.328105, + "nauc_cv_recall_at_20_diff1": -0.056326, + "nauc_cv_recall_at_100_max": 0.162219, + "nauc_cv_recall_at_100_std": 0.3533, + "nauc_cv_recall_at_100_diff1": 0.149759, + "nauc_cv_recall_at_1000_max": -0.552063, + "nauc_cv_recall_at_1000_std": 0.869837, + "nauc_cv_recall_at_1000_diff1": -0.552063, + "nauc_mrr_at_1_max": 0.012653, + "nauc_mrr_at_1_std": -0.078005, + "nauc_mrr_at_1_diff1": 0.339808, + "nauc_mrr_at_3_max": 0.082729, + "nauc_mrr_at_3_std": -0.027633, + "nauc_mrr_at_3_diff1": 0.290167, + "nauc_mrr_at_5_max": 0.083772, + "nauc_mrr_at_5_std": -0.003569, + "nauc_mrr_at_5_diff1": 0.282122, + "nauc_mrr_at_10_max": 0.083759, + "nauc_mrr_at_10_std": 0.000167, + "nauc_mrr_at_10_diff1": 0.271762, + "nauc_mrr_at_20_max": 0.080172, + "nauc_mrr_at_20_std": 0.006912, + "nauc_mrr_at_20_diff1": 0.259596, + "nauc_mrr_at_100_max": 0.076937, + "nauc_mrr_at_100_std": 0.003378, + "nauc_mrr_at_100_diff1": 0.263333, + "nauc_mrr_at_1000_max": 0.076924, + "nauc_mrr_at_1000_std": 0.002142, + "nauc_mrr_at_1000_diff1": 0.263416, + "main_score": 0.25332, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 138.12463974952698, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/Vidore2ESGReportsRetrieval.json b/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..a005167036 --- /dev/null +++ b/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.14035, + "ndcg_at_3": 0.12804, + "ndcg_at_5": 0.15097, + "ndcg_at_10": 0.17756, + "ndcg_at_20": 0.20338, + "ndcg_at_100": 0.27123, + "ndcg_at_1000": 0.3313, + "map_at_1": 0.04225, + "map_at_3": 0.06781, + "map_at_5": 0.09083, + "map_at_10": 0.10348, + "map_at_20": 0.11339, + "map_at_100": 0.12615, + "map_at_1000": 0.1299, + "recall_at_1": 0.04225, + "recall_at_3": 0.10414, + "recall_at_5": 0.17285, + "recall_at_10": 0.26203, + "recall_at_20": 0.34808, + "recall_at_100": 0.64315, + "recall_at_1000": 0.94893, + "cv_recall_at_1": 0.14035, + "cv_recall_at_3": 0.26316, + "cv_recall_at_5": 0.35088, + "cv_recall_at_10": 0.50877, + "cv_recall_at_20": 0.5614, + "cv_recall_at_100": 0.84211, + "cv_recall_at_1000": 0.96491, + "precision_at_1": 0.14035, + "precision_at_3": 0.10526, + "precision_at_5": 0.09825, + "precision_at_10": 0.07018, + "precision_at_20": 0.04825, + "precision_at_100": 0.01737, + "precision_at_1000": 0.0036, + "mrr_at_1": 0.140351, + "mrr_at_3": 0.192982, + "mrr_at_5": 0.214035, + "mrr_at_10": 0.236104, + "mrr_at_20": 0.240084, + "mrr_at_100": 0.246923, + "mrr_at_1000": 0.247512, + "nauc_ndcg_at_1_max": 0.119058, + "nauc_ndcg_at_1_std": 0.10422, + "nauc_ndcg_at_1_diff1": -0.156457, + "nauc_ndcg_at_3_max": 0.170408, + "nauc_ndcg_at_3_std": 0.117116, + "nauc_ndcg_at_3_diff1": -0.167538, + "nauc_ndcg_at_5_max": 0.197046, + "nauc_ndcg_at_5_std": 0.094016, + "nauc_ndcg_at_5_diff1": -0.19499, + "nauc_ndcg_at_10_max": 0.172382, + "nauc_ndcg_at_10_std": 0.161901, + "nauc_ndcg_at_10_diff1": -0.209244, + "nauc_ndcg_at_20_max": 0.181682, + "nauc_ndcg_at_20_std": 0.223511, + "nauc_ndcg_at_20_diff1": -0.178423, + "nauc_ndcg_at_100_max": 0.086533, + "nauc_ndcg_at_100_std": 0.2032, + "nauc_ndcg_at_100_diff1": -0.27886, + "nauc_ndcg_at_1000_max": 0.191941, + "nauc_ndcg_at_1000_std": 0.212045, + "nauc_ndcg_at_1000_diff1": -0.207993, + "nauc_map_at_1_max": 0.018043, + "nauc_map_at_1_std": 0.005236, + "nauc_map_at_1_diff1": -0.179406, + "nauc_map_at_3_max": 0.066676, + "nauc_map_at_3_std": 0.070348, + "nauc_map_at_3_diff1": -0.183815, + "nauc_map_at_5_max": 0.096499, + "nauc_map_at_5_std": 0.102982, + "nauc_map_at_5_diff1": -0.196399, + "nauc_map_at_10_max": 0.103178, + "nauc_map_at_10_std": 0.127865, + "nauc_map_at_10_diff1": -0.201834, + "nauc_map_at_20_max": 0.126772, + "nauc_map_at_20_std": 0.1535, + "nauc_map_at_20_diff1": -0.192011, + "nauc_map_at_100_max": 0.117896, + "nauc_map_at_100_std": 0.153142, + "nauc_map_at_100_diff1": -0.225653, + "nauc_map_at_1000_max": 0.127785, + "nauc_map_at_1000_std": 0.155794, + "nauc_map_at_1000_diff1": -0.215475, + "nauc_recall_at_1_max": 0.018043, + "nauc_recall_at_1_std": 0.005236, + "nauc_recall_at_1_diff1": -0.179406, + "nauc_recall_at_3_max": 0.071966, + "nauc_recall_at_3_std": 0.095576, + "nauc_recall_at_3_diff1": -0.137167, + "nauc_recall_at_5_max": 0.103244, + "nauc_recall_at_5_std": 0.096935, + "nauc_recall_at_5_diff1": -0.135655, + "nauc_recall_at_10_max": 0.124546, + "nauc_recall_at_10_std": 0.122966, + "nauc_recall_at_10_diff1": -0.140701, + "nauc_recall_at_20_max": 0.113962, + "nauc_recall_at_20_std": 0.226906, + "nauc_recall_at_20_diff1": -0.085589, + "nauc_recall_at_100_max": -0.223381, + "nauc_recall_at_100_std": 0.187894, + "nauc_recall_at_100_diff1": -0.290551, + "nauc_recall_at_1000_max": 0.299163, + "nauc_recall_at_1000_std": 0.837343, + "nauc_recall_at_1000_diff1": 0.442715, + "nauc_precision_at_1_max": 0.119058, + "nauc_precision_at_1_std": 0.10422, + "nauc_precision_at_1_diff1": -0.156457, + "nauc_precision_at_3_max": 0.196612, + "nauc_precision_at_3_std": 0.157458, + "nauc_precision_at_3_diff1": -0.179016, + "nauc_precision_at_5_max": 0.313966, + "nauc_precision_at_5_std": 0.131618, + "nauc_precision_at_5_diff1": -0.245032, + "nauc_precision_at_10_max": 0.345968, + "nauc_precision_at_10_std": 0.283988, + "nauc_precision_at_10_diff1": -0.277447, + "nauc_precision_at_20_max": 0.371139, + "nauc_precision_at_20_std": 0.297849, + "nauc_precision_at_20_diff1": -0.140208, + "nauc_precision_at_100_max": 0.264039, + "nauc_precision_at_100_std": 0.204442, + "nauc_precision_at_100_diff1": -0.317103, + "nauc_precision_at_1000_max": 0.22363, + "nauc_precision_at_1000_std": -0.024445, + "nauc_precision_at_1000_diff1": -0.003112, + "nauc_cv_recall_at_1_max": 0.119058, + "nauc_cv_recall_at_1_std": 0.10422, + "nauc_cv_recall_at_1_diff1": -0.156457, + "nauc_cv_recall_at_3_max": 0.141912, + "nauc_cv_recall_at_3_std": 0.134304, + "nauc_cv_recall_at_3_diff1": -0.140657, + "nauc_cv_recall_at_5_max": 0.272026, + "nauc_cv_recall_at_5_std": 0.107967, + "nauc_cv_recall_at_5_diff1": -0.206771, + "nauc_cv_recall_at_10_max": 0.340839, + "nauc_cv_recall_at_10_std": 0.3358, + "nauc_cv_recall_at_10_diff1": -0.231771, + "nauc_cv_recall_at_20_max": 0.323025, + "nauc_cv_recall_at_20_std": 0.426511, + "nauc_cv_recall_at_20_diff1": -0.205928, + "nauc_cv_recall_at_100_max": -0.078763, + "nauc_cv_recall_at_100_std": 0.265569, + "nauc_cv_recall_at_100_diff1": -0.407252, + "nauc_cv_recall_at_1000_max": 0.417528, + "nauc_cv_recall_at_1000_std": 0.934256, + "nauc_cv_recall_at_1000_diff1": 0.497564, + "nauc_mrr_at_1_max": 0.119058, + "nauc_mrr_at_1_std": 0.10422, + "nauc_mrr_at_1_diff1": -0.156457, + "nauc_mrr_at_3_max": 0.132226, + "nauc_mrr_at_3_std": 0.109693, + "nauc_mrr_at_3_diff1": -0.149165, + "nauc_mrr_at_5_max": 0.170408, + "nauc_mrr_at_5_std": 0.102368, + "nauc_mrr_at_5_diff1": -0.16697, + "nauc_mrr_at_10_max": 0.177533, + "nauc_mrr_at_10_std": 0.150873, + "nauc_mrr_at_10_diff1": -0.170648, + "nauc_mrr_at_20_max": 0.176997, + "nauc_mrr_at_20_std": 0.157073, + "nauc_mrr_at_20_diff1": -0.166533, + "nauc_mrr_at_100_max": 0.16622, + "nauc_mrr_at_100_std": 0.148436, + "nauc_mrr_at_100_diff1": -0.169572, + "nauc_mrr_at_1000_max": 0.167246, + "nauc_mrr_at_1000_std": 0.14873, + "nauc_mrr_at_1000_diff1": -0.168431, + "main_score": 0.15097, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.19298, + "ndcg_at_3": 0.16418, + "ndcg_at_5": 0.16327, + "ndcg_at_10": 0.1917, + "ndcg_at_20": 0.21764, + "ndcg_at_100": 0.28217, + "ndcg_at_1000": 0.34763, + "map_at_1": 0.05322, + "map_at_3": 0.08866, + "map_at_5": 0.10152, + "map_at_10": 0.11703, + "map_at_20": 0.12606, + "map_at_100": 0.13925, + "map_at_1000": 0.14359, + "recall_at_1": 0.05322, + "recall_at_3": 0.13703, + "recall_at_5": 0.17139, + "recall_at_10": 0.27007, + "recall_at_20": 0.35637, + "recall_at_100": 0.62502, + "recall_at_1000": 0.96794, + "cv_recall_at_1": 0.19298, + "cv_recall_at_3": 0.33333, + "cv_recall_at_5": 0.35088, + "cv_recall_at_10": 0.47368, + "cv_recall_at_20": 0.57895, + "cv_recall_at_100": 0.80702, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.19298, + "precision_at_3": 0.12865, + "precision_at_5": 0.09474, + "precision_at_10": 0.06842, + "precision_at_20": 0.04912, + "precision_at_100": 0.01737, + "precision_at_1000": 0.00365, + "mrr_at_1": 0.192982, + "mrr_at_3": 0.245614, + "mrr_at_5": 0.25, + "mrr_at_10": 0.266514, + "mrr_at_20": 0.273803, + "mrr_at_100": 0.280504, + "mrr_at_1000": 0.281434, + "nauc_ndcg_at_1_max": 0.462524, + "nauc_ndcg_at_1_std": 0.23998, + "nauc_ndcg_at_1_diff1": -0.253105, + "nauc_ndcg_at_3_max": 0.336152, + "nauc_ndcg_at_3_std": 0.128853, + "nauc_ndcg_at_3_diff1": -0.194425, + "nauc_ndcg_at_5_max": 0.318217, + "nauc_ndcg_at_5_std": 0.137437, + "nauc_ndcg_at_5_diff1": -0.263231, + "nauc_ndcg_at_10_max": 0.297646, + "nauc_ndcg_at_10_std": 0.214868, + "nauc_ndcg_at_10_diff1": -0.207114, + "nauc_ndcg_at_20_max": 0.27333, + "nauc_ndcg_at_20_std": 0.285303, + "nauc_ndcg_at_20_diff1": -0.242873, + "nauc_ndcg_at_100_max": 0.21585, + "nauc_ndcg_at_100_std": 0.27754, + "nauc_ndcg_at_100_diff1": -0.239639, + "nauc_ndcg_at_1000_max": 0.301577, + "nauc_ndcg_at_1000_std": 0.261458, + "nauc_ndcg_at_1000_diff1": -0.259641, + "nauc_map_at_1_max": 0.367276, + "nauc_map_at_1_std": 0.239016, + "nauc_map_at_1_diff1": -0.27739, + "nauc_map_at_3_max": 0.221979, + "nauc_map_at_3_std": 0.111172, + "nauc_map_at_3_diff1": -0.174184, + "nauc_map_at_5_max": 0.248542, + "nauc_map_at_5_std": 0.150898, + "nauc_map_at_5_diff1": -0.250433, + "nauc_map_at_10_max": 0.268638, + "nauc_map_at_10_std": 0.181364, + "nauc_map_at_10_diff1": -0.218426, + "nauc_map_at_20_max": 0.273658, + "nauc_map_at_20_std": 0.205169, + "nauc_map_at_20_diff1": -0.238624, + "nauc_map_at_100_max": 0.271757, + "nauc_map_at_100_std": 0.206753, + "nauc_map_at_100_diff1": -0.245398, + "nauc_map_at_1000_max": 0.28055, + "nauc_map_at_1000_std": 0.207091, + "nauc_map_at_1000_diff1": -0.250162, + "nauc_recall_at_1_max": 0.367276, + "nauc_recall_at_1_std": 0.239016, + "nauc_recall_at_1_diff1": -0.27739, + "nauc_recall_at_3_max": 0.04859, + "nauc_recall_at_3_std": 0.006134, + "nauc_recall_at_3_diff1": -0.046108, + "nauc_recall_at_5_max": 0.095789, + "nauc_recall_at_5_std": 0.091114, + "nauc_recall_at_5_diff1": -0.177327, + "nauc_recall_at_10_max": 0.110337, + "nauc_recall_at_10_std": 0.188672, + "nauc_recall_at_10_diff1": -0.085118, + "nauc_recall_at_20_max": 0.050847, + "nauc_recall_at_20_std": 0.293732, + "nauc_recall_at_20_diff1": -0.141382, + "nauc_recall_at_100_max": -0.17372, + "nauc_recall_at_100_std": 0.286407, + "nauc_recall_at_100_diff1": -0.093228, + "nauc_recall_at_1000_max": -0.083429, + "nauc_recall_at_1000_std": 0.865788, + "nauc_recall_at_1000_diff1": 0.226944, + "nauc_precision_at_1_max": 0.462524, + "nauc_precision_at_1_std": 0.23998, + "nauc_precision_at_1_diff1": -0.253105, + "nauc_precision_at_3_max": 0.333523, + "nauc_precision_at_3_std": 0.137543, + "nauc_precision_at_3_diff1": -0.225262, + "nauc_precision_at_5_max": 0.359735, + "nauc_precision_at_5_std": 0.179112, + "nauc_precision_at_5_diff1": -0.322996, + "nauc_precision_at_10_max": 0.393081, + "nauc_precision_at_10_std": 0.242002, + "nauc_precision_at_10_diff1": -0.215018, + "nauc_precision_at_20_max": 0.393991, + "nauc_precision_at_20_std": 0.311586, + "nauc_precision_at_20_diff1": -0.299693, + "nauc_precision_at_100_max": 0.279602, + "nauc_precision_at_100_std": 0.222073, + "nauc_precision_at_100_diff1": -0.268433, + "nauc_precision_at_1000_max": 0.155621, + "nauc_precision_at_1000_std": -0.046321, + "nauc_precision_at_1000_diff1": -0.146248, + "nauc_cv_recall_at_1_max": 0.462524, + "nauc_cv_recall_at_1_std": 0.23998, + "nauc_cv_recall_at_1_diff1": -0.253105, + "nauc_cv_recall_at_3_max": 0.302719, + "nauc_cv_recall_at_3_std": 0.106147, + "nauc_cv_recall_at_3_diff1": -0.168753, + "nauc_cv_recall_at_5_max": 0.293919, + "nauc_cv_recall_at_5_std": 0.163541, + "nauc_cv_recall_at_5_diff1": -0.217207, + "nauc_cv_recall_at_10_max": 0.272809, + "nauc_cv_recall_at_10_std": 0.337039, + "nauc_cv_recall_at_10_diff1": -0.184769, + "nauc_cv_recall_at_20_max": 0.306843, + "nauc_cv_recall_at_20_std": 0.512891, + "nauc_cv_recall_at_20_diff1": -0.237699, + "nauc_cv_recall_at_100_max": -0.124711, + "nauc_cv_recall_at_100_std": 0.394736, + "nauc_cv_recall_at_100_diff1": -0.2311, + "nauc_cv_recall_at_1000_max": -0.164944, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_1000_diff1": 0.555085, + "nauc_mrr_at_1_max": 0.462524, + "nauc_mrr_at_1_std": 0.23998, + "nauc_mrr_at_1_diff1": -0.253105, + "nauc_mrr_at_3_max": 0.403305, + "nauc_mrr_at_3_std": 0.176866, + "nauc_mrr_at_3_diff1": -0.222854, + "nauc_mrr_at_5_max": 0.400572, + "nauc_mrr_at_5_std": 0.192994, + "nauc_mrr_at_5_diff1": -0.236483, + "nauc_mrr_at_10_max": 0.400342, + "nauc_mrr_at_10_std": 0.221002, + "nauc_mrr_at_10_diff1": -0.224432, + "nauc_mrr_at_20_max": 0.403142, + "nauc_mrr_at_20_std": 0.233936, + "nauc_mrr_at_20_diff1": -0.227999, + "nauc_mrr_at_100_max": 0.394848, + "nauc_mrr_at_100_std": 0.226158, + "nauc_mrr_at_100_diff1": -0.229113, + "nauc_mrr_at_1000_max": 0.396669, + "nauc_mrr_at_1000_std": 0.226244, + "nauc_mrr_at_1000_diff1": -0.229396, + "main_score": 0.16327, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.21053, + "ndcg_at_3": 0.17086, + "ndcg_at_5": 0.1825, + "ndcg_at_10": 0.21839, + "ndcg_at_20": 0.24465, + "ndcg_at_100": 0.30496, + "ndcg_at_1000": 0.36757, + "map_at_1": 0.06857, + "map_at_3": 0.10536, + "map_at_5": 0.1229, + "map_at_10": 0.14097, + "map_at_20": 0.14969, + "map_at_100": 0.16187, + "map_at_1000": 0.16589, + "recall_at_1": 0.06857, + "recall_at_3": 0.13728, + "recall_at_5": 0.20015, + "recall_at_10": 0.31539, + "recall_at_20": 0.39701, + "recall_at_100": 0.64607, + "recall_at_1000": 0.96765, + "cv_recall_at_1": 0.17544, + "cv_recall_at_3": 0.35088, + "cv_recall_at_5": 0.36842, + "cv_recall_at_10": 0.52632, + "cv_recall_at_20": 0.63158, + "cv_recall_at_100": 0.84211, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.21053, + "precision_at_3": 0.1345, + "precision_at_5": 0.09825, + "precision_at_10": 0.07193, + "precision_at_20": 0.04912, + "precision_at_100": 0.01754, + "precision_at_1000": 0.00367, + "mrr_at_1": 0.175439, + "mrr_at_3": 0.248538, + "mrr_at_5": 0.252047, + "mrr_at_10": 0.272208, + "mrr_at_20": 0.280278, + "mrr_at_100": 0.285629, + "mrr_at_1000": 0.286348, + "nauc_ndcg_at_1_max": 0.257929, + "nauc_ndcg_at_1_std": 0.089002, + "nauc_ndcg_at_1_diff1": -0.093081, + "nauc_ndcg_at_3_max": 0.297727, + "nauc_ndcg_at_3_std": 0.018506, + "nauc_ndcg_at_3_diff1": -0.00041, + "nauc_ndcg_at_5_max": 0.260293, + "nauc_ndcg_at_5_std": -0.036916, + "nauc_ndcg_at_5_diff1": 0.092299, + "nauc_ndcg_at_10_max": 0.202574, + "nauc_ndcg_at_10_std": -0.022538, + "nauc_ndcg_at_10_diff1": 0.161795, + "nauc_ndcg_at_20_max": 0.259026, + "nauc_ndcg_at_20_std": 0.11347, + "nauc_ndcg_at_20_diff1": 0.151904, + "nauc_ndcg_at_100_max": 0.25985, + "nauc_ndcg_at_100_std": 0.157286, + "nauc_ndcg_at_100_diff1": 0.124498, + "nauc_ndcg_at_1000_max": 0.272861, + "nauc_ndcg_at_1000_std": 0.0975, + "nauc_ndcg_at_1000_diff1": 0.068985, + "nauc_map_at_1_max": 0.197736, + "nauc_map_at_1_std": 0.098394, + "nauc_map_at_1_diff1": -0.017538, + "nauc_map_at_3_max": 0.25391, + "nauc_map_at_3_std": 0.036046, + "nauc_map_at_3_diff1": 0.061024, + "nauc_map_at_5_max": 0.219591, + "nauc_map_at_5_std": -0.000973, + "nauc_map_at_5_diff1": 0.122922, + "nauc_map_at_10_max": 0.19566, + "nauc_map_at_10_std": -0.009611, + "nauc_map_at_10_diff1": 0.140936, + "nauc_map_at_20_max": 0.225191, + "nauc_map_at_20_std": 0.04546, + "nauc_map_at_20_diff1": 0.136762, + "nauc_map_at_100_max": 0.228096, + "nauc_map_at_100_std": 0.057388, + "nauc_map_at_100_diff1": 0.117574, + "nauc_map_at_1000_max": 0.231743, + "nauc_map_at_1000_std": 0.054926, + "nauc_map_at_1000_diff1": 0.115168, + "nauc_recall_at_1_max": 0.197736, + "nauc_recall_at_1_std": 0.098394, + "nauc_recall_at_1_diff1": -0.017538, + "nauc_recall_at_3_max": 0.260073, + "nauc_recall_at_3_std": 0.014232, + "nauc_recall_at_3_diff1": 0.17123, + "nauc_recall_at_5_max": 0.188041, + "nauc_recall_at_5_std": -0.090308, + "nauc_recall_at_5_diff1": 0.268591, + "nauc_recall_at_10_max": 0.093123, + "nauc_recall_at_10_std": -0.098808, + "nauc_recall_at_10_diff1": 0.350354, + "nauc_recall_at_20_max": 0.183697, + "nauc_recall_at_20_std": 0.19332, + "nauc_recall_at_20_diff1": 0.289765, + "nauc_recall_at_100_max": 0.209982, + "nauc_recall_at_100_std": 0.416323, + "nauc_recall_at_100_diff1": 0.273584, + "nauc_recall_at_1000_max": 0.074913, + "nauc_recall_at_1000_std": 0.88837, + "nauc_recall_at_1000_diff1": 0.562565, + "nauc_precision_at_1_max": 0.257929, + "nauc_precision_at_1_std": 0.089002, + "nauc_precision_at_1_diff1": -0.093081, + "nauc_precision_at_3_max": 0.338126, + "nauc_precision_at_3_std": -0.022918, + "nauc_precision_at_3_diff1": -0.021108, + "nauc_precision_at_5_max": 0.301638, + "nauc_precision_at_5_std": -0.073129, + "nauc_precision_at_5_diff1": 0.050022, + "nauc_precision_at_10_max": 0.265632, + "nauc_precision_at_10_std": 0.005046, + "nauc_precision_at_10_diff1": 0.061607, + "nauc_precision_at_20_max": 0.429772, + "nauc_precision_at_20_std": 0.256546, + "nauc_precision_at_20_diff1": 0.063493, + "nauc_precision_at_100_max": 0.384858, + "nauc_precision_at_100_std": 0.176283, + "nauc_precision_at_100_diff1": -0.033545, + "nauc_precision_at_1000_max": 0.159305, + "nauc_precision_at_1000_std": -0.121839, + "nauc_precision_at_1000_diff1": -0.251423, + "nauc_cv_recall_at_1_max": 0.198888, + "nauc_cv_recall_at_1_std": 0.021268, + "nauc_cv_recall_at_1_diff1": 0.029173, + "nauc_cv_recall_at_3_max": 0.283901, + "nauc_cv_recall_at_3_std": -0.041936, + "nauc_cv_recall_at_3_diff1": 0.032709, + "nauc_cv_recall_at_5_max": 0.339377, + "nauc_cv_recall_at_5_std": -0.047854, + "nauc_cv_recall_at_5_diff1": 0.090234, + "nauc_cv_recall_at_10_max": 0.358987, + "nauc_cv_recall_at_10_std": -0.001644, + "nauc_cv_recall_at_10_diff1": 0.25798, + "nauc_cv_recall_at_20_max": 0.320815, + "nauc_cv_recall_at_20_std": 0.250855, + "nauc_cv_recall_at_20_diff1": 0.078167, + "nauc_cv_recall_at_100_max": 0.263526, + "nauc_cv_recall_at_100_std": 0.511375, + "nauc_cv_recall_at_100_diff1": 0.236565, + "nauc_cv_recall_at_1000_max": 0.555085, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_1000_diff1": 0.722732, + "nauc_mrr_at_1_max": 0.198888, + "nauc_mrr_at_1_std": 0.021268, + "nauc_mrr_at_1_diff1": 0.029173, + "nauc_mrr_at_3_max": 0.235031, + "nauc_mrr_at_3_std": -0.00408, + "nauc_mrr_at_3_diff1": 0.016192, + "nauc_mrr_at_5_max": 0.248084, + "nauc_mrr_at_5_std": -0.005535, + "nauc_mrr_at_5_diff1": 0.029662, + "nauc_mrr_at_10_max": 0.243713, + "nauc_mrr_at_10_std": -0.008767, + "nauc_mrr_at_10_diff1": 0.050357, + "nauc_mrr_at_20_max": 0.239243, + "nauc_mrr_at_20_std": 0.00967, + "nauc_mrr_at_20_diff1": 0.033804, + "nauc_mrr_at_100_max": 0.233447, + "nauc_mrr_at_100_std": 0.011466, + "nauc_mrr_at_100_diff1": 0.036558, + "nauc_mrr_at_1000_max": 0.233578, + "nauc_mrr_at_1000_std": 0.010685, + "nauc_mrr_at_1000_diff1": 0.036181, + "main_score": 0.1825, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.10526, + "ndcg_at_3": 0.11236, + "ndcg_at_5": 0.14119, + "ndcg_at_10": 0.16111, + "ndcg_at_20": 0.19245, + "ndcg_at_100": 0.25574, + "ndcg_at_1000": 0.32437, + "map_at_1": 0.02997, + "map_at_3": 0.06257, + "map_at_5": 0.08521, + "map_at_10": 0.09574, + "map_at_20": 0.10579, + "map_at_100": 0.1177, + "map_at_1000": 0.12212, + "recall_at_1": 0.02997, + "recall_at_3": 0.09561, + "recall_at_5": 0.16773, + "recall_at_10": 0.22611, + "recall_at_20": 0.33697, + "recall_at_100": 0.60748, + "recall_at_1000": 0.96711, + "cv_recall_at_1": 0.12281, + "cv_recall_at_3": 0.26316, + "cv_recall_at_5": 0.35088, + "cv_recall_at_10": 0.45614, + "cv_recall_at_20": 0.5614, + "cv_recall_at_100": 0.80702, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.10526, + "precision_at_3": 0.09357, + "precision_at_5": 0.09123, + "precision_at_10": 0.06491, + "precision_at_20": 0.04474, + "precision_at_100": 0.01632, + "precision_at_1000": 0.00363, + "mrr_at_1": 0.122807, + "mrr_at_3": 0.187135, + "mrr_at_5": 0.208187, + "mrr_at_10": 0.222146, + "mrr_at_20": 0.22963, + "mrr_at_100": 0.236316, + "mrr_at_1000": 0.237152, + "nauc_ndcg_at_1_max": 0.497303, + "nauc_ndcg_at_1_std": 0.123504, + "nauc_ndcg_at_1_diff1": 0.128335, + "nauc_ndcg_at_3_max": 0.388936, + "nauc_ndcg_at_3_std": 0.183182, + "nauc_ndcg_at_3_diff1": 0.012738, + "nauc_ndcg_at_5_max": 0.328644, + "nauc_ndcg_at_5_std": 0.170781, + "nauc_ndcg_at_5_diff1": -0.02429, + "nauc_ndcg_at_10_max": 0.328436, + "nauc_ndcg_at_10_std": 0.228763, + "nauc_ndcg_at_10_diff1": -0.010366, + "nauc_ndcg_at_20_max": 0.276267, + "nauc_ndcg_at_20_std": 0.348259, + "nauc_ndcg_at_20_diff1": -0.025857, + "nauc_ndcg_at_100_max": 0.309772, + "nauc_ndcg_at_100_std": 0.336038, + "nauc_ndcg_at_100_diff1": -0.045887, + "nauc_ndcg_at_1000_max": 0.339366, + "nauc_ndcg_at_1000_std": 0.294938, + "nauc_ndcg_at_1000_diff1": -0.061935, + "nauc_map_at_1_max": 0.543848, + "nauc_map_at_1_std": 0.180173, + "nauc_map_at_1_diff1": 0.140491, + "nauc_map_at_3_max": 0.415574, + "nauc_map_at_3_std": 0.179738, + "nauc_map_at_3_diff1": 0.089521, + "nauc_map_at_5_max": 0.350927, + "nauc_map_at_5_std": 0.191062, + "nauc_map_at_5_diff1": 0.025545, + "nauc_map_at_10_max": 0.352374, + "nauc_map_at_10_std": 0.21421, + "nauc_map_at_10_diff1": 0.01473, + "nauc_map_at_20_max": 0.327337, + "nauc_map_at_20_std": 0.268065, + "nauc_map_at_20_diff1": -0.000704, + "nauc_map_at_100_max": 0.341336, + "nauc_map_at_100_std": 0.270476, + "nauc_map_at_100_diff1": -0.014517, + "nauc_map_at_1000_max": 0.344467, + "nauc_map_at_1000_std": 0.267158, + "nauc_map_at_1000_diff1": -0.017336, + "nauc_recall_at_1_max": 0.543848, + "nauc_recall_at_1_std": 0.180173, + "nauc_recall_at_1_diff1": 0.140491, + "nauc_recall_at_3_max": 0.31875, + "nauc_recall_at_3_std": 0.175122, + "nauc_recall_at_3_diff1": 0.079548, + "nauc_recall_at_5_max": 0.201852, + "nauc_recall_at_5_std": 0.131664, + "nauc_recall_at_5_diff1": 0.019911, + "nauc_recall_at_10_max": 0.202587, + "nauc_recall_at_10_std": 0.182861, + "nauc_recall_at_10_diff1": 0.015528, + "nauc_recall_at_20_max": 0.064851, + "nauc_recall_at_20_std": 0.389516, + "nauc_recall_at_20_diff1": 0.003207, + "nauc_recall_at_100_max": 0.148505, + "nauc_recall_at_100_std": 0.389477, + "nauc_recall_at_100_diff1": 0.023278, + "nauc_recall_at_1000_max": -0.200905, + "nauc_recall_at_1000_std": 0.725284, + "nauc_recall_at_1000_diff1": 0.273886, + "nauc_precision_at_1_max": 0.497303, + "nauc_precision_at_1_std": 0.123504, + "nauc_precision_at_1_diff1": 0.128335, + "nauc_precision_at_3_max": 0.320638, + "nauc_precision_at_3_std": 0.189162, + "nauc_precision_at_3_diff1": -0.090098, + "nauc_precision_at_5_max": 0.266881, + "nauc_precision_at_5_std": 0.201544, + "nauc_precision_at_5_diff1": -0.177285, + "nauc_precision_at_10_max": 0.342253, + "nauc_precision_at_10_std": 0.287035, + "nauc_precision_at_10_diff1": -0.178648, + "nauc_precision_at_20_max": 0.26849, + "nauc_precision_at_20_std": 0.369248, + "nauc_precision_at_20_diff1": -0.195397, + "nauc_precision_at_100_max": 0.312387, + "nauc_precision_at_100_std": 0.281076, + "nauc_precision_at_100_diff1": -0.236313, + "nauc_precision_at_1000_max": 0.219851, + "nauc_precision_at_1000_std": -0.015703, + "nauc_precision_at_1000_diff1": -0.101746, + "nauc_cv_recall_at_1_max": 0.415888, + "nauc_cv_recall_at_1_std": 0.212325, + "nauc_cv_recall_at_1_diff1": 0.028646, + "nauc_cv_recall_at_3_max": 0.257015, + "nauc_cv_recall_at_3_std": 0.197938, + "nauc_cv_recall_at_3_diff1": -0.051462, + "nauc_cv_recall_at_5_max": 0.206594, + "nauc_cv_recall_at_5_std": 0.198716, + "nauc_cv_recall_at_5_diff1": -0.046852, + "nauc_cv_recall_at_10_max": 0.339634, + "nauc_cv_recall_at_10_std": 0.368949, + "nauc_cv_recall_at_10_diff1": -0.079339, + "nauc_cv_recall_at_20_max": 0.273536, + "nauc_cv_recall_at_20_std": 0.565103, + "nauc_cv_recall_at_20_diff1": -0.04347, + "nauc_cv_recall_at_100_max": 0.183744, + "nauc_cv_recall_at_100_std": 0.391252, + "nauc_cv_recall_at_100_diff1": -0.158609, + "nauc_cv_recall_at_1000_max": -0.164944, + "nauc_cv_recall_at_1000_std": 0.868512, + "nauc_cv_recall_at_1000_diff1": 0.126616, + "nauc_mrr_at_1_max": 0.415888, + "nauc_mrr_at_1_std": 0.212325, + "nauc_mrr_at_1_diff1": 0.028646, + "nauc_mrr_at_3_max": 0.323981, + "nauc_mrr_at_3_std": 0.194721, + "nauc_mrr_at_3_diff1": -0.019065, + "nauc_mrr_at_5_max": 0.303903, + "nauc_mrr_at_5_std": 0.194626, + "nauc_mrr_at_5_diff1": -0.020254, + "nauc_mrr_at_10_max": 0.323841, + "nauc_mrr_at_10_std": 0.227754, + "nauc_mrr_at_10_diff1": -0.02832, + "nauc_mrr_at_20_max": 0.317897, + "nauc_mrr_at_20_std": 0.244413, + "nauc_mrr_at_20_diff1": -0.025887, + "nauc_mrr_at_100_max": 0.316792, + "nauc_mrr_at_100_std": 0.232309, + "nauc_mrr_at_100_diff1": -0.027583, + "nauc_mrr_at_1000_max": 0.317412, + "nauc_mrr_at_1000_std": 0.232704, + "nauc_mrr_at_1000_diff1": -0.027063, + "main_score": 0.14119, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 577.0602626800537, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/Vidore2EconomicsReportsRetrieval.json b/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..9501a21a33 --- /dev/null +++ b/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.44828, + "ndcg_at_3": 0.38634, + "ndcg_at_5": 0.36799, + "ndcg_at_10": 0.34306, + "ndcg_at_20": 0.36335, + "ndcg_at_100": 0.50195, + "ndcg_at_1000": 0.57682, + "map_at_1": 0.05682, + "map_at_3": 0.09304, + "map_at_5": 0.11895, + "map_at_10": 0.15162, + "map_at_20": 0.18368, + "map_at_100": 0.25198, + "map_at_1000": 0.27612, + "recall_at_1": 0.05682, + "recall_at_3": 0.1099, + "recall_at_5": 0.15515, + "recall_at_10": 0.23542, + "recall_at_20": 0.34416, + "recall_at_100": 0.77373, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.44828, + "cv_recall_at_3": 0.68966, + "cv_recall_at_5": 0.74138, + "cv_recall_at_10": 0.81034, + "cv_recall_at_20": 0.87931, + "cv_recall_at_100": 0.98276, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.44828, + "precision_at_3": 0.35632, + "precision_at_5": 0.33103, + "precision_at_10": 0.26207, + "precision_at_20": 0.21293, + "precision_at_100": 0.10655, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.448276, + "mrr_at_3": 0.551724, + "mrr_at_5": 0.562931, + "mrr_at_10": 0.5716, + "mrr_at_20": 0.576097, + "mrr_at_100": 0.578901, + "mrr_at_1000": 0.579063, + "nauc_ndcg_at_1_max": 0.31867, + "nauc_ndcg_at_1_std": 0.126741, + "nauc_ndcg_at_1_diff1": 0.227163, + "nauc_ndcg_at_3_max": 0.366199, + "nauc_ndcg_at_3_std": 0.080302, + "nauc_ndcg_at_3_diff1": 0.029382, + "nauc_ndcg_at_5_max": 0.418086, + "nauc_ndcg_at_5_std": 0.121647, + "nauc_ndcg_at_5_diff1": -0.020454, + "nauc_ndcg_at_10_max": 0.416244, + "nauc_ndcg_at_10_std": 0.089649, + "nauc_ndcg_at_10_diff1": -0.068465, + "nauc_ndcg_at_20_max": 0.389062, + "nauc_ndcg_at_20_std": 0.096632, + "nauc_ndcg_at_20_diff1": -0.131227, + "nauc_ndcg_at_100_max": 0.326832, + "nauc_ndcg_at_100_std": 0.113442, + "nauc_ndcg_at_100_diff1": -0.095855, + "nauc_ndcg_at_1000_max": 0.401622, + "nauc_ndcg_at_1000_std": 0.150891, + "nauc_ndcg_at_1000_diff1": -0.086558, + "nauc_map_at_1_max": -0.171065, + "nauc_map_at_1_std": -0.190032, + "nauc_map_at_1_diff1": -0.147725, + "nauc_map_at_3_max": 0.012287, + "nauc_map_at_3_std": -0.133405, + "nauc_map_at_3_diff1": -0.141354, + "nauc_map_at_5_max": 0.151332, + "nauc_map_at_5_std": -0.05713, + "nauc_map_at_5_diff1": -0.119357, + "nauc_map_at_10_max": 0.210187, + "nauc_map_at_10_std": -0.039805, + "nauc_map_at_10_diff1": -0.099466, + "nauc_map_at_20_max": 0.205127, + "nauc_map_at_20_std": -0.026618, + "nauc_map_at_20_diff1": -0.118816, + "nauc_map_at_100_max": 0.307176, + "nauc_map_at_100_std": 0.05433, + "nauc_map_at_100_diff1": -0.12712, + "nauc_map_at_1000_max": 0.380523, + "nauc_map_at_1000_std": 0.095304, + "nauc_map_at_1000_diff1": -0.130342, + "nauc_recall_at_1_max": -0.171065, + "nauc_recall_at_1_std": -0.190032, + "nauc_recall_at_1_diff1": -0.147725, + "nauc_recall_at_3_max": -0.013252, + "nauc_recall_at_3_std": -0.15499, + "nauc_recall_at_3_diff1": -0.184775, + "nauc_recall_at_5_max": 0.140941, + "nauc_recall_at_5_std": -0.036798, + "nauc_recall_at_5_diff1": -0.147004, + "nauc_recall_at_10_max": 0.159726, + "nauc_recall_at_10_std": -0.047714, + "nauc_recall_at_10_diff1": -0.148598, + "nauc_recall_at_20_max": 0.106861, + "nauc_recall_at_20_std": -0.050911, + "nauc_recall_at_20_diff1": -0.18758, + "nauc_recall_at_100_max": 0.004823, + "nauc_recall_at_100_std": 0.003834, + "nauc_recall_at_100_diff1": -0.06136, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.31867, + "nauc_precision_at_1_std": 0.126741, + "nauc_precision_at_1_diff1": 0.227163, + "nauc_precision_at_3_max": 0.43256, + "nauc_precision_at_3_std": 0.110621, + "nauc_precision_at_3_diff1": -9e-06, + "nauc_precision_at_5_max": 0.528126, + "nauc_precision_at_5_std": 0.206711, + "nauc_precision_at_5_diff1": -0.03484, + "nauc_precision_at_10_max": 0.529786, + "nauc_precision_at_10_std": 0.194617, + "nauc_precision_at_10_diff1": -0.002276, + "nauc_precision_at_20_max": 0.442469, + "nauc_precision_at_20_std": 0.189162, + "nauc_precision_at_20_diff1": -0.067928, + "nauc_precision_at_100_max": 0.537321, + "nauc_precision_at_100_std": 0.30287, + "nauc_precision_at_100_diff1": -0.074572, + "nauc_precision_at_1000_max": 0.580938, + "nauc_precision_at_1000_std": 0.30333, + "nauc_precision_at_1000_diff1": -0.075791, + "nauc_cv_recall_at_1_max": 0.31867, + "nauc_cv_recall_at_1_std": 0.126741, + "nauc_cv_recall_at_1_diff1": 0.227163, + "nauc_cv_recall_at_3_max": 0.207773, + "nauc_cv_recall_at_3_std": 0.005851, + "nauc_cv_recall_at_3_diff1": -0.069918, + "nauc_cv_recall_at_5_max": 0.132865, + "nauc_cv_recall_at_5_std": -0.132451, + "nauc_cv_recall_at_5_diff1": -0.173803, + "nauc_cv_recall_at_10_max": 0.397523, + "nauc_cv_recall_at_10_std": 0.176945, + "nauc_cv_recall_at_10_diff1": -0.092002, + "nauc_cv_recall_at_20_max": 0.521882, + "nauc_cv_recall_at_20_std": 0.296721, + "nauc_cv_recall_at_20_diff1": -0.2992, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 0.720441, + "nauc_cv_recall_at_100_diff1": -0.172914, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.31867, + "nauc_mrr_at_1_std": 0.126741, + "nauc_mrr_at_1_diff1": 0.227163, + "nauc_mrr_at_3_max": 0.288461, + "nauc_mrr_at_3_std": 0.101035, + "nauc_mrr_at_3_diff1": 0.10451, + "nauc_mrr_at_5_max": 0.277219, + "nauc_mrr_at_5_std": 0.080463, + "nauc_mrr_at_5_diff1": 0.093647, + "nauc_mrr_at_10_max": 0.302109, + "nauc_mrr_at_10_std": 0.109835, + "nauc_mrr_at_10_diff1": 0.110397, + "nauc_mrr_at_20_max": 0.305539, + "nauc_mrr_at_20_std": 0.114256, + "nauc_mrr_at_20_diff1": 0.109461, + "nauc_mrr_at_100_max": 0.304544, + "nauc_mrr_at_100_std": 0.113906, + "nauc_mrr_at_100_diff1": 0.111474, + "nauc_mrr_at_1000_max": 0.304165, + "nauc_mrr_at_1000_std": 0.113576, + "nauc_mrr_at_1000_diff1": 0.111628, + "main_score": 0.36799, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.37931, + "ndcg_at_3": 0.33639, + "ndcg_at_5": 0.30869, + "ndcg_at_10": 0.28945, + "ndcg_at_20": 0.32467, + "ndcg_at_100": 0.47126, + "ndcg_at_1000": 0.55023, + "map_at_1": 0.03041, + "map_at_3": 0.07264, + "map_at_5": 0.09283, + "map_at_10": 0.1186, + "map_at_20": 0.15031, + "map_at_100": 0.21732, + "map_at_1000": 0.24277, + "recall_at_1": 0.03041, + "recall_at_3": 0.10426, + "recall_at_5": 0.14386, + "recall_at_10": 0.2046, + "recall_at_20": 0.33517, + "recall_at_100": 0.77416, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.37931, + "cv_recall_at_3": 0.62069, + "cv_recall_at_5": 0.68966, + "cv_recall_at_10": 0.75862, + "cv_recall_at_20": 0.87931, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.37931, + "precision_at_3": 0.31609, + "precision_at_5": 0.27586, + "precision_at_10": 0.22414, + "precision_at_20": 0.19741, + "precision_at_100": 0.10241, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.37931, + "mrr_at_3": 0.488506, + "mrr_at_5": 0.505747, + "mrr_at_10": 0.51486, + "mrr_at_20": 0.523989, + "mrr_at_100": 0.526923, + "mrr_at_1000": 0.526923, + "nauc_ndcg_at_1_max": 0.407391, + "nauc_ndcg_at_1_std": 0.2357, + "nauc_ndcg_at_1_diff1": 0.27957, + "nauc_ndcg_at_3_max": 0.454263, + "nauc_ndcg_at_3_std": 0.234156, + "nauc_ndcg_at_3_diff1": 0.203659, + "nauc_ndcg_at_5_max": 0.385459, + "nauc_ndcg_at_5_std": 0.169229, + "nauc_ndcg_at_5_diff1": 0.135271, + "nauc_ndcg_at_10_max": 0.436656, + "nauc_ndcg_at_10_std": 0.168226, + "nauc_ndcg_at_10_diff1": 0.127982, + "nauc_ndcg_at_20_max": 0.458135, + "nauc_ndcg_at_20_std": 0.213645, + "nauc_ndcg_at_20_diff1": 0.039717, + "nauc_ndcg_at_100_max": 0.395508, + "nauc_ndcg_at_100_std": 0.241376, + "nauc_ndcg_at_100_diff1": 0.049265, + "nauc_ndcg_at_1000_max": 0.495008, + "nauc_ndcg_at_1000_std": 0.240793, + "nauc_ndcg_at_1000_diff1": 0.097959, + "nauc_map_at_1_max": 0.242846, + "nauc_map_at_1_std": 0.221813, + "nauc_map_at_1_diff1": 0.168062, + "nauc_map_at_3_max": 0.235385, + "nauc_map_at_3_std": 0.130441, + "nauc_map_at_3_diff1": 0.046397, + "nauc_map_at_5_max": 0.228239, + "nauc_map_at_5_std": 0.129055, + "nauc_map_at_5_diff1": 0.042628, + "nauc_map_at_10_max": 0.299458, + "nauc_map_at_10_std": 0.150806, + "nauc_map_at_10_diff1": 0.060596, + "nauc_map_at_20_max": 0.309877, + "nauc_map_at_20_std": 0.180334, + "nauc_map_at_20_diff1": 0.017561, + "nauc_map_at_100_max": 0.382505, + "nauc_map_at_100_std": 0.19398, + "nauc_map_at_100_diff1": 0.010267, + "nauc_map_at_1000_max": 0.463613, + "nauc_map_at_1000_std": 0.200077, + "nauc_map_at_1000_diff1": 0.035311, + "nauc_recall_at_1_max": 0.242846, + "nauc_recall_at_1_std": 0.221813, + "nauc_recall_at_1_diff1": 0.168062, + "nauc_recall_at_3_max": 0.045651, + "nauc_recall_at_3_std": -0.026627, + "nauc_recall_at_3_diff1": -0.085714, + "nauc_recall_at_5_max": 0.023857, + "nauc_recall_at_5_std": -0.015402, + "nauc_recall_at_5_diff1": -0.088703, + "nauc_recall_at_10_max": 0.113994, + "nauc_recall_at_10_std": 0.028334, + "nauc_recall_at_10_diff1": -0.0663, + "nauc_recall_at_20_max": 0.11176, + "nauc_recall_at_20_std": 0.082786, + "nauc_recall_at_20_diff1": -0.185691, + "nauc_recall_at_100_max": -0.13464, + "nauc_recall_at_100_std": 0.018102, + "nauc_recall_at_100_diff1": -0.071886, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.407391, + "nauc_precision_at_1_std": 0.2357, + "nauc_precision_at_1_diff1": 0.27957, + "nauc_precision_at_3_max": 0.500683, + "nauc_precision_at_3_std": 0.257184, + "nauc_precision_at_3_diff1": 0.202604, + "nauc_precision_at_5_max": 0.392709, + "nauc_precision_at_5_std": 0.176073, + "nauc_precision_at_5_diff1": 0.102572, + "nauc_precision_at_10_max": 0.471247, + "nauc_precision_at_10_std": 0.178373, + "nauc_precision_at_10_diff1": 0.142392, + "nauc_precision_at_20_max": 0.442122, + "nauc_precision_at_20_std": 0.160194, + "nauc_precision_at_20_diff1": 0.026507, + "nauc_precision_at_100_max": 0.5015, + "nauc_precision_at_100_std": 0.154753, + "nauc_precision_at_100_diff1": 0.075766, + "nauc_precision_at_1000_max": 0.55268, + "nauc_precision_at_1000_std": 0.133865, + "nauc_precision_at_1000_diff1": 0.099913, + "nauc_cv_recall_at_1_max": 0.407391, + "nauc_cv_recall_at_1_std": 0.2357, + "nauc_cv_recall_at_1_diff1": 0.27957, + "nauc_cv_recall_at_3_max": 0.467971, + "nauc_cv_recall_at_3_std": 0.30732, + "nauc_cv_recall_at_3_diff1": 0.155712, + "nauc_cv_recall_at_5_max": 0.382284, + "nauc_cv_recall_at_5_std": 0.227642, + "nauc_cv_recall_at_5_diff1": 0.138354, + "nauc_cv_recall_at_10_max": 0.251267, + "nauc_cv_recall_at_10_std": 0.095904, + "nauc_cv_recall_at_10_diff1": 0.029168, + "nauc_cv_recall_at_20_max": 0.563706, + "nauc_cv_recall_at_20_std": 0.267591, + "nauc_cv_recall_at_20_diff1": -0.151747, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.407391, + "nauc_mrr_at_1_std": 0.2357, + "nauc_mrr_at_1_diff1": 0.27957, + "nauc_mrr_at_3_max": 0.426587, + "nauc_mrr_at_3_std": 0.257409, + "nauc_mrr_at_3_diff1": 0.2341, + "nauc_mrr_at_5_max": 0.407996, + "nauc_mrr_at_5_std": 0.239786, + "nauc_mrr_at_5_diff1": 0.232872, + "nauc_mrr_at_10_max": 0.397417, + "nauc_mrr_at_10_std": 0.228831, + "nauc_mrr_at_10_diff1": 0.224666, + "nauc_mrr_at_20_max": 0.409517, + "nauc_mrr_at_20_std": 0.236348, + "nauc_mrr_at_20_diff1": 0.226507, + "nauc_mrr_at_100_max": 0.408039, + "nauc_mrr_at_100_std": 0.235759, + "nauc_mrr_at_100_diff1": 0.229633, + "nauc_mrr_at_1000_max": 0.408039, + "nauc_mrr_at_1000_std": 0.235759, + "nauc_mrr_at_1000_diff1": 0.229633, + "main_score": 0.30869, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.5, + "ndcg_at_3": 0.42434, + "ndcg_at_5": 0.39616, + "ndcg_at_10": 0.37403, + "ndcg_at_20": 0.40056, + "ndcg_at_100": 0.52819, + "ndcg_at_1000": 0.59491, + "map_at_1": 0.04378, + "map_at_3": 0.0918, + "map_at_5": 0.1246, + "map_at_10": 0.16239, + "map_at_20": 0.20246, + "map_at_100": 0.26981, + "map_at_1000": 0.29321, + "recall_at_1": 0.04378, + "recall_at_3": 0.11675, + "recall_at_5": 0.16573, + "recall_at_10": 0.26108, + "recall_at_20": 0.40127, + "recall_at_100": 0.81769, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.5, + "cv_recall_at_3": 0.67241, + "cv_recall_at_5": 0.7069, + "cv_recall_at_10": 0.84483, + "cv_recall_at_20": 0.91379, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.5, + "precision_at_3": 0.39655, + "precision_at_5": 0.36207, + "precision_at_10": 0.29655, + "precision_at_20": 0.23966, + "precision_at_100": 0.10776, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.5, + "mrr_at_3": 0.574713, + "mrr_at_5": 0.583333, + "mrr_at_10": 0.601628, + "mrr_at_20": 0.606474, + "mrr_at_100": 0.609277, + "mrr_at_1000": 0.609277, + "nauc_ndcg_at_1_max": 0.393092, + "nauc_ndcg_at_1_std": 0.203248, + "nauc_ndcg_at_1_diff1": 0.197768, + "nauc_ndcg_at_3_max": 0.398003, + "nauc_ndcg_at_3_std": 0.120242, + "nauc_ndcg_at_3_diff1": 0.146288, + "nauc_ndcg_at_5_max": 0.461624, + "nauc_ndcg_at_5_std": 0.132251, + "nauc_ndcg_at_5_diff1": 0.167285, + "nauc_ndcg_at_10_max": 0.435353, + "nauc_ndcg_at_10_std": 0.069813, + "nauc_ndcg_at_10_diff1": 0.133594, + "nauc_ndcg_at_20_max": 0.539854, + "nauc_ndcg_at_20_std": 0.157893, + "nauc_ndcg_at_20_diff1": 0.130977, + "nauc_ndcg_at_100_max": 0.528567, + "nauc_ndcg_at_100_std": 0.240582, + "nauc_ndcg_at_100_diff1": 0.071707, + "nauc_ndcg_at_1000_max": 0.498056, + "nauc_ndcg_at_1000_std": 0.194488, + "nauc_ndcg_at_1000_diff1": 0.067314, + "nauc_map_at_1_max": 0.297242, + "nauc_map_at_1_std": 0.28271, + "nauc_map_at_1_diff1": 0.044419, + "nauc_map_at_3_max": 0.336065, + "nauc_map_at_3_std": 0.162595, + "nauc_map_at_3_diff1": 0.094697, + "nauc_map_at_5_max": 0.442024, + "nauc_map_at_5_std": 0.185374, + "nauc_map_at_5_diff1": 0.058028, + "nauc_map_at_10_max": 0.408609, + "nauc_map_at_10_std": 0.120444, + "nauc_map_at_10_diff1": 0.045319, + "nauc_map_at_20_max": 0.444679, + "nauc_map_at_20_std": 0.125366, + "nauc_map_at_20_diff1": 0.059751, + "nauc_map_at_100_max": 0.507824, + "nauc_map_at_100_std": 0.158763, + "nauc_map_at_100_diff1": 0.043882, + "nauc_map_at_1000_max": 0.523444, + "nauc_map_at_1000_std": 0.157036, + "nauc_map_at_1000_diff1": 0.031397, + "nauc_recall_at_1_max": 0.297242, + "nauc_recall_at_1_std": 0.28271, + "nauc_recall_at_1_diff1": 0.044419, + "nauc_recall_at_3_max": 0.224823, + "nauc_recall_at_3_std": 0.018695, + "nauc_recall_at_3_diff1": 0.285418, + "nauc_recall_at_5_max": 0.35642, + "nauc_recall_at_5_std": 0.085706, + "nauc_recall_at_5_diff1": 0.223953, + "nauc_recall_at_10_max": 0.212703, + "nauc_recall_at_10_std": -0.083405, + "nauc_recall_at_10_diff1": 0.162397, + "nauc_recall_at_20_max": 0.263617, + "nauc_recall_at_20_std": 0.005591, + "nauc_recall_at_20_diff1": 0.226555, + "nauc_recall_at_100_max": 0.213659, + "nauc_recall_at_100_std": 0.23779, + "nauc_recall_at_100_diff1": -0.058046, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.393092, + "nauc_precision_at_1_std": 0.203248, + "nauc_precision_at_1_diff1": 0.197768, + "nauc_precision_at_3_max": 0.392152, + "nauc_precision_at_3_std": 0.083457, + "nauc_precision_at_3_diff1": 0.097186, + "nauc_precision_at_5_max": 0.478312, + "nauc_precision_at_5_std": 0.124125, + "nauc_precision_at_5_diff1": 0.110312, + "nauc_precision_at_10_max": 0.365656, + "nauc_precision_at_10_std": 0.016228, + "nauc_precision_at_10_diff1": 0.04205, + "nauc_precision_at_20_max": 0.368368, + "nauc_precision_at_20_std": 0.059635, + "nauc_precision_at_20_diff1": 0.019031, + "nauc_precision_at_100_max": 0.363438, + "nauc_precision_at_100_std": 0.123854, + "nauc_precision_at_100_diff1": -0.018026, + "nauc_precision_at_1000_max": 0.335146, + "nauc_precision_at_1000_std": 0.078673, + "nauc_precision_at_1000_diff1": -0.021342, + "nauc_cv_recall_at_1_max": 0.393092, + "nauc_cv_recall_at_1_std": 0.203248, + "nauc_cv_recall_at_1_diff1": 0.197768, + "nauc_cv_recall_at_3_max": 0.625368, + "nauc_cv_recall_at_3_std": 0.356335, + "nauc_cv_recall_at_3_diff1": 0.310324, + "nauc_cv_recall_at_5_max": 0.615156, + "nauc_cv_recall_at_5_std": 0.31012, + "nauc_cv_recall_at_5_diff1": 0.27655, + "nauc_cv_recall_at_10_max": 0.377875, + "nauc_cv_recall_at_10_std": 0.015812, + "nauc_cv_recall_at_10_diff1": 0.246351, + "nauc_cv_recall_at_20_max": 0.795361, + "nauc_cv_recall_at_20_std": 0.264938, + "nauc_cv_recall_at_20_diff1": 0.50459, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.393092, + "nauc_mrr_at_1_std": 0.203248, + "nauc_mrr_at_1_diff1": 0.197768, + "nauc_mrr_at_3_max": 0.482237, + "nauc_mrr_at_3_std": 0.278721, + "nauc_mrr_at_3_diff1": 0.217712, + "nauc_mrr_at_5_max": 0.477981, + "nauc_mrr_at_5_std": 0.268135, + "nauc_mrr_at_5_diff1": 0.20943, + "nauc_mrr_at_10_max": 0.453932, + "nauc_mrr_at_10_std": 0.242071, + "nauc_mrr_at_10_diff1": 0.208404, + "nauc_mrr_at_20_max": 0.465559, + "nauc_mrr_at_20_std": 0.252364, + "nauc_mrr_at_20_diff1": 0.213755, + "nauc_mrr_at_100_max": 0.462858, + "nauc_mrr_at_100_std": 0.25218, + "nauc_mrr_at_100_diff1": 0.210453, + "nauc_mrr_at_1000_max": 0.462858, + "nauc_mrr_at_1000_std": 0.25218, + "nauc_mrr_at_1000_diff1": 0.210453, + "main_score": 0.39616, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.37931, + "ndcg_at_3": 0.37562, + "ndcg_at_5": 0.34321, + "ndcg_at_10": 0.33104, + "ndcg_at_20": 0.36574, + "ndcg_at_100": 0.5029, + "ndcg_at_1000": 0.57306, + "map_at_1": 0.03795, + "map_at_3": 0.08798, + "map_at_5": 0.11084, + "map_at_10": 0.13945, + "map_at_20": 0.17697, + "map_at_100": 0.24817, + "map_at_1000": 0.27255, + "recall_at_1": 0.03795, + "recall_at_3": 0.11329, + "recall_at_5": 0.15106, + "recall_at_10": 0.23292, + "recall_at_20": 0.38496, + "recall_at_100": 0.80115, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.36207, + "cv_recall_at_3": 0.62069, + "cv_recall_at_5": 0.68966, + "cv_recall_at_10": 0.82759, + "cv_recall_at_20": 0.94828, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.37931, + "precision_at_3": 0.35632, + "precision_at_5": 0.3069, + "precision_at_10": 0.2569, + "precision_at_20": 0.21638, + "precision_at_100": 0.1081, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.362069, + "mrr_at_3": 0.485632, + "mrr_at_5": 0.501149, + "mrr_at_10": 0.518507, + "mrr_at_20": 0.526212, + "mrr_at_100": 0.526948, + "mrr_at_1000": 0.526948, + "nauc_ndcg_at_1_max": 0.498368, + "nauc_ndcg_at_1_std": 0.212887, + "nauc_ndcg_at_1_diff1": 0.139725, + "nauc_ndcg_at_3_max": 0.511586, + "nauc_ndcg_at_3_std": 0.233041, + "nauc_ndcg_at_3_diff1": 0.155033, + "nauc_ndcg_at_5_max": 0.469307, + "nauc_ndcg_at_5_std": 0.169167, + "nauc_ndcg_at_5_diff1": 0.112327, + "nauc_ndcg_at_10_max": 0.532938, + "nauc_ndcg_at_10_std": 0.208341, + "nauc_ndcg_at_10_diff1": 0.135431, + "nauc_ndcg_at_20_max": 0.523084, + "nauc_ndcg_at_20_std": 0.233664, + "nauc_ndcg_at_20_diff1": 0.069335, + "nauc_ndcg_at_100_max": 0.515384, + "nauc_ndcg_at_100_std": 0.265667, + "nauc_ndcg_at_100_diff1": 0.094941, + "nauc_ndcg_at_1000_max": 0.507515, + "nauc_ndcg_at_1000_std": 0.211994, + "nauc_ndcg_at_1000_diff1": 0.113437, + "nauc_map_at_1_max": 0.099643, + "nauc_map_at_1_std": 0.054092, + "nauc_map_at_1_diff1": 0.037999, + "nauc_map_at_3_max": 0.18404, + "nauc_map_at_3_std": 0.126573, + "nauc_map_at_3_diff1": 0.168677, + "nauc_map_at_5_max": 0.198707, + "nauc_map_at_5_std": 0.103036, + "nauc_map_at_5_diff1": 0.105747, + "nauc_map_at_10_max": 0.30959, + "nauc_map_at_10_std": 0.193762, + "nauc_map_at_10_diff1": 0.120793, + "nauc_map_at_20_max": 0.31749, + "nauc_map_at_20_std": 0.189957, + "nauc_map_at_20_diff1": 0.044882, + "nauc_map_at_100_max": 0.423209, + "nauc_map_at_100_std": 0.198003, + "nauc_map_at_100_diff1": 0.057039, + "nauc_map_at_1000_max": 0.46742, + "nauc_map_at_1000_std": 0.188047, + "nauc_map_at_1000_diff1": 0.071728, + "nauc_recall_at_1_max": 0.099643, + "nauc_recall_at_1_std": 0.054092, + "nauc_recall_at_1_diff1": 0.037999, + "nauc_recall_at_3_max": 0.079012, + "nauc_recall_at_3_std": 0.041641, + "nauc_recall_at_3_diff1": 0.228613, + "nauc_recall_at_5_max": 0.074886, + "nauc_recall_at_5_std": 0.000867, + "nauc_recall_at_5_diff1": 0.13647, + "nauc_recall_at_10_max": 0.263712, + "nauc_recall_at_10_std": 0.166527, + "nauc_recall_at_10_diff1": 0.11715, + "nauc_recall_at_20_max": 0.091716, + "nauc_recall_at_20_std": 0.094987, + "nauc_recall_at_20_diff1": -0.091808, + "nauc_recall_at_100_max": 0.149457, + "nauc_recall_at_100_std": 0.206087, + "nauc_recall_at_100_diff1": -0.075317, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.498368, + "nauc_precision_at_1_std": 0.212887, + "nauc_precision_at_1_diff1": 0.139725, + "nauc_precision_at_3_max": 0.530257, + "nauc_precision_at_3_std": 0.258644, + "nauc_precision_at_3_diff1": 0.119084, + "nauc_precision_at_5_max": 0.487014, + "nauc_precision_at_5_std": 0.189843, + "nauc_precision_at_5_diff1": 0.045461, + "nauc_precision_at_10_max": 0.577574, + "nauc_precision_at_10_std": 0.244907, + "nauc_precision_at_10_diff1": 0.071858, + "nauc_precision_at_20_max": 0.479778, + "nauc_precision_at_20_std": 0.164368, + "nauc_precision_at_20_diff1": -0.013625, + "nauc_precision_at_100_max": 0.465224, + "nauc_precision_at_100_std": 0.094441, + "nauc_precision_at_100_diff1": 0.088306, + "nauc_precision_at_1000_max": 0.481692, + "nauc_precision_at_1000_std": 0.060631, + "nauc_precision_at_1000_diff1": 0.113519, + "nauc_cv_recall_at_1_max": 0.52931, + "nauc_cv_recall_at_1_std": 0.256656, + "nauc_cv_recall_at_1_diff1": 0.189225, + "nauc_cv_recall_at_3_max": 0.562554, + "nauc_cv_recall_at_3_std": 0.289212, + "nauc_cv_recall_at_3_diff1": 0.271242, + "nauc_cv_recall_at_5_max": 0.506992, + "nauc_cv_recall_at_5_std": 0.259879, + "nauc_cv_recall_at_5_diff1": 0.138999, + "nauc_cv_recall_at_10_max": 0.577781, + "nauc_cv_recall_at_10_std": 0.22396, + "nauc_cv_recall_at_10_diff1": -0.076058, + "nauc_cv_recall_at_20_max": 0.807597, + "nauc_cv_recall_at_20_std": 0.093289, + "nauc_cv_recall_at_20_diff1": -0.207817, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.52931, + "nauc_mrr_at_1_std": 0.256656, + "nauc_mrr_at_1_diff1": 0.189225, + "nauc_mrr_at_3_max": 0.550896, + "nauc_mrr_at_3_std": 0.273149, + "nauc_mrr_at_3_diff1": 0.227661, + "nauc_mrr_at_5_max": 0.540515, + "nauc_mrr_at_5_std": 0.266531, + "nauc_mrr_at_5_diff1": 0.2028, + "nauc_mrr_at_10_max": 0.547881, + "nauc_mrr_at_10_std": 0.266024, + "nauc_mrr_at_10_diff1": 0.191218, + "nauc_mrr_at_20_max": 0.548566, + "nauc_mrr_at_20_std": 0.26504, + "nauc_mrr_at_20_diff1": 0.197543, + "nauc_mrr_at_100_max": 0.547932, + "nauc_mrr_at_100_std": 0.265434, + "nauc_mrr_at_100_diff1": 0.198539, + "nauc_mrr_at_1000_max": 0.547932, + "nauc_mrr_at_1000_std": 0.265434, + "nauc_mrr_at_1000_diff1": 0.198539, + "main_score": 0.34321, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 65.3086256980896, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/model_meta.json b/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/model_meta.json new file mode 100644 index 0000000000..c3a74a98dc --- /dev/null +++ b/results/jinaai__jina-clip-v2/344d954da76eb8ad47a7aaff42d012e30c15b8fe/model_meta.json @@ -0,0 +1 @@ +{"name": "jinaai/jina-clip-v2", "revision": "344d954da76eb8ad47a7aaff42d012e30c15b8fe", "release_date": "2024-10-09", "languages": ["eng-Latn"], "n_parameters": 865000000, "memory_usage_mb": 3000.0, "max_tokens": 8192.0, "embed_dim": 768, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["PyTorch"], "reference": "https://huggingface.co/jinaai/jina-clip-v2", "similarity_fn_name": null, "use_instructions": true, "training_datasets": {"MSMARCO": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "JinaCLIPModelWrapper"} \ No newline at end of file diff --git a/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/Vidore2BioMedicalLecturesRetrieval.json b/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..7bdfcd4768 --- /dev/null +++ b/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.21875, + "ndcg_at_3": 0.26096, + "ndcg_at_5": 0.28104, + "ndcg_at_10": 0.3118, + "ndcg_at_20": 0.34352, + "ndcg_at_100": 0.40418, + "ndcg_at_1000": 0.45113, + "map_at_1": 0.12292, + "map_at_3": 0.1983, + "map_at_5": 0.2167, + "map_at_10": 0.23397, + "map_at_20": 0.24678, + "map_at_100": 0.25983, + "map_at_1000": 0.2633, + "recall_at_1": 0.12292, + "recall_at_3": 0.26677, + "recall_at_5": 0.33015, + "recall_at_10": 0.42307, + "recall_at_20": 0.52042, + "recall_at_100": 0.74935, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.21875, + "cv_recall_at_3": 0.425, + "cv_recall_at_5": 0.51875, + "cv_recall_at_10": 0.625, + "cv_recall_at_20": 0.725, + "cv_recall_at_100": 0.9, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.21875, + "precision_at_3": 0.16667, + "precision_at_5": 0.1325, + "precision_at_10": 0.08875, + "precision_at_20": 0.06031, + "precision_at_100": 0.0205, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.21875, + "mrr_at_3": 0.308333, + "mrr_at_5": 0.329896, + "mrr_at_10": 0.344377, + "mrr_at_20": 0.351653, + "mrr_at_100": 0.356525, + "mrr_at_1000": 0.357015, + "nauc_ndcg_at_1_max": 0.190937, + "nauc_ndcg_at_1_std": 0.022361, + "nauc_ndcg_at_1_diff1": 0.250329, + "nauc_ndcg_at_3_max": 0.137752, + "nauc_ndcg_at_3_std": 0.026886, + "nauc_ndcg_at_3_diff1": 0.183335, + "nauc_ndcg_at_5_max": 0.148828, + "nauc_ndcg_at_5_std": 0.015866, + "nauc_ndcg_at_5_diff1": 0.202661, + "nauc_ndcg_at_10_max": 0.152466, + "nauc_ndcg_at_10_std": 0.01809, + "nauc_ndcg_at_10_diff1": 0.194593, + "nauc_ndcg_at_20_max": 0.16251, + "nauc_ndcg_at_20_std": 0.010339, + "nauc_ndcg_at_20_diff1": 0.21014, + "nauc_ndcg_at_100_max": 0.194142, + "nauc_ndcg_at_100_std": 0.040072, + "nauc_ndcg_at_100_diff1": 0.201372, + "nauc_ndcg_at_1000_max": 0.162285, + "nauc_ndcg_at_1000_std": 0.00993, + "nauc_ndcg_at_1000_diff1": 0.194929, + "nauc_map_at_1_max": 0.159602, + "nauc_map_at_1_std": 0.044069, + "nauc_map_at_1_diff1": 0.303299, + "nauc_map_at_3_max": 0.148131, + "nauc_map_at_3_std": 0.034801, + "nauc_map_at_3_diff1": 0.23101, + "nauc_map_at_5_max": 0.156675, + "nauc_map_at_5_std": 0.03137, + "nauc_map_at_5_diff1": 0.238733, + "nauc_map_at_10_max": 0.159293, + "nauc_map_at_10_std": 0.027565, + "nauc_map_at_10_diff1": 0.230231, + "nauc_map_at_20_max": 0.162608, + "nauc_map_at_20_std": 0.020626, + "nauc_map_at_20_diff1": 0.231431, + "nauc_map_at_100_max": 0.168323, + "nauc_map_at_100_std": 0.021779, + "nauc_map_at_100_diff1": 0.228003, + "nauc_map_at_1000_max": 0.167813, + "nauc_map_at_1000_std": 0.02079, + "nauc_map_at_1000_diff1": 0.227541, + "nauc_recall_at_1_max": 0.159602, + "nauc_recall_at_1_std": 0.044069, + "nauc_recall_at_1_diff1": 0.303299, + "nauc_recall_at_3_max": 0.07209, + "nauc_recall_at_3_std": 0.005475, + "nauc_recall_at_3_diff1": 0.129343, + "nauc_recall_at_5_max": 0.105377, + "nauc_recall_at_5_std": 0.003252, + "nauc_recall_at_5_diff1": 0.15683, + "nauc_recall_at_10_max": 0.112053, + "nauc_recall_at_10_std": 0.017703, + "nauc_recall_at_10_diff1": 0.108651, + "nauc_recall_at_20_max": 0.137503, + "nauc_recall_at_20_std": -0.005962, + "nauc_recall_at_20_diff1": 0.146729, + "nauc_recall_at_100_max": 0.326476, + "nauc_recall_at_100_std": 0.201408, + "nauc_recall_at_100_diff1": 0.11702, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.190937, + "nauc_precision_at_1_std": 0.022361, + "nauc_precision_at_1_diff1": 0.250329, + "nauc_precision_at_3_max": 0.172118, + "nauc_precision_at_3_std": 0.039325, + "nauc_precision_at_3_diff1": 0.126715, + "nauc_precision_at_5_max": 0.172416, + "nauc_precision_at_5_std": 0.000145, + "nauc_precision_at_5_diff1": 0.126567, + "nauc_precision_at_10_max": 0.140336, + "nauc_precision_at_10_std": -0.052421, + "nauc_precision_at_10_diff1": 0.052822, + "nauc_precision_at_20_max": 0.10397, + "nauc_precision_at_20_std": -0.078371, + "nauc_precision_at_20_diff1": 0.013337, + "nauc_precision_at_100_max": 0.052196, + "nauc_precision_at_100_std": -0.040333, + "nauc_precision_at_100_diff1": -0.093392, + "nauc_precision_at_1000_max": -0.131954, + "nauc_precision_at_1000_std": -0.138286, + "nauc_precision_at_1000_diff1": -0.202521, + "nauc_cv_recall_at_1_max": 0.190937, + "nauc_cv_recall_at_1_std": 0.022361, + "nauc_cv_recall_at_1_diff1": 0.250329, + "nauc_cv_recall_at_3_max": 0.086403, + "nauc_cv_recall_at_3_std": 0.010922, + "nauc_cv_recall_at_3_diff1": 0.124168, + "nauc_cv_recall_at_5_max": 0.126067, + "nauc_cv_recall_at_5_std": -0.001673, + "nauc_cv_recall_at_5_diff1": 0.168655, + "nauc_cv_recall_at_10_max": 0.105637, + "nauc_cv_recall_at_10_std": -0.004036, + "nauc_cv_recall_at_10_diff1": 0.064196, + "nauc_cv_recall_at_20_max": 0.156436, + "nauc_cv_recall_at_20_std": 0.003147, + "nauc_cv_recall_at_20_diff1": 0.142845, + "nauc_cv_recall_at_100_max": 0.569444, + "nauc_cv_recall_at_100_std": 0.428717, + "nauc_cv_recall_at_100_diff1": 0.17755, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.190937, + "nauc_mrr_at_1_std": 0.022361, + "nauc_mrr_at_1_diff1": 0.250329, + "nauc_mrr_at_3_max": 0.15072, + "nauc_mrr_at_3_std": 0.026396, + "nauc_mrr_at_3_diff1": 0.195561, + "nauc_mrr_at_5_max": 0.160341, + "nauc_mrr_at_5_std": 0.023207, + "nauc_mrr_at_5_diff1": 0.206522, + "nauc_mrr_at_10_max": 0.157727, + "nauc_mrr_at_10_std": 0.022619, + "nauc_mrr_at_10_diff1": 0.19177, + "nauc_mrr_at_20_max": 0.161501, + "nauc_mrr_at_20_std": 0.023521, + "nauc_mrr_at_20_diff1": 0.198724, + "nauc_mrr_at_100_max": 0.165216, + "nauc_mrr_at_100_std": 0.026069, + "nauc_mrr_at_100_diff1": 0.200489, + "nauc_mrr_at_1000_max": 0.164564, + "nauc_mrr_at_1000_std": 0.025395, + "nauc_mrr_at_1000_diff1": 0.200423, + "main_score": 0.28104, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.16875, + "ndcg_at_3": 0.21395, + "ndcg_at_5": 0.24512, + "ndcg_at_10": 0.28699, + "ndcg_at_20": 0.31871, + "ndcg_at_100": 0.37462, + "ndcg_at_1000": 0.42506, + "map_at_1": 0.11214, + "map_at_3": 0.16528, + "map_at_5": 0.18871, + "map_at_10": 0.21051, + "map_at_20": 0.22122, + "map_at_100": 0.23236, + "map_at_1000": 0.2359, + "recall_at_1": 0.11214, + "recall_at_3": 0.22131, + "recall_at_5": 0.29969, + "recall_at_10": 0.41454, + "recall_at_20": 0.51161, + "recall_at_100": 0.7286, + "recall_at_1000": 0.99911, + "cv_recall_at_1": 0.16875, + "cv_recall_at_3": 0.36875, + "cv_recall_at_5": 0.46875, + "cv_recall_at_10": 0.60625, + "cv_recall_at_20": 0.725, + "cv_recall_at_100": 0.8875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.16875, + "precision_at_3": 0.13958, + "precision_at_5": 0.11875, + "precision_at_10": 0.0875, + "precision_at_20": 0.05844, + "precision_at_100": 0.01963, + "precision_at_1000": 0.00321, + "mrr_at_1": 0.16875, + "mrr_at_3": 0.253125, + "mrr_at_5": 0.27625, + "mrr_at_10": 0.294417, + "mrr_at_20": 0.302315, + "mrr_at_100": 0.306394, + "mrr_at_1000": 0.306971, + "nauc_ndcg_at_1_max": 0.390935, + "nauc_ndcg_at_1_std": 0.060194, + "nauc_ndcg_at_1_diff1": 0.234254, + "nauc_ndcg_at_3_max": 0.2361, + "nauc_ndcg_at_3_std": 0.024503, + "nauc_ndcg_at_3_diff1": 0.110341, + "nauc_ndcg_at_5_max": 0.229049, + "nauc_ndcg_at_5_std": 0.019579, + "nauc_ndcg_at_5_diff1": 0.108329, + "nauc_ndcg_at_10_max": 0.241309, + "nauc_ndcg_at_10_std": 0.07348, + "nauc_ndcg_at_10_diff1": 0.097094, + "nauc_ndcg_at_20_max": 0.250892, + "nauc_ndcg_at_20_std": 0.077672, + "nauc_ndcg_at_20_diff1": 0.083189, + "nauc_ndcg_at_100_max": 0.254571, + "nauc_ndcg_at_100_std": 0.060531, + "nauc_ndcg_at_100_diff1": 0.106828, + "nauc_ndcg_at_1000_max": 0.247069, + "nauc_ndcg_at_1000_std": 0.039805, + "nauc_ndcg_at_1000_diff1": 0.113632, + "nauc_map_at_1_max": 0.352306, + "nauc_map_at_1_std": 0.130323, + "nauc_map_at_1_diff1": 0.262123, + "nauc_map_at_3_max": 0.269403, + "nauc_map_at_3_std": 0.086389, + "nauc_map_at_3_diff1": 0.161025, + "nauc_map_at_5_max": 0.262348, + "nauc_map_at_5_std": 0.071841, + "nauc_map_at_5_diff1": 0.146286, + "nauc_map_at_10_max": 0.265358, + "nauc_map_at_10_std": 0.091759, + "nauc_map_at_10_diff1": 0.140967, + "nauc_map_at_20_max": 0.267451, + "nauc_map_at_20_std": 0.090665, + "nauc_map_at_20_diff1": 0.135271, + "nauc_map_at_100_max": 0.27011, + "nauc_map_at_100_std": 0.086851, + "nauc_map_at_100_diff1": 0.13841, + "nauc_map_at_1000_max": 0.270052, + "nauc_map_at_1000_std": 0.085556, + "nauc_map_at_1000_diff1": 0.140676, + "nauc_recall_at_1_max": 0.352306, + "nauc_recall_at_1_std": 0.130323, + "nauc_recall_at_1_diff1": 0.262123, + "nauc_recall_at_3_max": 0.186498, + "nauc_recall_at_3_std": 0.049617, + "nauc_recall_at_3_diff1": 0.076321, + "nauc_recall_at_5_max": 0.159068, + "nauc_recall_at_5_std": -0.002942, + "nauc_recall_at_5_diff1": 0.066297, + "nauc_recall_at_10_max": 0.168337, + "nauc_recall_at_10_std": 0.104731, + "nauc_recall_at_10_diff1": 0.03606, + "nauc_recall_at_20_max": 0.186425, + "nauc_recall_at_20_std": 0.124724, + "nauc_recall_at_20_diff1": -0.010391, + "nauc_recall_at_100_max": 0.194405, + "nauc_recall_at_100_std": 0.08391, + "nauc_recall_at_100_diff1": 0.063422, + "nauc_recall_at_1000_max": 0.869281, + "nauc_recall_at_1000_std": 0.869281, + "nauc_recall_at_1000_diff1": 1.0, + "nauc_precision_at_1_max": 0.390935, + "nauc_precision_at_1_std": 0.060194, + "nauc_precision_at_1_diff1": 0.234254, + "nauc_precision_at_3_max": 0.141263, + "nauc_precision_at_3_std": -0.074132, + "nauc_precision_at_3_diff1": -0.014901, + "nauc_precision_at_5_max": 0.112287, + "nauc_precision_at_5_std": -0.103101, + "nauc_precision_at_5_diff1": -0.032817, + "nauc_precision_at_10_max": 0.111386, + "nauc_precision_at_10_std": -0.025513, + "nauc_precision_at_10_diff1": -0.04764, + "nauc_precision_at_20_max": 0.051783, + "nauc_precision_at_20_std": -0.071588, + "nauc_precision_at_20_diff1": -0.071658, + "nauc_precision_at_100_max": -0.047249, + "nauc_precision_at_100_std": -0.141877, + "nauc_precision_at_100_diff1": -0.005648, + "nauc_precision_at_1000_max": -0.180987, + "nauc_precision_at_1000_std": -0.225309, + "nauc_precision_at_1000_diff1": -0.050048, + "nauc_cv_recall_at_1_max": 0.390935, + "nauc_cv_recall_at_1_std": 0.060194, + "nauc_cv_recall_at_1_diff1": 0.234254, + "nauc_cv_recall_at_3_max": 0.149478, + "nauc_cv_recall_at_3_std": -0.055695, + "nauc_cv_recall_at_3_diff1": -0.029718, + "nauc_cv_recall_at_5_max": 0.097747, + "nauc_cv_recall_at_5_std": -0.10708, + "nauc_cv_recall_at_5_diff1": -0.038766, + "nauc_cv_recall_at_10_max": 0.156902, + "nauc_cv_recall_at_10_std": 0.034505, + "nauc_cv_recall_at_10_diff1": -0.050103, + "nauc_cv_recall_at_20_max": 0.285787, + "nauc_cv_recall_at_20_std": 0.138464, + "nauc_cv_recall_at_20_diff1": -0.043472, + "nauc_cv_recall_at_100_max": 0.151858, + "nauc_cv_recall_at_100_std": -0.024081, + "nauc_cv_recall_at_100_diff1": 0.117354, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.390935, + "nauc_mrr_at_1_std": 0.060194, + "nauc_mrr_at_1_diff1": 0.234254, + "nauc_mrr_at_3_max": 0.273933, + "nauc_mrr_at_3_std": 0.000664, + "nauc_mrr_at_3_diff1": 0.104361, + "nauc_mrr_at_5_max": 0.26124, + "nauc_mrr_at_5_std": -0.010409, + "nauc_mrr_at_5_diff1": 0.10138, + "nauc_mrr_at_10_max": 0.269673, + "nauc_mrr_at_10_std": 0.010483, + "nauc_mrr_at_10_diff1": 0.102169, + "nauc_mrr_at_20_max": 0.280423, + "nauc_mrr_at_20_std": 0.017314, + "nauc_mrr_at_20_diff1": 0.105655, + "nauc_mrr_at_100_max": 0.278987, + "nauc_mrr_at_100_std": 0.014578, + "nauc_mrr_at_100_diff1": 0.108249, + "nauc_mrr_at_1000_max": 0.279314, + "nauc_mrr_at_1000_std": 0.014816, + "nauc_mrr_at_1000_diff1": 0.108364, + "main_score": 0.24512, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.3625, + "ndcg_at_3": 0.38058, + "ndcg_at_5": 0.41431, + "ndcg_at_10": 0.455, + "ndcg_at_20": 0.47779, + "ndcg_at_100": 0.54368, + "ndcg_at_1000": 0.56943, + "map_at_1": 0.2296, + "map_at_3": 0.3082, + "map_at_5": 0.33914, + "map_at_10": 0.36747, + "map_at_20": 0.3785, + "map_at_100": 0.39515, + "map_at_1000": 0.39737, + "recall_at_1": 0.2296, + "recall_at_3": 0.37091, + "recall_at_5": 0.45783, + "recall_at_10": 0.56557, + "recall_at_20": 0.62838, + "recall_at_100": 0.87172, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.36875, + "cv_recall_at_3": 0.56875, + "cv_recall_at_5": 0.68125, + "cv_recall_at_10": 0.78125, + "cv_recall_at_20": 0.80625, + "cv_recall_at_100": 0.96875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.3625, + "precision_at_3": 0.22917, + "precision_at_5": 0.18625, + "precision_at_10": 0.12937, + "precision_at_20": 0.07844, + "precision_at_100": 0.02494, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.36875, + "mrr_at_3": 0.457292, + "mrr_at_5": 0.483854, + "mrr_at_10": 0.497555, + "mrr_at_20": 0.499579, + "mrr_at_100": 0.504722, + "mrr_at_1000": 0.504868, + "nauc_ndcg_at_1_max": 0.077937, + "nauc_ndcg_at_1_std": -0.098857, + "nauc_ndcg_at_1_diff1": 0.236268, + "nauc_ndcg_at_3_max": 0.117023, + "nauc_ndcg_at_3_std": -0.043105, + "nauc_ndcg_at_3_diff1": 0.261959, + "nauc_ndcg_at_5_max": 0.124056, + "nauc_ndcg_at_5_std": -0.005346, + "nauc_ndcg_at_5_diff1": 0.248618, + "nauc_ndcg_at_10_max": 0.119609, + "nauc_ndcg_at_10_std": -0.020785, + "nauc_ndcg_at_10_diff1": 0.254243, + "nauc_ndcg_at_20_max": 0.128022, + "nauc_ndcg_at_20_std": -0.025819, + "nauc_ndcg_at_20_diff1": 0.252211, + "nauc_ndcg_at_100_max": 0.14863, + "nauc_ndcg_at_100_std": -0.021586, + "nauc_ndcg_at_100_diff1": 0.251882, + "nauc_ndcg_at_1000_max": 0.143416, + "nauc_ndcg_at_1000_std": -0.026983, + "nauc_ndcg_at_1000_diff1": 0.240192, + "nauc_map_at_1_max": 0.063411, + "nauc_map_at_1_std": -0.075398, + "nauc_map_at_1_diff1": 0.267142, + "nauc_map_at_3_max": 0.106209, + "nauc_map_at_3_std": -0.030444, + "nauc_map_at_3_diff1": 0.27363, + "nauc_map_at_5_max": 0.111042, + "nauc_map_at_5_std": -0.012705, + "nauc_map_at_5_diff1": 0.252642, + "nauc_map_at_10_max": 0.110868, + "nauc_map_at_10_std": -0.024831, + "nauc_map_at_10_diff1": 0.249913, + "nauc_map_at_20_max": 0.115333, + "nauc_map_at_20_std": -0.029345, + "nauc_map_at_20_diff1": 0.247349, + "nauc_map_at_100_max": 0.117973, + "nauc_map_at_100_std": -0.028569, + "nauc_map_at_100_diff1": 0.243364, + "nauc_map_at_1000_max": 0.118256, + "nauc_map_at_1000_std": -0.028804, + "nauc_map_at_1000_diff1": 0.242837, + "nauc_recall_at_1_max": 0.063411, + "nauc_recall_at_1_std": -0.075398, + "nauc_recall_at_1_diff1": 0.267142, + "nauc_recall_at_3_max": 0.094542, + "nauc_recall_at_3_std": -0.011925, + "nauc_recall_at_3_diff1": 0.270882, + "nauc_recall_at_5_max": 0.115206, + "nauc_recall_at_5_std": 0.048465, + "nauc_recall_at_5_diff1": 0.232799, + "nauc_recall_at_10_max": 0.086282, + "nauc_recall_at_10_std": 0.016761, + "nauc_recall_at_10_diff1": 0.220064, + "nauc_recall_at_20_max": 0.103539, + "nauc_recall_at_20_std": -0.0104, + "nauc_recall_at_20_diff1": 0.227265, + "nauc_recall_at_100_max": 0.225072, + "nauc_recall_at_100_std": 0.044651, + "nauc_recall_at_100_diff1": 0.314183, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.077937, + "nauc_precision_at_1_std": -0.098857, + "nauc_precision_at_1_diff1": 0.236268, + "nauc_precision_at_3_max": 0.155174, + "nauc_precision_at_3_std": -0.022415, + "nauc_precision_at_3_diff1": 0.161411, + "nauc_precision_at_5_max": 0.17798, + "nauc_precision_at_5_std": 0.043906, + "nauc_precision_at_5_diff1": 0.040788, + "nauc_precision_at_10_max": 0.112668, + "nauc_precision_at_10_std": 0.000722, + "nauc_precision_at_10_diff1": -0.014368, + "nauc_precision_at_20_max": 0.1076, + "nauc_precision_at_20_std": -0.017145, + "nauc_precision_at_20_diff1": -0.06967, + "nauc_precision_at_100_max": 0.105925, + "nauc_precision_at_100_std": -0.012734, + "nauc_precision_at_100_diff1": -0.10489, + "nauc_precision_at_1000_max": 0.027815, + "nauc_precision_at_1000_std": -0.030316, + "nauc_precision_at_1000_diff1": -0.153119, + "nauc_cv_recall_at_1_max": 0.08756, + "nauc_cv_recall_at_1_std": -0.088703, + "nauc_cv_recall_at_1_diff1": 0.218067, + "nauc_cv_recall_at_3_max": 0.128983, + "nauc_cv_recall_at_3_std": -0.047177, + "nauc_cv_recall_at_3_diff1": 0.2976, + "nauc_cv_recall_at_5_max": 0.210821, + "nauc_cv_recall_at_5_std": 0.061877, + "nauc_cv_recall_at_5_diff1": 0.292345, + "nauc_cv_recall_at_10_max": 0.192592, + "nauc_cv_recall_at_10_std": 0.018223, + "nauc_cv_recall_at_10_diff1": 0.299352, + "nauc_cv_recall_at_20_max": 0.16403, + "nauc_cv_recall_at_20_std": -0.027218, + "nauc_cv_recall_at_20_diff1": 0.287258, + "nauc_cv_recall_at_100_max": 0.789916, + "nauc_cv_recall_at_100_std": 0.274697, + "nauc_cv_recall_at_100_diff1": 0.149113, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.08756, + "nauc_mrr_at_1_std": -0.088703, + "nauc_mrr_at_1_diff1": 0.218067, + "nauc_mrr_at_3_max": 0.116406, + "nauc_mrr_at_3_std": -0.062629, + "nauc_mrr_at_3_diff1": 0.261375, + "nauc_mrr_at_5_max": 0.133572, + "nauc_mrr_at_5_std": -0.039309, + "nauc_mrr_at_5_diff1": 0.257182, + "nauc_mrr_at_10_max": 0.128775, + "nauc_mrr_at_10_std": -0.047035, + "nauc_mrr_at_10_diff1": 0.255788, + "nauc_mrr_at_20_max": 0.127147, + "nauc_mrr_at_20_std": -0.049322, + "nauc_mrr_at_20_diff1": 0.255135, + "nauc_mrr_at_100_max": 0.12807, + "nauc_mrr_at_100_std": -0.04988, + "nauc_mrr_at_100_diff1": 0.252892, + "nauc_mrr_at_1000_max": 0.1278, + "nauc_mrr_at_1000_std": -0.050081, + "nauc_mrr_at_1000_diff1": 0.253066, + "main_score": 0.41431, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.14375, + "ndcg_at_3": 0.159, + "ndcg_at_5": 0.17041, + "ndcg_at_10": 0.19572, + "ndcg_at_20": 0.21804, + "ndcg_at_100": 0.27853, + "ndcg_at_1000": 0.35136, + "map_at_1": 0.08188, + "map_at_3": 0.12351, + "map_at_5": 0.13252, + "map_at_10": 0.14446, + "map_at_20": 0.15181, + "map_at_100": 0.1623, + "map_at_1000": 0.16701, + "recall_at_1": 0.08188, + "recall_at_3": 0.17079, + "recall_at_5": 0.20188, + "recall_at_10": 0.274, + "recall_at_20": 0.34238, + "recall_at_100": 0.59331, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.14375, + "cv_recall_at_3": 0.25, + "cv_recall_at_5": 0.3125, + "cv_recall_at_10": 0.425, + "cv_recall_at_20": 0.5125, + "cv_recall_at_100": 0.75, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.14375, + "precision_at_3": 0.09792, + "precision_at_5": 0.07875, + "precision_at_10": 0.05875, + "precision_at_20": 0.04063, + "precision_at_100": 0.01619, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.14375, + "mrr_at_3": 0.186458, + "mrr_at_5": 0.199896, + "mrr_at_10": 0.214482, + "mrr_at_20": 0.220694, + "mrr_at_100": 0.226458, + "mrr_at_1000": 0.227789, + "nauc_ndcg_at_1_max": 0.32616, + "nauc_ndcg_at_1_std": 0.158633, + "nauc_ndcg_at_1_diff1": 0.246356, + "nauc_ndcg_at_3_max": 0.302261, + "nauc_ndcg_at_3_std": 0.176984, + "nauc_ndcg_at_3_diff1": 0.201215, + "nauc_ndcg_at_5_max": 0.312773, + "nauc_ndcg_at_5_std": 0.190524, + "nauc_ndcg_at_5_diff1": 0.172511, + "nauc_ndcg_at_10_max": 0.334609, + "nauc_ndcg_at_10_std": 0.207045, + "nauc_ndcg_at_10_diff1": 0.180395, + "nauc_ndcg_at_20_max": 0.352017, + "nauc_ndcg_at_20_std": 0.209724, + "nauc_ndcg_at_20_diff1": 0.16407, + "nauc_ndcg_at_100_max": 0.344196, + "nauc_ndcg_at_100_std": 0.208781, + "nauc_ndcg_at_100_diff1": 0.151857, + "nauc_ndcg_at_1000_max": 0.325945, + "nauc_ndcg_at_1000_std": 0.200246, + "nauc_ndcg_at_1000_diff1": 0.17917, + "nauc_map_at_1_max": 0.319575, + "nauc_map_at_1_std": 0.140096, + "nauc_map_at_1_diff1": 0.211761, + "nauc_map_at_3_max": 0.317163, + "nauc_map_at_3_std": 0.160104, + "nauc_map_at_3_diff1": 0.234326, + "nauc_map_at_5_max": 0.323267, + "nauc_map_at_5_std": 0.178291, + "nauc_map_at_5_diff1": 0.213092, + "nauc_map_at_10_max": 0.332829, + "nauc_map_at_10_std": 0.181814, + "nauc_map_at_10_diff1": 0.211031, + "nauc_map_at_20_max": 0.34157, + "nauc_map_at_20_std": 0.185129, + "nauc_map_at_20_diff1": 0.207357, + "nauc_map_at_100_max": 0.341622, + "nauc_map_at_100_std": 0.186796, + "nauc_map_at_100_diff1": 0.198857, + "nauc_map_at_1000_max": 0.341969, + "nauc_map_at_1000_std": 0.188439, + "nauc_map_at_1000_diff1": 0.202589, + "nauc_recall_at_1_max": 0.319575, + "nauc_recall_at_1_std": 0.140096, + "nauc_recall_at_1_diff1": 0.211761, + "nauc_recall_at_3_max": 0.266786, + "nauc_recall_at_3_std": 0.164749, + "nauc_recall_at_3_diff1": 0.158367, + "nauc_recall_at_5_max": 0.282806, + "nauc_recall_at_5_std": 0.188351, + "nauc_recall_at_5_diff1": 0.112766, + "nauc_recall_at_10_max": 0.308856, + "nauc_recall_at_10_std": 0.209306, + "nauc_recall_at_10_diff1": 0.125776, + "nauc_recall_at_20_max": 0.343593, + "nauc_recall_at_20_std": 0.205141, + "nauc_recall_at_20_diff1": 0.105236, + "nauc_recall_at_100_max": 0.302487, + "nauc_recall_at_100_std": 0.182085, + "nauc_recall_at_100_diff1": 0.071873, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.32616, + "nauc_precision_at_1_std": 0.158633, + "nauc_precision_at_1_diff1": 0.246356, + "nauc_precision_at_3_max": 0.278141, + "nauc_precision_at_3_std": 0.168611, + "nauc_precision_at_3_diff1": 0.196961, + "nauc_precision_at_5_max": 0.282325, + "nauc_precision_at_5_std": 0.203805, + "nauc_precision_at_5_diff1": 0.11352, + "nauc_precision_at_10_max": 0.312311, + "nauc_precision_at_10_std": 0.240268, + "nauc_precision_at_10_diff1": 0.099853, + "nauc_precision_at_20_max": 0.317032, + "nauc_precision_at_20_std": 0.237251, + "nauc_precision_at_20_diff1": 0.046815, + "nauc_precision_at_100_max": 0.178, + "nauc_precision_at_100_std": 0.172158, + "nauc_precision_at_100_diff1": 0.048091, + "nauc_precision_at_1000_max": -0.068299, + "nauc_precision_at_1000_std": -0.02424, + "nauc_precision_at_1000_diff1": 0.132902, + "nauc_cv_recall_at_1_max": 0.32616, + "nauc_cv_recall_at_1_std": 0.158633, + "nauc_cv_recall_at_1_diff1": 0.246356, + "nauc_cv_recall_at_3_max": 0.273189, + "nauc_cv_recall_at_3_std": 0.219888, + "nauc_cv_recall_at_3_diff1": 0.131666, + "nauc_cv_recall_at_5_max": 0.308342, + "nauc_cv_recall_at_5_std": 0.23854, + "nauc_cv_recall_at_5_diff1": 0.085136, + "nauc_cv_recall_at_10_max": 0.313374, + "nauc_cv_recall_at_10_std": 0.290491, + "nauc_cv_recall_at_10_diff1": 0.084415, + "nauc_cv_recall_at_20_max": 0.362636, + "nauc_cv_recall_at_20_std": 0.295784, + "nauc_cv_recall_at_20_diff1": 0.057446, + "nauc_cv_recall_at_100_max": 0.288383, + "nauc_cv_recall_at_100_std": 0.244623, + "nauc_cv_recall_at_100_diff1": 0.126635, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.32616, + "nauc_mrr_at_1_std": 0.158633, + "nauc_mrr_at_1_diff1": 0.246356, + "nauc_mrr_at_3_max": 0.3072, + "nauc_mrr_at_3_std": 0.194261, + "nauc_mrr_at_3_diff1": 0.20383, + "nauc_mrr_at_5_max": 0.314973, + "nauc_mrr_at_5_std": 0.198758, + "nauc_mrr_at_5_diff1": 0.187544, + "nauc_mrr_at_10_max": 0.313165, + "nauc_mrr_at_10_std": 0.20607, + "nauc_mrr_at_10_diff1": 0.1837, + "nauc_mrr_at_20_max": 0.316325, + "nauc_mrr_at_20_std": 0.205474, + "nauc_mrr_at_20_diff1": 0.181604, + "nauc_mrr_at_100_max": 0.314375, + "nauc_mrr_at_100_std": 0.202586, + "nauc_mrr_at_100_diff1": 0.181848, + "nauc_mrr_at_1000_max": 0.314502, + "nauc_mrr_at_1000_std": 0.202658, + "nauc_mrr_at_1000_diff1": 0.182132, + "main_score": 0.17041, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 180.30777716636658, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/Vidore2ESGReportsHLRetrieval.json b/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..4755001038 --- /dev/null +++ b/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.07692, + "ndcg_at_3": 0.14396, + "ndcg_at_5": 0.16317, + "ndcg_at_10": 0.2162, + "ndcg_at_20": 0.2478, + "ndcg_at_100": 0.30942, + "ndcg_at_1000": 0.35123, + "map_at_1": 0.05944, + "map_at_3": 0.10516, + "map_at_5": 0.12071, + "map_at_10": 0.14404, + "map_at_20": 0.15447, + "map_at_100": 0.16671, + "map_at_1000": 0.16922, + "recall_at_1": 0.05944, + "recall_at_3": 0.1676, + "recall_at_5": 0.22227, + "recall_at_10": 0.37818, + "recall_at_20": 0.48236, + "recall_at_100": 0.754, + "recall_at_1000": 0.99359, + "cv_recall_at_1": 0.07692, + "cv_recall_at_3": 0.25, + "cv_recall_at_5": 0.32692, + "cv_recall_at_10": 0.55769, + "cv_recall_at_20": 0.63462, + "cv_recall_at_100": 0.86538, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.07692, + "precision_at_3": 0.08974, + "precision_at_5": 0.07692, + "precision_at_10": 0.06731, + "precision_at_20": 0.04615, + "precision_at_100": 0.01635, + "precision_at_1000": 0.00244, + "mrr_at_1": 0.076923, + "mrr_at_3": 0.150641, + "mrr_at_5": 0.16891, + "mrr_at_10": 0.198764, + "mrr_at_20": 0.204258, + "mrr_at_100": 0.210443, + "mrr_at_1000": 0.211086, + "nauc_ndcg_at_1_max": 0.343218, + "nauc_ndcg_at_1_std": 0.560607, + "nauc_ndcg_at_1_diff1": -0.006766, + "nauc_ndcg_at_3_max": 0.175865, + "nauc_ndcg_at_3_std": 0.382367, + "nauc_ndcg_at_3_diff1": -0.011122, + "nauc_ndcg_at_5_max": 0.166548, + "nauc_ndcg_at_5_std": 0.394304, + "nauc_ndcg_at_5_diff1": 0.018772, + "nauc_ndcg_at_10_max": 0.184898, + "nauc_ndcg_at_10_std": 0.40182, + "nauc_ndcg_at_10_diff1": -0.010405, + "nauc_ndcg_at_20_max": 0.182964, + "nauc_ndcg_at_20_std": 0.408488, + "nauc_ndcg_at_20_diff1": 0.076735, + "nauc_ndcg_at_100_max": 0.150869, + "nauc_ndcg_at_100_std": 0.377033, + "nauc_ndcg_at_100_diff1": 0.042464, + "nauc_ndcg_at_1000_max": 0.192525, + "nauc_ndcg_at_1000_std": 0.405418, + "nauc_ndcg_at_1000_diff1": 0.05336, + "nauc_map_at_1_max": 0.416053, + "nauc_map_at_1_std": 0.745635, + "nauc_map_at_1_diff1": 0.011388, + "nauc_map_at_3_max": 0.211032, + "nauc_map_at_3_std": 0.453337, + "nauc_map_at_3_diff1": 0.02035, + "nauc_map_at_5_max": 0.206918, + "nauc_map_at_5_std": 0.453569, + "nauc_map_at_5_diff1": 0.039695, + "nauc_map_at_10_max": 0.230638, + "nauc_map_at_10_std": 0.463937, + "nauc_map_at_10_diff1": 0.03211, + "nauc_map_at_20_max": 0.231073, + "nauc_map_at_20_std": 0.466152, + "nauc_map_at_20_diff1": 0.065323, + "nauc_map_at_100_max": 0.223212, + "nauc_map_at_100_std": 0.456406, + "nauc_map_at_100_diff1": 0.058091, + "nauc_map_at_1000_max": 0.227737, + "nauc_map_at_1000_std": 0.458627, + "nauc_map_at_1000_diff1": 0.059983, + "nauc_recall_at_1_max": 0.416053, + "nauc_recall_at_1_std": 0.745635, + "nauc_recall_at_1_diff1": 0.011388, + "nauc_recall_at_3_max": 0.096519, + "nauc_recall_at_3_std": 0.331585, + "nauc_recall_at_3_diff1": -0.021913, + "nauc_recall_at_5_max": 0.096502, + "nauc_recall_at_5_std": 0.335567, + "nauc_recall_at_5_diff1": 0.010106, + "nauc_recall_at_10_max": 0.106188, + "nauc_recall_at_10_std": 0.287561, + "nauc_recall_at_10_diff1": -0.088795, + "nauc_recall_at_20_max": 0.083372, + "nauc_recall_at_20_std": 0.28444, + "nauc_recall_at_20_diff1": 0.118909, + "nauc_recall_at_100_max": -0.055632, + "nauc_recall_at_100_std": 0.17598, + "nauc_recall_at_100_diff1": -0.006239, + "nauc_recall_at_1000_max": 0.724179, + "nauc_recall_at_1000_std": 0.869837, + "nauc_recall_at_1000_diff1": 0.554244, + "nauc_precision_at_1_max": 0.343218, + "nauc_precision_at_1_std": 0.560607, + "nauc_precision_at_1_diff1": -0.006766, + "nauc_precision_at_3_max": 0.118686, + "nauc_precision_at_3_std": 0.302465, + "nauc_precision_at_3_diff1": -0.049014, + "nauc_precision_at_5_max": 0.20248, + "nauc_precision_at_5_std": 0.330379, + "nauc_precision_at_5_diff1": 0.14119, + "nauc_precision_at_10_max": 0.210751, + "nauc_precision_at_10_std": 0.273014, + "nauc_precision_at_10_diff1": 0.077273, + "nauc_precision_at_20_max": 0.218149, + "nauc_precision_at_20_std": 0.228501, + "nauc_precision_at_20_diff1": 0.197222, + "nauc_precision_at_100_max": 0.101526, + "nauc_precision_at_100_std": 0.004436, + "nauc_precision_at_100_diff1": 0.005721, + "nauc_precision_at_1000_max": 0.099777, + "nauc_precision_at_1000_std": -0.039699, + "nauc_precision_at_1000_diff1": -0.006973, + "nauc_cv_recall_at_1_max": 0.343218, + "nauc_cv_recall_at_1_std": 0.560607, + "nauc_cv_recall_at_1_diff1": -0.006766, + "nauc_cv_recall_at_3_max": 0.121194, + "nauc_cv_recall_at_3_std": 0.342742, + "nauc_cv_recall_at_3_diff1": -0.046899, + "nauc_cv_recall_at_5_max": 0.161492, + "nauc_cv_recall_at_5_std": 0.326868, + "nauc_cv_recall_at_5_diff1": 0.061211, + "nauc_cv_recall_at_10_max": 0.112175, + "nauc_cv_recall_at_10_std": 0.260662, + "nauc_cv_recall_at_10_diff1": -0.041288, + "nauc_cv_recall_at_20_max": 0.118699, + "nauc_cv_recall_at_20_std": 0.269685, + "nauc_cv_recall_at_20_diff1": 0.197763, + "nauc_cv_recall_at_100_max": -0.063036, + "nauc_cv_recall_at_100_std": 0.095856, + "nauc_cv_recall_at_100_diff1": 0.022769, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.343218, + "nauc_mrr_at_1_std": 0.560607, + "nauc_mrr_at_1_diff1": -0.006766, + "nauc_mrr_at_3_max": 0.18484, + "nauc_mrr_at_3_std": 0.377906, + "nauc_mrr_at_3_diff1": -0.026238, + "nauc_mrr_at_5_max": 0.190553, + "nauc_mrr_at_5_std": 0.36992, + "nauc_mrr_at_5_diff1": 0.007327, + "nauc_mrr_at_10_max": 0.190896, + "nauc_mrr_at_10_std": 0.362131, + "nauc_mrr_at_10_diff1": -0.0048, + "nauc_mrr_at_20_max": 0.194205, + "nauc_mrr_at_20_std": 0.363195, + "nauc_mrr_at_20_diff1": 0.021943, + "nauc_mrr_at_100_max": 0.187194, + "nauc_mrr_at_100_std": 0.356359, + "nauc_mrr_at_100_diff1": 0.014858, + "nauc_mrr_at_1000_max": 0.188014, + "nauc_mrr_at_1000_std": 0.35735, + "nauc_mrr_at_1000_diff1": 0.014936, + "main_score": 0.16317, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 126.76670837402344, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/Vidore2ESGReportsRetrieval.json b/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..e1293c4c18 --- /dev/null +++ b/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.10526, + "ndcg_at_3": 0.09198, + "ndcg_at_5": 0.10218, + "ndcg_at_10": 0.12799, + "ndcg_at_20": 0.15919, + "ndcg_at_100": 0.23781, + "ndcg_at_1000": 0.29991, + "map_at_1": 0.03436, + "map_at_3": 0.04868, + "map_at_5": 0.05742, + "map_at_10": 0.0728, + "map_at_20": 0.08323, + "map_at_100": 0.09848, + "map_at_1000": 0.10182, + "recall_at_1": 0.03436, + "recall_at_3": 0.06495, + "recall_at_5": 0.09492, + "recall_at_10": 0.17753, + "recall_at_20": 0.2863, + "recall_at_100": 0.61426, + "recall_at_1000": 0.96337, + "cv_recall_at_1": 0.10526, + "cv_recall_at_3": 0.21053, + "cv_recall_at_5": 0.2807, + "cv_recall_at_10": 0.40351, + "cv_recall_at_20": 0.52632, + "cv_recall_at_100": 0.80702, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.10526, + "precision_at_3": 0.07018, + "precision_at_5": 0.06667, + "precision_at_10": 0.05789, + "precision_at_20": 0.04211, + "precision_at_100": 0.01772, + "precision_at_1000": 0.00361, + "mrr_at_1": 0.105263, + "mrr_at_3": 0.154971, + "mrr_at_5": 0.171637, + "mrr_at_10": 0.187886, + "mrr_at_20": 0.196649, + "mrr_at_100": 0.204262, + "mrr_at_1000": 0.204985, + "nauc_ndcg_at_1_max": 0.020343, + "nauc_ndcg_at_1_std": 0.019172, + "nauc_ndcg_at_1_diff1": 0.031331, + "nauc_ndcg_at_3_max": 0.166606, + "nauc_ndcg_at_3_std": -0.09226, + "nauc_ndcg_at_3_diff1": 0.18921, + "nauc_ndcg_at_5_max": 0.225129, + "nauc_ndcg_at_5_std": -0.126003, + "nauc_ndcg_at_5_diff1": 0.295951, + "nauc_ndcg_at_10_max": 0.257848, + "nauc_ndcg_at_10_std": -0.151084, + "nauc_ndcg_at_10_diff1": 0.350261, + "nauc_ndcg_at_20_max": 0.219923, + "nauc_ndcg_at_20_std": -0.134952, + "nauc_ndcg_at_20_diff1": 0.274534, + "nauc_ndcg_at_100_max": 0.253152, + "nauc_ndcg_at_100_std": 0.015222, + "nauc_ndcg_at_100_diff1": 0.146554, + "nauc_ndcg_at_1000_max": 0.181739, + "nauc_ndcg_at_1000_std": -0.045755, + "nauc_ndcg_at_1000_diff1": 0.166295, + "nauc_map_at_1_max": 0.306377, + "nauc_map_at_1_std": -0.137259, + "nauc_map_at_1_diff1": 0.494198, + "nauc_map_at_3_max": 0.327405, + "nauc_map_at_3_std": -0.141256, + "nauc_map_at_3_diff1": 0.462927, + "nauc_map_at_5_max": 0.339318, + "nauc_map_at_5_std": -0.139414, + "nauc_map_at_5_diff1": 0.476639, + "nauc_map_at_10_max": 0.314036, + "nauc_map_at_10_std": -0.131293, + "nauc_map_at_10_diff1": 0.427841, + "nauc_map_at_20_max": 0.286645, + "nauc_map_at_20_std": -0.132965, + "nauc_map_at_20_diff1": 0.373285, + "nauc_map_at_100_max": 0.285478, + "nauc_map_at_100_std": -0.09607, + "nauc_map_at_100_diff1": 0.314865, + "nauc_map_at_1000_max": 0.279327, + "nauc_map_at_1000_std": -0.100251, + "nauc_map_at_1000_diff1": 0.314787, + "nauc_recall_at_1_max": 0.306377, + "nauc_recall_at_1_std": -0.137259, + "nauc_recall_at_1_diff1": 0.494198, + "nauc_recall_at_3_max": 0.33114, + "nauc_recall_at_3_std": -0.13237, + "nauc_recall_at_3_diff1": 0.399915, + "nauc_recall_at_5_max": 0.366242, + "nauc_recall_at_5_std": -0.147262, + "nauc_recall_at_5_diff1": 0.484876, + "nauc_recall_at_10_max": 0.278201, + "nauc_recall_at_10_std": -0.183768, + "nauc_recall_at_10_diff1": 0.366677, + "nauc_recall_at_20_max": 0.142742, + "nauc_recall_at_20_std": -0.135268, + "nauc_recall_at_20_diff1": 0.204004, + "nauc_recall_at_100_max": 0.236898, + "nauc_recall_at_100_std": 0.212329, + "nauc_recall_at_100_diff1": -0.071885, + "nauc_recall_at_1000_max": 0.137281, + "nauc_recall_at_1000_std": 0.807406, + "nauc_recall_at_1000_diff1": -0.847644, + "nauc_precision_at_1_max": 0.020343, + "nauc_precision_at_1_std": 0.019172, + "nauc_precision_at_1_diff1": 0.031331, + "nauc_precision_at_3_max": 0.124967, + "nauc_precision_at_3_std": -0.041343, + "nauc_precision_at_3_diff1": 0.059381, + "nauc_precision_at_5_max": 0.146599, + "nauc_precision_at_5_std": -0.089663, + "nauc_precision_at_5_diff1": 0.112274, + "nauc_precision_at_10_max": 0.089607, + "nauc_precision_at_10_std": -0.112356, + "nauc_precision_at_10_diff1": 0.121623, + "nauc_precision_at_20_max": 0.014381, + "nauc_precision_at_20_std": -0.118485, + "nauc_precision_at_20_diff1": -0.039874, + "nauc_precision_at_100_max": 0.016362, + "nauc_precision_at_100_std": 0.053416, + "nauc_precision_at_100_diff1": -0.20308, + "nauc_precision_at_1000_max": -0.290597, + "nauc_precision_at_1000_std": -0.223458, + "nauc_precision_at_1000_diff1": -0.049376, + "nauc_cv_recall_at_1_max": 0.020343, + "nauc_cv_recall_at_1_std": 0.019172, + "nauc_cv_recall_at_1_diff1": 0.031331, + "nauc_cv_recall_at_3_max": 0.124967, + "nauc_cv_recall_at_3_std": -0.041343, + "nauc_cv_recall_at_3_diff1": 0.059381, + "nauc_cv_recall_at_5_max": 0.191219, + "nauc_cv_recall_at_5_std": -0.143768, + "nauc_cv_recall_at_5_diff1": 0.272286, + "nauc_cv_recall_at_10_max": 0.218541, + "nauc_cv_recall_at_10_std": -0.35179, + "nauc_cv_recall_at_10_diff1": 0.392537, + "nauc_cv_recall_at_20_max": 0.20814, + "nauc_cv_recall_at_20_std": -0.2749, + "nauc_cv_recall_at_20_diff1": 0.328099, + "nauc_cv_recall_at_100_max": 0.226478, + "nauc_cv_recall_at_100_std": 0.312929, + "nauc_cv_recall_at_100_diff1": -0.138467, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.020343, + "nauc_mrr_at_1_std": 0.019172, + "nauc_mrr_at_1_diff1": 0.031331, + "nauc_mrr_at_3_max": 0.07867, + "nauc_mrr_at_3_std": -0.020392, + "nauc_mrr_at_3_diff1": 0.06137, + "nauc_mrr_at_5_max": 0.096985, + "nauc_mrr_at_5_std": -0.054038, + "nauc_mrr_at_5_diff1": 0.127477, + "nauc_mrr_at_10_max": 0.108845, + "nauc_mrr_at_10_std": -0.094454, + "nauc_mrr_at_10_diff1": 0.154171, + "nauc_mrr_at_20_max": 0.109285, + "nauc_mrr_at_20_std": -0.08522, + "nauc_mrr_at_20_diff1": 0.142986, + "nauc_mrr_at_100_max": 0.10834, + "nauc_mrr_at_100_std": -0.067297, + "nauc_mrr_at_100_diff1": 0.129551, + "nauc_mrr_at_1000_max": 0.108604, + "nauc_mrr_at_1000_std": -0.067591, + "nauc_mrr_at_1000_diff1": 0.129885, + "main_score": 0.10218, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.12281, + "ndcg_at_3": 0.12673, + "ndcg_at_5": 0.12761, + "ndcg_at_10": 0.15128, + "ndcg_at_20": 0.1828, + "ndcg_at_100": 0.24663, + "ndcg_at_1000": 0.31735, + "map_at_1": 0.05848, + "map_at_3": 0.08281, + "map_at_5": 0.08767, + "map_at_10": 0.09997, + "map_at_20": 0.11179, + "map_at_100": 0.12354, + "map_at_1000": 0.12749, + "recall_at_1": 0.05848, + "recall_at_3": 0.11393, + "recall_at_5": 0.13147, + "recall_at_10": 0.21247, + "recall_at_20": 0.30443, + "recall_at_100": 0.58513, + "recall_at_1000": 0.96431, + "cv_recall_at_1": 0.12281, + "cv_recall_at_3": 0.22807, + "cv_recall_at_5": 0.2807, + "cv_recall_at_10": 0.42105, + "cv_recall_at_20": 0.50877, + "cv_recall_at_100": 0.78947, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.12281, + "precision_at_3": 0.08772, + "precision_at_5": 0.06667, + "precision_at_10": 0.05263, + "precision_at_20": 0.04386, + "precision_at_100": 0.01561, + "precision_at_1000": 0.00354, + "mrr_at_1": 0.122807, + "mrr_at_3": 0.166667, + "mrr_at_5": 0.178947, + "mrr_at_10": 0.195732, + "mrr_at_20": 0.202448, + "mrr_at_100": 0.209576, + "mrr_at_1000": 0.210857, + "nauc_ndcg_at_1_max": 0.132694, + "nauc_ndcg_at_1_std": 0.065436, + "nauc_ndcg_at_1_diff1": 0.057271, + "nauc_ndcg_at_3_max": 0.12605, + "nauc_ndcg_at_3_std": 0.11086, + "nauc_ndcg_at_3_diff1": 0.060602, + "nauc_ndcg_at_5_max": 0.183853, + "nauc_ndcg_at_5_std": 0.09379, + "nauc_ndcg_at_5_diff1": 0.020397, + "nauc_ndcg_at_10_max": 0.2251, + "nauc_ndcg_at_10_std": 0.115739, + "nauc_ndcg_at_10_diff1": 0.058471, + "nauc_ndcg_at_20_max": 0.206255, + "nauc_ndcg_at_20_std": 0.125069, + "nauc_ndcg_at_20_diff1": 0.041733, + "nauc_ndcg_at_100_max": 0.207381, + "nauc_ndcg_at_100_std": 0.185488, + "nauc_ndcg_at_100_diff1": 0.014834, + "nauc_ndcg_at_1000_max": 0.169784, + "nauc_ndcg_at_1000_std": 0.168129, + "nauc_ndcg_at_1000_diff1": 0.03956, + "nauc_map_at_1_max": 0.212303, + "nauc_map_at_1_std": 0.10239, + "nauc_map_at_1_diff1": -0.022757, + "nauc_map_at_3_max": 0.177494, + "nauc_map_at_3_std": 0.073788, + "nauc_map_at_3_diff1": 0.007792, + "nauc_map_at_5_max": 0.210776, + "nauc_map_at_5_std": 0.086378, + "nauc_map_at_5_diff1": 0.00867, + "nauc_map_at_10_max": 0.210355, + "nauc_map_at_10_std": 0.100588, + "nauc_map_at_10_diff1": 0.02848, + "nauc_map_at_20_max": 0.205753, + "nauc_map_at_20_std": 0.116646, + "nauc_map_at_20_diff1": 0.029397, + "nauc_map_at_100_max": 0.204085, + "nauc_map_at_100_std": 0.136413, + "nauc_map_at_100_diff1": 0.02577, + "nauc_map_at_1000_max": 0.201672, + "nauc_map_at_1000_std": 0.134866, + "nauc_map_at_1000_diff1": 0.027318, + "nauc_recall_at_1_max": 0.212303, + "nauc_recall_at_1_std": 0.10239, + "nauc_recall_at_1_diff1": -0.022757, + "nauc_recall_at_3_max": 0.134789, + "nauc_recall_at_3_std": 0.033261, + "nauc_recall_at_3_diff1": 0.010762, + "nauc_recall_at_5_max": 0.231011, + "nauc_recall_at_5_std": 0.049712, + "nauc_recall_at_5_diff1": -5.7e-05, + "nauc_recall_at_10_max": 0.240143, + "nauc_recall_at_10_std": 0.105394, + "nauc_recall_at_10_diff1": 0.088139, + "nauc_recall_at_20_max": 0.178994, + "nauc_recall_at_20_std": 0.107315, + "nauc_recall_at_20_diff1": 0.007453, + "nauc_recall_at_100_max": 0.167952, + "nauc_recall_at_100_std": 0.244466, + "nauc_recall_at_100_diff1": -0.085381, + "nauc_recall_at_1000_max": 0.354375, + "nauc_recall_at_1000_std": 0.331515, + "nauc_recall_at_1000_diff1": -0.732038, + "nauc_precision_at_1_max": 0.132694, + "nauc_precision_at_1_std": 0.065436, + "nauc_precision_at_1_diff1": 0.057271, + "nauc_precision_at_3_max": 0.107542, + "nauc_precision_at_3_std": 0.156867, + "nauc_precision_at_3_diff1": 0.125417, + "nauc_precision_at_5_max": 0.220474, + "nauc_precision_at_5_std": 0.160337, + "nauc_precision_at_5_diff1": 0.084882, + "nauc_precision_at_10_max": 0.286073, + "nauc_precision_at_10_std": 0.241692, + "nauc_precision_at_10_diff1": 0.175628, + "nauc_precision_at_20_max": 0.150354, + "nauc_precision_at_20_std": 0.190235, + "nauc_precision_at_20_diff1": 0.113745, + "nauc_precision_at_100_max": 0.0745, + "nauc_precision_at_100_std": 0.187229, + "nauc_precision_at_100_diff1": 0.079998, + "nauc_precision_at_1000_max": -0.248817, + "nauc_precision_at_1000_std": 0.06991, + "nauc_precision_at_1000_diff1": 0.130585, + "nauc_cv_recall_at_1_max": 0.132694, + "nauc_cv_recall_at_1_std": 0.065436, + "nauc_cv_recall_at_1_diff1": 0.057271, + "nauc_cv_recall_at_3_max": -0.00299, + "nauc_cv_recall_at_3_std": 0.111175, + "nauc_cv_recall_at_3_diff1": 0.050694, + "nauc_cv_recall_at_5_max": 0.130842, + "nauc_cv_recall_at_5_std": 0.072386, + "nauc_cv_recall_at_5_diff1": -0.012325, + "nauc_cv_recall_at_10_max": 0.327063, + "nauc_cv_recall_at_10_std": 0.217969, + "nauc_cv_recall_at_10_diff1": 0.123511, + "nauc_cv_recall_at_20_max": 0.223304, + "nauc_cv_recall_at_20_std": 0.146476, + "nauc_cv_recall_at_20_diff1": 0.041812, + "nauc_cv_recall_at_100_max": 0.24551, + "nauc_cv_recall_at_100_std": 0.314341, + "nauc_cv_recall_at_100_diff1": -0.070172, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.132694, + "nauc_mrr_at_1_std": 0.065436, + "nauc_mrr_at_1_diff1": 0.057271, + "nauc_mrr_at_3_max": 0.062873, + "nauc_mrr_at_3_std": 0.087298, + "nauc_mrr_at_3_diff1": 0.055816, + "nauc_mrr_at_5_max": 0.099134, + "nauc_mrr_at_5_std": 0.07732, + "nauc_mrr_at_5_diff1": 0.037543, + "nauc_mrr_at_10_max": 0.132714, + "nauc_mrr_at_10_std": 0.098897, + "nauc_mrr_at_10_diff1": 0.053098, + "nauc_mrr_at_20_max": 0.121131, + "nauc_mrr_at_20_std": 0.091292, + "nauc_mrr_at_20_diff1": 0.047395, + "nauc_mrr_at_100_max": 0.120795, + "nauc_mrr_at_100_std": 0.099963, + "nauc_mrr_at_100_diff1": 0.042059, + "nauc_mrr_at_1000_max": 0.120161, + "nauc_mrr_at_1000_std": 0.099611, + "nauc_mrr_at_1000_diff1": 0.043492, + "main_score": 0.12761, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.19298, + "ndcg_at_3": 0.17881, + "ndcg_at_5": 0.1985, + "ndcg_at_10": 0.23948, + "ndcg_at_20": 0.27465, + "ndcg_at_100": 0.33301, + "ndcg_at_1000": 0.38159, + "map_at_1": 0.05764, + "map_at_3": 0.09846, + "map_at_5": 0.12448, + "map_at_10": 0.1474, + "map_at_20": 0.16185, + "map_at_100": 0.17423, + "map_at_1000": 0.17772, + "recall_at_1": 0.05764, + "recall_at_3": 0.14025, + "recall_at_5": 0.22285, + "recall_at_10": 0.36598, + "recall_at_20": 0.47928, + "recall_at_100": 0.7218, + "recall_at_1000": 0.95439, + "cv_recall_at_1": 0.19298, + "cv_recall_at_3": 0.33333, + "cv_recall_at_5": 0.42105, + "cv_recall_at_10": 0.61404, + "cv_recall_at_20": 0.7193, + "cv_recall_at_100": 0.87719, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.19298, + "precision_at_3": 0.1345, + "precision_at_5": 0.11228, + "precision_at_10": 0.08596, + "precision_at_20": 0.06053, + "precision_at_100": 0.01947, + "precision_at_1000": 0.00356, + "mrr_at_1": 0.192982, + "mrr_at_3": 0.254386, + "mrr_at_5": 0.275439, + "mrr_at_10": 0.29938, + "mrr_at_20": 0.306427, + "mrr_at_100": 0.311001, + "mrr_at_1000": 0.31175, + "nauc_ndcg_at_1_max": 0.047227, + "nauc_ndcg_at_1_std": -0.090323, + "nauc_ndcg_at_1_diff1": 0.168899, + "nauc_ndcg_at_3_max": 0.052669, + "nauc_ndcg_at_3_std": -0.166837, + "nauc_ndcg_at_3_diff1": 0.167502, + "nauc_ndcg_at_5_max": 0.140135, + "nauc_ndcg_at_5_std": -0.078813, + "nauc_ndcg_at_5_diff1": 0.268155, + "nauc_ndcg_at_10_max": 0.152508, + "nauc_ndcg_at_10_std": -0.054752, + "nauc_ndcg_at_10_diff1": 0.201965, + "nauc_ndcg_at_20_max": 0.152087, + "nauc_ndcg_at_20_std": -0.071608, + "nauc_ndcg_at_20_diff1": 0.236862, + "nauc_ndcg_at_100_max": 0.120451, + "nauc_ndcg_at_100_std": -0.025126, + "nauc_ndcg_at_100_diff1": 0.278001, + "nauc_ndcg_at_1000_max": 0.08099, + "nauc_ndcg_at_1000_std": -0.070304, + "nauc_ndcg_at_1000_diff1": 0.200897, + "nauc_map_at_1_max": 0.320295, + "nauc_map_at_1_std": -0.077273, + "nauc_map_at_1_diff1": 0.425231, + "nauc_map_at_3_max": 0.163081, + "nauc_map_at_3_std": -0.107356, + "nauc_map_at_3_diff1": 0.305448, + "nauc_map_at_5_max": 0.196591, + "nauc_map_at_5_std": -0.03257, + "nauc_map_at_5_diff1": 0.348688, + "nauc_map_at_10_max": 0.182105, + "nauc_map_at_10_std": -0.040832, + "nauc_map_at_10_diff1": 0.282374, + "nauc_map_at_20_max": 0.165655, + "nauc_map_at_20_std": -0.058573, + "nauc_map_at_20_diff1": 0.281591, + "nauc_map_at_100_max": 0.163955, + "nauc_map_at_100_std": -0.057743, + "nauc_map_at_100_diff1": 0.29273, + "nauc_map_at_1000_max": 0.161283, + "nauc_map_at_1000_std": -0.060016, + "nauc_map_at_1000_diff1": 0.285776, + "nauc_recall_at_1_max": 0.320295, + "nauc_recall_at_1_std": -0.077273, + "nauc_recall_at_1_diff1": 0.425231, + "nauc_recall_at_3_max": 0.076602, + "nauc_recall_at_3_std": -0.103224, + "nauc_recall_at_3_diff1": 0.232834, + "nauc_recall_at_5_max": 0.150227, + "nauc_recall_at_5_std": 0.01736, + "nauc_recall_at_5_diff1": 0.305024, + "nauc_recall_at_10_max": 0.127998, + "nauc_recall_at_10_std": 0.014705, + "nauc_recall_at_10_diff1": 0.09046, + "nauc_recall_at_20_max": 0.164751, + "nauc_recall_at_20_std": -0.011786, + "nauc_recall_at_20_diff1": 0.179855, + "nauc_recall_at_100_max": 0.067108, + "nauc_recall_at_100_std": 0.209819, + "nauc_recall_at_100_diff1": 0.386645, + "nauc_recall_at_1000_max": -0.351548, + "nauc_recall_at_1000_std": 0.539073, + "nauc_recall_at_1000_diff1": 0.085481, + "nauc_precision_at_1_max": 0.047227, + "nauc_precision_at_1_std": -0.090323, + "nauc_precision_at_1_diff1": 0.168899, + "nauc_precision_at_3_max": -0.025518, + "nauc_precision_at_3_std": -0.213514, + "nauc_precision_at_3_diff1": 0.059597, + "nauc_precision_at_5_max": 0.080714, + "nauc_precision_at_5_std": -0.122797, + "nauc_precision_at_5_diff1": 0.168074, + "nauc_precision_at_10_max": 0.045033, + "nauc_precision_at_10_std": -0.133393, + "nauc_precision_at_10_diff1": 0.040919, + "nauc_precision_at_20_max": -0.002345, + "nauc_precision_at_20_std": -0.174016, + "nauc_precision_at_20_diff1": 0.018199, + "nauc_precision_at_100_max": -0.104875, + "nauc_precision_at_100_std": -0.114813, + "nauc_precision_at_100_diff1": -0.021795, + "nauc_precision_at_1000_max": -0.249124, + "nauc_precision_at_1000_std": -0.248688, + "nauc_precision_at_1000_diff1": -0.263782, + "nauc_cv_recall_at_1_max": 0.047227, + "nauc_cv_recall_at_1_std": -0.090323, + "nauc_cv_recall_at_1_diff1": 0.168899, + "nauc_cv_recall_at_3_max": 0.023118, + "nauc_cv_recall_at_3_std": -0.152906, + "nauc_cv_recall_at_3_diff1": 0.068241, + "nauc_cv_recall_at_5_max": 0.115566, + "nauc_cv_recall_at_5_std": -0.124159, + "nauc_cv_recall_at_5_diff1": 0.17721, + "nauc_cv_recall_at_10_max": 0.132252, + "nauc_cv_recall_at_10_std": -0.035594, + "nauc_cv_recall_at_10_diff1": 0.069485, + "nauc_cv_recall_at_20_max": 0.281486, + "nauc_cv_recall_at_20_std": -0.126479, + "nauc_cv_recall_at_20_diff1": 0.267979, + "nauc_cv_recall_at_100_max": 0.047884, + "nauc_cv_recall_at_100_std": 0.394505, + "nauc_cv_recall_at_100_diff1": 0.441905, + "nauc_cv_recall_at_1000_max": -1.169034, + "nauc_cv_recall_at_1000_std": 0.555085, + "nauc_cv_recall_at_1000_diff1": -0.559408, + "nauc_mrr_at_1_max": 0.047227, + "nauc_mrr_at_1_std": -0.090323, + "nauc_mrr_at_1_diff1": 0.168899, + "nauc_mrr_at_3_max": 0.039032, + "nauc_mrr_at_3_std": -0.11732, + "nauc_mrr_at_3_diff1": 0.111396, + "nauc_mrr_at_5_max": 0.067838, + "nauc_mrr_at_5_std": -0.106999, + "nauc_mrr_at_5_diff1": 0.144651, + "nauc_mrr_at_10_max": 0.069965, + "nauc_mrr_at_10_std": -0.09606, + "nauc_mrr_at_10_diff1": 0.126392, + "nauc_mrr_at_20_max": 0.074049, + "nauc_mrr_at_20_std": -0.102549, + "nauc_mrr_at_20_diff1": 0.140604, + "nauc_mrr_at_100_max": 0.069745, + "nauc_mrr_at_100_std": -0.098701, + "nauc_mrr_at_100_diff1": 0.143107, + "nauc_mrr_at_1000_max": 0.069385, + "nauc_mrr_at_1000_std": -0.099713, + "nauc_mrr_at_1000_diff1": 0.141979, + "main_score": 0.1985, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.07018, + "ndcg_at_3": 0.06765, + "ndcg_at_5": 0.0702, + "ndcg_at_10": 0.09319, + "ndcg_at_20": 0.11923, + "ndcg_at_100": 0.20029, + "ndcg_at_1000": 0.25763, + "map_at_1": 0.02924, + "map_at_3": 0.04057, + "map_at_5": 0.0451, + "map_at_10": 0.0543, + "map_at_20": 0.06255, + "map_at_100": 0.07572, + "map_at_1000": 0.07889, + "recall_at_1": 0.02924, + "recall_at_3": 0.05044, + "recall_at_5": 0.06652, + "recall_at_10": 0.13001, + "recall_at_20": 0.22431, + "recall_at_100": 0.57803, + "recall_at_1000": 0.89563, + "cv_recall_at_1": 0.07018, + "cv_recall_at_3": 0.12281, + "cv_recall_at_5": 0.15789, + "cv_recall_at_10": 0.31579, + "cv_recall_at_20": 0.42105, + "cv_recall_at_100": 0.78947, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.07018, + "precision_at_3": 0.05263, + "precision_at_5": 0.04211, + "precision_at_10": 0.04035, + "precision_at_20": 0.02982, + "precision_at_100": 0.01649, + "precision_at_1000": 0.00325, + "mrr_at_1": 0.070175, + "mrr_at_3": 0.093567, + "mrr_at_5": 0.102339, + "mrr_at_10": 0.123914, + "mrr_at_20": 0.130938, + "mrr_at_100": 0.13979, + "mrr_at_1000": 0.140542, + "nauc_ndcg_at_1_max": 0.507976, + "nauc_ndcg_at_1_std": -0.018085, + "nauc_ndcg_at_1_diff1": 0.260843, + "nauc_ndcg_at_3_max": 0.44272, + "nauc_ndcg_at_3_std": -0.07188, + "nauc_ndcg_at_3_diff1": 0.204612, + "nauc_ndcg_at_5_max": 0.506931, + "nauc_ndcg_at_5_std": -0.099126, + "nauc_ndcg_at_5_diff1": 0.197988, + "nauc_ndcg_at_10_max": 0.442758, + "nauc_ndcg_at_10_std": -0.089236, + "nauc_ndcg_at_10_diff1": 0.227958, + "nauc_ndcg_at_20_max": 0.399891, + "nauc_ndcg_at_20_std": -0.052721, + "nauc_ndcg_at_20_diff1": 0.211268, + "nauc_ndcg_at_100_max": 0.285259, + "nauc_ndcg_at_100_std": 0.003019, + "nauc_ndcg_at_100_diff1": 0.042251, + "nauc_ndcg_at_1000_max": 0.272528, + "nauc_ndcg_at_1000_std": -0.07865, + "nauc_ndcg_at_1000_diff1": 0.063546, + "nauc_map_at_1_max": 0.525526, + "nauc_map_at_1_std": -0.175562, + "nauc_map_at_1_diff1": 0.160429, + "nauc_map_at_3_max": 0.491627, + "nauc_map_at_3_std": -0.130121, + "nauc_map_at_3_diff1": 0.244188, + "nauc_map_at_5_max": 0.532173, + "nauc_map_at_5_std": -0.139464, + "nauc_map_at_5_diff1": 0.226545, + "nauc_map_at_10_max": 0.499467, + "nauc_map_at_10_std": -0.135549, + "nauc_map_at_10_diff1": 0.221707, + "nauc_map_at_20_max": 0.470848, + "nauc_map_at_20_std": -0.111498, + "nauc_map_at_20_diff1": 0.211633, + "nauc_map_at_100_max": 0.427744, + "nauc_map_at_100_std": -0.103813, + "nauc_map_at_100_diff1": 0.148886, + "nauc_map_at_1000_max": 0.422532, + "nauc_map_at_1000_std": -0.109001, + "nauc_map_at_1000_diff1": 0.148303, + "nauc_recall_at_1_max": 0.525526, + "nauc_recall_at_1_std": -0.175562, + "nauc_recall_at_1_diff1": 0.160429, + "nauc_recall_at_3_max": 0.427415, + "nauc_recall_at_3_std": -0.087989, + "nauc_recall_at_3_diff1": 0.275151, + "nauc_recall_at_5_max": 0.540333, + "nauc_recall_at_5_std": -0.122738, + "nauc_recall_at_5_diff1": 0.230553, + "nauc_recall_at_10_max": 0.400904, + "nauc_recall_at_10_std": -0.074311, + "nauc_recall_at_10_diff1": 0.227676, + "nauc_recall_at_20_max": 0.277028, + "nauc_recall_at_20_std": 0.030301, + "nauc_recall_at_20_diff1": 0.1453, + "nauc_recall_at_100_max": 0.072414, + "nauc_recall_at_100_std": 0.251059, + "nauc_recall_at_100_diff1": -0.171127, + "nauc_recall_at_1000_max": 0.140068, + "nauc_recall_at_1000_std": 0.485447, + "nauc_recall_at_1000_diff1": -0.446635, + "nauc_precision_at_1_max": 0.507976, + "nauc_precision_at_1_std": -0.018085, + "nauc_precision_at_1_diff1": 0.260843, + "nauc_precision_at_3_max": 0.379493, + "nauc_precision_at_3_std": 0.011301, + "nauc_precision_at_3_diff1": 0.244141, + "nauc_precision_at_5_max": 0.511876, + "nauc_precision_at_5_std": -0.033524, + "nauc_precision_at_5_diff1": 0.205492, + "nauc_precision_at_10_max": 0.368419, + "nauc_precision_at_10_std": -0.051687, + "nauc_precision_at_10_diff1": 0.160998, + "nauc_precision_at_20_max": 0.28489, + "nauc_precision_at_20_std": -0.052145, + "nauc_precision_at_20_diff1": 0.127064, + "nauc_precision_at_100_max": -0.03038, + "nauc_precision_at_100_std": -0.138446, + "nauc_precision_at_100_diff1": -0.143967, + "nauc_precision_at_1000_max": -0.212755, + "nauc_precision_at_1000_std": -0.316526, + "nauc_precision_at_1000_diff1": -0.084699, + "nauc_cv_recall_at_1_max": 0.507976, + "nauc_cv_recall_at_1_std": -0.018085, + "nauc_cv_recall_at_1_diff1": 0.260843, + "nauc_cv_recall_at_3_max": 0.242059, + "nauc_cv_recall_at_3_std": 0.011364, + "nauc_cv_recall_at_3_diff1": 0.198517, + "nauc_cv_recall_at_5_max": 0.410064, + "nauc_cv_recall_at_5_std": -0.039685, + "nauc_cv_recall_at_5_diff1": 0.209314, + "nauc_cv_recall_at_10_max": 0.301054, + "nauc_cv_recall_at_10_std": -0.035296, + "nauc_cv_recall_at_10_diff1": 0.226454, + "nauc_cv_recall_at_20_max": 0.316623, + "nauc_cv_recall_at_20_std": -0.046904, + "nauc_cv_recall_at_20_diff1": 0.266903, + "nauc_cv_recall_at_100_max": -0.107112, + "nauc_cv_recall_at_100_std": 0.283013, + "nauc_cv_recall_at_100_diff1": -0.171279, + "nauc_cv_recall_at_1000_max": 0.722732, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_1000_diff1": -0.559408, + "nauc_mrr_at_1_max": 0.507976, + "nauc_mrr_at_1_std": -0.018085, + "nauc_mrr_at_1_diff1": 0.260843, + "nauc_mrr_at_3_max": 0.359305, + "nauc_mrr_at_3_std": -0.021241, + "nauc_mrr_at_3_diff1": 0.228993, + "nauc_mrr_at_5_max": 0.412792, + "nauc_mrr_at_5_std": -0.037678, + "nauc_mrr_at_5_diff1": 0.231035, + "nauc_mrr_at_10_max": 0.38025, + "nauc_mrr_at_10_std": -0.048502, + "nauc_mrr_at_10_diff1": 0.231839, + "nauc_mrr_at_20_max": 0.380518, + "nauc_mrr_at_20_std": -0.052816, + "nauc_mrr_at_20_diff1": 0.237412, + "nauc_mrr_at_100_max": 0.367165, + "nauc_mrr_at_100_std": -0.048361, + "nauc_mrr_at_100_diff1": 0.21749, + "nauc_mrr_at_1000_max": 0.370162, + "nauc_mrr_at_1000_std": -0.049289, + "nauc_mrr_at_1000_diff1": 0.219032, + "main_score": 0.0702, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 511.3586630821228, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/Vidore2EconomicsReportsRetrieval.json b/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..b78e5e281a --- /dev/null +++ b/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.2069, + "ndcg_at_3": 0.16818, + "ndcg_at_5": 0.15685, + "ndcg_at_10": 0.16738, + "ndcg_at_20": 0.19752, + "ndcg_at_100": 0.32304, + "ndcg_at_1000": 0.4582, + "map_at_1": 0.02106, + "map_at_3": 0.03913, + "map_at_5": 0.05159, + "map_at_10": 0.07191, + "map_at_20": 0.09604, + "map_at_100": 0.12893, + "map_at_1000": 0.15661, + "recall_at_1": 0.02106, + "recall_at_3": 0.04873, + "recall_at_5": 0.07532, + "recall_at_10": 0.13861, + "recall_at_20": 0.23067, + "recall_at_100": 0.59672, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.2069, + "cv_recall_at_3": 0.31034, + "cv_recall_at_5": 0.36207, + "cv_recall_at_10": 0.56897, + "cv_recall_at_20": 0.58621, + "cv_recall_at_100": 0.96552, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.2069, + "precision_at_3": 0.16092, + "precision_at_5": 0.14483, + "precision_at_10": 0.13621, + "precision_at_20": 0.11638, + "precision_at_100": 0.065, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.206897, + "mrr_at_3": 0.244253, + "mrr_at_5": 0.25546, + "mrr_at_10": 0.28412, + "mrr_at_20": 0.285688, + "mrr_at_100": 0.296099, + "mrr_at_1000": 0.296293, + "nauc_ndcg_at_1_max": 0.376783, + "nauc_ndcg_at_1_std": 0.155527, + "nauc_ndcg_at_1_diff1": 0.452022, + "nauc_ndcg_at_3_max": 0.42803, + "nauc_ndcg_at_3_std": 0.237008, + "nauc_ndcg_at_3_diff1": 0.386183, + "nauc_ndcg_at_5_max": 0.42163, + "nauc_ndcg_at_5_std": 0.252132, + "nauc_ndcg_at_5_diff1": 0.356582, + "nauc_ndcg_at_10_max": 0.389174, + "nauc_ndcg_at_10_std": 0.269334, + "nauc_ndcg_at_10_diff1": 0.288215, + "nauc_ndcg_at_20_max": 0.442829, + "nauc_ndcg_at_20_std": 0.321381, + "nauc_ndcg_at_20_diff1": 0.284395, + "nauc_ndcg_at_100_max": 0.441612, + "nauc_ndcg_at_100_std": 0.43281, + "nauc_ndcg_at_100_diff1": 0.234006, + "nauc_ndcg_at_1000_max": 0.435971, + "nauc_ndcg_at_1000_std": 0.221122, + "nauc_ndcg_at_1000_diff1": 0.257046, + "nauc_map_at_1_max": 0.294679, + "nauc_map_at_1_std": 0.002285, + "nauc_map_at_1_diff1": 0.313393, + "nauc_map_at_3_max": 0.353943, + "nauc_map_at_3_std": 0.072212, + "nauc_map_at_3_diff1": 0.290327, + "nauc_map_at_5_max": 0.372067, + "nauc_map_at_5_std": 0.097018, + "nauc_map_at_5_diff1": 0.29603, + "nauc_map_at_10_max": 0.357264, + "nauc_map_at_10_std": 0.155415, + "nauc_map_at_10_diff1": 0.253203, + "nauc_map_at_20_max": 0.384565, + "nauc_map_at_20_std": 0.22939, + "nauc_map_at_20_diff1": 0.273226, + "nauc_map_at_100_max": 0.434861, + "nauc_map_at_100_std": 0.304099, + "nauc_map_at_100_diff1": 0.263103, + "nauc_map_at_1000_max": 0.463791, + "nauc_map_at_1000_std": 0.236585, + "nauc_map_at_1000_diff1": 0.280816, + "nauc_recall_at_1_max": 0.294679, + "nauc_recall_at_1_std": 0.002285, + "nauc_recall_at_1_diff1": 0.313393, + "nauc_recall_at_3_max": 0.407466, + "nauc_recall_at_3_std": 0.105729, + "nauc_recall_at_3_diff1": 0.251306, + "nauc_recall_at_5_max": 0.367337, + "nauc_recall_at_5_std": 0.153698, + "nauc_recall_at_5_diff1": 0.232339, + "nauc_recall_at_10_max": 0.287415, + "nauc_recall_at_10_std": 0.31563, + "nauc_recall_at_10_diff1": 0.123181, + "nauc_recall_at_20_max": 0.315496, + "nauc_recall_at_20_std": 0.406578, + "nauc_recall_at_20_diff1": 0.144264, + "nauc_recall_at_100_max": 0.183753, + "nauc_recall_at_100_std": 0.51327, + "nauc_recall_at_100_diff1": 0.017309, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.376783, + "nauc_precision_at_1_std": 0.155527, + "nauc_precision_at_1_diff1": 0.452022, + "nauc_precision_at_3_max": 0.477409, + "nauc_precision_at_3_std": 0.276618, + "nauc_precision_at_3_diff1": 0.355343, + "nauc_precision_at_5_max": 0.439392, + "nauc_precision_at_5_std": 0.282617, + "nauc_precision_at_5_diff1": 0.315575, + "nauc_precision_at_10_max": 0.401109, + "nauc_precision_at_10_std": 0.296421, + "nauc_precision_at_10_diff1": 0.266378, + "nauc_precision_at_20_max": 0.507589, + "nauc_precision_at_20_std": 0.384382, + "nauc_precision_at_20_diff1": 0.28278, + "nauc_precision_at_100_max": 0.263709, + "nauc_precision_at_100_std": 0.230843, + "nauc_precision_at_100_diff1": 0.051079, + "nauc_precision_at_1000_max": 0.176422, + "nauc_precision_at_1000_std": -0.20241, + "nauc_precision_at_1000_diff1": 0.087107, + "nauc_cv_recall_at_1_max": 0.376783, + "nauc_cv_recall_at_1_std": 0.155527, + "nauc_cv_recall_at_1_diff1": 0.452022, + "nauc_cv_recall_at_3_max": 0.48481, + "nauc_cv_recall_at_3_std": 0.25544, + "nauc_cv_recall_at_3_diff1": 0.3745, + "nauc_cv_recall_at_5_max": 0.428837, + "nauc_cv_recall_at_5_std": 0.283788, + "nauc_cv_recall_at_5_diff1": 0.375844, + "nauc_cv_recall_at_10_max": 0.565661, + "nauc_cv_recall_at_10_std": 0.342314, + "nauc_cv_recall_at_10_diff1": 0.416998, + "nauc_cv_recall_at_20_max": 0.607687, + "nauc_cv_recall_at_20_std": 0.410745, + "nauc_cv_recall_at_20_diff1": 0.386146, + "nauc_cv_recall_at_100_max": 0.338887, + "nauc_cv_recall_at_100_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.093071, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.376783, + "nauc_mrr_at_1_std": 0.155527, + "nauc_mrr_at_1_diff1": 0.452022, + "nauc_mrr_at_3_max": 0.413502, + "nauc_mrr_at_3_std": 0.196795, + "nauc_mrr_at_3_diff1": 0.417787, + "nauc_mrr_at_5_max": 0.40011, + "nauc_mrr_at_5_std": 0.203089, + "nauc_mrr_at_5_diff1": 0.41812, + "nauc_mrr_at_10_max": 0.41671, + "nauc_mrr_at_10_std": 0.208452, + "nauc_mrr_at_10_diff1": 0.42247, + "nauc_mrr_at_20_max": 0.420344, + "nauc_mrr_at_20_std": 0.214566, + "nauc_mrr_at_20_diff1": 0.419614, + "nauc_mrr_at_100_max": 0.41632, + "nauc_mrr_at_100_std": 0.210183, + "nauc_mrr_at_100_diff1": 0.419906, + "nauc_mrr_at_1000_max": 0.416399, + "nauc_mrr_at_1000_std": 0.209678, + "nauc_mrr_at_1000_diff1": 0.420079, + "main_score": 0.15685, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.17241, + "ndcg_at_3": 0.18667, + "ndcg_at_5": 0.18774, + "ndcg_at_10": 0.18421, + "ndcg_at_20": 0.20983, + "ndcg_at_100": 0.33458, + "ndcg_at_1000": 0.46715, + "map_at_1": 0.01677, + "map_at_3": 0.04509, + "map_at_5": 0.06221, + "map_at_10": 0.08332, + "map_at_20": 0.1018, + "map_at_100": 0.13553, + "map_at_1000": 0.16321, + "recall_at_1": 0.01677, + "recall_at_3": 0.05883, + "recall_at_5": 0.1004, + "recall_at_10": 0.16378, + "recall_at_20": 0.24423, + "recall_at_100": 0.60276, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.17241, + "cv_recall_at_3": 0.36207, + "cv_recall_at_5": 0.51724, + "cv_recall_at_10": 0.63793, + "cv_recall_at_20": 0.74138, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.17241, + "precision_at_3": 0.18966, + "precision_at_5": 0.18621, + "precision_at_10": 0.14828, + "precision_at_20": 0.11897, + "precision_at_100": 0.06879, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.172414, + "mrr_at_3": 0.252874, + "mrr_at_5": 0.28908, + "mrr_at_10": 0.305768, + "mrr_at_20": 0.313386, + "mrr_at_100": 0.319244, + "mrr_at_1000": 0.319244, + "nauc_ndcg_at_1_max": 0.042699, + "nauc_ndcg_at_1_std": 0.044913, + "nauc_ndcg_at_1_diff1": 0.013421, + "nauc_ndcg_at_3_max": 0.053757, + "nauc_ndcg_at_3_std": 0.207865, + "nauc_ndcg_at_3_diff1": -0.139425, + "nauc_ndcg_at_5_max": 0.169419, + "nauc_ndcg_at_5_std": 0.177364, + "nauc_ndcg_at_5_diff1": -0.104498, + "nauc_ndcg_at_10_max": 0.263654, + "nauc_ndcg_at_10_std": 0.273945, + "nauc_ndcg_at_10_diff1": -0.087878, + "nauc_ndcg_at_20_max": 0.397371, + "nauc_ndcg_at_20_std": 0.289983, + "nauc_ndcg_at_20_diff1": -0.065502, + "nauc_ndcg_at_100_max": 0.366565, + "nauc_ndcg_at_100_std": 0.347883, + "nauc_ndcg_at_100_diff1": -0.05622, + "nauc_ndcg_at_1000_max": 0.341518, + "nauc_ndcg_at_1000_std": 0.117171, + "nauc_ndcg_at_1000_diff1": -0.094713, + "nauc_map_at_1_max": 0.092857, + "nauc_map_at_1_std": 0.20037, + "nauc_map_at_1_diff1": -0.108501, + "nauc_map_at_3_max": 0.06417, + "nauc_map_at_3_std": 0.324497, + "nauc_map_at_3_diff1": -0.181215, + "nauc_map_at_5_max": 0.193792, + "nauc_map_at_5_std": 0.265306, + "nauc_map_at_5_diff1": -0.147306, + "nauc_map_at_10_max": 0.236685, + "nauc_map_at_10_std": 0.304962, + "nauc_map_at_10_diff1": -0.163089, + "nauc_map_at_20_max": 0.30131, + "nauc_map_at_20_std": 0.300608, + "nauc_map_at_20_diff1": -0.134174, + "nauc_map_at_100_max": 0.347537, + "nauc_map_at_100_std": 0.342214, + "nauc_map_at_100_diff1": -0.129176, + "nauc_map_at_1000_max": 0.364484, + "nauc_map_at_1000_std": 0.274222, + "nauc_map_at_1000_diff1": -0.136103, + "nauc_recall_at_1_max": 0.092857, + "nauc_recall_at_1_std": 0.20037, + "nauc_recall_at_1_diff1": -0.108501, + "nauc_recall_at_3_max": 0.10521, + "nauc_recall_at_3_std": 0.380469, + "nauc_recall_at_3_diff1": -0.207291, + "nauc_recall_at_5_max": 0.289394, + "nauc_recall_at_5_std": 0.235036, + "nauc_recall_at_5_diff1": -0.09346, + "nauc_recall_at_10_max": 0.369334, + "nauc_recall_at_10_std": 0.278847, + "nauc_recall_at_10_diff1": -0.017673, + "nauc_recall_at_20_max": 0.463789, + "nauc_recall_at_20_std": 0.276485, + "nauc_recall_at_20_diff1": 0.00939, + "nauc_recall_at_100_max": 0.187957, + "nauc_recall_at_100_std": 0.500896, + "nauc_recall_at_100_diff1": 0.09464, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.042699, + "nauc_precision_at_1_std": 0.044913, + "nauc_precision_at_1_diff1": 0.013421, + "nauc_precision_at_3_max": 0.05818, + "nauc_precision_at_3_std": 0.257272, + "nauc_precision_at_3_diff1": -0.189015, + "nauc_precision_at_5_max": 0.23227, + "nauc_precision_at_5_std": 0.175915, + "nauc_precision_at_5_diff1": -0.111739, + "nauc_precision_at_10_max": 0.318894, + "nauc_precision_at_10_std": 0.276864, + "nauc_precision_at_10_diff1": -0.105069, + "nauc_precision_at_20_max": 0.485481, + "nauc_precision_at_20_std": 0.244708, + "nauc_precision_at_20_diff1": -0.018023, + "nauc_precision_at_100_max": 0.289831, + "nauc_precision_at_100_std": 0.017262, + "nauc_precision_at_100_diff1": -0.041936, + "nauc_precision_at_1000_max": 0.147484, + "nauc_precision_at_1000_std": -0.246692, + "nauc_precision_at_1000_diff1": -0.031335, + "nauc_cv_recall_at_1_max": 0.042699, + "nauc_cv_recall_at_1_std": 0.044913, + "nauc_cv_recall_at_1_diff1": 0.013421, + "nauc_cv_recall_at_3_max": 0.081564, + "nauc_cv_recall_at_3_std": 0.27819, + "nauc_cv_recall_at_3_diff1": -0.220206, + "nauc_cv_recall_at_5_max": 0.243348, + "nauc_cv_recall_at_5_std": 0.17263, + "nauc_cv_recall_at_5_diff1": -0.101067, + "nauc_cv_recall_at_10_max": 0.606368, + "nauc_cv_recall_at_10_std": 0.242206, + "nauc_cv_recall_at_10_diff1": 0.228307, + "nauc_cv_recall_at_20_max": 0.611177, + "nauc_cv_recall_at_20_std": 0.030486, + "nauc_cv_recall_at_20_diff1": 0.102719, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.042699, + "nauc_mrr_at_1_std": 0.044913, + "nauc_mrr_at_1_diff1": 0.013421, + "nauc_mrr_at_3_max": 0.060514, + "nauc_mrr_at_3_std": 0.174741, + "nauc_mrr_at_3_diff1": -0.116471, + "nauc_mrr_at_5_max": 0.105243, + "nauc_mrr_at_5_std": 0.147767, + "nauc_mrr_at_5_diff1": -0.082908, + "nauc_mrr_at_10_max": 0.150709, + "nauc_mrr_at_10_std": 0.154883, + "nauc_mrr_at_10_diff1": -0.033878, + "nauc_mrr_at_20_max": 0.144113, + "nauc_mrr_at_20_std": 0.142146, + "nauc_mrr_at_20_diff1": -0.046179, + "nauc_mrr_at_100_max": 0.136369, + "nauc_mrr_at_100_std": 0.140803, + "nauc_mrr_at_100_diff1": -0.046223, + "nauc_mrr_at_1000_max": 0.136369, + "nauc_mrr_at_1000_std": 0.140803, + "nauc_mrr_at_1000_diff1": -0.046223, + "main_score": 0.18774, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.41379, + "ndcg_at_3": 0.38372, + "ndcg_at_5": 0.35614, + "ndcg_at_10": 0.34302, + "ndcg_at_20": 0.38819, + "ndcg_at_100": 0.5033, + "ndcg_at_1000": 0.57906, + "map_at_1": 0.04318, + "map_at_3": 0.08265, + "map_at_5": 0.10751, + "map_at_10": 0.14179, + "map_at_20": 0.18874, + "map_at_100": 0.24756, + "map_at_1000": 0.272, + "recall_at_1": 0.04318, + "recall_at_3": 0.10052, + "recall_at_5": 0.15761, + "recall_at_10": 0.26242, + "recall_at_20": 0.44276, + "recall_at_100": 0.78671, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.41379, + "cv_recall_at_3": 0.62069, + "cv_recall_at_5": 0.74138, + "cv_recall_at_10": 0.86207, + "cv_recall_at_20": 0.94828, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.41379, + "precision_at_3": 0.37356, + "precision_at_5": 0.32069, + "precision_at_10": 0.26207, + "precision_at_20": 0.225, + "precision_at_100": 0.10241, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.413793, + "mrr_at_3": 0.5, + "mrr_at_5": 0.527586, + "mrr_at_10": 0.543336, + "mrr_at_20": 0.549394, + "mrr_at_100": 0.551167, + "mrr_at_1000": 0.551167, + "nauc_ndcg_at_1_max": 0.398065, + "nauc_ndcg_at_1_std": 0.065584, + "nauc_ndcg_at_1_diff1": 0.083249, + "nauc_ndcg_at_3_max": 0.329709, + "nauc_ndcg_at_3_std": 0.014447, + "nauc_ndcg_at_3_diff1": 0.094704, + "nauc_ndcg_at_5_max": 0.278739, + "nauc_ndcg_at_5_std": 0.012801, + "nauc_ndcg_at_5_diff1": 0.010163, + "nauc_ndcg_at_10_max": 0.319912, + "nauc_ndcg_at_10_std": -0.024888, + "nauc_ndcg_at_10_diff1": 0.002758, + "nauc_ndcg_at_20_max": 0.313411, + "nauc_ndcg_at_20_std": 0.002135, + "nauc_ndcg_at_20_diff1": -0.03522, + "nauc_ndcg_at_100_max": 0.472529, + "nauc_ndcg_at_100_std": 0.16055, + "nauc_ndcg_at_100_diff1": 0.092483, + "nauc_ndcg_at_1000_max": 0.389118, + "nauc_ndcg_at_1000_std": 0.036766, + "nauc_ndcg_at_1000_diff1": 0.034002, + "nauc_map_at_1_max": 0.22384, + "nauc_map_at_1_std": 0.101916, + "nauc_map_at_1_diff1": 0.139749, + "nauc_map_at_3_max": 0.264928, + "nauc_map_at_3_std": 0.044157, + "nauc_map_at_3_diff1": 0.116003, + "nauc_map_at_5_max": 0.264909, + "nauc_map_at_5_std": 0.08226, + "nauc_map_at_5_diff1": 0.011942, + "nauc_map_at_10_max": 0.294946, + "nauc_map_at_10_std": 0.03853, + "nauc_map_at_10_diff1": 0.013103, + "nauc_map_at_20_max": 0.305621, + "nauc_map_at_20_std": 0.040893, + "nauc_map_at_20_diff1": 0.008856, + "nauc_map_at_100_max": 0.383026, + "nauc_map_at_100_std": 0.069007, + "nauc_map_at_100_diff1": 0.036242, + "nauc_map_at_1000_max": 0.350294, + "nauc_map_at_1000_std": 0.013542, + "nauc_map_at_1000_diff1": 0.000994, + "nauc_recall_at_1_max": 0.22384, + "nauc_recall_at_1_std": 0.101916, + "nauc_recall_at_1_diff1": 0.139749, + "nauc_recall_at_3_max": 0.220856, + "nauc_recall_at_3_std": 0.034277, + "nauc_recall_at_3_diff1": 0.08151, + "nauc_recall_at_5_max": 0.103198, + "nauc_recall_at_5_std": 0.020102, + "nauc_recall_at_5_diff1": -0.129853, + "nauc_recall_at_10_max": 0.121048, + "nauc_recall_at_10_std": -0.027504, + "nauc_recall_at_10_diff1": -0.099437, + "nauc_recall_at_20_max": 0.069246, + "nauc_recall_at_20_std": -0.026099, + "nauc_recall_at_20_diff1": -0.110855, + "nauc_recall_at_100_max": 0.279448, + "nauc_recall_at_100_std": 0.34628, + "nauc_recall_at_100_diff1": 0.106412, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.398065, + "nauc_precision_at_1_std": 0.065584, + "nauc_precision_at_1_diff1": 0.083249, + "nauc_precision_at_3_max": 0.308817, + "nauc_precision_at_3_std": -4.4e-05, + "nauc_precision_at_3_diff1": 0.087717, + "nauc_precision_at_5_max": 0.259787, + "nauc_precision_at_5_std": 0.014309, + "nauc_precision_at_5_diff1": -0.030967, + "nauc_precision_at_10_max": 0.269555, + "nauc_precision_at_10_std": -0.068954, + "nauc_precision_at_10_diff1": 0.028796, + "nauc_precision_at_20_max": 0.222802, + "nauc_precision_at_20_std": -0.06622, + "nauc_precision_at_20_diff1": 0.006661, + "nauc_precision_at_100_max": 0.148858, + "nauc_precision_at_100_std": -0.077562, + "nauc_precision_at_100_diff1": -0.047978, + "nauc_precision_at_1000_max": 0.028596, + "nauc_precision_at_1000_std": -0.192816, + "nauc_precision_at_1000_diff1": -0.13058, + "nauc_cv_recall_at_1_max": 0.398065, + "nauc_cv_recall_at_1_std": 0.065584, + "nauc_cv_recall_at_1_diff1": 0.083249, + "nauc_cv_recall_at_3_max": 0.363969, + "nauc_cv_recall_at_3_std": 0.082651, + "nauc_cv_recall_at_3_diff1": 0.023413, + "nauc_cv_recall_at_5_max": 0.219018, + "nauc_cv_recall_at_5_std": -0.094587, + "nauc_cv_recall_at_5_diff1": -0.047726, + "nauc_cv_recall_at_10_max": 0.569801, + "nauc_cv_recall_at_10_std": 0.28553, + "nauc_cv_recall_at_10_diff1": 0.071156, + "nauc_cv_recall_at_20_max": 0.479517, + "nauc_cv_recall_at_20_std": -0.0644, + "nauc_cv_recall_at_20_diff1": -0.1457, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.398065, + "nauc_mrr_at_1_std": 0.065584, + "nauc_mrr_at_1_diff1": 0.083249, + "nauc_mrr_at_3_max": 0.37823, + "nauc_mrr_at_3_std": 0.059121, + "nauc_mrr_at_3_diff1": 0.058702, + "nauc_mrr_at_5_max": 0.355618, + "nauc_mrr_at_5_std": 0.032696, + "nauc_mrr_at_5_diff1": 0.047033, + "nauc_mrr_at_10_max": 0.384231, + "nauc_mrr_at_10_std": 0.060279, + "nauc_mrr_at_10_diff1": 0.054654, + "nauc_mrr_at_20_max": 0.379124, + "nauc_mrr_at_20_std": 0.051264, + "nauc_mrr_at_20_diff1": 0.051872, + "nauc_mrr_at_100_max": 0.378383, + "nauc_mrr_at_100_std": 0.05215, + "nauc_mrr_at_100_diff1": 0.05405, + "nauc_mrr_at_1000_max": 0.378383, + "nauc_mrr_at_1000_std": 0.05215, + "nauc_mrr_at_1000_diff1": 0.05405, + "main_score": 0.35614, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.17241, + "ndcg_at_3": 0.14496, + "ndcg_at_5": 0.13388, + "ndcg_at_10": 0.12057, + "ndcg_at_20": 0.12302, + "ndcg_at_100": 0.23823, + "ndcg_at_1000": 0.4032, + "map_at_1": 0.01205, + "map_at_3": 0.02534, + "map_at_5": 0.03423, + "map_at_10": 0.04456, + "map_at_20": 0.05361, + "map_at_100": 0.08005, + "map_at_1000": 0.10856, + "recall_at_1": 0.01205, + "recall_at_3": 0.03402, + "recall_at_5": 0.05179, + "recall_at_10": 0.08465, + "recall_at_20": 0.12279, + "recall_at_100": 0.46135, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.17241, + "cv_recall_at_3": 0.25862, + "cv_recall_at_5": 0.2931, + "cv_recall_at_10": 0.34483, + "cv_recall_at_20": 0.37931, + "cv_recall_at_100": 0.86207, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.17241, + "precision_at_3": 0.13793, + "precision_at_5": 0.12414, + "precision_at_10": 0.09138, + "precision_at_20": 0.06724, + "precision_at_100": 0.05552, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.172414, + "mrr_at_3": 0.212644, + "mrr_at_5": 0.220402, + "mrr_at_10": 0.227894, + "mrr_at_20": 0.231029, + "mrr_at_100": 0.24297, + "mrr_at_1000": 0.243854, + "nauc_ndcg_at_1_max": 0.278301, + "nauc_ndcg_at_1_std": 0.040229, + "nauc_ndcg_at_1_diff1": 0.134422, + "nauc_ndcg_at_3_max": 0.388992, + "nauc_ndcg_at_3_std": 0.040669, + "nauc_ndcg_at_3_diff1": 0.122837, + "nauc_ndcg_at_5_max": 0.471213, + "nauc_ndcg_at_5_std": 0.037357, + "nauc_ndcg_at_5_diff1": 0.120538, + "nauc_ndcg_at_10_max": 0.589819, + "nauc_ndcg_at_10_std": 0.11967, + "nauc_ndcg_at_10_diff1": 0.213857, + "nauc_ndcg_at_20_max": 0.620445, + "nauc_ndcg_at_20_std": 0.202598, + "nauc_ndcg_at_20_diff1": 0.266407, + "nauc_ndcg_at_100_max": 0.581014, + "nauc_ndcg_at_100_std": 0.394193, + "nauc_ndcg_at_100_diff1": 0.323896, + "nauc_ndcg_at_1000_max": 0.339597, + "nauc_ndcg_at_1000_std": 0.096678, + "nauc_ndcg_at_1000_diff1": 0.047111, + "nauc_map_at_1_max": 0.276011, + "nauc_map_at_1_std": 0.067827, + "nauc_map_at_1_diff1": 0.235488, + "nauc_map_at_3_max": 0.449848, + "nauc_map_at_3_std": 0.071555, + "nauc_map_at_3_diff1": 0.228299, + "nauc_map_at_5_max": 0.568682, + "nauc_map_at_5_std": 0.066678, + "nauc_map_at_5_diff1": 0.239424, + "nauc_map_at_10_max": 0.652123, + "nauc_map_at_10_std": 0.147956, + "nauc_map_at_10_diff1": 0.267818, + "nauc_map_at_20_max": 0.670419, + "nauc_map_at_20_std": 0.183208, + "nauc_map_at_20_diff1": 0.285642, + "nauc_map_at_100_max": 0.661031, + "nauc_map_at_100_std": 0.246663, + "nauc_map_at_100_diff1": 0.307762, + "nauc_map_at_1000_max": 0.536713, + "nauc_map_at_1000_std": 0.137209, + "nauc_map_at_1000_diff1": 0.182543, + "nauc_recall_at_1_max": 0.276011, + "nauc_recall_at_1_std": 0.067827, + "nauc_recall_at_1_diff1": 0.235488, + "nauc_recall_at_3_max": 0.462349, + "nauc_recall_at_3_std": 0.071344, + "nauc_recall_at_3_diff1": 0.235921, + "nauc_recall_at_5_max": 0.590067, + "nauc_recall_at_5_std": 0.061886, + "nauc_recall_at_5_diff1": 0.256404, + "nauc_recall_at_10_max": 0.725705, + "nauc_recall_at_10_std": 0.334905, + "nauc_recall_at_10_diff1": 0.323421, + "nauc_recall_at_20_max": 0.668749, + "nauc_recall_at_20_std": 0.411106, + "nauc_recall_at_20_diff1": 0.301582, + "nauc_recall_at_100_max": 0.582045, + "nauc_recall_at_100_std": 0.615084, + "nauc_recall_at_100_diff1": 0.411745, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.278301, + "nauc_precision_at_1_std": 0.040229, + "nauc_precision_at_1_diff1": 0.134422, + "nauc_precision_at_3_max": 0.416275, + "nauc_precision_at_3_std": 0.050391, + "nauc_precision_at_3_diff1": 0.109391, + "nauc_precision_at_5_max": 0.51571, + "nauc_precision_at_5_std": 0.045644, + "nauc_precision_at_5_diff1": 0.096957, + "nauc_precision_at_10_max": 0.625206, + "nauc_precision_at_10_std": 0.129702, + "nauc_precision_at_10_diff1": 0.2095, + "nauc_precision_at_20_max": 0.595529, + "nauc_precision_at_20_std": 0.170641, + "nauc_precision_at_20_diff1": 0.217554, + "nauc_precision_at_100_max": 0.14483, + "nauc_precision_at_100_std": 0.101944, + "nauc_precision_at_100_diff1": 0.000291, + "nauc_precision_at_1000_max": -0.196895, + "nauc_precision_at_1000_std": -0.250213, + "nauc_precision_at_1000_diff1": -0.325623, + "nauc_cv_recall_at_1_max": 0.278301, + "nauc_cv_recall_at_1_std": 0.040229, + "nauc_cv_recall_at_1_diff1": 0.134422, + "nauc_cv_recall_at_3_max": 0.391141, + "nauc_cv_recall_at_3_std": 0.016309, + "nauc_cv_recall_at_3_diff1": 0.180494, + "nauc_cv_recall_at_5_max": 0.419638, + "nauc_cv_recall_at_5_std": 0.036499, + "nauc_cv_recall_at_5_diff1": 0.127914, + "nauc_cv_recall_at_10_max": 0.603861, + "nauc_cv_recall_at_10_std": 0.230409, + "nauc_cv_recall_at_10_diff1": 0.274021, + "nauc_cv_recall_at_20_max": 0.596207, + "nauc_cv_recall_at_20_std": 0.36341, + "nauc_cv_recall_at_20_diff1": 0.271835, + "nauc_cv_recall_at_100_max": 0.473515, + "nauc_cv_recall_at_100_std": 0.565125, + "nauc_cv_recall_at_100_diff1": -0.001177, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.278301, + "nauc_mrr_at_1_std": 0.040229, + "nauc_mrr_at_1_diff1": 0.134422, + "nauc_mrr_at_3_max": 0.341665, + "nauc_mrr_at_3_std": 0.021285, + "nauc_mrr_at_3_diff1": 0.162346, + "nauc_mrr_at_5_max": 0.351817, + "nauc_mrr_at_5_std": 0.02929, + "nauc_mrr_at_5_diff1": 0.147834, + "nauc_mrr_at_10_max": 0.383502, + "nauc_mrr_at_10_std": 0.0616, + "nauc_mrr_at_10_diff1": 0.175568, + "nauc_mrr_at_20_max": 0.383058, + "nauc_mrr_at_20_std": 0.076649, + "nauc_mrr_at_20_diff1": 0.175506, + "nauc_mrr_at_100_max": 0.378379, + "nauc_mrr_at_100_std": 0.0774, + "nauc_mrr_at_100_diff1": 0.171423, + "nauc_mrr_at_1000_max": 0.378183, + "nauc_mrr_at_1000_std": 0.076031, + "nauc_mrr_at_1000_diff1": 0.172255, + "main_score": 0.13388, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 65.99775815010071, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/model_meta.json b/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/model_meta.json index 0221e0f329..b9f850786f 100644 --- a/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/model_meta.json +++ b/results/laion__CLIP-ViT-bigG-14-laion2B-39B-b160k/bc7788f151930d91b58474715fdce5524ad9a189/model_meta.json @@ -1 +1 @@ -{"name": "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "revision": "bc7788f151930d91b58474715fdce5524ad9a189", "release_date": "2023-01-23", "languages": ["eng_Latn"], "n_parameters": 2540000000, "memory_usage_mb": 9689.0, "max_tokens": 77.0, "embed_dim": 1280, "license": "mit", "open_weights": true, "public_training_code": "https://github.com/mlfoundations/open_clip", "public_training_data": "https://laion.ai/blog/laion-5b/", "framework": ["PyTorch"], "reference": "https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "similarity_fn_name": null, "use_instructions": false, "training_datasets": {}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "openclip_loader"} \ No newline at end of file +{"name": "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "revision": "bc7788f151930d91b58474715fdce5524ad9a189", "release_date": "2023-01-23", "languages": ["eng-Latn"], "n_parameters": 2540000000, "memory_usage_mb": 9689.0, "max_tokens": 77.0, "embed_dim": 1280, "license": "mit", "open_weights": true, "public_training_code": "https://github.com/mlfoundations/open_clip", "public_training_data": "https://laion.ai/blog/laion-5b/", "framework": ["PyTorch"], "reference": "https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "similarity_fn_name": null, "use_instructions": false, "training_datasets": {}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "openclip_loader"} \ No newline at end of file diff --git a/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Vidore2BioMedicalLecturesRetrieval.json b/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..e18083ac33 --- /dev/null +++ b/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.05625, + "ndcg_at_3": 0.07594, + "ndcg_at_5": 0.08872, + "ndcg_at_10": 0.11467, + "ndcg_at_20": 0.13803, + "ndcg_at_100": 0.20183, + "ndcg_at_1000": 0.28225, + "map_at_1": 0.03683, + "map_at_3": 0.05548, + "map_at_5": 0.06334, + "map_at_10": 0.07706, + "map_at_20": 0.0845, + "map_at_100": 0.09505, + "map_at_1000": 0.09913, + "recall_at_1": 0.03683, + "recall_at_3": 0.07782, + "recall_at_5": 0.10792, + "recall_at_10": 0.18031, + "recall_at_20": 0.25422, + "recall_at_100": 0.52609, + "recall_at_1000": 0.99811, + "cv_recall_at_1": 0.05625, + "cv_recall_at_3": 0.13125, + "cv_recall_at_5": 0.19375, + "cv_recall_at_10": 0.2875, + "cv_recall_at_20": 0.39375, + "cv_recall_at_100": 0.66875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.05625, + "precision_at_3": 0.04792, + "precision_at_5": 0.045, + "precision_at_10": 0.04, + "precision_at_20": 0.02969, + "precision_at_100": 0.01369, + "precision_at_1000": 0.0032, + "mrr_at_1": 0.05625, + "mrr_at_3": 0.089583, + "mrr_at_5": 0.103958, + "mrr_at_10": 0.116233, + "mrr_at_20": 0.123624, + "mrr_at_100": 0.130396, + "mrr_at_1000": 0.131754, + "nauc_ndcg_at_1_max": 0.537042, + "nauc_ndcg_at_1_std": 0.137286, + "nauc_ndcg_at_1_diff1": 0.519184, + "nauc_ndcg_at_3_max": 0.341886, + "nauc_ndcg_at_3_std": 0.141076, + "nauc_ndcg_at_3_diff1": 0.273409, + "nauc_ndcg_at_5_max": 0.307399, + "nauc_ndcg_at_5_std": 0.139317, + "nauc_ndcg_at_5_diff1": 0.222696, + "nauc_ndcg_at_10_max": 0.302754, + "nauc_ndcg_at_10_std": 0.148846, + "nauc_ndcg_at_10_diff1": 0.141747, + "nauc_ndcg_at_20_max": 0.290061, + "nauc_ndcg_at_20_std": 0.122132, + "nauc_ndcg_at_20_diff1": 0.155522, + "nauc_ndcg_at_100_max": 0.319651, + "nauc_ndcg_at_100_std": 0.144195, + "nauc_ndcg_at_100_diff1": 0.189774, + "nauc_ndcg_at_1000_max": 0.315857, + "nauc_ndcg_at_1000_std": 0.15012, + "nauc_ndcg_at_1000_diff1": 0.20033, + "nauc_map_at_1_max": 0.371821, + "nauc_map_at_1_std": 0.11353, + "nauc_map_at_1_diff1": 0.442554, + "nauc_map_at_3_max": 0.297346, + "nauc_map_at_3_std": 0.132937, + "nauc_map_at_3_diff1": 0.266463, + "nauc_map_at_5_max": 0.290595, + "nauc_map_at_5_std": 0.123415, + "nauc_map_at_5_diff1": 0.24429, + "nauc_map_at_10_max": 0.304774, + "nauc_map_at_10_std": 0.131793, + "nauc_map_at_10_diff1": 0.197692, + "nauc_map_at_20_max": 0.301036, + "nauc_map_at_20_std": 0.12449, + "nauc_map_at_20_diff1": 0.201677, + "nauc_map_at_100_max": 0.306989, + "nauc_map_at_100_std": 0.131824, + "nauc_map_at_100_diff1": 0.208607, + "nauc_map_at_1000_max": 0.309303, + "nauc_map_at_1000_std": 0.135929, + "nauc_map_at_1000_diff1": 0.210127, + "nauc_recall_at_1_max": 0.371821, + "nauc_recall_at_1_std": 0.11353, + "nauc_recall_at_1_diff1": 0.442554, + "nauc_recall_at_3_max": 0.22752, + "nauc_recall_at_3_std": 0.128478, + "nauc_recall_at_3_diff1": 0.132103, + "nauc_recall_at_5_max": 0.215472, + "nauc_recall_at_5_std": 0.101149, + "nauc_recall_at_5_diff1": 0.095897, + "nauc_recall_at_10_max": 0.240658, + "nauc_recall_at_10_std": 0.138387, + "nauc_recall_at_10_diff1": 0.02533, + "nauc_recall_at_20_max": 0.205308, + "nauc_recall_at_20_std": 0.084731, + "nauc_recall_at_20_diff1": 0.044164, + "nauc_recall_at_100_max": 0.280187, + "nauc_recall_at_100_std": 0.132425, + "nauc_recall_at_100_diff1": 0.139045, + "nauc_recall_at_1000_max": 0.869736, + "nauc_recall_at_1000_std": 0.736878, + "nauc_recall_at_1000_diff1": -1.01409, + "nauc_precision_at_1_max": 0.537042, + "nauc_precision_at_1_std": 0.137286, + "nauc_precision_at_1_diff1": 0.519184, + "nauc_precision_at_3_max": 0.36785, + "nauc_precision_at_3_std": 0.122298, + "nauc_precision_at_3_diff1": 0.262789, + "nauc_precision_at_5_max": 0.355108, + "nauc_precision_at_5_std": 0.186021, + "nauc_precision_at_5_diff1": 0.236024, + "nauc_precision_at_10_max": 0.342584, + "nauc_precision_at_10_std": 0.206425, + "nauc_precision_at_10_diff1": 0.074374, + "nauc_precision_at_20_max": 0.328209, + "nauc_precision_at_20_std": 0.17789, + "nauc_precision_at_20_diff1": 0.134939, + "nauc_precision_at_100_max": 0.298904, + "nauc_precision_at_100_std": 0.214145, + "nauc_precision_at_100_diff1": 0.119655, + "nauc_precision_at_1000_max": -0.012604, + "nauc_precision_at_1000_std": 0.047249, + "nauc_precision_at_1000_diff1": 0.025431, + "nauc_cv_recall_at_1_max": 0.537042, + "nauc_cv_recall_at_1_std": 0.137286, + "nauc_cv_recall_at_1_diff1": 0.519184, + "nauc_cv_recall_at_3_max": 0.319883, + "nauc_cv_recall_at_3_std": 0.13352, + "nauc_cv_recall_at_3_diff1": 0.186433, + "nauc_cv_recall_at_5_max": 0.282904, + "nauc_cv_recall_at_5_std": 0.146404, + "nauc_cv_recall_at_5_diff1": 0.190403, + "nauc_cv_recall_at_10_max": 0.243711, + "nauc_cv_recall_at_10_std": 0.157756, + "nauc_cv_recall_at_10_diff1": 0.058801, + "nauc_cv_recall_at_20_max": 0.286126, + "nauc_cv_recall_at_20_std": 0.111619, + "nauc_cv_recall_at_20_diff1": 0.131987, + "nauc_cv_recall_at_100_max": 0.326688, + "nauc_cv_recall_at_100_std": 0.177813, + "nauc_cv_recall_at_100_diff1": 0.197729, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.537042, + "nauc_mrr_at_1_std": 0.137286, + "nauc_mrr_at_1_diff1": 0.519184, + "nauc_mrr_at_3_max": 0.396384, + "nauc_mrr_at_3_std": 0.134079, + "nauc_mrr_at_3_diff1": 0.315735, + "nauc_mrr_at_5_max": 0.372164, + "nauc_mrr_at_5_std": 0.136044, + "nauc_mrr_at_5_diff1": 0.30545, + "nauc_mrr_at_10_max": 0.359661, + "nauc_mrr_at_10_std": 0.141945, + "nauc_mrr_at_10_diff1": 0.267387, + "nauc_mrr_at_20_max": 0.363482, + "nauc_mrr_at_20_std": 0.133115, + "nauc_mrr_at_20_diff1": 0.274393, + "nauc_mrr_at_100_max": 0.362654, + "nauc_mrr_at_100_std": 0.133523, + "nauc_mrr_at_100_diff1": 0.275888, + "nauc_mrr_at_1000_max": 0.364285, + "nauc_mrr_at_1000_std": 0.133975, + "nauc_mrr_at_1000_diff1": 0.276492, + "main_score": 0.08872, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.06875, + "ndcg_at_3": 0.08469, + "ndcg_at_5": 0.09896, + "ndcg_at_10": 0.12244, + "ndcg_at_20": 0.14408, + "ndcg_at_100": 0.19162, + "ndcg_at_1000": 0.28256, + "map_at_1": 0.03531, + "map_at_3": 0.05715, + "map_at_5": 0.06874, + "map_at_10": 0.07991, + "map_at_20": 0.08646, + "map_at_100": 0.09425, + "map_at_1000": 0.0988, + "recall_at_1": 0.03531, + "recall_at_3": 0.08547, + "recall_at_5": 0.12495, + "recall_at_10": 0.18751, + "recall_at_20": 0.25392, + "recall_at_100": 0.45854, + "recall_at_1000": 0.99807, + "cv_recall_at_1": 0.06875, + "cv_recall_at_3": 0.1625, + "cv_recall_at_5": 0.21875, + "cv_recall_at_10": 0.325, + "cv_recall_at_20": 0.43125, + "cv_recall_at_100": 0.61875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.06875, + "precision_at_3": 0.05833, + "precision_at_5": 0.0525, + "precision_at_10": 0.04313, + "precision_at_20": 0.03094, + "precision_at_100": 0.01194, + "precision_at_1000": 0.00321, + "mrr_at_1": 0.06875, + "mrr_at_3": 0.109375, + "mrr_at_5": 0.122812, + "mrr_at_10": 0.136567, + "mrr_at_20": 0.143937, + "mrr_at_100": 0.148088, + "mrr_at_1000": 0.149663, + "nauc_ndcg_at_1_max": 0.38737, + "nauc_ndcg_at_1_std": 0.149621, + "nauc_ndcg_at_1_diff1": 0.32917, + "nauc_ndcg_at_3_max": 0.236155, + "nauc_ndcg_at_3_std": 0.150712, + "nauc_ndcg_at_3_diff1": 0.215274, + "nauc_ndcg_at_5_max": 0.241326, + "nauc_ndcg_at_5_std": 0.12007, + "nauc_ndcg_at_5_diff1": 0.238007, + "nauc_ndcg_at_10_max": 0.266128, + "nauc_ndcg_at_10_std": 0.124726, + "nauc_ndcg_at_10_diff1": 0.198546, + "nauc_ndcg_at_20_max": 0.261798, + "nauc_ndcg_at_20_std": 0.148469, + "nauc_ndcg_at_20_diff1": 0.161141, + "nauc_ndcg_at_100_max": 0.258405, + "nauc_ndcg_at_100_std": 0.146058, + "nauc_ndcg_at_100_diff1": 0.134363, + "nauc_ndcg_at_1000_max": 0.226387, + "nauc_ndcg_at_1000_std": 0.155259, + "nauc_ndcg_at_1000_diff1": 0.121584, + "nauc_map_at_1_max": 0.232304, + "nauc_map_at_1_std": -0.000829, + "nauc_map_at_1_diff1": 0.194289, + "nauc_map_at_3_max": 0.205899, + "nauc_map_at_3_std": 0.054405, + "nauc_map_at_3_diff1": 0.203547, + "nauc_map_at_5_max": 0.213902, + "nauc_map_at_5_std": 0.061411, + "nauc_map_at_5_diff1": 0.214676, + "nauc_map_at_10_max": 0.253033, + "nauc_map_at_10_std": 0.078991, + "nauc_map_at_10_diff1": 0.214121, + "nauc_map_at_20_max": 0.263041, + "nauc_map_at_20_std": 0.098675, + "nauc_map_at_20_diff1": 0.201078, + "nauc_map_at_100_max": 0.262392, + "nauc_map_at_100_std": 0.101935, + "nauc_map_at_100_diff1": 0.190486, + "nauc_map_at_1000_max": 0.258589, + "nauc_map_at_1000_std": 0.103594, + "nauc_map_at_1000_diff1": 0.186452, + "nauc_recall_at_1_max": 0.232304, + "nauc_recall_at_1_std": -0.000829, + "nauc_recall_at_1_diff1": 0.194289, + "nauc_recall_at_3_max": 0.155769, + "nauc_recall_at_3_std": 0.066752, + "nauc_recall_at_3_diff1": 0.182577, + "nauc_recall_at_5_max": 0.178975, + "nauc_recall_at_5_std": 0.08122, + "nauc_recall_at_5_diff1": 0.201528, + "nauc_recall_at_10_max": 0.230416, + "nauc_recall_at_10_std": 0.099082, + "nauc_recall_at_10_diff1": 0.159417, + "nauc_recall_at_20_max": 0.195824, + "nauc_recall_at_20_std": 0.131467, + "nauc_recall_at_20_diff1": 0.072628, + "nauc_recall_at_100_max": 0.200711, + "nauc_recall_at_100_std": 0.121483, + "nauc_recall_at_100_diff1": 0.044145, + "nauc_recall_at_1000_max": 0.869281, + "nauc_recall_at_1000_std": 0.59513, + "nauc_recall_at_1000_diff1": 0.790096, + "nauc_precision_at_1_max": 0.38737, + "nauc_precision_at_1_std": 0.149621, + "nauc_precision_at_1_diff1": 0.32917, + "nauc_precision_at_3_max": 0.228737, + "nauc_precision_at_3_std": 0.242742, + "nauc_precision_at_3_diff1": 0.208989, + "nauc_precision_at_5_max": 0.296396, + "nauc_precision_at_5_std": 0.240222, + "nauc_precision_at_5_diff1": 0.274318, + "nauc_precision_at_10_max": 0.341334, + "nauc_precision_at_10_std": 0.254989, + "nauc_precision_at_10_diff1": 0.168192, + "nauc_precision_at_20_max": 0.316314, + "nauc_precision_at_20_std": 0.280809, + "nauc_precision_at_20_diff1": 0.110828, + "nauc_precision_at_100_max": 0.173476, + "nauc_precision_at_100_std": 0.209008, + "nauc_precision_at_100_diff1": -0.046956, + "nauc_precision_at_1000_max": -0.14657, + "nauc_precision_at_1000_std": 0.047799, + "nauc_precision_at_1000_diff1": -0.16334, + "nauc_cv_recall_at_1_max": 0.38737, + "nauc_cv_recall_at_1_std": 0.149621, + "nauc_cv_recall_at_1_diff1": 0.32917, + "nauc_cv_recall_at_3_max": 0.21385, + "nauc_cv_recall_at_3_std": 0.23557, + "nauc_cv_recall_at_3_diff1": 0.186378, + "nauc_cv_recall_at_5_max": 0.254261, + "nauc_cv_recall_at_5_std": 0.224327, + "nauc_cv_recall_at_5_diff1": 0.212675, + "nauc_cv_recall_at_10_max": 0.261086, + "nauc_cv_recall_at_10_std": 0.212664, + "nauc_cv_recall_at_10_diff1": 0.090905, + "nauc_cv_recall_at_20_max": 0.135485, + "nauc_cv_recall_at_20_std": 0.142072, + "nauc_cv_recall_at_20_diff1": 0.013353, + "nauc_cv_recall_at_100_max": 0.117154, + "nauc_cv_recall_at_100_std": 0.107907, + "nauc_cv_recall_at_100_diff1": 0.011108, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.38737, + "nauc_mrr_at_1_std": 0.149621, + "nauc_mrr_at_1_diff1": 0.32917, + "nauc_mrr_at_3_max": 0.289885, + "nauc_mrr_at_3_std": 0.205043, + "nauc_mrr_at_3_diff1": 0.248746, + "nauc_mrr_at_5_max": 0.298218, + "nauc_mrr_at_5_std": 0.201269, + "nauc_mrr_at_5_diff1": 0.255618, + "nauc_mrr_at_10_max": 0.299549, + "nauc_mrr_at_10_std": 0.199986, + "nauc_mrr_at_10_diff1": 0.224102, + "nauc_mrr_at_20_max": 0.283622, + "nauc_mrr_at_20_std": 0.191601, + "nauc_mrr_at_20_diff1": 0.214271, + "nauc_mrr_at_100_max": 0.28446, + "nauc_mrr_at_100_std": 0.188873, + "nauc_mrr_at_100_diff1": 0.214474, + "nauc_mrr_at_1000_max": 0.285728, + "nauc_mrr_at_1000_std": 0.190488, + "nauc_mrr_at_1000_diff1": 0.215038, + "main_score": 0.09896, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.28125, + "ndcg_at_3": 0.31514, + "ndcg_at_5": 0.33202, + "ndcg_at_10": 0.36397, + "ndcg_at_20": 0.39097, + "ndcg_at_100": 0.44946, + "ndcg_at_1000": 0.49517, + "map_at_1": 0.17996, + "map_at_3": 0.24899, + "map_at_5": 0.2697, + "map_at_10": 0.29105, + "map_at_20": 0.30106, + "map_at_100": 0.31383, + "map_at_1000": 0.31768, + "recall_at_1": 0.17996, + "recall_at_3": 0.31696, + "recall_at_5": 0.36922, + "recall_at_10": 0.46079, + "recall_at_20": 0.54455, + "recall_at_100": 0.76759, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.28125, + "cv_recall_at_3": 0.475, + "cv_recall_at_5": 0.525, + "cv_recall_at_10": 0.64375, + "cv_recall_at_20": 0.725, + "cv_recall_at_100": 0.91875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.28125, + "precision_at_3": 0.19375, + "precision_at_5": 0.15, + "precision_at_10": 0.1025, + "precision_at_20": 0.065, + "precision_at_100": 0.02081, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.28125, + "mrr_at_3": 0.365625, + "mrr_at_5": 0.376875, + "mrr_at_10": 0.393943, + "mrr_at_20": 0.399546, + "mrr_at_100": 0.404677, + "mrr_at_1000": 0.405081, + "nauc_ndcg_at_1_max": 0.155329, + "nauc_ndcg_at_1_std": -0.11213, + "nauc_ndcg_at_1_diff1": 0.381767, + "nauc_ndcg_at_3_max": 0.167631, + "nauc_ndcg_at_3_std": -0.001127, + "nauc_ndcg_at_3_diff1": 0.256369, + "nauc_ndcg_at_5_max": 0.199649, + "nauc_ndcg_at_5_std": 0.059875, + "nauc_ndcg_at_5_diff1": 0.247347, + "nauc_ndcg_at_10_max": 0.175618, + "nauc_ndcg_at_10_std": 0.07331, + "nauc_ndcg_at_10_diff1": 0.245241, + "nauc_ndcg_at_20_max": 0.19649, + "nauc_ndcg_at_20_std": 0.111987, + "nauc_ndcg_at_20_diff1": 0.228283, + "nauc_ndcg_at_100_max": 0.190114, + "nauc_ndcg_at_100_std": 0.085947, + "nauc_ndcg_at_100_diff1": 0.256032, + "nauc_ndcg_at_1000_max": 0.19062, + "nauc_ndcg_at_1000_std": 0.040558, + "nauc_ndcg_at_1000_diff1": 0.257428, + "nauc_map_at_1_max": 0.159555, + "nauc_map_at_1_std": -0.056941, + "nauc_map_at_1_diff1": 0.437189, + "nauc_map_at_3_max": 0.134302, + "nauc_map_at_3_std": -0.004236, + "nauc_map_at_3_diff1": 0.297791, + "nauc_map_at_5_max": 0.161844, + "nauc_map_at_5_std": 0.048994, + "nauc_map_at_5_diff1": 0.278281, + "nauc_map_at_10_max": 0.155382, + "nauc_map_at_10_std": 0.059923, + "nauc_map_at_10_diff1": 0.275617, + "nauc_map_at_20_max": 0.164658, + "nauc_map_at_20_std": 0.070637, + "nauc_map_at_20_diff1": 0.271311, + "nauc_map_at_100_max": 0.16629, + "nauc_map_at_100_std": 0.063682, + "nauc_map_at_100_diff1": 0.27442, + "nauc_map_at_1000_max": 0.1673, + "nauc_map_at_1000_std": 0.060478, + "nauc_map_at_1000_diff1": 0.27537, + "nauc_recall_at_1_max": 0.159555, + "nauc_recall_at_1_std": -0.056941, + "nauc_recall_at_1_diff1": 0.437189, + "nauc_recall_at_3_max": 0.116071, + "nauc_recall_at_3_std": 0.054659, + "nauc_recall_at_3_diff1": 0.183831, + "nauc_recall_at_5_max": 0.18652, + "nauc_recall_at_5_std": 0.174419, + "nauc_recall_at_5_diff1": 0.154855, + "nauc_recall_at_10_max": 0.16973, + "nauc_recall_at_10_std": 0.194646, + "nauc_recall_at_10_diff1": 0.148677, + "nauc_recall_at_20_max": 0.241268, + "nauc_recall_at_20_std": 0.32841, + "nauc_recall_at_20_diff1": 0.085363, + "nauc_recall_at_100_max": 0.205202, + "nauc_recall_at_100_std": 0.363882, + "nauc_recall_at_100_diff1": 0.176042, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.155329, + "nauc_precision_at_1_std": -0.11213, + "nauc_precision_at_1_diff1": 0.381767, + "nauc_precision_at_3_max": 0.151444, + "nauc_precision_at_3_std": 0.025795, + "nauc_precision_at_3_diff1": 0.066177, + "nauc_precision_at_5_max": 0.229647, + "nauc_precision_at_5_std": 0.127737, + "nauc_precision_at_5_diff1": 0.02216, + "nauc_precision_at_10_max": 0.155854, + "nauc_precision_at_10_std": 0.122697, + "nauc_precision_at_10_diff1": 0.00345, + "nauc_precision_at_20_max": 0.188981, + "nauc_precision_at_20_std": 0.123869, + "nauc_precision_at_20_diff1": -0.040928, + "nauc_precision_at_100_max": 0.125504, + "nauc_precision_at_100_std": -0.053063, + "nauc_precision_at_100_diff1": -0.034175, + "nauc_precision_at_1000_max": 0.066276, + "nauc_precision_at_1000_std": -0.153914, + "nauc_precision_at_1000_diff1": -0.121846, + "nauc_cv_recall_at_1_max": 0.155329, + "nauc_cv_recall_at_1_std": -0.11213, + "nauc_cv_recall_at_1_diff1": 0.381767, + "nauc_cv_recall_at_3_max": 0.184633, + "nauc_cv_recall_at_3_std": 0.009832, + "nauc_cv_recall_at_3_diff1": 0.183469, + "nauc_cv_recall_at_5_max": 0.277219, + "nauc_cv_recall_at_5_std": 0.096264, + "nauc_cv_recall_at_5_diff1": 0.160325, + "nauc_cv_recall_at_10_max": 0.25955, + "nauc_cv_recall_at_10_std": 0.157559, + "nauc_cv_recall_at_10_diff1": 0.141197, + "nauc_cv_recall_at_20_max": 0.347558, + "nauc_cv_recall_at_20_std": 0.307414, + "nauc_cv_recall_at_20_diff1": 0.036344, + "nauc_cv_recall_at_100_max": 0.245852, + "nauc_cv_recall_at_100_std": 0.374776, + "nauc_cv_recall_at_100_diff1": 0.453746, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.155329, + "nauc_mrr_at_1_std": -0.11213, + "nauc_mrr_at_1_diff1": 0.381767, + "nauc_mrr_at_3_max": 0.166027, + "nauc_mrr_at_3_std": -0.069266, + "nauc_mrr_at_3_diff1": 0.285985, + "nauc_mrr_at_5_max": 0.187696, + "nauc_mrr_at_5_std": -0.049833, + "nauc_mrr_at_5_diff1": 0.281786, + "nauc_mrr_at_10_max": 0.180481, + "nauc_mrr_at_10_std": -0.045912, + "nauc_mrr_at_10_diff1": 0.281243, + "nauc_mrr_at_20_max": 0.18345, + "nauc_mrr_at_20_std": -0.041436, + "nauc_mrr_at_20_diff1": 0.277205, + "nauc_mrr_at_100_max": 0.179655, + "nauc_mrr_at_100_std": -0.044456, + "nauc_mrr_at_100_diff1": 0.281499, + "nauc_mrr_at_1000_max": 0.179759, + "nauc_mrr_at_1000_std": -0.044934, + "nauc_mrr_at_1000_diff1": 0.281375, + "main_score": 0.33202, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.0625, + "ndcg_at_3": 0.05058, + "ndcg_at_5": 0.05182, + "ndcg_at_10": 0.06029, + "ndcg_at_20": 0.07584, + "ndcg_at_100": 0.11761, + "ndcg_at_1000": 0.22891, + "map_at_1": 0.02354, + "map_at_3": 0.03154, + "map_at_5": 0.03341, + "map_at_10": 0.03929, + "map_at_20": 0.04425, + "map_at_100": 0.05091, + "map_at_1000": 0.05595, + "recall_at_1": 0.02354, + "recall_at_3": 0.04224, + "recall_at_5": 0.04968, + "recall_at_10": 0.07994, + "recall_at_20": 0.13378, + "recall_at_100": 0.30285, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.0625, + "cv_recall_at_3": 0.0875, + "cv_recall_at_5": 0.10625, + "cv_recall_at_10": 0.1375, + "cv_recall_at_20": 0.225, + "cv_recall_at_100": 0.45, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.0625, + "precision_at_3": 0.03333, + "precision_at_5": 0.0275, + "precision_at_10": 0.02187, + "precision_at_20": 0.01875, + "precision_at_100": 0.00969, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.0625, + "mrr_at_3": 0.071875, + "mrr_at_5": 0.075625, + "mrr_at_10": 0.080387, + "mrr_at_20": 0.086411, + "mrr_at_100": 0.091428, + "mrr_at_1000": 0.093634, + "nauc_ndcg_at_1_max": 0.402012, + "nauc_ndcg_at_1_std": 0.156829, + "nauc_ndcg_at_1_diff1": 0.313711, + "nauc_ndcg_at_3_max": 0.300529, + "nauc_ndcg_at_3_std": 0.14835, + "nauc_ndcg_at_3_diff1": 0.174406, + "nauc_ndcg_at_5_max": 0.284687, + "nauc_ndcg_at_5_std": 0.149708, + "nauc_ndcg_at_5_diff1": 0.171658, + "nauc_ndcg_at_10_max": 0.259475, + "nauc_ndcg_at_10_std": 0.21485, + "nauc_ndcg_at_10_diff1": 0.113289, + "nauc_ndcg_at_20_max": 0.301777, + "nauc_ndcg_at_20_std": 0.281017, + "nauc_ndcg_at_20_diff1": 0.140018, + "nauc_ndcg_at_100_max": 0.316611, + "nauc_ndcg_at_100_std": 0.277362, + "nauc_ndcg_at_100_diff1": 0.13875, + "nauc_ndcg_at_1000_max": 0.295571, + "nauc_ndcg_at_1000_std": 0.267347, + "nauc_ndcg_at_1000_diff1": 0.09351, + "nauc_map_at_1_max": 0.125067, + "nauc_map_at_1_std": 0.067901, + "nauc_map_at_1_diff1": 0.21157, + "nauc_map_at_3_max": 0.153939, + "nauc_map_at_3_std": 0.103681, + "nauc_map_at_3_diff1": 0.109774, + "nauc_map_at_5_max": 0.165431, + "nauc_map_at_5_std": 0.121529, + "nauc_map_at_5_diff1": 0.119846, + "nauc_map_at_10_max": 0.19066, + "nauc_map_at_10_std": 0.179445, + "nauc_map_at_10_diff1": 0.100797, + "nauc_map_at_20_max": 0.232631, + "nauc_map_at_20_std": 0.21149, + "nauc_map_at_20_diff1": 0.12619, + "nauc_map_at_100_max": 0.2554, + "nauc_map_at_100_std": 0.223868, + "nauc_map_at_100_diff1": 0.125055, + "nauc_map_at_1000_max": 0.260717, + "nauc_map_at_1000_std": 0.226951, + "nauc_map_at_1000_diff1": 0.120701, + "nauc_recall_at_1_max": 0.125067, + "nauc_recall_at_1_std": 0.067901, + "nauc_recall_at_1_diff1": 0.21157, + "nauc_recall_at_3_max": 0.086951, + "nauc_recall_at_3_std": 0.042655, + "nauc_recall_at_3_diff1": 0.042168, + "nauc_recall_at_5_max": 0.101125, + "nauc_recall_at_5_std": 0.114004, + "nauc_recall_at_5_diff1": 0.073215, + "nauc_recall_at_10_max": 0.153015, + "nauc_recall_at_10_std": 0.248647, + "nauc_recall_at_10_diff1": 0.015437, + "nauc_recall_at_20_max": 0.252135, + "nauc_recall_at_20_std": 0.337708, + "nauc_recall_at_20_diff1": 0.110335, + "nauc_recall_at_100_max": 0.293254, + "nauc_recall_at_100_std": 0.295446, + "nauc_recall_at_100_diff1": 0.148809, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.402012, + "nauc_precision_at_1_std": 0.156829, + "nauc_precision_at_1_diff1": 0.313711, + "nauc_precision_at_3_max": 0.39343, + "nauc_precision_at_3_std": 0.219412, + "nauc_precision_at_3_diff1": 0.142323, + "nauc_precision_at_5_max": 0.442688, + "nauc_precision_at_5_std": 0.223673, + "nauc_precision_at_5_diff1": 0.20679, + "nauc_precision_at_10_max": 0.50876, + "nauc_precision_at_10_std": 0.334793, + "nauc_precision_at_10_diff1": 0.166945, + "nauc_precision_at_20_max": 0.573565, + "nauc_precision_at_20_std": 0.390289, + "nauc_precision_at_20_diff1": 0.17884, + "nauc_precision_at_100_max": 0.43053, + "nauc_precision_at_100_std": 0.303218, + "nauc_precision_at_100_diff1": 0.106556, + "nauc_precision_at_1000_max": 0.098768, + "nauc_precision_at_1000_std": 0.073824, + "nauc_precision_at_1000_diff1": -0.033988, + "nauc_cv_recall_at_1_max": 0.402012, + "nauc_cv_recall_at_1_std": 0.156829, + "nauc_cv_recall_at_1_diff1": 0.313711, + "nauc_cv_recall_at_3_max": 0.330925, + "nauc_cv_recall_at_3_std": 0.132048, + "nauc_cv_recall_at_3_diff1": 0.190444, + "nauc_cv_recall_at_5_max": 0.295737, + "nauc_cv_recall_at_5_std": 0.196316, + "nauc_cv_recall_at_5_diff1": 0.206831, + "nauc_cv_recall_at_10_max": 0.300159, + "nauc_cv_recall_at_10_std": 0.269124, + "nauc_cv_recall_at_10_diff1": 0.116721, + "nauc_cv_recall_at_20_max": 0.358908, + "nauc_cv_recall_at_20_std": 0.377959, + "nauc_cv_recall_at_20_diff1": 0.066188, + "nauc_cv_recall_at_100_max": 0.279726, + "nauc_cv_recall_at_100_std": 0.283851, + "nauc_cv_recall_at_100_diff1": 0.086713, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.402012, + "nauc_mrr_at_1_std": 0.156829, + "nauc_mrr_at_1_diff1": 0.313711, + "nauc_mrr_at_3_max": 0.381832, + "nauc_mrr_at_3_std": 0.149193, + "nauc_mrr_at_3_diff1": 0.26057, + "nauc_mrr_at_5_max": 0.369834, + "nauc_mrr_at_5_std": 0.166234, + "nauc_mrr_at_5_diff1": 0.262006, + "nauc_mrr_at_10_max": 0.367982, + "nauc_mrr_at_10_std": 0.185606, + "nauc_mrr_at_10_diff1": 0.237205, + "nauc_mrr_at_20_max": 0.375709, + "nauc_mrr_at_20_std": 0.206528, + "nauc_mrr_at_20_diff1": 0.226186, + "nauc_mrr_at_100_max": 0.368298, + "nauc_mrr_at_100_std": 0.201229, + "nauc_mrr_at_100_diff1": 0.220714, + "nauc_mrr_at_1000_max": 0.369677, + "nauc_mrr_at_1000_std": 0.203122, + "nauc_mrr_at_1000_diff1": 0.22079, + "main_score": 0.05182, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 41.99298143386841, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Vidore2ESGReportsHLRetrieval.json b/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..5c9e902542 --- /dev/null +++ b/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.05769, + "ndcg_at_3": 0.10509, + "ndcg_at_5": 0.12837, + "ndcg_at_10": 0.1406, + "ndcg_at_20": 0.17528, + "ndcg_at_100": 0.21247, + "ndcg_at_1000": 0.28655, + "map_at_1": 0.04021, + "map_at_3": 0.08042, + "map_at_5": 0.09305, + "map_at_10": 0.10079, + "map_at_20": 0.10988, + "map_at_100": 0.1157, + "map_at_1000": 0.11954, + "recall_at_1": 0.04021, + "recall_at_3": 0.1285, + "recall_at_5": 0.18202, + "recall_at_10": 0.22048, + "recall_at_20": 0.34933, + "recall_at_100": 0.50959, + "recall_at_1000": 0.97692, + "cv_recall_at_1": 0.05769, + "cv_recall_at_3": 0.15385, + "cv_recall_at_5": 0.26923, + "cv_recall_at_10": 0.30769, + "cv_recall_at_20": 0.46154, + "cv_recall_at_100": 0.67308, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.05769, + "precision_at_3": 0.05769, + "precision_at_5": 0.05769, + "precision_at_10": 0.03654, + "precision_at_20": 0.02788, + "precision_at_100": 0.00942, + "precision_at_1000": 0.00238, + "mrr_at_1": 0.057692, + "mrr_at_3": 0.099359, + "mrr_at_5": 0.125321, + "mrr_at_10": 0.129861, + "mrr_at_20": 0.140327, + "mrr_at_100": 0.145655, + "mrr_at_1000": 0.146827, + "nauc_ndcg_at_1_max": 0.770306, + "nauc_ndcg_at_1_std": 0.885153, + "nauc_ndcg_at_1_diff1": 0.770306, + "nauc_ndcg_at_3_max": 0.318762, + "nauc_ndcg_at_3_std": 0.359423, + "nauc_ndcg_at_3_diff1": 0.358998, + "nauc_ndcg_at_5_max": 0.242368, + "nauc_ndcg_at_5_std": 0.37572, + "nauc_ndcg_at_5_diff1": 0.252308, + "nauc_ndcg_at_10_max": 0.152434, + "nauc_ndcg_at_10_std": 0.302444, + "nauc_ndcg_at_10_diff1": 0.171601, + "nauc_ndcg_at_20_max": 0.14571, + "nauc_ndcg_at_20_std": 0.211007, + "nauc_ndcg_at_20_diff1": 0.147125, + "nauc_ndcg_at_100_max": 0.140895, + "nauc_ndcg_at_100_std": 0.26794, + "nauc_ndcg_at_100_diff1": 0.158598, + "nauc_ndcg_at_1000_max": 0.249987, + "nauc_ndcg_at_1000_std": 0.32498, + "nauc_ndcg_at_1000_diff1": 0.237859, + "nauc_map_at_1_max": 0.670439, + "nauc_map_at_1_std": 0.835219, + "nauc_map_at_1_diff1": 0.820239, + "nauc_map_at_3_max": 0.283889, + "nauc_map_at_3_std": 0.355448, + "nauc_map_at_3_diff1": 0.36791, + "nauc_map_at_5_max": 0.258477, + "nauc_map_at_5_std": 0.388452, + "nauc_map_at_5_diff1": 0.310743, + "nauc_map_at_10_max": 0.209188, + "nauc_map_at_10_std": 0.343512, + "nauc_map_at_10_diff1": 0.263855, + "nauc_map_at_20_max": 0.204162, + "nauc_map_at_20_std": 0.313882, + "nauc_map_at_20_diff1": 0.254091, + "nauc_map_at_100_max": 0.20418, + "nauc_map_at_100_std": 0.332839, + "nauc_map_at_100_diff1": 0.253386, + "nauc_map_at_1000_max": 0.216373, + "nauc_map_at_1000_std": 0.336297, + "nauc_map_at_1000_diff1": 0.261486, + "nauc_recall_at_1_max": 0.670439, + "nauc_recall_at_1_std": 0.835219, + "nauc_recall_at_1_diff1": 0.820239, + "nauc_recall_at_3_max": 0.126217, + "nauc_recall_at_3_std": 0.121059, + "nauc_recall_at_3_diff1": 0.204699, + "nauc_recall_at_5_max": 0.096292, + "nauc_recall_at_5_std": 0.23698, + "nauc_recall_at_5_diff1": 0.094319, + "nauc_recall_at_10_max": -0.006232, + "nauc_recall_at_10_std": 0.154778, + "nauc_recall_at_10_diff1": -0.005625, + "nauc_recall_at_20_max": -0.025684, + "nauc_recall_at_20_std": -0.056772, + "nauc_recall_at_20_diff1": -0.040951, + "nauc_recall_at_100_max": -0.120999, + "nauc_recall_at_100_std": 0.057148, + "nauc_recall_at_100_diff1": -0.032484, + "nauc_recall_at_1000_max": 0.344351, + "nauc_recall_at_1000_std": -0.086866, + "nauc_recall_at_1000_diff1": 0.851243, + "nauc_precision_at_1_max": 0.770306, + "nauc_precision_at_1_std": 0.885153, + "nauc_precision_at_1_diff1": 0.770306, + "nauc_precision_at_3_max": 0.294169, + "nauc_precision_at_3_std": 0.285617, + "nauc_precision_at_3_diff1": 0.312398, + "nauc_precision_at_5_max": 0.19853, + "nauc_precision_at_5_std": 0.359672, + "nauc_precision_at_5_diff1": 0.133282, + "nauc_precision_at_10_max": 0.05608, + "nauc_precision_at_10_std": 0.245737, + "nauc_precision_at_10_diff1": -0.009113, + "nauc_precision_at_20_max": 0.1518, + "nauc_precision_at_20_std": 0.096553, + "nauc_precision_at_20_diff1": -0.032296, + "nauc_precision_at_100_max": 0.349326, + "nauc_precision_at_100_std": 0.166892, + "nauc_precision_at_100_diff1": 0.082308, + "nauc_precision_at_1000_max": 0.53319, + "nauc_precision_at_1000_std": 0.222023, + "nauc_precision_at_1000_diff1": 0.219573, + "nauc_cv_recall_at_1_max": 0.770306, + "nauc_cv_recall_at_1_std": 0.885153, + "nauc_cv_recall_at_1_diff1": 0.770306, + "nauc_cv_recall_at_3_max": 0.19285, + "nauc_cv_recall_at_3_std": 0.18307, + "nauc_cv_recall_at_3_diff1": 0.263153, + "nauc_cv_recall_at_5_max": 0.114351, + "nauc_cv_recall_at_5_std": 0.292417, + "nauc_cv_recall_at_5_diff1": 0.078437, + "nauc_cv_recall_at_10_max": 0.060377, + "nauc_cv_recall_at_10_std": 0.308337, + "nauc_cv_recall_at_10_diff1": 0.002642, + "nauc_cv_recall_at_20_max": 0.08684, + "nauc_cv_recall_at_20_std": 0.056372, + "nauc_cv_recall_at_20_diff1": -0.057004, + "nauc_cv_recall_at_100_max": 0.108664, + "nauc_cv_recall_at_100_std": 0.026902, + "nauc_cv_recall_at_100_diff1": 0.145805, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.770306, + "nauc_mrr_at_1_std": 0.885153, + "nauc_mrr_at_1_diff1": 0.770306, + "nauc_mrr_at_3_max": 0.389745, + "nauc_mrr_at_3_std": 0.446457, + "nauc_mrr_at_3_diff1": 0.42066, + "nauc_mrr_at_5_max": 0.331892, + "nauc_mrr_at_5_std": 0.461877, + "nauc_mrr_at_5_diff1": 0.329709, + "nauc_mrr_at_10_max": 0.316615, + "nauc_mrr_at_10_std": 0.461859, + "nauc_mrr_at_10_diff1": 0.309476, + "nauc_mrr_at_20_max": 0.318502, + "nauc_mrr_at_20_std": 0.428229, + "nauc_mrr_at_20_diff1": 0.298778, + "nauc_mrr_at_100_max": 0.31619, + "nauc_mrr_at_100_std": 0.434102, + "nauc_mrr_at_100_diff1": 0.304247, + "nauc_mrr_at_1000_max": 0.31668, + "nauc_mrr_at_1000_std": 0.435707, + "nauc_mrr_at_1000_diff1": 0.305194, + "main_score": 0.12837, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 75.18479609489441, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Vidore2ESGReportsRetrieval.json b/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..a90d11c740 --- /dev/null +++ b/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.07018, + "ndcg_at_3": 0.06638, + "ndcg_at_5": 0.06497, + "ndcg_at_10": 0.07394, + "ndcg_at_20": 0.09136, + "ndcg_at_100": 0.13421, + "ndcg_at_1000": 0.23095, + "map_at_1": 0.01316, + "map_at_3": 0.03289, + "map_at_5": 0.03523, + "map_at_10": 0.03981, + "map_at_20": 0.04476, + "map_at_100": 0.05122, + "map_at_1000": 0.05512, + "recall_at_1": 0.01316, + "recall_at_3": 0.0636, + "recall_at_5": 0.07529, + "recall_at_10": 0.10746, + "recall_at_20": 0.16656, + "recall_at_100": 0.36514, + "recall_at_1000": 0.94541, + "cv_recall_at_1": 0.07018, + "cv_recall_at_3": 0.15789, + "cv_recall_at_5": 0.19298, + "cv_recall_at_10": 0.26316, + "cv_recall_at_20": 0.36842, + "cv_recall_at_100": 0.59649, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.07018, + "precision_at_3": 0.05263, + "precision_at_5": 0.0386, + "precision_at_10": 0.02807, + "precision_at_20": 0.02193, + "precision_at_100": 0.00877, + "precision_at_1000": 0.00328, + "mrr_at_1": 0.070175, + "mrr_at_3": 0.102339, + "mrr_at_5": 0.109357, + "mrr_at_10": 0.117565, + "mrr_at_20": 0.124472, + "mrr_at_100": 0.130203, + "mrr_at_1000": 0.131554, + "nauc_ndcg_at_1_max": -0.060548, + "nauc_ndcg_at_1_std": -0.057696, + "nauc_ndcg_at_1_diff1": 0.003514, + "nauc_ndcg_at_3_max": 0.120145, + "nauc_ndcg_at_3_std": 0.145776, + "nauc_ndcg_at_3_diff1": -0.032578, + "nauc_ndcg_at_5_max": 0.154235, + "nauc_ndcg_at_5_std": 0.09525, + "nauc_ndcg_at_5_diff1": -0.021679, + "nauc_ndcg_at_10_max": 0.222184, + "nauc_ndcg_at_10_std": 0.096664, + "nauc_ndcg_at_10_diff1": 0.121881, + "nauc_ndcg_at_20_max": 0.246265, + "nauc_ndcg_at_20_std": 0.193902, + "nauc_ndcg_at_20_diff1": 0.077124, + "nauc_ndcg_at_100_max": 0.268226, + "nauc_ndcg_at_100_std": 0.314881, + "nauc_ndcg_at_100_diff1": 0.029908, + "nauc_ndcg_at_1000_max": 0.156408, + "nauc_ndcg_at_1000_std": 0.173864, + "nauc_ndcg_at_1000_diff1": 0.033653, + "nauc_map_at_1_max": 0.092914, + "nauc_map_at_1_std": 0.031744, + "nauc_map_at_1_diff1": 0.251377, + "nauc_map_at_3_max": 0.192015, + "nauc_map_at_3_std": 0.173375, + "nauc_map_at_3_diff1": 0.084278, + "nauc_map_at_5_max": 0.197575, + "nauc_map_at_5_std": 0.13307, + "nauc_map_at_5_diff1": 0.076406, + "nauc_map_at_10_max": 0.198232, + "nauc_map_at_10_std": 0.102609, + "nauc_map_at_10_diff1": 0.12285, + "nauc_map_at_20_max": 0.208623, + "nauc_map_at_20_std": 0.153875, + "nauc_map_at_20_diff1": 0.093803, + "nauc_map_at_100_max": 0.216576, + "nauc_map_at_100_std": 0.201698, + "nauc_map_at_100_diff1": 0.066887, + "nauc_map_at_1000_max": 0.209218, + "nauc_map_at_1000_std": 0.189165, + "nauc_map_at_1000_diff1": 0.069903, + "nauc_recall_at_1_max": 0.092914, + "nauc_recall_at_1_std": 0.031744, + "nauc_recall_at_1_diff1": 0.251377, + "nauc_recall_at_3_max": 0.303142, + "nauc_recall_at_3_std": 0.300286, + "nauc_recall_at_3_diff1": -0.022236, + "nauc_recall_at_5_max": 0.299838, + "nauc_recall_at_5_std": 0.186645, + "nauc_recall_at_5_diff1": -0.028675, + "nauc_recall_at_10_max": 0.327041, + "nauc_recall_at_10_std": 0.134539, + "nauc_recall_at_10_diff1": 0.166875, + "nauc_recall_at_20_max": 0.316953, + "nauc_recall_at_20_std": 0.253982, + "nauc_recall_at_20_diff1": 0.062659, + "nauc_recall_at_100_max": 0.317395, + "nauc_recall_at_100_std": 0.412922, + "nauc_recall_at_100_diff1": -0.014138, + "nauc_recall_at_1000_max": 0.454586, + "nauc_recall_at_1000_std": 0.305481, + "nauc_recall_at_1000_diff1": 0.251357, + "nauc_precision_at_1_max": -0.060548, + "nauc_precision_at_1_std": -0.057696, + "nauc_precision_at_1_diff1": 0.003514, + "nauc_precision_at_3_max": 0.16982, + "nauc_precision_at_3_std": 0.236195, + "nauc_precision_at_3_diff1": -0.049929, + "nauc_precision_at_5_max": 0.190009, + "nauc_precision_at_5_std": 0.103982, + "nauc_precision_at_5_diff1": -0.054628, + "nauc_precision_at_10_max": 0.219177, + "nauc_precision_at_10_std": 0.040404, + "nauc_precision_at_10_diff1": 0.157664, + "nauc_precision_at_20_max": 0.158239, + "nauc_precision_at_20_std": 0.149281, + "nauc_precision_at_20_diff1": -0.015696, + "nauc_precision_at_100_max": 0.079763, + "nauc_precision_at_100_std": 0.245978, + "nauc_precision_at_100_diff1": -0.134462, + "nauc_precision_at_1000_max": -0.32447, + "nauc_precision_at_1000_std": -0.233373, + "nauc_precision_at_1000_diff1": -0.214325, + "nauc_cv_recall_at_1_max": -0.060548, + "nauc_cv_recall_at_1_std": -0.057696, + "nauc_cv_recall_at_1_diff1": 0.003514, + "nauc_cv_recall_at_3_max": 0.16982, + "nauc_cv_recall_at_3_std": 0.236195, + "nauc_cv_recall_at_3_diff1": -0.049929, + "nauc_cv_recall_at_5_max": 0.190009, + "nauc_cv_recall_at_5_std": 0.103982, + "nauc_cv_recall_at_5_diff1": -0.054628, + "nauc_cv_recall_at_10_max": 0.26979, + "nauc_cv_recall_at_10_std": 0.098388, + "nauc_cv_recall_at_10_diff1": 0.16349, + "nauc_cv_recall_at_20_max": 0.224609, + "nauc_cv_recall_at_20_std": 0.147996, + "nauc_cv_recall_at_20_diff1": 0.059934, + "nauc_cv_recall_at_100_max": 0.267188, + "nauc_cv_recall_at_100_std": 0.368492, + "nauc_cv_recall_at_100_diff1": 0.080932, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.060548, + "nauc_mrr_at_1_std": -0.057696, + "nauc_mrr_at_1_diff1": 0.003514, + "nauc_mrr_at_3_max": 0.043473, + "nauc_mrr_at_3_std": 0.077097, + "nauc_mrr_at_3_diff1": -0.01276, + "nauc_mrr_at_5_max": 0.055942, + "nauc_mrr_at_5_std": 0.041624, + "nauc_mrr_at_5_diff1": -0.016026, + "nauc_mrr_at_10_max": 0.071104, + "nauc_mrr_at_10_std": 0.039195, + "nauc_mrr_at_10_diff1": 0.025578, + "nauc_mrr_at_20_max": 0.067453, + "nauc_mrr_at_20_std": 0.048706, + "nauc_mrr_at_20_diff1": 0.013906, + "nauc_mrr_at_100_max": 0.069205, + "nauc_mrr_at_100_std": 0.060942, + "nauc_mrr_at_100_diff1": 0.015092, + "nauc_mrr_at_1000_max": 0.067959, + "nauc_mrr_at_1000_std": 0.058999, + "nauc_mrr_at_1000_diff1": 0.014633, + "main_score": 0.06497, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.01754, + "ndcg_at_3": 0.03811, + "ndcg_at_5": 0.05175, + "ndcg_at_10": 0.06054, + "ndcg_at_20": 0.08547, + "ndcg_at_100": 0.13407, + "ndcg_at_1000": 0.22295, + "map_at_1": 0.01754, + "map_at_3": 0.02997, + "map_at_5": 0.0349, + "map_at_10": 0.03832, + "map_at_20": 0.04519, + "map_at_100": 0.05214, + "map_at_1000": 0.05567, + "recall_at_1": 0.01754, + "recall_at_3": 0.04605, + "recall_at_5": 0.06871, + "recall_at_10": 0.09064, + "recall_at_20": 0.17325, + "recall_at_100": 0.38162, + "recall_at_1000": 0.92335, + "cv_recall_at_1": 0.01754, + "cv_recall_at_3": 0.07018, + "cv_recall_at_5": 0.15789, + "cv_recall_at_10": 0.21053, + "cv_recall_at_20": 0.33333, + "cv_recall_at_100": 0.66667, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.01754, + "precision_at_3": 0.02339, + "precision_at_5": 0.03158, + "precision_at_10": 0.02456, + "precision_at_20": 0.02193, + "precision_at_100": 0.01018, + "precision_at_1000": 0.00321, + "mrr_at_1": 0.017544, + "mrr_at_3": 0.038012, + "mrr_at_5": 0.05731, + "mrr_at_10": 0.06345, + "mrr_at_20": 0.071535, + "mrr_at_100": 0.079252, + "mrr_at_1000": 0.080262, + "nauc_ndcg_at_1_max": 0.323758, + "nauc_ndcg_at_1_std": -0.073279, + "nauc_ndcg_at_1_diff1": -0.578752, + "nauc_ndcg_at_3_max": 0.138757, + "nauc_ndcg_at_3_std": 0.051415, + "nauc_ndcg_at_3_diff1": 0.002792, + "nauc_ndcg_at_5_max": 0.093899, + "nauc_ndcg_at_5_std": 0.019223, + "nauc_ndcg_at_5_diff1": -0.011607, + "nauc_ndcg_at_10_max": 0.07505, + "nauc_ndcg_at_10_std": 0.039965, + "nauc_ndcg_at_10_diff1": 0.004931, + "nauc_ndcg_at_20_max": 0.180225, + "nauc_ndcg_at_20_std": 0.210732, + "nauc_ndcg_at_20_diff1": -0.009101, + "nauc_ndcg_at_100_max": 0.169358, + "nauc_ndcg_at_100_std": 0.260951, + "nauc_ndcg_at_100_diff1": -0.007592, + "nauc_ndcg_at_1000_max": 0.11902, + "nauc_ndcg_at_1000_std": 0.20549, + "nauc_ndcg_at_1000_diff1": -0.07342, + "nauc_map_at_1_max": 0.323758, + "nauc_map_at_1_std": -0.073279, + "nauc_map_at_1_diff1": -0.578752, + "nauc_map_at_3_max": 0.19182, + "nauc_map_at_3_std": 0.025383, + "nauc_map_at_3_diff1": -0.044621, + "nauc_map_at_5_max": 0.155724, + "nauc_map_at_5_std": -0.002485, + "nauc_map_at_5_diff1": -0.052754, + "nauc_map_at_10_max": 0.140771, + "nauc_map_at_10_std": 0.009041, + "nauc_map_at_10_diff1": -0.04665, + "nauc_map_at_20_max": 0.186166, + "nauc_map_at_20_std": 0.095741, + "nauc_map_at_20_diff1": -0.05171, + "nauc_map_at_100_max": 0.18978, + "nauc_map_at_100_std": 0.128393, + "nauc_map_at_100_diff1": -0.043464, + "nauc_map_at_1000_max": 0.185847, + "nauc_map_at_1000_std": 0.125094, + "nauc_map_at_1000_diff1": -0.049229, + "nauc_recall_at_1_max": 0.323758, + "nauc_recall_at_1_std": -0.073279, + "nauc_recall_at_1_diff1": -0.578752, + "nauc_recall_at_3_max": 0.141791, + "nauc_recall_at_3_std": 0.119347, + "nauc_recall_at_3_diff1": 0.163361, + "nauc_recall_at_5_max": 0.072124, + "nauc_recall_at_5_std": 0.026247, + "nauc_recall_at_5_diff1": 0.087876, + "nauc_recall_at_10_max": 0.045686, + "nauc_recall_at_10_std": 0.062377, + "nauc_recall_at_10_diff1": 0.101888, + "nauc_recall_at_20_max": 0.218335, + "nauc_recall_at_20_std": 0.314003, + "nauc_recall_at_20_diff1": 0.018429, + "nauc_recall_at_100_max": 0.141767, + "nauc_recall_at_100_std": 0.314679, + "nauc_recall_at_100_diff1": -0.00218, + "nauc_recall_at_1000_max": 0.458561, + "nauc_recall_at_1000_std": 0.602695, + "nauc_recall_at_1000_diff1": -0.308069, + "nauc_precision_at_1_max": 0.323758, + "nauc_precision_at_1_std": -0.073279, + "nauc_precision_at_1_diff1": -0.578752, + "nauc_precision_at_3_max": 0.063245, + "nauc_precision_at_3_std": 0.133046, + "nauc_precision_at_3_diff1": 0.048863, + "nauc_precision_at_5_max": -0.016482, + "nauc_precision_at_5_std": 0.034416, + "nauc_precision_at_5_diff1": -0.017805, + "nauc_precision_at_10_max": -0.044927, + "nauc_precision_at_10_std": 0.067975, + "nauc_precision_at_10_diff1": -0.009902, + "nauc_precision_at_20_max": 0.147381, + "nauc_precision_at_20_std": 0.327954, + "nauc_precision_at_20_diff1": 0.027549, + "nauc_precision_at_100_max": 0.147397, + "nauc_precision_at_100_std": 0.372382, + "nauc_precision_at_100_diff1": 0.073271, + "nauc_precision_at_1000_max": -0.228078, + "nauc_precision_at_1000_std": -0.11888, + "nauc_precision_at_1000_diff1": -0.089531, + "nauc_cv_recall_at_1_max": 0.323758, + "nauc_cv_recall_at_1_std": -0.073279, + "nauc_cv_recall_at_1_diff1": -0.578752, + "nauc_cv_recall_at_3_max": 0.063245, + "nauc_cv_recall_at_3_std": 0.133046, + "nauc_cv_recall_at_3_diff1": 0.048863, + "nauc_cv_recall_at_5_max": -0.016482, + "nauc_cv_recall_at_5_std": 0.034416, + "nauc_cv_recall_at_5_diff1": -0.017805, + "nauc_cv_recall_at_10_max": -0.054441, + "nauc_cv_recall_at_10_std": 0.071903, + "nauc_cv_recall_at_10_diff1": 0.014328, + "nauc_cv_recall_at_20_max": 0.136869, + "nauc_cv_recall_at_20_std": 0.313887, + "nauc_cv_recall_at_20_diff1": 0.053697, + "nauc_cv_recall_at_100_max": -0.120233, + "nauc_cv_recall_at_100_std": 0.325978, + "nauc_cv_recall_at_100_diff1": -0.028805, + "nauc_cv_recall_at_1000_max": 0.126616, + "nauc_cv_recall_at_1000_std": 0.868512, + "nauc_cv_recall_at_1000_diff1": -1.727857, + "nauc_mrr_at_1_max": 0.323758, + "nauc_mrr_at_1_std": -0.073279, + "nauc_mrr_at_1_diff1": -0.578752, + "nauc_mrr_at_3_max": 0.132901, + "nauc_mrr_at_3_std": 0.05369, + "nauc_mrr_at_3_diff1": -0.071084, + "nauc_mrr_at_5_max": 0.068675, + "nauc_mrr_at_5_std": 0.022799, + "nauc_mrr_at_5_diff1": -0.076304, + "nauc_mrr_at_10_max": 0.048597, + "nauc_mrr_at_10_std": 0.035019, + "nauc_mrr_at_10_diff1": -0.066005, + "nauc_mrr_at_20_max": 0.090549, + "nauc_mrr_at_20_std": 0.094952, + "nauc_mrr_at_20_diff1": -0.059208, + "nauc_mrr_at_100_max": 0.080373, + "nauc_mrr_at_100_std": 0.097059, + "nauc_mrr_at_100_diff1": -0.067555, + "nauc_mrr_at_1000_max": 0.083569, + "nauc_mrr_at_1000_std": 0.096535, + "nauc_mrr_at_1000_diff1": -0.071115, + "main_score": 0.05175, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.10526, + "ndcg_at_3": 0.10117, + "ndcg_at_5": 0.13295, + "ndcg_at_10": 0.1494, + "ndcg_at_20": 0.19097, + "ndcg_at_100": 0.2409, + "ndcg_at_1000": 0.31024, + "map_at_1": 0.04898, + "map_at_3": 0.06981, + "map_at_5": 0.08553, + "map_at_10": 0.09614, + "map_at_20": 0.10851, + "map_at_100": 0.11608, + "map_at_1000": 0.11981, + "recall_at_1": 0.04898, + "recall_at_3": 0.09064, + "recall_at_5": 0.15936, + "recall_at_10": 0.21404, + "recall_at_20": 0.3534, + "recall_at_100": 0.5699, + "recall_at_1000": 0.95081, + "cv_recall_at_1": 0.10526, + "cv_recall_at_3": 0.19298, + "cv_recall_at_5": 0.33333, + "cv_recall_at_10": 0.40351, + "cv_recall_at_20": 0.59649, + "cv_recall_at_100": 0.80702, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.10526, + "precision_at_3": 0.07018, + "precision_at_5": 0.07719, + "precision_at_10": 0.05088, + "precision_at_20": 0.04123, + "precision_at_100": 0.01456, + "precision_at_1000": 0.00347, + "mrr_at_1": 0.105263, + "mrr_at_3": 0.146199, + "mrr_at_5": 0.176901, + "mrr_at_10": 0.186786, + "mrr_at_20": 0.200394, + "mrr_at_100": 0.205695, + "mrr_at_1000": 0.206478, + "nauc_ndcg_at_1_max": 0.116617, + "nauc_ndcg_at_1_std": -0.010261, + "nauc_ndcg_at_1_diff1": -0.020312, + "nauc_ndcg_at_3_max": 0.29326, + "nauc_ndcg_at_3_std": -0.085154, + "nauc_ndcg_at_3_diff1": 0.185365, + "nauc_ndcg_at_5_max": 0.243374, + "nauc_ndcg_at_5_std": -0.113443, + "nauc_ndcg_at_5_diff1": 0.134031, + "nauc_ndcg_at_10_max": 0.153235, + "nauc_ndcg_at_10_std": -0.040657, + "nauc_ndcg_at_10_diff1": 0.060512, + "nauc_ndcg_at_20_max": 0.080492, + "nauc_ndcg_at_20_std": 0.027769, + "nauc_ndcg_at_20_diff1": -0.046042, + "nauc_ndcg_at_100_max": 0.081504, + "nauc_ndcg_at_100_std": 0.062802, + "nauc_ndcg_at_100_diff1": -0.027224, + "nauc_ndcg_at_1000_max": 0.095385, + "nauc_ndcg_at_1000_std": 0.000436, + "nauc_ndcg_at_1000_diff1": -0.036083, + "nauc_map_at_1_max": 0.23235, + "nauc_map_at_1_std": -0.083202, + "nauc_map_at_1_diff1": 0.044678, + "nauc_map_at_3_max": 0.270657, + "nauc_map_at_3_std": -0.10147, + "nauc_map_at_3_diff1": 0.163426, + "nauc_map_at_5_max": 0.236385, + "nauc_map_at_5_std": -0.115611, + "nauc_map_at_5_diff1": 0.132818, + "nauc_map_at_10_max": 0.193539, + "nauc_map_at_10_std": -0.063838, + "nauc_map_at_10_diff1": 0.088637, + "nauc_map_at_20_max": 0.161144, + "nauc_map_at_20_std": -0.043722, + "nauc_map_at_20_diff1": 0.046236, + "nauc_map_at_100_max": 0.166694, + "nauc_map_at_100_std": -0.036079, + "nauc_map_at_100_diff1": 0.051975, + "nauc_map_at_1000_max": 0.168528, + "nauc_map_at_1000_std": -0.036858, + "nauc_map_at_1000_diff1": 0.049857, + "nauc_recall_at_1_max": 0.23235, + "nauc_recall_at_1_std": -0.083202, + "nauc_recall_at_1_diff1": 0.044678, + "nauc_recall_at_3_max": 0.297446, + "nauc_recall_at_3_std": -0.111047, + "nauc_recall_at_3_diff1": 0.23821, + "nauc_recall_at_5_max": 0.197704, + "nauc_recall_at_5_std": -0.127222, + "nauc_recall_at_5_diff1": 0.151998, + "nauc_recall_at_10_max": 0.056866, + "nauc_recall_at_10_std": 0.016098, + "nauc_recall_at_10_diff1": 0.016358, + "nauc_recall_at_20_max": -0.076435, + "nauc_recall_at_20_std": 0.153515, + "nauc_recall_at_20_diff1": -0.195932, + "nauc_recall_at_100_max": -0.086616, + "nauc_recall_at_100_std": 0.28484, + "nauc_recall_at_100_diff1": -0.131965, + "nauc_recall_at_1000_max": -0.138815, + "nauc_recall_at_1000_std": 0.398774, + "nauc_recall_at_1000_diff1": -0.203387, + "nauc_precision_at_1_max": 0.116617, + "nauc_precision_at_1_std": -0.010261, + "nauc_precision_at_1_diff1": -0.020312, + "nauc_precision_at_3_max": 0.240157, + "nauc_precision_at_3_std": -0.07117, + "nauc_precision_at_3_diff1": 0.17711, + "nauc_precision_at_5_max": 0.199136, + "nauc_precision_at_5_std": -0.094463, + "nauc_precision_at_5_diff1": 0.103619, + "nauc_precision_at_10_max": 0.056278, + "nauc_precision_at_10_std": 0.004029, + "nauc_precision_at_10_diff1": -0.047131, + "nauc_precision_at_20_max": -0.057287, + "nauc_precision_at_20_std": 0.104665, + "nauc_precision_at_20_diff1": -0.252883, + "nauc_precision_at_100_max": -0.095829, + "nauc_precision_at_100_std": 0.092824, + "nauc_precision_at_100_diff1": -0.219792, + "nauc_precision_at_1000_max": -0.13551, + "nauc_precision_at_1000_std": -0.172365, + "nauc_precision_at_1000_diff1": -0.222178, + "nauc_cv_recall_at_1_max": 0.116617, + "nauc_cv_recall_at_1_std": -0.010261, + "nauc_cv_recall_at_1_diff1": -0.020312, + "nauc_cv_recall_at_3_max": 0.310332, + "nauc_cv_recall_at_3_std": -0.088929, + "nauc_cv_recall_at_3_diff1": 0.230224, + "nauc_cv_recall_at_5_max": 0.250929, + "nauc_cv_recall_at_5_std": -0.127577, + "nauc_cv_recall_at_5_diff1": 0.192766, + "nauc_cv_recall_at_10_max": 0.097988, + "nauc_cv_recall_at_10_std": -0.007952, + "nauc_cv_recall_at_10_diff1": 0.050221, + "nauc_cv_recall_at_20_max": -0.002672, + "nauc_cv_recall_at_20_std": 0.168434, + "nauc_cv_recall_at_20_diff1": -0.268078, + "nauc_cv_recall_at_100_max": -0.159431, + "nauc_cv_recall_at_100_std": 0.466184, + "nauc_cv_recall_at_100_diff1": -0.294362, + "nauc_cv_recall_at_1000_max": -0.164944, + "nauc_cv_recall_at_1000_std": -0.164944, + "nauc_cv_recall_at_1000_diff1": -0.559408, + "nauc_mrr_at_1_max": 0.116617, + "nauc_mrr_at_1_std": -0.010261, + "nauc_mrr_at_1_diff1": -0.020312, + "nauc_mrr_at_3_max": 0.242822, + "nauc_mrr_at_3_std": -0.061211, + "nauc_mrr_at_3_diff1": 0.136087, + "nauc_mrr_at_5_max": 0.220306, + "nauc_mrr_at_5_std": -0.080532, + "nauc_mrr_at_5_diff1": 0.119235, + "nauc_mrr_at_10_max": 0.188784, + "nauc_mrr_at_10_std": -0.051203, + "nauc_mrr_at_10_diff1": 0.090887, + "nauc_mrr_at_20_max": 0.181007, + "nauc_mrr_at_20_std": -0.037971, + "nauc_mrr_at_20_diff1": 0.060112, + "nauc_mrr_at_100_max": 0.180586, + "nauc_mrr_at_100_std": -0.036423, + "nauc_mrr_at_100_diff1": 0.067288, + "nauc_mrr_at_1000_max": 0.181759, + "nauc_mrr_at_1000_std": -0.038119, + "nauc_mrr_at_1000_diff1": 0.068462, + "main_score": 0.13295, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.0, + "ndcg_at_3": 0.02146, + "ndcg_at_5": 0.03232, + "ndcg_at_10": 0.04674, + "ndcg_at_20": 0.08173, + "ndcg_at_100": 0.10928, + "ndcg_at_1000": 0.20245, + "map_at_1": 0.0, + "map_at_3": 0.01243, + "map_at_5": 0.01758, + "map_at_10": 0.02308, + "map_at_20": 0.03262, + "map_at_100": 0.03643, + "map_at_1000": 0.04006, + "recall_at_1": 0.0, + "recall_at_3": 0.02485, + "recall_at_5": 0.04898, + "recall_at_10": 0.08699, + "recall_at_20": 0.19664, + "recall_at_100": 0.31009, + "recall_at_1000": 0.88411, + "cv_recall_at_1": 0.0, + "cv_recall_at_3": 0.05263, + "cv_recall_at_5": 0.10526, + "cv_recall_at_10": 0.19298, + "cv_recall_at_20": 0.38596, + "cv_recall_at_100": 0.54386, + "cv_recall_at_1000": 0.96491, + "precision_at_1": 0.0, + "precision_at_3": 0.01754, + "precision_at_5": 0.02105, + "precision_at_10": 0.02105, + "precision_at_20": 0.02368, + "precision_at_100": 0.00825, + "precision_at_1000": 0.00309, + "mrr_at_1": 0.0, + "mrr_at_3": 0.026316, + "mrr_at_5": 0.038596, + "mrr_at_10": 0.050118, + "mrr_at_20": 0.064043, + "mrr_at_100": 0.068229, + "mrr_at_1000": 0.069712, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_3_max": 0.51332, + "nauc_ndcg_at_3_std": 0.369694, + "nauc_ndcg_at_3_diff1": 0.35895, + "nauc_ndcg_at_5_max": 0.252666, + "nauc_ndcg_at_5_std": 0.125246, + "nauc_ndcg_at_5_diff1": 0.041618, + "nauc_ndcg_at_10_max": 0.11175, + "nauc_ndcg_at_10_std": 0.032633, + "nauc_ndcg_at_10_diff1": 0.00222, + "nauc_ndcg_at_20_max": 0.162842, + "nauc_ndcg_at_20_std": 0.158314, + "nauc_ndcg_at_20_diff1": 0.027572, + "nauc_ndcg_at_100_max": 0.236193, + "nauc_ndcg_at_100_std": 0.265566, + "nauc_ndcg_at_100_diff1": 0.044936, + "nauc_ndcg_at_1000_max": 0.128048, + "nauc_ndcg_at_1000_std": 0.206443, + "nauc_ndcg_at_1000_diff1": 0.014987, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_3_max": 0.46082, + "nauc_map_at_3_std": 0.202387, + "nauc_map_at_3_diff1": 0.238627, + "nauc_map_at_5_max": 0.308681, + "nauc_map_at_5_std": 0.087135, + "nauc_map_at_5_diff1": 0.030055, + "nauc_map_at_10_max": 0.217195, + "nauc_map_at_10_std": 0.049092, + "nauc_map_at_10_diff1": 0.039348, + "nauc_map_at_20_max": 0.221219, + "nauc_map_at_20_std": 0.120694, + "nauc_map_at_20_diff1": 0.035445, + "nauc_map_at_100_max": 0.248767, + "nauc_map_at_100_std": 0.170498, + "nauc_map_at_100_diff1": 0.056881, + "nauc_map_at_1000_max": 0.242837, + "nauc_map_at_1000_std": 0.177921, + "nauc_map_at_1000_diff1": 0.052821, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_3_max": 0.46082, + "nauc_recall_at_3_std": 0.202387, + "nauc_recall_at_3_diff1": 0.238627, + "nauc_recall_at_5_max": 0.216201, + "nauc_recall_at_5_std": 0.007569, + "nauc_recall_at_5_diff1": -0.119403, + "nauc_recall_at_10_max": 0.070695, + "nauc_recall_at_10_std": -0.033628, + "nauc_recall_at_10_diff1": -0.05795, + "nauc_recall_at_20_max": 0.163465, + "nauc_recall_at_20_std": 0.173531, + "nauc_recall_at_20_diff1": 0.028841, + "nauc_recall_at_100_max": 0.246441, + "nauc_recall_at_100_std": 0.292003, + "nauc_recall_at_100_diff1": 0.034066, + "nauc_recall_at_1000_max": 0.016761, + "nauc_recall_at_1000_std": -0.093899, + "nauc_recall_at_1000_diff1": 0.030319, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_3_max": 0.451775, + "nauc_precision_at_3_std": 0.473073, + "nauc_precision_at_3_diff1": 0.372217, + "nauc_precision_at_5_max": 0.118541, + "nauc_precision_at_5_std": 0.136263, + "nauc_precision_at_5_diff1": 0.035846, + "nauc_precision_at_10_max": -0.028088, + "nauc_precision_at_10_std": -0.016922, + "nauc_precision_at_10_diff1": 0.046388, + "nauc_precision_at_20_max": 0.024085, + "nauc_precision_at_20_std": 0.123483, + "nauc_precision_at_20_diff1": -0.007915, + "nauc_precision_at_100_max": 0.142509, + "nauc_precision_at_100_std": 0.251878, + "nauc_precision_at_100_diff1": -0.020382, + "nauc_precision_at_1000_max": -0.200934, + "nauc_precision_at_1000_std": -0.015031, + "nauc_precision_at_1000_diff1": -0.112648, + "nauc_cv_recall_at_1_max": NaN, + "nauc_cv_recall_at_1_std": NaN, + "nauc_cv_recall_at_1_diff1": NaN, + "nauc_cv_recall_at_3_max": 0.451775, + "nauc_cv_recall_at_3_std": 0.473073, + "nauc_cv_recall_at_3_diff1": 0.372217, + "nauc_cv_recall_at_5_max": 0.118541, + "nauc_cv_recall_at_5_std": 0.136263, + "nauc_cv_recall_at_5_diff1": 0.035846, + "nauc_cv_recall_at_10_max": -0.023271, + "nauc_cv_recall_at_10_std": 0.025854, + "nauc_cv_recall_at_10_diff1": -0.055245, + "nauc_cv_recall_at_20_max": 0.128362, + "nauc_cv_recall_at_20_std": 0.22725, + "nauc_cv_recall_at_20_diff1": 0.055764, + "nauc_cv_recall_at_100_max": 0.241346, + "nauc_cv_recall_at_100_std": 0.45821, + "nauc_cv_recall_at_100_diff1": 0.043712, + "nauc_cv_recall_at_1000_max": -0.864221, + "nauc_cv_recall_at_1000_std": -0.800621, + "nauc_cv_recall_at_1000_diff1": -0.019164, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_3_max": 0.451775, + "nauc_mrr_at_3_std": 0.473073, + "nauc_mrr_at_3_diff1": 0.372217, + "nauc_mrr_at_5_max": 0.232668, + "nauc_mrr_at_5_std": 0.258888, + "nauc_mrr_at_5_diff1": 0.164486, + "nauc_mrr_at_10_max": 0.150101, + "nauc_mrr_at_10_std": 0.194951, + "nauc_mrr_at_10_diff1": 0.092174, + "nauc_mrr_at_20_max": 0.189908, + "nauc_mrr_at_20_std": 0.238532, + "nauc_mrr_at_20_diff1": 0.119556, + "nauc_mrr_at_100_max": 0.205077, + "nauc_mrr_at_100_std": 0.263, + "nauc_mrr_at_100_diff1": 0.128924, + "nauc_mrr_at_1000_max": 0.202053, + "nauc_mrr_at_1000_std": 0.260022, + "nauc_mrr_at_1000_diff1": 0.131162, + "main_score": 0.03232, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 286.14110589027405, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Vidore2EconomicsReportsRetrieval.json b/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..94198c5e29 --- /dev/null +++ b/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.05172, + "ndcg_at_3": 0.05683, + "ndcg_at_5": 0.05005, + "ndcg_at_10": 0.0617, + "ndcg_at_20": 0.0783, + "ndcg_at_100": 0.1579, + "ndcg_at_1000": 0.35959, + "map_at_1": 0.00613, + "map_at_3": 0.01709, + "map_at_5": 0.02096, + "map_at_10": 0.02591, + "map_at_20": 0.03158, + "map_at_100": 0.04421, + "map_at_1000": 0.07396, + "recall_at_1": 0.00613, + "recall_at_3": 0.02491, + "recall_at_5": 0.0324, + "recall_at_10": 0.0616, + "recall_at_20": 0.1008, + "recall_at_100": 0.31922, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.05172, + "cv_recall_at_3": 0.12069, + "cv_recall_at_5": 0.15517, + "cv_recall_at_10": 0.32759, + "cv_recall_at_20": 0.44828, + "cv_recall_at_100": 0.86207, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.05172, + "precision_at_3": 0.05747, + "precision_at_5": 0.04483, + "precision_at_10": 0.05, + "precision_at_20": 0.04569, + "precision_at_100": 0.0381, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.051724, + "mrr_at_3": 0.083333, + "mrr_at_5": 0.09023, + "mrr_at_10": 0.110728, + "mrr_at_20": 0.11855, + "mrr_at_100": 0.12744, + "mrr_at_1000": 0.12829, + "nauc_ndcg_at_1_max": -0.21453, + "nauc_ndcg_at_1_std": 0.037626, + "nauc_ndcg_at_1_diff1": -0.209783, + "nauc_ndcg_at_3_max": -0.226013, + "nauc_ndcg_at_3_std": 0.004268, + "nauc_ndcg_at_3_diff1": -0.17301, + "nauc_ndcg_at_5_max": -0.188062, + "nauc_ndcg_at_5_std": 0.072124, + "nauc_ndcg_at_5_diff1": -0.090126, + "nauc_ndcg_at_10_max": 0.002404, + "nauc_ndcg_at_10_std": 0.121031, + "nauc_ndcg_at_10_diff1": 0.030073, + "nauc_ndcg_at_20_max": 0.088098, + "nauc_ndcg_at_20_std": 0.053444, + "nauc_ndcg_at_20_diff1": 0.051681, + "nauc_ndcg_at_100_max": 0.116997, + "nauc_ndcg_at_100_std": 0.003596, + "nauc_ndcg_at_100_diff1": 0.025619, + "nauc_ndcg_at_1000_max": 0.117651, + "nauc_ndcg_at_1000_std": -0.062317, + "nauc_ndcg_at_1000_diff1": 0.055606, + "nauc_map_at_1_max": -0.219432, + "nauc_map_at_1_std": 0.018219, + "nauc_map_at_1_diff1": -0.209783, + "nauc_map_at_3_max": -0.22307, + "nauc_map_at_3_std": 0.081009, + "nauc_map_at_3_diff1": -0.143185, + "nauc_map_at_5_max": -0.196061, + "nauc_map_at_5_std": 0.191054, + "nauc_map_at_5_diff1": -0.077411, + "nauc_map_at_10_max": -0.096865, + "nauc_map_at_10_std": 0.215504, + "nauc_map_at_10_diff1": -0.010529, + "nauc_map_at_20_max": -0.050402, + "nauc_map_at_20_std": 0.182368, + "nauc_map_at_20_diff1": 0.014768, + "nauc_map_at_100_max": 0.001951, + "nauc_map_at_100_std": 0.144741, + "nauc_map_at_100_diff1": 0.036439, + "nauc_map_at_1000_max": 0.089627, + "nauc_map_at_1000_std": 0.078376, + "nauc_map_at_1000_diff1": 0.061038, + "nauc_recall_at_1_max": -0.219432, + "nauc_recall_at_1_std": 0.018219, + "nauc_recall_at_1_diff1": -0.209783, + "nauc_recall_at_3_max": -0.241482, + "nauc_recall_at_3_std": 0.082255, + "nauc_recall_at_3_diff1": -0.134992, + "nauc_recall_at_5_max": -0.151974, + "nauc_recall_at_5_std": 0.222591, + "nauc_recall_at_5_diff1": -0.011107, + "nauc_recall_at_10_max": 0.125481, + "nauc_recall_at_10_std": 0.223761, + "nauc_recall_at_10_diff1": 0.142954, + "nauc_recall_at_20_max": 0.194942, + "nauc_recall_at_20_std": 0.064903, + "nauc_recall_at_20_diff1": 0.117779, + "nauc_recall_at_100_max": 0.134154, + "nauc_recall_at_100_std": -0.055932, + "nauc_recall_at_100_diff1": -0.023448, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.21453, + "nauc_precision_at_1_std": 0.037626, + "nauc_precision_at_1_diff1": -0.209783, + "nauc_precision_at_3_max": -0.226524, + "nauc_precision_at_3_std": 0.012556, + "nauc_precision_at_3_diff1": -0.160314, + "nauc_precision_at_5_max": -0.158903, + "nauc_precision_at_5_std": 0.10038, + "nauc_precision_at_5_diff1": -0.028978, + "nauc_precision_at_10_max": 0.159009, + "nauc_precision_at_10_std": 0.109959, + "nauc_precision_at_10_diff1": 0.101412, + "nauc_precision_at_20_max": 0.261029, + "nauc_precision_at_20_std": -0.026966, + "nauc_precision_at_20_diff1": 0.126793, + "nauc_precision_at_100_max": 0.295529, + "nauc_precision_at_100_std": -0.0934, + "nauc_precision_at_100_diff1": 0.16109, + "nauc_precision_at_1000_max": 0.224444, + "nauc_precision_at_1000_std": -0.121221, + "nauc_precision_at_1000_diff1": 0.115944, + "nauc_cv_recall_at_1_max": -0.21453, + "nauc_cv_recall_at_1_std": 0.037626, + "nauc_cv_recall_at_1_diff1": -0.209783, + "nauc_cv_recall_at_3_max": -0.249004, + "nauc_cv_recall_at_3_std": -0.083721, + "nauc_cv_recall_at_3_diff1": -0.184671, + "nauc_cv_recall_at_5_max": -0.145927, + "nauc_cv_recall_at_5_std": -0.017805, + "nauc_cv_recall_at_5_diff1": -0.016283, + "nauc_cv_recall_at_10_max": 0.259411, + "nauc_cv_recall_at_10_std": -0.085635, + "nauc_cv_recall_at_10_diff1": 0.16109, + "nauc_cv_recall_at_20_max": 0.505067, + "nauc_cv_recall_at_20_std": -0.120194, + "nauc_cv_recall_at_20_diff1": 0.097953, + "nauc_cv_recall_at_100_max": 0.2215, + "nauc_cv_recall_at_100_std": -0.054208, + "nauc_cv_recall_at_100_diff1": -0.342271, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.21453, + "nauc_mrr_at_1_std": 0.037626, + "nauc_mrr_at_1_diff1": -0.209783, + "nauc_mrr_at_3_max": -0.247488, + "nauc_mrr_at_3_std": -0.035728, + "nauc_mrr_at_3_diff1": -0.194447, + "nauc_mrr_at_5_max": -0.214503, + "nauc_mrr_at_5_std": -0.017152, + "nauc_mrr_at_5_diff1": -0.139906, + "nauc_mrr_at_10_max": -0.107561, + "nauc_mrr_at_10_std": -0.03924, + "nauc_mrr_at_10_diff1": -0.083365, + "nauc_mrr_at_20_max": -0.066191, + "nauc_mrr_at_20_std": -0.047359, + "nauc_mrr_at_20_diff1": -0.087438, + "nauc_mrr_at_100_max": -0.091757, + "nauc_mrr_at_100_std": -0.044684, + "nauc_mrr_at_100_diff1": -0.102365, + "nauc_mrr_at_1000_max": -0.093243, + "nauc_mrr_at_1000_std": -0.045621, + "nauc_mrr_at_1000_diff1": -0.100556, + "main_score": 0.05005, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.13793, + "ndcg_at_3": 0.09834, + "ndcg_at_5": 0.08899, + "ndcg_at_10": 0.09386, + "ndcg_at_20": 0.11946, + "ndcg_at_100": 0.21907, + "ndcg_at_1000": 0.3969, + "map_at_1": 0.01981, + "map_at_3": 0.03003, + "map_at_5": 0.03198, + "map_at_10": 0.03852, + "map_at_20": 0.04807, + "map_at_100": 0.06631, + "map_at_1000": 0.09659, + "recall_at_1": 0.01981, + "recall_at_3": 0.03632, + "recall_at_5": 0.04182, + "recall_at_10": 0.07109, + "recall_at_20": 0.14153, + "recall_at_100": 0.41804, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.13793, + "cv_recall_at_3": 0.18966, + "cv_recall_at_5": 0.25862, + "cv_recall_at_10": 0.41379, + "cv_recall_at_20": 0.62069, + "cv_recall_at_100": 0.93103, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.13793, + "precision_at_3": 0.08621, + "precision_at_5": 0.07241, + "precision_at_10": 0.06724, + "precision_at_20": 0.06466, + "precision_at_100": 0.05052, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.137931, + "mrr_at_3": 0.163793, + "mrr_at_5": 0.17931, + "mrr_at_10": 0.201526, + "mrr_at_20": 0.216996, + "mrr_at_100": 0.22475, + "mrr_at_1000": 0.225263, + "nauc_ndcg_at_1_max": -0.29897, + "nauc_ndcg_at_1_std": -0.310901, + "nauc_ndcg_at_1_diff1": 0.022308, + "nauc_ndcg_at_3_max": -0.193554, + "nauc_ndcg_at_3_std": -0.155671, + "nauc_ndcg_at_3_diff1": 0.157076, + "nauc_ndcg_at_5_max": -0.156219, + "nauc_ndcg_at_5_std": -0.174537, + "nauc_ndcg_at_5_diff1": 0.223857, + "nauc_ndcg_at_10_max": -0.097003, + "nauc_ndcg_at_10_std": -0.138384, + "nauc_ndcg_at_10_diff1": 0.246483, + "nauc_ndcg_at_20_max": -0.0083, + "nauc_ndcg_at_20_std": -0.145758, + "nauc_ndcg_at_20_diff1": 0.292166, + "nauc_ndcg_at_100_max": 0.032676, + "nauc_ndcg_at_100_std": -0.131498, + "nauc_ndcg_at_100_diff1": 0.311449, + "nauc_ndcg_at_1000_max": -0.023502, + "nauc_ndcg_at_1000_std": -0.262974, + "nauc_ndcg_at_1000_diff1": 0.160447, + "nauc_map_at_1_max": -0.100435, + "nauc_map_at_1_std": -0.198838, + "nauc_map_at_1_diff1": 0.024282, + "nauc_map_at_3_max": -0.064537, + "nauc_map_at_3_std": -0.031855, + "nauc_map_at_3_diff1": 0.159712, + "nauc_map_at_5_max": -0.071463, + "nauc_map_at_5_std": -0.040507, + "nauc_map_at_5_diff1": 0.199064, + "nauc_map_at_10_max": -0.079078, + "nauc_map_at_10_std": -0.064581, + "nauc_map_at_10_diff1": 0.213311, + "nauc_map_at_20_max": -0.05197, + "nauc_map_at_20_std": -0.091948, + "nauc_map_at_20_diff1": 0.237149, + "nauc_map_at_100_max": -0.035793, + "nauc_map_at_100_std": -0.089642, + "nauc_map_at_100_diff1": 0.256592, + "nauc_map_at_1000_max": -0.005023, + "nauc_map_at_1000_std": -0.150484, + "nauc_map_at_1000_diff1": 0.170542, + "nauc_recall_at_1_max": -0.100435, + "nauc_recall_at_1_std": -0.198838, + "nauc_recall_at_1_diff1": 0.024282, + "nauc_recall_at_3_max": -0.036904, + "nauc_recall_at_3_std": 0.049652, + "nauc_recall_at_3_diff1": 0.221783, + "nauc_recall_at_5_max": -0.053377, + "nauc_recall_at_5_std": 0.024208, + "nauc_recall_at_5_diff1": 0.30537, + "nauc_recall_at_10_max": -0.057033, + "nauc_recall_at_10_std": -0.027601, + "nauc_recall_at_10_diff1": 0.315012, + "nauc_recall_at_20_max": 0.135716, + "nauc_recall_at_20_std": -0.053155, + "nauc_recall_at_20_diff1": 0.373956, + "nauc_recall_at_100_max": 0.150021, + "nauc_recall_at_100_std": -0.051015, + "nauc_recall_at_100_diff1": 0.312886, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.29897, + "nauc_precision_at_1_std": -0.310901, + "nauc_precision_at_1_diff1": 0.022308, + "nauc_precision_at_3_max": -0.14179, + "nauc_precision_at_3_std": -0.083014, + "nauc_precision_at_3_diff1": 0.207687, + "nauc_precision_at_5_max": -0.118668, + "nauc_precision_at_5_std": -0.146356, + "nauc_precision_at_5_diff1": 0.320557, + "nauc_precision_at_10_max": -0.029943, + "nauc_precision_at_10_std": -0.148315, + "nauc_precision_at_10_diff1": 0.33465, + "nauc_precision_at_20_max": 0.019647, + "nauc_precision_at_20_std": -0.266696, + "nauc_precision_at_20_diff1": 0.320833, + "nauc_precision_at_100_max": 0.139408, + "nauc_precision_at_100_std": -0.187161, + "nauc_precision_at_100_diff1": 0.011648, + "nauc_precision_at_1000_max": 0.041926, + "nauc_precision_at_1000_std": -0.232722, + "nauc_precision_at_1000_diff1": -0.1514, + "nauc_cv_recall_at_1_max": -0.29897, + "nauc_cv_recall_at_1_std": -0.310901, + "nauc_cv_recall_at_1_diff1": 0.022308, + "nauc_cv_recall_at_3_max": -0.179502, + "nauc_cv_recall_at_3_std": -0.146272, + "nauc_cv_recall_at_3_diff1": 0.188364, + "nauc_cv_recall_at_5_max": -0.138185, + "nauc_cv_recall_at_5_std": -0.196924, + "nauc_cv_recall_at_5_diff1": 0.282113, + "nauc_cv_recall_at_10_max": 0.016931, + "nauc_cv_recall_at_10_std": -0.107211, + "nauc_cv_recall_at_10_diff1": 0.326645, + "nauc_cv_recall_at_20_max": 0.305994, + "nauc_cv_recall_at_20_std": -0.182889, + "nauc_cv_recall_at_20_diff1": 0.496265, + "nauc_cv_recall_at_100_max": 0.269241, + "nauc_cv_recall_at_100_std": -1.252725, + "nauc_cv_recall_at_100_diff1": 0.562469, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.29897, + "nauc_mrr_at_1_std": -0.310901, + "nauc_mrr_at_1_diff1": 0.022308, + "nauc_mrr_at_3_max": -0.23206, + "nauc_mrr_at_3_std": -0.218697, + "nauc_mrr_at_3_diff1": 0.115311, + "nauc_mrr_at_5_max": -0.21989, + "nauc_mrr_at_5_std": -0.228436, + "nauc_mrr_at_5_diff1": 0.150688, + "nauc_mrr_at_10_max": -0.183448, + "nauc_mrr_at_10_std": -0.210707, + "nauc_mrr_at_10_diff1": 0.158942, + "nauc_mrr_at_20_max": -0.163594, + "nauc_mrr_at_20_std": -0.22223, + "nauc_mrr_at_20_diff1": 0.168349, + "nauc_mrr_at_100_max": -0.174018, + "nauc_mrr_at_100_std": -0.229543, + "nauc_mrr_at_100_diff1": 0.165007, + "nauc_mrr_at_1000_max": -0.174943, + "nauc_mrr_at_1000_std": -0.227565, + "nauc_mrr_at_1000_diff1": 0.164258, + "main_score": 0.08899, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.2069, + "ndcg_at_3": 0.24509, + "ndcg_at_5": 0.22422, + "ndcg_at_10": 0.2183, + "ndcg_at_20": 0.23857, + "ndcg_at_100": 0.3471, + "ndcg_at_1000": 0.47905, + "map_at_1": 0.02101, + "map_at_3": 0.05234, + "map_at_5": 0.06161, + "map_at_10": 0.07857, + "map_at_20": 0.09827, + "map_at_100": 0.13987, + "map_at_1000": 0.16961, + "recall_at_1": 0.02101, + "recall_at_3": 0.07723, + "recall_at_5": 0.10126, + "recall_at_10": 0.16294, + "recall_at_20": 0.25609, + "recall_at_100": 0.55994, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.2069, + "cv_recall_at_3": 0.48276, + "cv_recall_at_5": 0.56897, + "cv_recall_at_10": 0.7069, + "cv_recall_at_20": 0.84483, + "cv_recall_at_100": 0.94828, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.2069, + "precision_at_3": 0.24713, + "precision_at_5": 0.2069, + "precision_at_10": 0.17414, + "precision_at_20": 0.14569, + "precision_at_100": 0.08172, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.206897, + "mrr_at_3": 0.327586, + "mrr_at_5": 0.346552, + "mrr_at_10": 0.362445, + "mrr_at_20": 0.371945, + "mrr_at_100": 0.375258, + "mrr_at_1000": 0.375706, + "nauc_ndcg_at_1_max": 0.288283, + "nauc_ndcg_at_1_std": 0.37698, + "nauc_ndcg_at_1_diff1": 0.309828, + "nauc_ndcg_at_3_max": 0.295775, + "nauc_ndcg_at_3_std": 0.262102, + "nauc_ndcg_at_3_diff1": 0.254609, + "nauc_ndcg_at_5_max": 0.323436, + "nauc_ndcg_at_5_std": 0.278642, + "nauc_ndcg_at_5_diff1": 0.261015, + "nauc_ndcg_at_10_max": 0.358121, + "nauc_ndcg_at_10_std": 0.284695, + "nauc_ndcg_at_10_diff1": 0.212432, + "nauc_ndcg_at_20_max": 0.339842, + "nauc_ndcg_at_20_std": 0.292414, + "nauc_ndcg_at_20_diff1": 0.129578, + "nauc_ndcg_at_100_max": 0.323615, + "nauc_ndcg_at_100_std": 0.235507, + "nauc_ndcg_at_100_diff1": 0.14392, + "nauc_ndcg_at_1000_max": 0.34894, + "nauc_ndcg_at_1000_std": 0.278572, + "nauc_ndcg_at_1000_diff1": 0.170962, + "nauc_map_at_1_max": 0.012678, + "nauc_map_at_1_std": 0.386478, + "nauc_map_at_1_diff1": 0.120111, + "nauc_map_at_3_max": 0.041031, + "nauc_map_at_3_std": 0.200589, + "nauc_map_at_3_diff1": 0.044439, + "nauc_map_at_5_max": 0.098745, + "nauc_map_at_5_std": 0.219628, + "nauc_map_at_5_diff1": 0.09997, + "nauc_map_at_10_max": 0.166942, + "nauc_map_at_10_std": 0.224557, + "nauc_map_at_10_diff1": 0.098288, + "nauc_map_at_20_max": 0.228317, + "nauc_map_at_20_std": 0.249261, + "nauc_map_at_20_diff1": 0.113298, + "nauc_map_at_100_max": 0.301358, + "nauc_map_at_100_std": 0.260948, + "nauc_map_at_100_diff1": 0.130537, + "nauc_map_at_1000_max": 0.318138, + "nauc_map_at_1000_std": 0.27639, + "nauc_map_at_1000_diff1": 0.118474, + "nauc_recall_at_1_max": 0.012678, + "nauc_recall_at_1_std": 0.386478, + "nauc_recall_at_1_diff1": 0.120111, + "nauc_recall_at_3_max": -0.100897, + "nauc_recall_at_3_std": -0.008025, + "nauc_recall_at_3_diff1": -0.116833, + "nauc_recall_at_5_max": 0.00279, + "nauc_recall_at_5_std": 0.023068, + "nauc_recall_at_5_diff1": 0.000709, + "nauc_recall_at_10_max": 0.114916, + "nauc_recall_at_10_std": 0.085869, + "nauc_recall_at_10_diff1": -0.064084, + "nauc_recall_at_20_max": 0.163122, + "nauc_recall_at_20_std": 0.131684, + "nauc_recall_at_20_diff1": -0.039479, + "nauc_recall_at_100_max": 0.106266, + "nauc_recall_at_100_std": -0.016441, + "nauc_recall_at_100_diff1": -0.07844, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.288283, + "nauc_precision_at_1_std": 0.37698, + "nauc_precision_at_1_diff1": 0.309828, + "nauc_precision_at_3_max": 0.293633, + "nauc_precision_at_3_std": 0.232285, + "nauc_precision_at_3_diff1": 0.242643, + "nauc_precision_at_5_max": 0.350732, + "nauc_precision_at_5_std": 0.250525, + "nauc_precision_at_5_diff1": 0.253462, + "nauc_precision_at_10_max": 0.44946, + "nauc_precision_at_10_std": 0.248854, + "nauc_precision_at_10_diff1": 0.253556, + "nauc_precision_at_20_max": 0.407715, + "nauc_precision_at_20_std": 0.2035, + "nauc_precision_at_20_diff1": 0.161004, + "nauc_precision_at_100_max": 0.291799, + "nauc_precision_at_100_std": 0.164381, + "nauc_precision_at_100_diff1": 0.057254, + "nauc_precision_at_1000_max": 0.238522, + "nauc_precision_at_1000_std": 0.144076, + "nauc_precision_at_1000_diff1": -0.00754, + "nauc_cv_recall_at_1_max": 0.288283, + "nauc_cv_recall_at_1_std": 0.37698, + "nauc_cv_recall_at_1_diff1": 0.309828, + "nauc_cv_recall_at_3_max": 0.19356, + "nauc_cv_recall_at_3_std": 0.165436, + "nauc_cv_recall_at_3_diff1": 0.119136, + "nauc_cv_recall_at_5_max": 0.32483, + "nauc_cv_recall_at_5_std": 0.204865, + "nauc_cv_recall_at_5_diff1": 0.270796, + "nauc_cv_recall_at_10_max": 0.472482, + "nauc_cv_recall_at_10_std": 0.386535, + "nauc_cv_recall_at_10_diff1": 0.269429, + "nauc_cv_recall_at_20_max": 0.438107, + "nauc_cv_recall_at_20_std": 0.287667, + "nauc_cv_recall_at_20_diff1": 0.264418, + "nauc_cv_recall_at_100_max": 0.103301, + "nauc_cv_recall_at_100_std": -0.129386, + "nauc_cv_recall_at_100_diff1": -0.306709, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.288283, + "nauc_mrr_at_1_std": 0.37698, + "nauc_mrr_at_1_diff1": 0.309828, + "nauc_mrr_at_3_max": 0.273332, + "nauc_mrr_at_3_std": 0.288911, + "nauc_mrr_at_3_diff1": 0.237932, + "nauc_mrr_at_5_max": 0.304882, + "nauc_mrr_at_5_std": 0.297571, + "nauc_mrr_at_5_diff1": 0.271726, + "nauc_mrr_at_10_max": 0.319591, + "nauc_mrr_at_10_std": 0.31678, + "nauc_mrr_at_10_diff1": 0.266855, + "nauc_mrr_at_20_max": 0.311899, + "nauc_mrr_at_20_std": 0.310946, + "nauc_mrr_at_20_diff1": 0.265351, + "nauc_mrr_at_100_max": 0.309901, + "nauc_mrr_at_100_std": 0.310265, + "nauc_mrr_at_100_diff1": 0.261384, + "nauc_mrr_at_1000_max": 0.310238, + "nauc_mrr_at_1000_std": 0.310977, + "nauc_mrr_at_1000_diff1": 0.262197, + "main_score": 0.22422, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.06897, + "ndcg_at_3": 0.07002, + "ndcg_at_5": 0.07184, + "ndcg_at_10": 0.07742, + "ndcg_at_20": 0.08832, + "ndcg_at_100": 0.14931, + "ndcg_at_1000": 0.35968, + "map_at_1": 0.01118, + "map_at_3": 0.01787, + "map_at_5": 0.02168, + "map_at_10": 0.02661, + "map_at_20": 0.03148, + "map_at_100": 0.04371, + "map_at_1000": 0.07193, + "recall_at_1": 0.01118, + "recall_at_3": 0.02183, + "recall_at_5": 0.03462, + "recall_at_10": 0.05762, + "recall_at_20": 0.09313, + "recall_at_100": 0.28106, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.06897, + "cv_recall_at_3": 0.17241, + "cv_recall_at_5": 0.24138, + "cv_recall_at_10": 0.34483, + "cv_recall_at_20": 0.48276, + "cv_recall_at_100": 0.75862, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.06897, + "precision_at_3": 0.06897, + "precision_at_5": 0.06897, + "precision_at_10": 0.06379, + "precision_at_20": 0.05345, + "precision_at_100": 0.03466, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.068966, + "mrr_at_3": 0.114943, + "mrr_at_5": 0.13046, + "mrr_at_10": 0.144896, + "mrr_at_20": 0.153477, + "mrr_at_100": 0.15965, + "mrr_at_1000": 0.161287, + "nauc_ndcg_at_1_max": -0.362655, + "nauc_ndcg_at_1_std": 0.482751, + "nauc_ndcg_at_1_diff1": 0.218431, + "nauc_ndcg_at_3_max": -0.115359, + "nauc_ndcg_at_3_std": 0.198353, + "nauc_ndcg_at_3_diff1": 0.172005, + "nauc_ndcg_at_5_max": -0.04242, + "nauc_ndcg_at_5_std": 0.079908, + "nauc_ndcg_at_5_diff1": 0.15589, + "nauc_ndcg_at_10_max": -0.012734, + "nauc_ndcg_at_10_std": 0.003278, + "nauc_ndcg_at_10_diff1": 0.14952, + "nauc_ndcg_at_20_max": -0.032231, + "nauc_ndcg_at_20_std": 0.013128, + "nauc_ndcg_at_20_diff1": 0.153715, + "nauc_ndcg_at_100_max": 0.052061, + "nauc_ndcg_at_100_std": 0.042682, + "nauc_ndcg_at_100_diff1": 0.090611, + "nauc_ndcg_at_1000_max": 0.082206, + "nauc_ndcg_at_1000_std": 0.018806, + "nauc_ndcg_at_1000_diff1": 0.205497, + "nauc_map_at_1_max": -0.396017, + "nauc_map_at_1_std": 0.578727, + "nauc_map_at_1_diff1": 0.419211, + "nauc_map_at_3_max": -0.372577, + "nauc_map_at_3_std": 0.553035, + "nauc_map_at_3_diff1": 0.428206, + "nauc_map_at_5_max": -0.333498, + "nauc_map_at_5_std": 0.437879, + "nauc_map_at_5_diff1": 0.365305, + "nauc_map_at_10_max": -0.268685, + "nauc_map_at_10_std": 0.325804, + "nauc_map_at_10_diff1": 0.281179, + "nauc_map_at_20_max": -0.246117, + "nauc_map_at_20_std": 0.28269, + "nauc_map_at_20_diff1": 0.240549, + "nauc_map_at_100_max": -0.121293, + "nauc_map_at_100_std": 0.195929, + "nauc_map_at_100_diff1": 0.195362, + "nauc_map_at_1000_max": 0.052851, + "nauc_map_at_1000_std": 0.092891, + "nauc_map_at_1000_diff1": 0.266213, + "nauc_recall_at_1_max": -0.396017, + "nauc_recall_at_1_std": 0.578727, + "nauc_recall_at_1_diff1": 0.419211, + "nauc_recall_at_3_max": -0.297952, + "nauc_recall_at_3_std": 0.432348, + "nauc_recall_at_3_diff1": 0.308482, + "nauc_recall_at_5_max": -0.21801, + "nauc_recall_at_5_std": 0.174408, + "nauc_recall_at_5_diff1": 0.219153, + "nauc_recall_at_10_max": -0.096217, + "nauc_recall_at_10_std": -0.046421, + "nauc_recall_at_10_diff1": 0.053084, + "nauc_recall_at_20_max": -0.075277, + "nauc_recall_at_20_std": -0.025065, + "nauc_recall_at_20_diff1": 0.052435, + "nauc_recall_at_100_max": 0.14964, + "nauc_recall_at_100_std": 0.041717, + "nauc_recall_at_100_diff1": 0.044736, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.362655, + "nauc_precision_at_1_std": 0.482751, + "nauc_precision_at_1_diff1": 0.218431, + "nauc_precision_at_3_max": -0.016936, + "nauc_precision_at_3_std": 0.099456, + "nauc_precision_at_3_diff1": 0.15441, + "nauc_precision_at_5_max": 0.070147, + "nauc_precision_at_5_std": -0.048022, + "nauc_precision_at_5_diff1": 0.122589, + "nauc_precision_at_10_max": 0.176902, + "nauc_precision_at_10_std": -0.225198, + "nauc_precision_at_10_diff1": 0.026033, + "nauc_precision_at_20_max": 0.198158, + "nauc_precision_at_20_std": -0.270701, + "nauc_precision_at_20_diff1": 0.071932, + "nauc_precision_at_100_max": 0.454024, + "nauc_precision_at_100_std": -0.207897, + "nauc_precision_at_100_diff1": 0.096669, + "nauc_precision_at_1000_max": 0.38785, + "nauc_precision_at_1000_std": -0.206007, + "nauc_precision_at_1000_diff1": 0.24017, + "nauc_cv_recall_at_1_max": -0.362655, + "nauc_cv_recall_at_1_std": 0.482751, + "nauc_cv_recall_at_1_diff1": 0.218431, + "nauc_cv_recall_at_3_max": -0.030473, + "nauc_cv_recall_at_3_std": 0.065609, + "nauc_cv_recall_at_3_diff1": -0.002312, + "nauc_cv_recall_at_5_max": 0.051949, + "nauc_cv_recall_at_5_std": -0.098938, + "nauc_cv_recall_at_5_diff1": 0.062126, + "nauc_cv_recall_at_10_max": 0.126126, + "nauc_cv_recall_at_10_std": -0.259597, + "nauc_cv_recall_at_10_diff1": 0.050434, + "nauc_cv_recall_at_20_max": 0.123144, + "nauc_cv_recall_at_20_std": -0.350638, + "nauc_cv_recall_at_20_diff1": 0.063424, + "nauc_cv_recall_at_100_max": 0.280588, + "nauc_cv_recall_at_100_std": -0.152317, + "nauc_cv_recall_at_100_diff1": 0.296214, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.362655, + "nauc_mrr_at_1_std": 0.482751, + "nauc_mrr_at_1_diff1": 0.218431, + "nauc_mrr_at_3_max": -0.176677, + "nauc_mrr_at_3_std": 0.229231, + "nauc_mrr_at_3_diff1": 0.087475, + "nauc_mrr_at_5_max": -0.141366, + "nauc_mrr_at_5_std": 0.157216, + "nauc_mrr_at_5_diff1": 0.107863, + "nauc_mrr_at_10_max": -0.118459, + "nauc_mrr_at_10_std": 0.109765, + "nauc_mrr_at_10_diff1": 0.110209, + "nauc_mrr_at_20_max": -0.117934, + "nauc_mrr_at_20_std": 0.099944, + "nauc_mrr_at_20_diff1": 0.109429, + "nauc_mrr_at_100_max": -0.113988, + "nauc_mrr_at_100_std": 0.11707, + "nauc_mrr_at_100_diff1": 0.122414, + "nauc_mrr_at_1000_max": -0.116679, + "nauc_mrr_at_1000_std": 0.119747, + "nauc_mrr_at_1000_diff1": 0.121037, + "main_score": 0.07184, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 19.46478581428528, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json b/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json index 2102a72c61..b7968349a2 100644 --- a/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json +++ b/results/openai__clip-vit-base-patch16/57c216476eefef5ab752ec549e440a49ae4ae5f3/model_meta.json @@ -1 +1 @@ -{"name": "openai/clip-vit-base-patch16", "revision": "57c216476eefef5ab752ec549e440a49ae4ae5f3", "release_date": "2021-02-26", "languages": ["eng_Latn"], "n_parameters": 151000000, "memory_usage_mb": 576.0, "max_tokens": 77.0, "embed_dim": 512, "license": null, "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["PyTorch"], "reference": "https://huggingface.co/openai/clip-vit-base-patch16", "similarity_fn_name": null, "use_instructions": false, "training_datasets": null, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "CLIPModelWrapper"} \ No newline at end of file +{"name": "openai/clip-vit-base-patch16", "revision": "57c216476eefef5ab752ec549e440a49ae4ae5f3", "release_date": "2021-02-26", "languages": ["eng-Latn"], "n_parameters": 151000000, "memory_usage_mb": 576.0, "max_tokens": 77.0, "embed_dim": 512, "license": null, "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["PyTorch"], "reference": "https://huggingface.co/openai/clip-vit-base-patch16", "similarity_fn_name": null, "use_instructions": false, "training_datasets": null, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "CLIPModelWrapper"} \ No newline at end of file diff --git a/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/Vidore2BioMedicalLecturesRetrieval.json b/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..5e68ed8b2f --- /dev/null +++ b/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.4375, + "ndcg_at_3": 0.45466, + "ndcg_at_5": 0.48488, + "ndcg_at_10": 0.5147, + "ndcg_at_20": 0.54531, + "ndcg_at_100": 0.5905, + "ndcg_at_1000": 0.61286, + "map_at_1": 0.24297, + "map_at_3": 0.34369, + "map_at_5": 0.38738, + "map_at_10": 0.41553, + "map_at_20": 0.43036, + "map_at_100": 0.44224, + "map_at_1000": 0.44445, + "recall_at_1": 0.24297, + "recall_at_3": 0.42334, + "recall_at_5": 0.53476, + "recall_at_10": 0.63369, + "recall_at_20": 0.73402, + "recall_at_100": 0.89529, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.43125, + "cv_recall_at_3": 0.66875, + "cv_recall_at_5": 0.75, + "cv_recall_at_10": 0.8375, + "cv_recall_at_20": 0.88125, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.4375, + "precision_at_3": 0.3, + "precision_at_5": 0.2375, + "precision_at_10": 0.1525, + "precision_at_20": 0.09125, + "precision_at_100": 0.0255, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.43125, + "mrr_at_3": 0.532292, + "mrr_at_5": 0.551667, + "mrr_at_10": 0.563557, + "mrr_at_20": 0.566532, + "mrr_at_100": 0.569144, + "mrr_at_1000": 0.569277, + "nauc_ndcg_at_1_max": 0.23888, + "nauc_ndcg_at_1_std": -0.127512, + "nauc_ndcg_at_1_diff1": 0.421702, + "nauc_ndcg_at_3_max": 0.185226, + "nauc_ndcg_at_3_std": -0.075851, + "nauc_ndcg_at_3_diff1": 0.358588, + "nauc_ndcg_at_5_max": 0.160605, + "nauc_ndcg_at_5_std": -0.109257, + "nauc_ndcg_at_5_diff1": 0.342192, + "nauc_ndcg_at_10_max": 0.127075, + "nauc_ndcg_at_10_std": -0.123216, + "nauc_ndcg_at_10_diff1": 0.365131, + "nauc_ndcg_at_20_max": 0.109814, + "nauc_ndcg_at_20_std": -0.120217, + "nauc_ndcg_at_20_diff1": 0.344609, + "nauc_ndcg_at_100_max": 0.133465, + "nauc_ndcg_at_100_std": -0.095078, + "nauc_ndcg_at_100_diff1": 0.331464, + "nauc_ndcg_at_1000_max": 0.167749, + "nauc_ndcg_at_1000_std": -0.083126, + "nauc_ndcg_at_1000_diff1": 0.335919, + "nauc_map_at_1_max": 0.116726, + "nauc_map_at_1_std": -0.106595, + "nauc_map_at_1_diff1": 0.403785, + "nauc_map_at_3_max": 0.119159, + "nauc_map_at_3_std": -0.07773, + "nauc_map_at_3_diff1": 0.361282, + "nauc_map_at_5_max": 0.112084, + "nauc_map_at_5_std": -0.109937, + "nauc_map_at_5_diff1": 0.352975, + "nauc_map_at_10_max": 0.11289, + "nauc_map_at_10_std": -0.121375, + "nauc_map_at_10_diff1": 0.361759, + "nauc_map_at_20_max": 0.113229, + "nauc_map_at_20_std": -0.120865, + "nauc_map_at_20_diff1": 0.354499, + "nauc_map_at_100_max": 0.123744, + "nauc_map_at_100_std": -0.111462, + "nauc_map_at_100_diff1": 0.351969, + "nauc_map_at_1000_max": 0.127361, + "nauc_map_at_1000_std": -0.110272, + "nauc_map_at_1000_diff1": 0.351648, + "nauc_recall_at_1_max": 0.116726, + "nauc_recall_at_1_std": -0.106595, + "nauc_recall_at_1_diff1": 0.403785, + "nauc_recall_at_3_max": 0.079168, + "nauc_recall_at_3_std": -0.050988, + "nauc_recall_at_3_diff1": 0.316928, + "nauc_recall_at_5_max": 0.074603, + "nauc_recall_at_5_std": -0.105477, + "nauc_recall_at_5_diff1": 0.254885, + "nauc_recall_at_10_max": 0.016692, + "nauc_recall_at_10_std": -0.155054, + "nauc_recall_at_10_diff1": 0.269423, + "nauc_recall_at_20_max": -0.049936, + "nauc_recall_at_20_std": -0.16919, + "nauc_recall_at_20_diff1": 0.211524, + "nauc_recall_at_100_max": -0.201162, + "nauc_recall_at_100_std": -0.165606, + "nauc_recall_at_100_diff1": 0.089493, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.23888, + "nauc_precision_at_1_std": -0.127512, + "nauc_precision_at_1_diff1": 0.421702, + "nauc_precision_at_3_max": 0.156354, + "nauc_precision_at_3_std": -0.030138, + "nauc_precision_at_3_diff1": 0.1976, + "nauc_precision_at_5_max": 0.125143, + "nauc_precision_at_5_std": -0.060235, + "nauc_precision_at_5_diff1": 0.112322, + "nauc_precision_at_10_max": 0.115952, + "nauc_precision_at_10_std": -0.027936, + "nauc_precision_at_10_diff1": 0.066143, + "nauc_precision_at_20_max": 0.125515, + "nauc_precision_at_20_std": 0.013374, + "nauc_precision_at_20_diff1": -0.012916, + "nauc_precision_at_100_max": 0.20347, + "nauc_precision_at_100_std": 0.112703, + "nauc_precision_at_100_diff1": -0.115701, + "nauc_precision_at_1000_max": 0.22887, + "nauc_precision_at_1000_std": 0.119834, + "nauc_precision_at_1000_diff1": -0.149199, + "nauc_cv_recall_at_1_max": 0.218308, + "nauc_cv_recall_at_1_std": -0.157547, + "nauc_cv_recall_at_1_diff1": 0.438218, + "nauc_cv_recall_at_3_max": 0.178025, + "nauc_cv_recall_at_3_std": -0.034332, + "nauc_cv_recall_at_3_diff1": 0.401167, + "nauc_cv_recall_at_5_max": 0.211643, + "nauc_cv_recall_at_5_std": -0.03213, + "nauc_cv_recall_at_5_diff1": 0.352172, + "nauc_cv_recall_at_10_max": 0.140172, + "nauc_cv_recall_at_10_std": -0.008999, + "nauc_cv_recall_at_10_diff1": 0.287662, + "nauc_cv_recall_at_20_max": -0.033293, + "nauc_cv_recall_at_20_std": -0.002961, + "nauc_cv_recall_at_20_diff1": 0.107908, + "nauc_cv_recall_at_100_max": -0.220005, + "nauc_cv_recall_at_100_std": 0.082283, + "nauc_cv_recall_at_100_diff1": -0.661531, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.218308, + "nauc_mrr_at_1_std": -0.157547, + "nauc_mrr_at_1_diff1": 0.438218, + "nauc_mrr_at_3_max": 0.206681, + "nauc_mrr_at_3_std": -0.092839, + "nauc_mrr_at_3_diff1": 0.405546, + "nauc_mrr_at_5_max": 0.21364, + "nauc_mrr_at_5_std": -0.094181, + "nauc_mrr_at_5_diff1": 0.398559, + "nauc_mrr_at_10_max": 0.206774, + "nauc_mrr_at_10_std": -0.096246, + "nauc_mrr_at_10_diff1": 0.396178, + "nauc_mrr_at_20_max": 0.202505, + "nauc_mrr_at_20_std": -0.097855, + "nauc_mrr_at_20_diff1": 0.392224, + "nauc_mrr_at_100_max": 0.20469, + "nauc_mrr_at_100_std": -0.097423, + "nauc_mrr_at_100_diff1": 0.392123, + "nauc_mrr_at_1000_max": 0.204739, + "nauc_mrr_at_1000_std": -0.097503, + "nauc_mrr_at_1000_diff1": 0.392491, + "main_score": 0.48488, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.41875, + "ndcg_at_3": 0.42627, + "ndcg_at_5": 0.45553, + "ndcg_at_10": 0.49786, + "ndcg_at_20": 0.519, + "ndcg_at_100": 0.57125, + "ndcg_at_1000": 0.5938, + "map_at_1": 0.22834, + "map_at_3": 0.31914, + "map_at_5": 0.36171, + "map_at_10": 0.39196, + "map_at_20": 0.403, + "map_at_100": 0.41646, + "map_at_1000": 0.41866, + "recall_at_1": 0.22834, + "recall_at_3": 0.39621, + "recall_at_5": 0.50679, + "recall_at_10": 0.64201, + "recall_at_20": 0.70349, + "recall_at_100": 0.89395, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.41875, + "cv_recall_at_3": 0.6375, + "cv_recall_at_5": 0.725, + "cv_recall_at_10": 0.8625, + "cv_recall_at_20": 0.875, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.41875, + "precision_at_3": 0.28125, + "precision_at_5": 0.22, + "precision_at_10": 0.14688, + "precision_at_20": 0.08688, + "precision_at_100": 0.0255, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.41875, + "mrr_at_3": 0.513542, + "mrr_at_5": 0.533854, + "mrr_at_10": 0.55119, + "mrr_at_20": 0.551885, + "mrr_at_100": 0.554387, + "mrr_at_1000": 0.554553, + "nauc_ndcg_at_1_max": 0.24631, + "nauc_ndcg_at_1_std": 0.029053, + "nauc_ndcg_at_1_diff1": 0.400894, + "nauc_ndcg_at_3_max": 0.180007, + "nauc_ndcg_at_3_std": 0.002033, + "nauc_ndcg_at_3_diff1": 0.246326, + "nauc_ndcg_at_5_max": 0.162116, + "nauc_ndcg_at_5_std": -0.032749, + "nauc_ndcg_at_5_diff1": 0.239126, + "nauc_ndcg_at_10_max": 0.14227, + "nauc_ndcg_at_10_std": -0.039253, + "nauc_ndcg_at_10_diff1": 0.264339, + "nauc_ndcg_at_20_max": 0.130473, + "nauc_ndcg_at_20_std": -0.024968, + "nauc_ndcg_at_20_diff1": 0.262326, + "nauc_ndcg_at_100_max": 0.144675, + "nauc_ndcg_at_100_std": 0.002034, + "nauc_ndcg_at_100_diff1": 0.242244, + "nauc_ndcg_at_1000_max": 0.176261, + "nauc_ndcg_at_1000_std": 0.00332, + "nauc_ndcg_at_1000_diff1": 0.238592, + "nauc_map_at_1_max": 0.088461, + "nauc_map_at_1_std": -0.012252, + "nauc_map_at_1_diff1": 0.34777, + "nauc_map_at_3_max": 0.119493, + "nauc_map_at_3_std": 0.010155, + "nauc_map_at_3_diff1": 0.278723, + "nauc_map_at_5_max": 0.128593, + "nauc_map_at_5_std": -0.024798, + "nauc_map_at_5_diff1": 0.271442, + "nauc_map_at_10_max": 0.137209, + "nauc_map_at_10_std": -0.031875, + "nauc_map_at_10_diff1": 0.278172, + "nauc_map_at_20_max": 0.139794, + "nauc_map_at_20_std": -0.026156, + "nauc_map_at_20_diff1": 0.280211, + "nauc_map_at_100_max": 0.146459, + "nauc_map_at_100_std": -0.019739, + "nauc_map_at_100_diff1": 0.275184, + "nauc_map_at_1000_max": 0.149594, + "nauc_map_at_1000_std": -0.01975, + "nauc_map_at_1000_diff1": 0.274303, + "nauc_recall_at_1_max": 0.088461, + "nauc_recall_at_1_std": -0.012252, + "nauc_recall_at_1_diff1": 0.34777, + "nauc_recall_at_3_max": 0.075622, + "nauc_recall_at_3_std": 0.006, + "nauc_recall_at_3_diff1": 0.184573, + "nauc_recall_at_5_max": 0.083844, + "nauc_recall_at_5_std": -0.052333, + "nauc_recall_at_5_diff1": 0.128337, + "nauc_recall_at_10_max": 0.0032, + "nauc_recall_at_10_std": -0.103157, + "nauc_recall_at_10_diff1": 0.145617, + "nauc_recall_at_20_max": -0.02706, + "nauc_recall_at_20_std": -0.10695, + "nauc_recall_at_20_diff1": 0.174984, + "nauc_recall_at_100_max": -0.156088, + "nauc_recall_at_100_std": -0.000272, + "nauc_recall_at_100_diff1": 0.094352, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.24631, + "nauc_precision_at_1_std": 0.029053, + "nauc_precision_at_1_diff1": 0.400894, + "nauc_precision_at_3_max": 0.216657, + "nauc_precision_at_3_std": 0.035347, + "nauc_precision_at_3_diff1": 0.091547, + "nauc_precision_at_5_max": 0.189531, + "nauc_precision_at_5_std": -0.049075, + "nauc_precision_at_5_diff1": 0.060092, + "nauc_precision_at_10_max": 0.211184, + "nauc_precision_at_10_std": -0.038606, + "nauc_precision_at_10_diff1": 0.028344, + "nauc_precision_at_20_max": 0.197825, + "nauc_precision_at_20_std": 0.026153, + "nauc_precision_at_20_diff1": -0.02049, + "nauc_precision_at_100_max": 0.191016, + "nauc_precision_at_100_std": 0.062152, + "nauc_precision_at_100_diff1": -0.139058, + "nauc_precision_at_1000_max": 0.237569, + "nauc_precision_at_1000_std": 0.035725, + "nauc_precision_at_1000_diff1": -0.176805, + "nauc_cv_recall_at_1_max": 0.24631, + "nauc_cv_recall_at_1_std": 0.029053, + "nauc_cv_recall_at_1_diff1": 0.400894, + "nauc_cv_recall_at_3_max": 0.128459, + "nauc_cv_recall_at_3_std": 0.003923, + "nauc_cv_recall_at_3_diff1": 0.175874, + "nauc_cv_recall_at_5_max": 0.144067, + "nauc_cv_recall_at_5_std": 0.014341, + "nauc_cv_recall_at_5_diff1": 0.104105, + "nauc_cv_recall_at_10_max": -0.026164, + "nauc_cv_recall_at_10_std": 0.018506, + "nauc_cv_recall_at_10_diff1": 0.024514, + "nauc_cv_recall_at_20_max": -0.086146, + "nauc_cv_recall_at_20_std": 0.027948, + "nauc_cv_recall_at_20_diff1": -0.030944, + "nauc_cv_recall_at_100_max": -0.364613, + "nauc_cv_recall_at_100_std": 0.172152, + "nauc_cv_recall_at_100_diff1": -0.68324, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.24631, + "nauc_mrr_at_1_std": 0.029053, + "nauc_mrr_at_1_diff1": 0.400894, + "nauc_mrr_at_3_max": 0.201667, + "nauc_mrr_at_3_std": 0.023692, + "nauc_mrr_at_3_diff1": 0.299805, + "nauc_mrr_at_5_max": 0.20838, + "nauc_mrr_at_5_std": 0.025946, + "nauc_mrr_at_5_diff1": 0.291482, + "nauc_mrr_at_10_max": 0.203853, + "nauc_mrr_at_10_std": 0.028169, + "nauc_mrr_at_10_diff1": 0.295569, + "nauc_mrr_at_20_max": 0.202985, + "nauc_mrr_at_20_std": 0.028395, + "nauc_mrr_at_20_diff1": 0.294881, + "nauc_mrr_at_100_max": 0.20388, + "nauc_mrr_at_100_std": 0.02915, + "nauc_mrr_at_100_diff1": 0.294929, + "nauc_mrr_at_1000_max": 0.2041, + "nauc_mrr_at_1000_std": 0.029191, + "nauc_mrr_at_1000_diff1": 0.29527, + "main_score": 0.45553, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.5125, + "ndcg_at_3": 0.50453, + "ndcg_at_5": 0.53803, + "ndcg_at_10": 0.57066, + "ndcg_at_20": 0.59153, + "ndcg_at_100": 0.63761, + "ndcg_at_1000": 0.6576, + "map_at_1": 0.29028, + "map_at_3": 0.39287, + "map_at_5": 0.43656, + "map_at_10": 0.46979, + "map_at_20": 0.48105, + "map_at_100": 0.49354, + "map_at_1000": 0.49566, + "recall_at_1": 0.29028, + "recall_at_3": 0.4546, + "recall_at_5": 0.57872, + "recall_at_10": 0.68416, + "recall_at_20": 0.74776, + "recall_at_100": 0.9102, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.5125, + "cv_recall_at_3": 0.70625, + "cv_recall_at_5": 0.81875, + "cv_recall_at_10": 0.86875, + "cv_recall_at_20": 0.9, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.5125, + "precision_at_3": 0.31458, + "precision_at_5": 0.25, + "precision_at_10": 0.15875, + "precision_at_20": 0.09219, + "precision_at_100": 0.02587, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.5125, + "mrr_at_3": 0.605208, + "mrr_at_5": 0.630833, + "mrr_at_10": 0.638105, + "mrr_at_20": 0.639926, + "mrr_at_100": 0.642077, + "mrr_at_1000": 0.642254, + "nauc_ndcg_at_1_max": 0.164923, + "nauc_ndcg_at_1_std": -0.128355, + "nauc_ndcg_at_1_diff1": 0.406526, + "nauc_ndcg_at_3_max": 0.13483, + "nauc_ndcg_at_3_std": -0.12875, + "nauc_ndcg_at_3_diff1": 0.312631, + "nauc_ndcg_at_5_max": 0.111951, + "nauc_ndcg_at_5_std": -0.196081, + "nauc_ndcg_at_5_diff1": 0.317219, + "nauc_ndcg_at_10_max": 0.108175, + "nauc_ndcg_at_10_std": -0.19904, + "nauc_ndcg_at_10_diff1": 0.353972, + "nauc_ndcg_at_20_max": 0.124044, + "nauc_ndcg_at_20_std": -0.156272, + "nauc_ndcg_at_20_diff1": 0.3417, + "nauc_ndcg_at_100_max": 0.120351, + "nauc_ndcg_at_100_std": -0.136262, + "nauc_ndcg_at_100_diff1": 0.330251, + "nauc_ndcg_at_1000_max": 0.152494, + "nauc_ndcg_at_1000_std": -0.126157, + "nauc_ndcg_at_1000_diff1": 0.322859, + "nauc_map_at_1_max": 0.013051, + "nauc_map_at_1_std": -0.194511, + "nauc_map_at_1_diff1": 0.316511, + "nauc_map_at_3_max": 0.08834, + "nauc_map_at_3_std": -0.147232, + "nauc_map_at_3_diff1": 0.310105, + "nauc_map_at_5_max": 0.098369, + "nauc_map_at_5_std": -0.185763, + "nauc_map_at_5_diff1": 0.327407, + "nauc_map_at_10_max": 0.104193, + "nauc_map_at_10_std": -0.198103, + "nauc_map_at_10_diff1": 0.341342, + "nauc_map_at_20_max": 0.11508, + "nauc_map_at_20_std": -0.183923, + "nauc_map_at_20_diff1": 0.336034, + "nauc_map_at_100_max": 0.115894, + "nauc_map_at_100_std": -0.176385, + "nauc_map_at_100_diff1": 0.332212, + "nauc_map_at_1000_max": 0.118845, + "nauc_map_at_1000_std": -0.175711, + "nauc_map_at_1000_diff1": 0.330957, + "nauc_recall_at_1_max": 0.013051, + "nauc_recall_at_1_std": -0.194511, + "nauc_recall_at_1_diff1": 0.316511, + "nauc_recall_at_3_max": 0.084376, + "nauc_recall_at_3_std": -0.130372, + "nauc_recall_at_3_diff1": 0.282107, + "nauc_recall_at_5_max": 0.070352, + "nauc_recall_at_5_std": -0.204931, + "nauc_recall_at_5_diff1": 0.252675, + "nauc_recall_at_10_max": 0.054649, + "nauc_recall_at_10_std": -0.234395, + "nauc_recall_at_10_diff1": 0.310441, + "nauc_recall_at_20_max": 0.075095, + "nauc_recall_at_20_std": -0.146179, + "nauc_recall_at_20_diff1": 0.248625, + "nauc_recall_at_100_max": -0.172055, + "nauc_recall_at_100_std": -0.143399, + "nauc_recall_at_100_diff1": 0.191277, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.164923, + "nauc_precision_at_1_std": -0.128355, + "nauc_precision_at_1_diff1": 0.406526, + "nauc_precision_at_3_max": 0.20584, + "nauc_precision_at_3_std": 0.023114, + "nauc_precision_at_3_diff1": 0.184194, + "nauc_precision_at_5_max": 0.161359, + "nauc_precision_at_5_std": -0.065552, + "nauc_precision_at_5_diff1": 0.109169, + "nauc_precision_at_10_max": 0.130006, + "nauc_precision_at_10_std": -0.051504, + "nauc_precision_at_10_diff1": 0.062252, + "nauc_precision_at_20_max": 0.158246, + "nauc_precision_at_20_std": 0.069713, + "nauc_precision_at_20_diff1": -0.024664, + "nauc_precision_at_100_max": 0.151344, + "nauc_precision_at_100_std": 0.156415, + "nauc_precision_at_100_diff1": -0.161554, + "nauc_precision_at_1000_max": 0.221648, + "nauc_precision_at_1000_std": 0.200012, + "nauc_precision_at_1000_diff1": -0.251463, + "nauc_cv_recall_at_1_max": 0.164923, + "nauc_cv_recall_at_1_std": -0.128355, + "nauc_cv_recall_at_1_diff1": 0.406526, + "nauc_cv_recall_at_3_max": 0.170255, + "nauc_cv_recall_at_3_std": -0.067779, + "nauc_cv_recall_at_3_diff1": 0.412827, + "nauc_cv_recall_at_5_max": 0.155289, + "nauc_cv_recall_at_5_std": -0.071343, + "nauc_cv_recall_at_5_diff1": 0.278515, + "nauc_cv_recall_at_10_max": 0.195287, + "nauc_cv_recall_at_10_std": 0.024506, + "nauc_cv_recall_at_10_diff1": 0.309394, + "nauc_cv_recall_at_20_max": 0.058648, + "nauc_cv_recall_at_20_std": 0.099323, + "nauc_cv_recall_at_20_diff1": 0.126167, + "nauc_cv_recall_at_100_max": -0.088002, + "nauc_cv_recall_at_100_std": 0.31676, + "nauc_cv_recall_at_100_diff1": -0.292367, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.164923, + "nauc_mrr_at_1_std": -0.128355, + "nauc_mrr_at_1_diff1": 0.406526, + "nauc_mrr_at_3_max": 0.171834, + "nauc_mrr_at_3_std": -0.093573, + "nauc_mrr_at_3_diff1": 0.404966, + "nauc_mrr_at_5_max": 0.168572, + "nauc_mrr_at_5_std": -0.095102, + "nauc_mrr_at_5_diff1": 0.386147, + "nauc_mrr_at_10_max": 0.171049, + "nauc_mrr_at_10_std": -0.089953, + "nauc_mrr_at_10_diff1": 0.392717, + "nauc_mrr_at_20_max": 0.168103, + "nauc_mrr_at_20_std": -0.088928, + "nauc_mrr_at_20_diff1": 0.389372, + "nauc_mrr_at_100_max": 0.167982, + "nauc_mrr_at_100_std": -0.090048, + "nauc_mrr_at_100_diff1": 0.389704, + "nauc_mrr_at_1000_max": 0.168003, + "nauc_mrr_at_1000_std": -0.090285, + "nauc_mrr_at_1000_diff1": 0.390006, + "main_score": 0.53803, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.4125, + "ndcg_at_3": 0.42385, + "ndcg_at_5": 0.4411, + "ndcg_at_10": 0.48659, + "ndcg_at_20": 0.51255, + "ndcg_at_100": 0.56513, + "ndcg_at_1000": 0.58899, + "map_at_1": 0.23282, + "map_at_3": 0.32867, + "map_at_5": 0.358, + "map_at_10": 0.39035, + "map_at_20": 0.40231, + "map_at_100": 0.41537, + "map_at_1000": 0.41776, + "recall_at_1": 0.23282, + "recall_at_3": 0.40867, + "recall_at_5": 0.48138, + "recall_at_10": 0.61713, + "recall_at_20": 0.69821, + "recall_at_100": 0.88776, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.4125, + "cv_recall_at_3": 0.6125, + "cv_recall_at_5": 0.675, + "cv_recall_at_10": 0.8, + "cv_recall_at_20": 0.85625, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.4125, + "precision_at_3": 0.26875, + "precision_at_5": 0.20625, + "precision_at_10": 0.14188, + "precision_at_20": 0.08438, + "precision_at_100": 0.02513, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.4125, + "mrr_at_3": 0.496875, + "mrr_at_5": 0.51, + "mrr_at_10": 0.526756, + "mrr_at_20": 0.530521, + "mrr_at_100": 0.533973, + "mrr_at_1000": 0.534091, + "nauc_ndcg_at_1_max": 0.298104, + "nauc_ndcg_at_1_std": 0.049426, + "nauc_ndcg_at_1_diff1": 0.199318, + "nauc_ndcg_at_3_max": 0.19455, + "nauc_ndcg_at_3_std": 0.043584, + "nauc_ndcg_at_3_diff1": 0.154918, + "nauc_ndcg_at_5_max": 0.180727, + "nauc_ndcg_at_5_std": 0.013002, + "nauc_ndcg_at_5_diff1": 0.159858, + "nauc_ndcg_at_10_max": 0.175892, + "nauc_ndcg_at_10_std": 0.036496, + "nauc_ndcg_at_10_diff1": 0.173449, + "nauc_ndcg_at_20_max": 0.174266, + "nauc_ndcg_at_20_std": 0.031716, + "nauc_ndcg_at_20_diff1": 0.156893, + "nauc_ndcg_at_100_max": 0.205574, + "nauc_ndcg_at_100_std": 0.055085, + "nauc_ndcg_at_100_diff1": 0.140152, + "nauc_ndcg_at_1000_max": 0.218293, + "nauc_ndcg_at_1000_std": 0.041215, + "nauc_ndcg_at_1000_diff1": 0.135981, + "nauc_map_at_1_max": 0.180051, + "nauc_map_at_1_std": 0.014574, + "nauc_map_at_1_diff1": 0.288203, + "nauc_map_at_3_max": 0.146641, + "nauc_map_at_3_std": 0.03324, + "nauc_map_at_3_diff1": 0.212195, + "nauc_map_at_5_max": 0.152769, + "nauc_map_at_5_std": 0.012347, + "nauc_map_at_5_diff1": 0.191026, + "nauc_map_at_10_max": 0.163611, + "nauc_map_at_10_std": 0.020617, + "nauc_map_at_10_diff1": 0.182245, + "nauc_map_at_20_max": 0.170786, + "nauc_map_at_20_std": 0.020047, + "nauc_map_at_20_diff1": 0.177665, + "nauc_map_at_100_max": 0.18012, + "nauc_map_at_100_std": 0.025792, + "nauc_map_at_100_diff1": 0.172883, + "nauc_map_at_1000_max": 0.181841, + "nauc_map_at_1000_std": 0.024894, + "nauc_map_at_1000_diff1": 0.171838, + "nauc_recall_at_1_max": 0.180051, + "nauc_recall_at_1_std": 0.014574, + "nauc_recall_at_1_diff1": 0.288203, + "nauc_recall_at_3_max": 0.068348, + "nauc_recall_at_3_std": 0.038, + "nauc_recall_at_3_diff1": 0.167268, + "nauc_recall_at_5_max": 0.097101, + "nauc_recall_at_5_std": -0.004535, + "nauc_recall_at_5_diff1": 0.144517, + "nauc_recall_at_10_max": 0.05386, + "nauc_recall_at_10_std": 0.041831, + "nauc_recall_at_10_diff1": 0.139867, + "nauc_recall_at_20_max": 0.020208, + "nauc_recall_at_20_std": -0.003433, + "nauc_recall_at_20_diff1": 0.096983, + "nauc_recall_at_100_max": 0.060761, + "nauc_recall_at_100_std": 0.182101, + "nauc_recall_at_100_diff1": 0.048614, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.298104, + "nauc_precision_at_1_std": 0.049426, + "nauc_precision_at_1_diff1": 0.199318, + "nauc_precision_at_3_max": 0.200433, + "nauc_precision_at_3_std": 0.074502, + "nauc_precision_at_3_diff1": -0.022735, + "nauc_precision_at_5_max": 0.159975, + "nauc_precision_at_5_std": 0.013526, + "nauc_precision_at_5_diff1": -0.089484, + "nauc_precision_at_10_max": 0.167884, + "nauc_precision_at_10_std": 0.044593, + "nauc_precision_at_10_diff1": -0.104567, + "nauc_precision_at_20_max": 0.182832, + "nauc_precision_at_20_std": 0.042221, + "nauc_precision_at_20_diff1": -0.148978, + "nauc_precision_at_100_max": 0.197932, + "nauc_precision_at_100_std": 0.023513, + "nauc_precision_at_100_diff1": -0.211229, + "nauc_precision_at_1000_max": 0.135668, + "nauc_precision_at_1000_std": -0.068238, + "nauc_precision_at_1000_diff1": -0.206873, + "nauc_cv_recall_at_1_max": 0.298104, + "nauc_cv_recall_at_1_std": 0.049426, + "nauc_cv_recall_at_1_diff1": 0.199318, + "nauc_cv_recall_at_3_max": 0.200419, + "nauc_cv_recall_at_3_std": 0.108049, + "nauc_cv_recall_at_3_diff1": 0.134416, + "nauc_cv_recall_at_5_max": 0.236918, + "nauc_cv_recall_at_5_std": 0.058879, + "nauc_cv_recall_at_5_diff1": 0.150346, + "nauc_cv_recall_at_10_max": 0.153222, + "nauc_cv_recall_at_10_std": 0.145152, + "nauc_cv_recall_at_10_diff1": 0.103147, + "nauc_cv_recall_at_20_max": 0.06014, + "nauc_cv_recall_at_20_std": 0.079229, + "nauc_cv_recall_at_20_diff1": 0.003636, + "nauc_cv_recall_at_100_max": 0.17612, + "nauc_cv_recall_at_100_std": 0.645892, + "nauc_cv_recall_at_100_diff1": -0.661531, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.298104, + "nauc_mrr_at_1_std": 0.049426, + "nauc_mrr_at_1_diff1": 0.199318, + "nauc_mrr_at_3_max": 0.25787, + "nauc_mrr_at_3_std": 0.067208, + "nauc_mrr_at_3_diff1": 0.164647, + "nauc_mrr_at_5_max": 0.266557, + "nauc_mrr_at_5_std": 0.057356, + "nauc_mrr_at_5_diff1": 0.169171, + "nauc_mrr_at_10_max": 0.26338, + "nauc_mrr_at_10_std": 0.065568, + "nauc_mrr_at_10_diff1": 0.167142, + "nauc_mrr_at_20_max": 0.262922, + "nauc_mrr_at_20_std": 0.063454, + "nauc_mrr_at_20_diff1": 0.165295, + "nauc_mrr_at_100_max": 0.264744, + "nauc_mrr_at_100_std": 0.065192, + "nauc_mrr_at_100_diff1": 0.163755, + "nauc_mrr_at_1000_max": 0.264702, + "nauc_mrr_at_1000_std": 0.065035, + "nauc_mrr_at_1000_diff1": 0.163961, + "main_score": 0.4411, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 543.9012343883514, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/Vidore2ESGReportsHLRetrieval.json b/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..2a0ecede77 --- /dev/null +++ b/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.42308, + "ndcg_at_3": 0.4168, + "ndcg_at_5": 0.45113, + "ndcg_at_10": 0.46097, + "ndcg_at_20": 0.4959, + "ndcg_at_100": 0.54895, + "ndcg_at_1000": 0.56891, + "map_at_1": 0.27963, + "map_at_3": 0.34164, + "map_at_5": 0.37662, + "map_at_10": 0.38532, + "map_at_20": 0.39754, + "map_at_100": 0.40895, + "map_at_1000": 0.41077, + "recall_at_1": 0.27963, + "recall_at_3": 0.39709, + "recall_at_5": 0.49791, + "recall_at_10": 0.54503, + "recall_at_20": 0.66746, + "recall_at_100": 0.89067, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.42308, + "cv_recall_at_3": 0.55769, + "cv_recall_at_5": 0.65385, + "cv_recall_at_10": 0.71154, + "cv_recall_at_20": 0.82692, + "cv_recall_at_100": 0.98077, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.42308, + "precision_at_3": 0.23718, + "precision_at_5": 0.19231, + "precision_at_10": 0.10769, + "precision_at_20": 0.06635, + "precision_at_100": 0.02, + "precision_at_1000": 0.00246, + "mrr_at_1": 0.423077, + "mrr_at_3": 0.477564, + "mrr_at_5": 0.498718, + "mrr_at_10": 0.506807, + "mrr_at_20": 0.514467, + "mrr_at_100": 0.519221, + "mrr_at_1000": 0.519284, + "nauc_ndcg_at_1_max": 0.266975, + "nauc_ndcg_at_1_std": -0.049691, + "nauc_ndcg_at_1_diff1": 0.369487, + "nauc_ndcg_at_3_max": 0.393921, + "nauc_ndcg_at_3_std": -0.026677, + "nauc_ndcg_at_3_diff1": 0.362511, + "nauc_ndcg_at_5_max": 0.34108, + "nauc_ndcg_at_5_std": 0.001656, + "nauc_ndcg_at_5_diff1": 0.354756, + "nauc_ndcg_at_10_max": 0.279275, + "nauc_ndcg_at_10_std": -0.043621, + "nauc_ndcg_at_10_diff1": 0.362819, + "nauc_ndcg_at_20_max": 0.254784, + "nauc_ndcg_at_20_std": -0.028778, + "nauc_ndcg_at_20_diff1": 0.348726, + "nauc_ndcg_at_100_max": 0.262054, + "nauc_ndcg_at_100_std": -0.029621, + "nauc_ndcg_at_100_diff1": 0.343581, + "nauc_ndcg_at_1000_max": 0.287208, + "nauc_ndcg_at_1000_std": -0.008685, + "nauc_ndcg_at_1000_diff1": 0.33801, + "nauc_map_at_1_max": 0.236283, + "nauc_map_at_1_std": -0.19524, + "nauc_map_at_1_diff1": 0.47933, + "nauc_map_at_3_max": 0.31584, + "nauc_map_at_3_std": -0.140164, + "nauc_map_at_3_diff1": 0.430676, + "nauc_map_at_5_max": 0.30604, + "nauc_map_at_5_std": -0.093651, + "nauc_map_at_5_diff1": 0.408773, + "nauc_map_at_10_max": 0.294869, + "nauc_map_at_10_std": -0.092582, + "nauc_map_at_10_diff1": 0.401763, + "nauc_map_at_20_max": 0.287398, + "nauc_map_at_20_std": -0.081537, + "nauc_map_at_20_diff1": 0.390294, + "nauc_map_at_100_max": 0.291431, + "nauc_map_at_100_std": -0.076828, + "nauc_map_at_100_diff1": 0.389681, + "nauc_map_at_1000_max": 0.293811, + "nauc_map_at_1000_std": -0.073931, + "nauc_map_at_1000_diff1": 0.388343, + "nauc_recall_at_1_max": 0.236283, + "nauc_recall_at_1_std": -0.19524, + "nauc_recall_at_1_diff1": 0.47933, + "nauc_recall_at_3_max": 0.322069, + "nauc_recall_at_3_std": -0.169203, + "nauc_recall_at_3_diff1": 0.39027, + "nauc_recall_at_5_max": 0.249355, + "nauc_recall_at_5_std": -0.031966, + "nauc_recall_at_5_diff1": 0.352366, + "nauc_recall_at_10_max": 0.184044, + "nauc_recall_at_10_std": -0.053334, + "nauc_recall_at_10_diff1": 0.316131, + "nauc_recall_at_20_max": 0.049945, + "nauc_recall_at_20_std": -0.047304, + "nauc_recall_at_20_diff1": 0.270678, + "nauc_recall_at_100_max": -0.220343, + "nauc_recall_at_100_std": -0.257377, + "nauc_recall_at_100_diff1": 0.196907, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.266975, + "nauc_precision_at_1_std": -0.049691, + "nauc_precision_at_1_diff1": 0.369487, + "nauc_precision_at_3_max": 0.344691, + "nauc_precision_at_3_std": 0.161945, + "nauc_precision_at_3_diff1": 0.052807, + "nauc_precision_at_5_max": 0.270949, + "nauc_precision_at_5_std": 0.30631, + "nauc_precision_at_5_diff1": -0.060992, + "nauc_precision_at_10_max": 0.186855, + "nauc_precision_at_10_std": 0.316502, + "nauc_precision_at_10_diff1": -0.115543, + "nauc_precision_at_20_max": 0.136241, + "nauc_precision_at_20_std": 0.417229, + "nauc_precision_at_20_diff1": -0.231378, + "nauc_precision_at_100_max": 0.122296, + "nauc_precision_at_100_std": 0.399824, + "nauc_precision_at_100_diff1": -0.318476, + "nauc_precision_at_1000_max": 0.143198, + "nauc_precision_at_1000_std": 0.413809, + "nauc_precision_at_1000_diff1": -0.357048, + "nauc_cv_recall_at_1_max": 0.266975, + "nauc_cv_recall_at_1_std": -0.049691, + "nauc_cv_recall_at_1_diff1": 0.369487, + "nauc_cv_recall_at_3_max": 0.28467, + "nauc_cv_recall_at_3_std": -0.044986, + "nauc_cv_recall_at_3_diff1": 0.295632, + "nauc_cv_recall_at_5_max": 0.208931, + "nauc_cv_recall_at_5_std": 0.194726, + "nauc_cv_recall_at_5_diff1": 0.232634, + "nauc_cv_recall_at_10_max": 0.12547, + "nauc_cv_recall_at_10_std": 0.109259, + "nauc_cv_recall_at_10_diff1": 0.157562, + "nauc_cv_recall_at_20_max": 0.143766, + "nauc_cv_recall_at_20_std": 0.336155, + "nauc_cv_recall_at_20_diff1": 0.184469, + "nauc_cv_recall_at_100_max": -1.163827, + "nauc_cv_recall_at_100_std": -0.16971, + "nauc_cv_recall_at_100_diff1": -0.552063, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.266975, + "nauc_mrr_at_1_std": -0.049691, + "nauc_mrr_at_1_diff1": 0.369487, + "nauc_mrr_at_3_max": 0.2937, + "nauc_mrr_at_3_std": -0.024668, + "nauc_mrr_at_3_diff1": 0.348191, + "nauc_mrr_at_5_max": 0.281141, + "nauc_mrr_at_5_std": 0.024377, + "nauc_mrr_at_5_diff1": 0.334074, + "nauc_mrr_at_10_max": 0.273702, + "nauc_mrr_at_10_std": 0.012588, + "nauc_mrr_at_10_diff1": 0.326879, + "nauc_mrr_at_20_max": 0.276394, + "nauc_mrr_at_20_std": 0.018476, + "nauc_mrr_at_20_diff1": 0.328809, + "nauc_mrr_at_100_max": 0.276091, + "nauc_mrr_at_100_std": 0.014087, + "nauc_mrr_at_100_diff1": 0.329045, + "nauc_mrr_at_1000_max": 0.27637, + "nauc_mrr_at_1000_std": 0.014122, + "nauc_mrr_at_1000_diff1": 0.329216, + "main_score": 0.45113, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 302.5255150794983, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/Vidore2ESGReportsRetrieval.json b/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..4bb88d0ae7 --- /dev/null +++ b/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.40351, + "ndcg_at_3": 0.39043, + "ndcg_at_5": 0.4328, + "ndcg_at_10": 0.4708, + "ndcg_at_20": 0.50314, + "ndcg_at_100": 0.53728, + "ndcg_at_1000": 0.57393, + "map_at_1": 0.20969, + "map_at_3": 0.28654, + "map_at_5": 0.3265, + "map_at_10": 0.35877, + "map_at_20": 0.3759, + "map_at_100": 0.38631, + "map_at_1000": 0.38974, + "recall_at_1": 0.20969, + "recall_at_3": 0.34029, + "recall_at_5": 0.46734, + "recall_at_10": 0.59804, + "recall_at_20": 0.70203, + "recall_at_100": 0.81581, + "recall_at_1000": 0.99353, + "cv_recall_at_1": 0.40351, + "cv_recall_at_3": 0.59649, + "cv_recall_at_5": 0.75439, + "cv_recall_at_10": 0.87719, + "cv_recall_at_20": 0.92982, + "cv_recall_at_100": 0.94737, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.40351, + "precision_at_3": 0.25731, + "precision_at_5": 0.21404, + "precision_at_10": 0.14386, + "precision_at_20": 0.09123, + "precision_at_100": 0.02561, + "precision_at_1000": 0.00382, + "mrr_at_1": 0.403509, + "mrr_at_3": 0.482456, + "mrr_at_5": 0.521053, + "mrr_at_10": 0.538346, + "mrr_at_20": 0.541651, + "mrr_at_100": 0.542199, + "mrr_at_1000": 0.54236, + "nauc_ndcg_at_1_max": 0.377209, + "nauc_ndcg_at_1_std": 0.141056, + "nauc_ndcg_at_1_diff1": 0.402854, + "nauc_ndcg_at_3_max": 0.380846, + "nauc_ndcg_at_3_std": 0.082232, + "nauc_ndcg_at_3_diff1": 0.258325, + "nauc_ndcg_at_5_max": 0.320782, + "nauc_ndcg_at_5_std": 0.079373, + "nauc_ndcg_at_5_diff1": 0.229671, + "nauc_ndcg_at_10_max": 0.217678, + "nauc_ndcg_at_10_std": 0.054874, + "nauc_ndcg_at_10_diff1": 0.21728, + "nauc_ndcg_at_20_max": 0.222216, + "nauc_ndcg_at_20_std": 0.100495, + "nauc_ndcg_at_20_diff1": 0.187504, + "nauc_ndcg_at_100_max": 0.25997, + "nauc_ndcg_at_100_std": 0.137182, + "nauc_ndcg_at_100_diff1": 0.132115, + "nauc_ndcg_at_1000_max": 0.329676, + "nauc_ndcg_at_1000_std": 0.143117, + "nauc_ndcg_at_1000_diff1": 0.166631, + "nauc_map_at_1_max": 0.171091, + "nauc_map_at_1_std": 0.053231, + "nauc_map_at_1_diff1": 0.194687, + "nauc_map_at_3_max": 0.225271, + "nauc_map_at_3_std": -0.001635, + "nauc_map_at_3_diff1": 0.234582, + "nauc_map_at_5_max": 0.224797, + "nauc_map_at_5_std": 0.022151, + "nauc_map_at_5_diff1": 0.234164, + "nauc_map_at_10_max": 0.225539, + "nauc_map_at_10_std": 0.045985, + "nauc_map_at_10_diff1": 0.241212, + "nauc_map_at_20_max": 0.245579, + "nauc_map_at_20_std": 0.072507, + "nauc_map_at_20_diff1": 0.238427, + "nauc_map_at_100_max": 0.262778, + "nauc_map_at_100_std": 0.086242, + "nauc_map_at_100_diff1": 0.226422, + "nauc_map_at_1000_max": 0.270743, + "nauc_map_at_1000_std": 0.089201, + "nauc_map_at_1000_diff1": 0.225718, + "nauc_recall_at_1_max": 0.171091, + "nauc_recall_at_1_std": 0.053231, + "nauc_recall_at_1_diff1": 0.194687, + "nauc_recall_at_3_max": 0.178115, + "nauc_recall_at_3_std": -0.073425, + "nauc_recall_at_3_diff1": 0.186714, + "nauc_recall_at_5_max": 0.05439, + "nauc_recall_at_5_std": -0.04201, + "nauc_recall_at_5_diff1": 0.120209, + "nauc_recall_at_10_max": -0.147872, + "nauc_recall_at_10_std": -0.066975, + "nauc_recall_at_10_diff1": 0.043328, + "nauc_recall_at_20_max": -0.211774, + "nauc_recall_at_20_std": 0.019102, + "nauc_recall_at_20_diff1": -0.110214, + "nauc_recall_at_100_max": -0.272275, + "nauc_recall_at_100_std": 0.125138, + "nauc_recall_at_100_diff1": -0.483543, + "nauc_recall_at_1000_max": -0.463723, + "nauc_recall_at_1000_std": 0.740892, + "nauc_recall_at_1000_diff1": -1.078762, + "nauc_precision_at_1_max": 0.377209, + "nauc_precision_at_1_std": 0.141056, + "nauc_precision_at_1_diff1": 0.402854, + "nauc_precision_at_3_max": 0.531327, + "nauc_precision_at_3_std": 0.116782, + "nauc_precision_at_3_diff1": 0.300366, + "nauc_precision_at_5_max": 0.588125, + "nauc_precision_at_5_std": 0.253561, + "nauc_precision_at_5_diff1": 0.209374, + "nauc_precision_at_10_max": 0.532546, + "nauc_precision_at_10_std": 0.364832, + "nauc_precision_at_10_diff1": 0.084897, + "nauc_precision_at_20_max": 0.507631, + "nauc_precision_at_20_std": 0.405755, + "nauc_precision_at_20_diff1": -0.015756, + "nauc_precision_at_100_max": 0.456732, + "nauc_precision_at_100_std": 0.352387, + "nauc_precision_at_100_diff1": -0.169802, + "nauc_precision_at_1000_max": 0.304555, + "nauc_precision_at_1000_std": 0.165429, + "nauc_precision_at_1000_diff1": -0.176083, + "nauc_cv_recall_at_1_max": 0.377209, + "nauc_cv_recall_at_1_std": 0.141056, + "nauc_cv_recall_at_1_diff1": 0.402854, + "nauc_cv_recall_at_3_max": 0.433095, + "nauc_cv_recall_at_3_std": -0.003655, + "nauc_cv_recall_at_3_diff1": 0.277599, + "nauc_cv_recall_at_5_max": 0.376746, + "nauc_cv_recall_at_5_std": 0.026257, + "nauc_cv_recall_at_5_diff1": 0.160001, + "nauc_cv_recall_at_10_max": 0.108071, + "nauc_cv_recall_at_10_std": 0.124988, + "nauc_cv_recall_at_10_diff1": -0.196102, + "nauc_cv_recall_at_20_max": 0.138366, + "nauc_cv_recall_at_20_std": 0.355553, + "nauc_cv_recall_at_20_diff1": -1.017416, + "nauc_cv_recall_at_100_max": 0.23947, + "nauc_cv_recall_at_100_std": 0.23316, + "nauc_cv_recall_at_100_diff1": -1.541583, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.377209, + "nauc_mrr_at_1_std": 0.141056, + "nauc_mrr_at_1_diff1": 0.402854, + "nauc_mrr_at_3_max": 0.389754, + "nauc_mrr_at_3_std": 0.085607, + "nauc_mrr_at_3_diff1": 0.348705, + "nauc_mrr_at_5_max": 0.37562, + "nauc_mrr_at_5_std": 0.094078, + "nauc_mrr_at_5_diff1": 0.335607, + "nauc_mrr_at_10_max": 0.360241, + "nauc_mrr_at_10_std": 0.103194, + "nauc_mrr_at_10_diff1": 0.325251, + "nauc_mrr_at_20_max": 0.363294, + "nauc_mrr_at_20_std": 0.106957, + "nauc_mrr_at_20_diff1": 0.318657, + "nauc_mrr_at_100_max": 0.364249, + "nauc_mrr_at_100_std": 0.105843, + "nauc_mrr_at_100_diff1": 0.31823, + "nauc_mrr_at_1000_max": 0.364407, + "nauc_mrr_at_1000_std": 0.105602, + "nauc_mrr_at_1000_diff1": 0.319264, + "main_score": 0.4328, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.40351, + "ndcg_at_3": 0.39724, + "ndcg_at_5": 0.42403, + "ndcg_at_10": 0.4667, + "ndcg_at_20": 0.49196, + "ndcg_at_100": 0.52596, + "ndcg_at_1000": 0.56571, + "map_at_1": 0.19799, + "map_at_3": 0.28732, + "map_at_5": 0.31902, + "map_at_10": 0.35218, + "map_at_20": 0.36641, + "map_at_100": 0.37712, + "map_at_1000": 0.38088, + "recall_at_1": 0.19799, + "recall_at_3": 0.34628, + "recall_at_5": 0.43371, + "recall_at_10": 0.60345, + "recall_at_20": 0.69077, + "recall_at_100": 0.80119, + "recall_at_1000": 0.99415, + "cv_recall_at_1": 0.40351, + "cv_recall_at_3": 0.57895, + "cv_recall_at_5": 0.70175, + "cv_recall_at_10": 0.85965, + "cv_recall_at_20": 0.91228, + "cv_recall_at_100": 0.94737, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.40351, + "precision_at_3": 0.26901, + "precision_at_5": 0.22456, + "precision_at_10": 0.14737, + "precision_at_20": 0.09035, + "precision_at_100": 0.02544, + "precision_at_1000": 0.00384, + "mrr_at_1": 0.403509, + "mrr_at_3": 0.479532, + "mrr_at_5": 0.506725, + "mrr_at_10": 0.52933, + "mrr_at_20": 0.533275, + "mrr_at_100": 0.534423, + "mrr_at_1000": 0.534605, + "nauc_ndcg_at_1_max": 0.422698, + "nauc_ndcg_at_1_std": 0.223689, + "nauc_ndcg_at_1_diff1": 0.341448, + "nauc_ndcg_at_3_max": 0.37961, + "nauc_ndcg_at_3_std": 0.183234, + "nauc_ndcg_at_3_diff1": 0.309155, + "nauc_ndcg_at_5_max": 0.380367, + "nauc_ndcg_at_5_std": 0.186828, + "nauc_ndcg_at_5_diff1": 0.249718, + "nauc_ndcg_at_10_max": 0.237409, + "nauc_ndcg_at_10_std": 0.156902, + "nauc_ndcg_at_10_diff1": 0.225982, + "nauc_ndcg_at_20_max": 0.233176, + "nauc_ndcg_at_20_std": 0.188192, + "nauc_ndcg_at_20_diff1": 0.205944, + "nauc_ndcg_at_100_max": 0.285795, + "nauc_ndcg_at_100_std": 0.220557, + "nauc_ndcg_at_100_diff1": 0.170219, + "nauc_ndcg_at_1000_max": 0.340767, + "nauc_ndcg_at_1000_std": 0.223431, + "nauc_ndcg_at_1000_diff1": 0.193411, + "nauc_map_at_1_max": 0.139851, + "nauc_map_at_1_std": 0.153661, + "nauc_map_at_1_diff1": 0.214014, + "nauc_map_at_3_max": 0.164642, + "nauc_map_at_3_std": 0.088292, + "nauc_map_at_3_diff1": 0.282773, + "nauc_map_at_5_max": 0.226203, + "nauc_map_at_5_std": 0.114437, + "nauc_map_at_5_diff1": 0.278948, + "nauc_map_at_10_max": 0.21441, + "nauc_map_at_10_std": 0.137223, + "nauc_map_at_10_diff1": 0.27873, + "nauc_map_at_20_max": 0.231542, + "nauc_map_at_20_std": 0.152005, + "nauc_map_at_20_diff1": 0.278099, + "nauc_map_at_100_max": 0.252177, + "nauc_map_at_100_std": 0.163862, + "nauc_map_at_100_diff1": 0.272044, + "nauc_map_at_1000_max": 0.257857, + "nauc_map_at_1000_std": 0.166748, + "nauc_map_at_1000_diff1": 0.271856, + "nauc_recall_at_1_max": 0.139851, + "nauc_recall_at_1_std": 0.153661, + "nauc_recall_at_1_diff1": 0.214014, + "nauc_recall_at_3_max": 0.122931, + "nauc_recall_at_3_std": 0.082651, + "nauc_recall_at_3_diff1": 0.221934, + "nauc_recall_at_5_max": 0.152062, + "nauc_recall_at_5_std": 0.094731, + "nauc_recall_at_5_diff1": 0.121968, + "nauc_recall_at_10_max": -0.135016, + "nauc_recall_at_10_std": 0.036002, + "nauc_recall_at_10_diff1": 0.023857, + "nauc_recall_at_20_max": -0.220682, + "nauc_recall_at_20_std": 0.088985, + "nauc_recall_at_20_diff1": -0.106161, + "nauc_recall_at_100_max": -0.169252, + "nauc_recall_at_100_std": 0.187719, + "nauc_recall_at_100_diff1": -0.350986, + "nauc_recall_at_1000_max": -0.526974, + "nauc_recall_at_1000_std": 0.755766, + "nauc_recall_at_1000_diff1": -0.897904, + "nauc_precision_at_1_max": 0.422698, + "nauc_precision_at_1_std": 0.223689, + "nauc_precision_at_1_diff1": 0.341448, + "nauc_precision_at_3_max": 0.540814, + "nauc_precision_at_3_std": 0.172638, + "nauc_precision_at_3_diff1": 0.354115, + "nauc_precision_at_5_max": 0.660239, + "nauc_precision_at_5_std": 0.287221, + "nauc_precision_at_5_diff1": 0.228365, + "nauc_precision_at_10_max": 0.578593, + "nauc_precision_at_10_std": 0.407442, + "nauc_precision_at_10_diff1": 0.132126, + "nauc_precision_at_20_max": 0.59161, + "nauc_precision_at_20_std": 0.418912, + "nauc_precision_at_20_diff1": 0.113927, + "nauc_precision_at_100_max": 0.549245, + "nauc_precision_at_100_std": 0.343672, + "nauc_precision_at_100_diff1": -0.018615, + "nauc_precision_at_1000_max": 0.327844, + "nauc_precision_at_1000_std": 0.114951, + "nauc_precision_at_1000_diff1": -0.128018, + "nauc_cv_recall_at_1_max": 0.422698, + "nauc_cv_recall_at_1_std": 0.223689, + "nauc_cv_recall_at_1_diff1": 0.341448, + "nauc_cv_recall_at_3_max": 0.503764, + "nauc_cv_recall_at_3_std": 0.169193, + "nauc_cv_recall_at_3_diff1": 0.302949, + "nauc_cv_recall_at_5_max": 0.556924, + "nauc_cv_recall_at_5_std": 0.248206, + "nauc_cv_recall_at_5_diff1": 0.07353, + "nauc_cv_recall_at_10_max": 0.257099, + "nauc_cv_recall_at_10_std": 0.193046, + "nauc_cv_recall_at_10_diff1": -0.200211, + "nauc_cv_recall_at_20_max": -0.001189, + "nauc_cv_recall_at_20_std": 0.349939, + "nauc_cv_recall_at_20_diff1": -0.699626, + "nauc_cv_recall_at_100_max": 0.23947, + "nauc_cv_recall_at_100_std": 0.480197, + "nauc_cv_recall_at_100_diff1": -1.541583, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.422698, + "nauc_mrr_at_1_std": 0.223689, + "nauc_mrr_at_1_diff1": 0.341448, + "nauc_mrr_at_3_max": 0.43558, + "nauc_mrr_at_3_std": 0.170747, + "nauc_mrr_at_3_diff1": 0.314001, + "nauc_mrr_at_5_max": 0.444684, + "nauc_mrr_at_5_std": 0.186848, + "nauc_mrr_at_5_diff1": 0.273235, + "nauc_mrr_at_10_max": 0.419286, + "nauc_mrr_at_10_std": 0.179751, + "nauc_mrr_at_10_diff1": 0.264824, + "nauc_mrr_at_20_max": 0.41502, + "nauc_mrr_at_20_std": 0.181867, + "nauc_mrr_at_20_diff1": 0.260049, + "nauc_mrr_at_100_max": 0.418237, + "nauc_mrr_at_100_std": 0.183173, + "nauc_mrr_at_100_diff1": 0.25966, + "nauc_mrr_at_1000_max": 0.418442, + "nauc_mrr_at_1000_std": 0.182834, + "nauc_mrr_at_1000_diff1": 0.260778, + "main_score": 0.42403, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.42105, + "ndcg_at_3": 0.42636, + "ndcg_at_5": 0.45616, + "ndcg_at_10": 0.47886, + "ndcg_at_20": 0.50869, + "ndcg_at_100": 0.55229, + "ndcg_at_1000": 0.58432, + "map_at_1": 0.19975, + "map_at_3": 0.30332, + "map_at_5": 0.34343, + "map_at_10": 0.3676, + "map_at_20": 0.38548, + "map_at_100": 0.39763, + "map_at_1000": 0.40065, + "recall_at_1": 0.19975, + "recall_at_3": 0.37289, + "recall_at_5": 0.4859, + "recall_at_10": 0.59467, + "recall_at_20": 0.68906, + "recall_at_100": 0.84198, + "recall_at_1000": 0.99927, + "cv_recall_at_1": 0.42105, + "cv_recall_at_3": 0.63158, + "cv_recall_at_5": 0.75439, + "cv_recall_at_10": 0.87719, + "cv_recall_at_20": 0.91228, + "cv_recall_at_100": 0.94737, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.42105, + "precision_at_3": 0.2924, + "precision_at_5": 0.23509, + "precision_at_10": 0.14561, + "precision_at_20": 0.09298, + "precision_at_100": 0.02719, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.421053, + "mrr_at_3": 0.508772, + "mrr_at_5": 0.538596, + "mrr_at_10": 0.556328, + "mrr_at_20": 0.558238, + "mrr_at_100": 0.559463, + "mrr_at_1000": 0.559591, + "nauc_ndcg_at_1_max": 0.285362, + "nauc_ndcg_at_1_std": 0.182568, + "nauc_ndcg_at_1_diff1": 0.273017, + "nauc_ndcg_at_3_max": 0.347021, + "nauc_ndcg_at_3_std": 0.235735, + "nauc_ndcg_at_3_diff1": 0.183815, + "nauc_ndcg_at_5_max": 0.281631, + "nauc_ndcg_at_5_std": 0.18724, + "nauc_ndcg_at_5_diff1": 0.094981, + "nauc_ndcg_at_10_max": 0.161669, + "nauc_ndcg_at_10_std": 0.117249, + "nauc_ndcg_at_10_diff1": 0.129759, + "nauc_ndcg_at_20_max": 0.146314, + "nauc_ndcg_at_20_std": 0.153554, + "nauc_ndcg_at_20_diff1": 0.081432, + "nauc_ndcg_at_100_max": 0.240315, + "nauc_ndcg_at_100_std": 0.210098, + "nauc_ndcg_at_100_diff1": 0.043935, + "nauc_ndcg_at_1000_max": 0.275507, + "nauc_ndcg_at_1000_std": 0.211095, + "nauc_ndcg_at_1000_diff1": 0.083367, + "nauc_map_at_1_max": 0.008028, + "nauc_map_at_1_std": 0.038239, + "nauc_map_at_1_diff1": 0.083772, + "nauc_map_at_3_max": 0.104494, + "nauc_map_at_3_std": 0.091996, + "nauc_map_at_3_diff1": 0.162982, + "nauc_map_at_5_max": 0.131699, + "nauc_map_at_5_std": 0.094058, + "nauc_map_at_5_diff1": 0.137788, + "nauc_map_at_10_max": 0.141132, + "nauc_map_at_10_std": 0.115849, + "nauc_map_at_10_diff1": 0.163138, + "nauc_map_at_20_max": 0.147375, + "nauc_map_at_20_std": 0.13058, + "nauc_map_at_20_diff1": 0.148331, + "nauc_map_at_100_max": 0.175805, + "nauc_map_at_100_std": 0.150438, + "nauc_map_at_100_diff1": 0.140493, + "nauc_map_at_1000_max": 0.180233, + "nauc_map_at_1000_std": 0.152112, + "nauc_map_at_1000_diff1": 0.140752, + "nauc_recall_at_1_max": 0.008028, + "nauc_recall_at_1_std": 0.038239, + "nauc_recall_at_1_diff1": 0.083772, + "nauc_recall_at_3_max": 0.056741, + "nauc_recall_at_3_std": 0.042704, + "nauc_recall_at_3_diff1": 0.083909, + "nauc_recall_at_5_max": 0.020809, + "nauc_recall_at_5_std": 0.019913, + "nauc_recall_at_5_diff1": -0.067564, + "nauc_recall_at_10_max": -0.154797, + "nauc_recall_at_10_std": -0.074038, + "nauc_recall_at_10_diff1": 0.016077, + "nauc_recall_at_20_max": -0.229665, + "nauc_recall_at_20_std": 0.02343, + "nauc_recall_at_20_diff1": -0.155813, + "nauc_recall_at_100_max": -0.007401, + "nauc_recall_at_100_std": 0.178774, + "nauc_recall_at_100_diff1": -0.496632, + "nauc_recall_at_1000_max": -1.727857, + "nauc_recall_at_1000_std": -0.164944, + "nauc_recall_at_1000_diff1": -0.164944, + "nauc_precision_at_1_max": 0.285362, + "nauc_precision_at_1_std": 0.182568, + "nauc_precision_at_1_diff1": 0.273017, + "nauc_precision_at_3_max": 0.561296, + "nauc_precision_at_3_std": 0.308845, + "nauc_precision_at_3_diff1": 0.256333, + "nauc_precision_at_5_max": 0.637945, + "nauc_precision_at_5_std": 0.349563, + "nauc_precision_at_5_diff1": 0.146493, + "nauc_precision_at_10_max": 0.628381, + "nauc_precision_at_10_std": 0.387953, + "nauc_precision_at_10_diff1": 0.179563, + "nauc_precision_at_20_max": 0.560612, + "nauc_precision_at_20_std": 0.375654, + "nauc_precision_at_20_diff1": 0.038659, + "nauc_precision_at_100_max": 0.628944, + "nauc_precision_at_100_std": 0.385327, + "nauc_precision_at_100_diff1": -0.068081, + "nauc_precision_at_1000_max": 0.442247, + "nauc_precision_at_1000_std": 0.184035, + "nauc_precision_at_1000_diff1": -0.036736, + "nauc_cv_recall_at_1_max": 0.285362, + "nauc_cv_recall_at_1_std": 0.182568, + "nauc_cv_recall_at_1_diff1": 0.273017, + "nauc_cv_recall_at_3_max": 0.374921, + "nauc_cv_recall_at_3_std": 0.151059, + "nauc_cv_recall_at_3_diff1": 0.084319, + "nauc_cv_recall_at_5_max": 0.487059, + "nauc_cv_recall_at_5_std": 0.275312, + "nauc_cv_recall_at_5_diff1": -0.167246, + "nauc_cv_recall_at_10_max": 0.143228, + "nauc_cv_recall_at_10_std": 0.046811, + "nauc_cv_recall_at_10_diff1": -0.198756, + "nauc_cv_recall_at_20_max": -0.02857, + "nauc_cv_recall_at_20_std": 0.26533, + "nauc_cv_recall_at_20_diff1": -0.557751, + "nauc_cv_recall_at_100_max": 0.325321, + "nauc_cv_recall_at_100_std": 0.387775, + "nauc_cv_recall_at_100_diff1": -1.338374, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.285362, + "nauc_mrr_at_1_std": 0.182568, + "nauc_mrr_at_1_diff1": 0.273017, + "nauc_mrr_at_3_max": 0.325513, + "nauc_mrr_at_3_std": 0.186421, + "nauc_mrr_at_3_diff1": 0.194664, + "nauc_mrr_at_5_max": 0.336516, + "nauc_mrr_at_5_std": 0.209299, + "nauc_mrr_at_5_diff1": 0.156888, + "nauc_mrr_at_10_max": 0.307899, + "nauc_mrr_at_10_std": 0.192148, + "nauc_mrr_at_10_diff1": 0.169589, + "nauc_mrr_at_20_max": 0.306361, + "nauc_mrr_at_20_std": 0.197109, + "nauc_mrr_at_20_diff1": 0.165986, + "nauc_mrr_at_100_max": 0.309907, + "nauc_mrr_at_100_std": 0.198536, + "nauc_mrr_at_100_diff1": 0.164534, + "nauc_mrr_at_1000_max": 0.309914, + "nauc_mrr_at_1000_std": 0.198394, + "nauc_mrr_at_1000_diff1": 0.165218, + "main_score": 0.45616, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.40351, + "ndcg_at_3": 0.39312, + "ndcg_at_5": 0.42426, + "ndcg_at_10": 0.46516, + "ndcg_at_20": 0.49273, + "ndcg_at_100": 0.53091, + "ndcg_at_1000": 0.56593, + "map_at_1": 0.19946, + "map_at_3": 0.2833, + "map_at_5": 0.31774, + "map_at_10": 0.35185, + "map_at_20": 0.36749, + "map_at_100": 0.37798, + "map_at_1000": 0.38124, + "recall_at_1": 0.19946, + "recall_at_3": 0.34336, + "recall_at_5": 0.4435, + "recall_at_10": 0.59355, + "recall_at_20": 0.6914, + "recall_at_100": 0.82469, + "recall_at_1000": 0.99342, + "cv_recall_at_1": 0.42105, + "cv_recall_at_3": 0.59649, + "cv_recall_at_5": 0.7193, + "cv_recall_at_10": 0.84211, + "cv_recall_at_20": 0.89474, + "cv_recall_at_100": 0.94737, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.40351, + "precision_at_3": 0.26316, + "precision_at_5": 0.21754, + "precision_at_10": 0.14737, + "precision_at_20": 0.09035, + "precision_at_100": 0.02614, + "precision_at_1000": 0.00382, + "mrr_at_1": 0.421053, + "mrr_at_3": 0.491228, + "mrr_at_5": 0.519298, + "mrr_at_10": 0.535498, + "mrr_at_20": 0.539359, + "mrr_at_100": 0.540696, + "mrr_at_1000": 0.540897, + "nauc_ndcg_at_1_max": 0.330161, + "nauc_ndcg_at_1_std": 0.044465, + "nauc_ndcg_at_1_diff1": 0.218518, + "nauc_ndcg_at_3_max": 0.325329, + "nauc_ndcg_at_3_std": 0.104726, + "nauc_ndcg_at_3_diff1": 0.040027, + "nauc_ndcg_at_5_max": 0.244912, + "nauc_ndcg_at_5_std": 0.068731, + "nauc_ndcg_at_5_diff1": 0.076705, + "nauc_ndcg_at_10_max": 0.134857, + "nauc_ndcg_at_10_std": 0.05554, + "nauc_ndcg_at_10_diff1": 0.077827, + "nauc_ndcg_at_20_max": 0.12854, + "nauc_ndcg_at_20_std": 0.068964, + "nauc_ndcg_at_20_diff1": 0.051445, + "nauc_ndcg_at_100_max": 0.225773, + "nauc_ndcg_at_100_std": 0.140175, + "nauc_ndcg_at_100_diff1": 0.024474, + "nauc_ndcg_at_1000_max": 0.257603, + "nauc_ndcg_at_1000_std": 0.123716, + "nauc_ndcg_at_1000_diff1": 0.014019, + "nauc_map_at_1_max": -0.028588, + "nauc_map_at_1_std": -0.013832, + "nauc_map_at_1_diff1": 0.043453, + "nauc_map_at_3_max": 0.09563, + "nauc_map_at_3_std": -0.00697, + "nauc_map_at_3_diff1": 0.027829, + "nauc_map_at_5_max": 0.124115, + "nauc_map_at_5_std": -3.5e-05, + "nauc_map_at_5_diff1": 0.090868, + "nauc_map_at_10_max": 0.134136, + "nauc_map_at_10_std": 0.050599, + "nauc_map_at_10_diff1": 0.100361, + "nauc_map_at_20_max": 0.145473, + "nauc_map_at_20_std": 0.058584, + "nauc_map_at_20_diff1": 0.089089, + "nauc_map_at_100_max": 0.17546, + "nauc_map_at_100_std": 0.078763, + "nauc_map_at_100_diff1": 0.082224, + "nauc_map_at_1000_max": 0.179882, + "nauc_map_at_1000_std": 0.079498, + "nauc_map_at_1000_diff1": 0.078516, + "nauc_recall_at_1_max": -0.028588, + "nauc_recall_at_1_std": -0.013832, + "nauc_recall_at_1_diff1": 0.043453, + "nauc_recall_at_3_max": 0.041628, + "nauc_recall_at_3_std": -0.04695, + "nauc_recall_at_3_diff1": -0.069378, + "nauc_recall_at_5_max": -0.044517, + "nauc_recall_at_5_std": -0.061751, + "nauc_recall_at_5_diff1": 0.069635, + "nauc_recall_at_10_max": -0.201665, + "nauc_recall_at_10_std": -0.043185, + "nauc_recall_at_10_diff1": 0.059353, + "nauc_recall_at_20_max": -0.293997, + "nauc_recall_at_20_std": -0.023146, + "nauc_recall_at_20_diff1": -0.072954, + "nauc_recall_at_100_max": -0.07904, + "nauc_recall_at_100_std": 0.263948, + "nauc_recall_at_100_diff1": -0.262329, + "nauc_recall_at_1000_max": -0.749118, + "nauc_recall_at_1000_std": 0.815299, + "nauc_recall_at_1000_diff1": -0.944834, + "nauc_precision_at_1_max": 0.330161, + "nauc_precision_at_1_std": 0.044465, + "nauc_precision_at_1_diff1": 0.218518, + "nauc_precision_at_3_max": 0.557268, + "nauc_precision_at_3_std": 0.170277, + "nauc_precision_at_3_diff1": 0.087445, + "nauc_precision_at_5_max": 0.615141, + "nauc_precision_at_5_std": 0.260855, + "nauc_precision_at_5_diff1": 0.122169, + "nauc_precision_at_10_max": 0.560942, + "nauc_precision_at_10_std": 0.365245, + "nauc_precision_at_10_diff1": 0.053583, + "nauc_precision_at_20_max": 0.571881, + "nauc_precision_at_20_std": 0.350758, + "nauc_precision_at_20_diff1": -0.026307, + "nauc_precision_at_100_max": 0.652671, + "nauc_precision_at_100_std": 0.391217, + "nauc_precision_at_100_diff1": -0.130366, + "nauc_precision_at_1000_max": 0.392913, + "nauc_precision_at_1000_std": 0.177773, + "nauc_precision_at_1000_diff1": -0.261273, + "nauc_cv_recall_at_1_max": 0.32601, + "nauc_cv_recall_at_1_std": 0.027226, + "nauc_cv_recall_at_1_diff1": 0.170929, + "nauc_cv_recall_at_3_max": 0.338657, + "nauc_cv_recall_at_3_std": -0.003235, + "nauc_cv_recall_at_3_diff1": -0.047727, + "nauc_cv_recall_at_5_max": 0.207461, + "nauc_cv_recall_at_5_std": 0.080073, + "nauc_cv_recall_at_5_diff1": -0.015451, + "nauc_cv_recall_at_10_max": -0.095704, + "nauc_cv_recall_at_10_std": -0.021374, + "nauc_cv_recall_at_10_diff1": -0.268195, + "nauc_cv_recall_at_20_max": -0.336082, + "nauc_cv_recall_at_20_std": 0.063299, + "nauc_cv_recall_at_20_diff1": -0.629331, + "nauc_cv_recall_at_100_max": 0.276728, + "nauc_cv_recall_at_100_std": 0.23316, + "nauc_cv_recall_at_100_diff1": -1.1521, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.32601, + "nauc_mrr_at_1_std": 0.027226, + "nauc_mrr_at_1_diff1": 0.170929, + "nauc_mrr_at_3_max": 0.343871, + "nauc_mrr_at_3_std": 0.03025, + "nauc_mrr_at_3_diff1": 0.098965, + "nauc_mrr_at_5_max": 0.32364, + "nauc_mrr_at_5_std": 0.045894, + "nauc_mrr_at_5_diff1": 0.110905, + "nauc_mrr_at_10_max": 0.307144, + "nauc_mrr_at_10_std": 0.037595, + "nauc_mrr_at_10_diff1": 0.099903, + "nauc_mrr_at_20_max": 0.304687, + "nauc_mrr_at_20_std": 0.041015, + "nauc_mrr_at_20_diff1": 0.094831, + "nauc_mrr_at_100_max": 0.309285, + "nauc_mrr_at_100_std": 0.04339, + "nauc_mrr_at_100_diff1": 0.095067, + "nauc_mrr_at_1000_max": 0.309277, + "nauc_mrr_at_1000_std": 0.042979, + "nauc_mrr_at_1000_diff1": 0.095878, + "main_score": 0.42426, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 1195.8371875286102, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/Vidore2EconomicsReportsRetrieval.json b/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..fb40534a78 --- /dev/null +++ b/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.13", + "scores": { + "test": [ + { + "ndcg_at_1": 0.58621, + "ndcg_at_3": 0.52726, + "ndcg_at_5": 0.50047, + "ndcg_at_10": 0.48557, + "ndcg_at_20": 0.51803, + "ndcg_at_100": 0.6416, + "ndcg_at_1000": 0.68757, + "map_at_1": 0.12103, + "map_at_3": 0.17835, + "map_at_5": 0.21256, + "map_at_10": 0.25958, + "map_at_20": 0.31006, + "map_at_100": 0.396, + "map_at_1000": 0.41476, + "recall_at_1": 0.12103, + "recall_at_3": 0.20211, + "recall_at_5": 0.25778, + "recall_at_10": 0.36038, + "recall_at_20": 0.5099, + "recall_at_100": 0.87344, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.58621, + "cv_recall_at_3": 0.7931, + "cv_recall_at_5": 0.84483, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.58621, + "precision_at_3": 0.45977, + "precision_at_5": 0.41034, + "precision_at_10": 0.33793, + "precision_at_20": 0.27069, + "precision_at_100": 0.12276, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.586207, + "mrr_at_3": 0.681034, + "mrr_at_5": 0.693103, + "mrr_at_10": 0.705651, + "mrr_at_20": 0.708414, + "mrr_at_100": 0.709198, + "mrr_at_1000": 0.709198, + "nauc_ndcg_at_1_max": 0.032663, + "nauc_ndcg_at_1_std": 0.058516, + "nauc_ndcg_at_1_diff1": 0.014971, + "nauc_ndcg_at_3_max": 0.175876, + "nauc_ndcg_at_3_std": 0.127305, + "nauc_ndcg_at_3_diff1": 0.135621, + "nauc_ndcg_at_5_max": 0.185358, + "nauc_ndcg_at_5_std": 0.138148, + "nauc_ndcg_at_5_diff1": 0.145896, + "nauc_ndcg_at_10_max": 0.171208, + "nauc_ndcg_at_10_std": 0.139156, + "nauc_ndcg_at_10_diff1": 0.146049, + "nauc_ndcg_at_20_max": 0.148743, + "nauc_ndcg_at_20_std": 0.090778, + "nauc_ndcg_at_20_diff1": 0.10611, + "nauc_ndcg_at_100_max": 0.162027, + "nauc_ndcg_at_100_std": 0.097575, + "nauc_ndcg_at_100_diff1": 0.162906, + "nauc_ndcg_at_1000_max": 0.124186, + "nauc_ndcg_at_1000_std": 0.092923, + "nauc_ndcg_at_1000_diff1": 0.126538, + "nauc_map_at_1_max": -0.263082, + "nauc_map_at_1_std": -0.326217, + "nauc_map_at_1_diff1": -0.108366, + "nauc_map_at_3_max": -0.137841, + "nauc_map_at_3_std": -0.24215, + "nauc_map_at_3_diff1": -0.00205, + "nauc_map_at_5_max": -0.117582, + "nauc_map_at_5_std": -0.228774, + "nauc_map_at_5_diff1": 0.026294, + "nauc_map_at_10_max": -0.044999, + "nauc_map_at_10_std": -0.157923, + "nauc_map_at_10_diff1": 0.082157, + "nauc_map_at_20_max": 0.005896, + "nauc_map_at_20_std": -0.109975, + "nauc_map_at_20_diff1": 0.118643, + "nauc_map_at_100_max": 0.065385, + "nauc_map_at_100_std": -0.001711, + "nauc_map_at_100_diff1": 0.098796, + "nauc_map_at_1000_max": 0.068437, + "nauc_map_at_1000_std": 0.023454, + "nauc_map_at_1000_diff1": 0.072296, + "nauc_recall_at_1_max": -0.263082, + "nauc_recall_at_1_std": -0.326217, + "nauc_recall_at_1_diff1": -0.108366, + "nauc_recall_at_3_max": -0.05442, + "nauc_recall_at_3_std": -0.18731, + "nauc_recall_at_3_diff1": 0.063149, + "nauc_recall_at_5_max": -0.037395, + "nauc_recall_at_5_std": -0.170284, + "nauc_recall_at_5_diff1": 0.102115, + "nauc_recall_at_10_max": 0.023529, + "nauc_recall_at_10_std": -0.105274, + "nauc_recall_at_10_diff1": 0.141584, + "nauc_recall_at_20_max": 0.123816, + "nauc_recall_at_20_std": -0.060541, + "nauc_recall_at_20_diff1": 0.199414, + "nauc_recall_at_100_max": 0.296069, + "nauc_recall_at_100_std": 0.149474, + "nauc_recall_at_100_diff1": 0.235668, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.032663, + "nauc_precision_at_1_std": 0.058516, + "nauc_precision_at_1_diff1": 0.014971, + "nauc_precision_at_3_max": 0.314105, + "nauc_precision_at_3_std": 0.272146, + "nauc_precision_at_3_diff1": 0.223154, + "nauc_precision_at_5_max": 0.339794, + "nauc_precision_at_5_std": 0.327564, + "nauc_precision_at_5_diff1": 0.218445, + "nauc_precision_at_10_max": 0.316449, + "nauc_precision_at_10_std": 0.374004, + "nauc_precision_at_10_diff1": 0.193822, + "nauc_precision_at_20_max": 0.270226, + "nauc_precision_at_20_std": 0.351258, + "nauc_precision_at_20_diff1": 0.097652, + "nauc_precision_at_100_max": 0.159899, + "nauc_precision_at_100_std": 0.321022, + "nauc_precision_at_100_diff1": -0.096297, + "nauc_precision_at_1000_max": 0.123791, + "nauc_precision_at_1000_std": 0.296764, + "nauc_precision_at_1000_diff1": -0.138147, + "nauc_cv_recall_at_1_max": 0.032663, + "nauc_cv_recall_at_1_std": 0.058516, + "nauc_cv_recall_at_1_diff1": 0.014971, + "nauc_cv_recall_at_3_max": 0.407328, + "nauc_cv_recall_at_3_std": 0.304706, + "nauc_cv_recall_at_3_diff1": 0.207849, + "nauc_cv_recall_at_5_max": 0.499322, + "nauc_cv_recall_at_5_std": 0.407613, + "nauc_cv_recall_at_5_diff1": 0.455944, + "nauc_cv_recall_at_10_max": 0.103301, + "nauc_cv_recall_at_10_std": 0.181732, + "nauc_cv_recall_at_10_diff1": 0.544503, + "nauc_cv_recall_at_20_max": 0.123761, + "nauc_cv_recall_at_20_std": -0.172914, + "nauc_cv_recall_at_20_diff1": 0.554013, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.032663, + "nauc_mrr_at_1_std": 0.058516, + "nauc_mrr_at_1_diff1": 0.014971, + "nauc_mrr_at_3_max": 0.161193, + "nauc_mrr_at_3_std": 0.141231, + "nauc_mrr_at_3_diff1": 0.075605, + "nauc_mrr_at_5_max": 0.163708, + "nauc_mrr_at_5_std": 0.1491, + "nauc_mrr_at_5_diff1": 0.102112, + "nauc_mrr_at_10_max": 0.1374, + "nauc_mrr_at_10_std": 0.129409, + "nauc_mrr_at_10_diff1": 0.094134, + "nauc_mrr_at_20_max": 0.138039, + "nauc_mrr_at_20_std": 0.126737, + "nauc_mrr_at_20_diff1": 0.089034, + "nauc_mrr_at_100_max": 0.138086, + "nauc_mrr_at_100_std": 0.12773, + "nauc_mrr_at_100_diff1": 0.087494, + "nauc_mrr_at_1000_max": 0.138086, + "nauc_mrr_at_1000_std": 0.12773, + "nauc_mrr_at_1000_diff1": 0.087494, + "main_score": 0.50047, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.46552, + "ndcg_at_3": 0.4827, + "ndcg_at_5": 0.46477, + "ndcg_at_10": 0.45359, + "ndcg_at_20": 0.48552, + "ndcg_at_100": 0.61955, + "ndcg_at_1000": 0.66084, + "map_at_1": 0.08543, + "map_at_3": 0.15411, + "map_at_5": 0.18957, + "map_at_10": 0.23519, + "map_at_20": 0.28088, + "map_at_100": 0.36709, + "map_at_1000": 0.38459, + "recall_at_1": 0.08543, + "recall_at_3": 0.19502, + "recall_at_5": 0.25139, + "recall_at_10": 0.36291, + "recall_at_20": 0.50566, + "recall_at_100": 0.88813, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.46552, + "cv_recall_at_3": 0.7931, + "cv_recall_at_5": 0.84483, + "cv_recall_at_10": 0.93103, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.46552, + "precision_at_3": 0.43678, + "precision_at_5": 0.39655, + "precision_at_10": 0.32241, + "precision_at_20": 0.25776, + "precision_at_100": 0.12397, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.465517, + "mrr_at_3": 0.614943, + "mrr_at_5": 0.626149, + "mrr_at_10": 0.637521, + "mrr_at_20": 0.641805, + "mrr_at_100": 0.642327, + "mrr_at_1000": 0.642327, + "nauc_ndcg_at_1_max": 0.187661, + "nauc_ndcg_at_1_std": 0.0731, + "nauc_ndcg_at_1_diff1": 0.280879, + "nauc_ndcg_at_3_max": 0.134106, + "nauc_ndcg_at_3_std": -0.041905, + "nauc_ndcg_at_3_diff1": 0.261946, + "nauc_ndcg_at_5_max": 0.187769, + "nauc_ndcg_at_5_std": 0.009003, + "nauc_ndcg_at_5_diff1": 0.268471, + "nauc_ndcg_at_10_max": 0.185736, + "nauc_ndcg_at_10_std": 0.033862, + "nauc_ndcg_at_10_diff1": 0.281348, + "nauc_ndcg_at_20_max": 0.136532, + "nauc_ndcg_at_20_std": -0.052043, + "nauc_ndcg_at_20_diff1": 0.263545, + "nauc_ndcg_at_100_max": 0.115004, + "nauc_ndcg_at_100_std": -0.065204, + "nauc_ndcg_at_100_diff1": 0.180795, + "nauc_ndcg_at_1000_max": 0.107428, + "nauc_ndcg_at_1000_std": -0.049538, + "nauc_ndcg_at_1000_diff1": 0.20021, + "nauc_map_at_1_max": -0.19997, + "nauc_map_at_1_std": -0.292716, + "nauc_map_at_1_diff1": -0.02322, + "nauc_map_at_3_max": -0.17943, + "nauc_map_at_3_std": -0.292771, + "nauc_map_at_3_diff1": -0.007636, + "nauc_map_at_5_max": -0.123353, + "nauc_map_at_5_std": -0.252424, + "nauc_map_at_5_diff1": 0.039618, + "nauc_map_at_10_max": -0.062562, + "nauc_map_at_10_std": -0.180657, + "nauc_map_at_10_diff1": 0.084199, + "nauc_map_at_20_max": -0.031404, + "nauc_map_at_20_std": -0.172796, + "nauc_map_at_20_diff1": 0.121605, + "nauc_map_at_100_max": 0.038452, + "nauc_map_at_100_std": -0.10806, + "nauc_map_at_100_diff1": 0.156485, + "nauc_map_at_1000_max": 0.057833, + "nauc_map_at_1000_std": -0.08908, + "nauc_map_at_1000_diff1": 0.182163, + "nauc_recall_at_1_max": -0.19997, + "nauc_recall_at_1_std": -0.292716, + "nauc_recall_at_1_diff1": -0.02322, + "nauc_recall_at_3_max": -0.144733, + "nauc_recall_at_3_std": -0.260567, + "nauc_recall_at_3_diff1": -0.020794, + "nauc_recall_at_5_max": -0.068647, + "nauc_recall_at_5_std": -0.206892, + "nauc_recall_at_5_diff1": 0.029231, + "nauc_recall_at_10_max": 0.019659, + "nauc_recall_at_10_std": -0.057902, + "nauc_recall_at_10_diff1": 0.080585, + "nauc_recall_at_20_max": 0.025934, + "nauc_recall_at_20_std": -0.103529, + "nauc_recall_at_20_diff1": 0.151237, + "nauc_recall_at_100_max": 0.103457, + "nauc_recall_at_100_std": -0.072982, + "nauc_recall_at_100_diff1": -0.010629, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.187661, + "nauc_precision_at_1_std": 0.0731, + "nauc_precision_at_1_diff1": 0.280879, + "nauc_precision_at_3_max": 0.221776, + "nauc_precision_at_3_std": 0.039299, + "nauc_precision_at_3_diff1": 0.327637, + "nauc_precision_at_5_max": 0.334042, + "nauc_precision_at_5_std": 0.161619, + "nauc_precision_at_5_diff1": 0.340922, + "nauc_precision_at_10_max": 0.314062, + "nauc_precision_at_10_std": 0.22331, + "nauc_precision_at_10_diff1": 0.305446, + "nauc_precision_at_20_max": 0.272898, + "nauc_precision_at_20_std": 0.179523, + "nauc_precision_at_20_diff1": 0.277016, + "nauc_precision_at_100_max": 0.205299, + "nauc_precision_at_100_std": 0.171948, + "nauc_precision_at_100_diff1": 0.179933, + "nauc_precision_at_1000_max": 0.20464, + "nauc_precision_at_1000_std": 0.158434, + "nauc_precision_at_1000_diff1": 0.22144, + "nauc_cv_recall_at_1_max": 0.187661, + "nauc_cv_recall_at_1_std": 0.0731, + "nauc_cv_recall_at_1_diff1": 0.280879, + "nauc_cv_recall_at_3_max": 0.218787, + "nauc_cv_recall_at_3_std": -0.101192, + "nauc_cv_recall_at_3_diff1": 0.318567, + "nauc_cv_recall_at_5_max": 0.373563, + "nauc_cv_recall_at_5_std": 0.092044, + "nauc_cv_recall_at_5_diff1": 0.301205, + "nauc_cv_recall_at_10_max": 0.115633, + "nauc_cv_recall_at_10_std": -0.127074, + "nauc_cv_recall_at_10_diff1": 0.12907, + "nauc_cv_recall_at_20_max": 0.359055, + "nauc_cv_recall_at_20_std": 0.359055, + "nauc_cv_recall_at_20_diff1": 0.868778, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.187661, + "nauc_mrr_at_1_std": 0.0731, + "nauc_mrr_at_1_diff1": 0.280879, + "nauc_mrr_at_3_max": 0.188005, + "nauc_mrr_at_3_std": 0.006042, + "nauc_mrr_at_3_diff1": 0.280725, + "nauc_mrr_at_5_max": 0.207844, + "nauc_mrr_at_5_std": 0.032279, + "nauc_mrr_at_5_diff1": 0.280178, + "nauc_mrr_at_10_max": 0.195692, + "nauc_mrr_at_10_std": 0.028362, + "nauc_mrr_at_10_diff1": 0.273473, + "nauc_mrr_at_20_max": 0.197371, + "nauc_mrr_at_20_std": 0.03131, + "nauc_mrr_at_20_diff1": 0.280691, + "nauc_mrr_at_100_max": 0.197045, + "nauc_mrr_at_100_std": 0.030649, + "nauc_mrr_at_100_diff1": 0.279505, + "nauc_mrr_at_1000_max": 0.197045, + "nauc_mrr_at_1000_std": 0.030649, + "nauc_mrr_at_1000_diff1": 0.279505, + "main_score": 0.46477, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.60345, + "ndcg_at_3": 0.55113, + "ndcg_at_5": 0.53126, + "ndcg_at_10": 0.50678, + "ndcg_at_20": 0.53807, + "ndcg_at_100": 0.65901, + "ndcg_at_1000": 0.69668, + "map_at_1": 0.09189, + "map_at_3": 0.17342, + "map_at_5": 0.21738, + "map_at_10": 0.26714, + "map_at_20": 0.31746, + "map_at_100": 0.40623, + "map_at_1000": 0.42305, + "recall_at_1": 0.09189, + "recall_at_3": 0.21548, + "recall_at_5": 0.28444, + "recall_at_10": 0.38668, + "recall_at_20": 0.54301, + "recall_at_100": 0.89824, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.60345, + "cv_recall_at_3": 0.84483, + "cv_recall_at_5": 0.89655, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.60345, + "precision_at_3": 0.48851, + "precision_at_5": 0.44828, + "precision_at_10": 0.3569, + "precision_at_20": 0.28103, + "precision_at_100": 0.12724, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.603448, + "mrr_at_3": 0.70977, + "mrr_at_5": 0.721839, + "mrr_at_10": 0.729331, + "mrr_at_20": 0.733484, + "mrr_at_100": 0.733484, + "mrr_at_1000": 0.733484, + "nauc_ndcg_at_1_max": 0.161312, + "nauc_ndcg_at_1_std": 0.177432, + "nauc_ndcg_at_1_diff1": 0.257717, + "nauc_ndcg_at_3_max": 0.27755, + "nauc_ndcg_at_3_std": 0.20543, + "nauc_ndcg_at_3_diff1": 0.289353, + "nauc_ndcg_at_5_max": 0.31156, + "nauc_ndcg_at_5_std": 0.233214, + "nauc_ndcg_at_5_diff1": 0.255843, + "nauc_ndcg_at_10_max": 0.289254, + "nauc_ndcg_at_10_std": 0.262358, + "nauc_ndcg_at_10_diff1": 0.282495, + "nauc_ndcg_at_20_max": 0.23914, + "nauc_ndcg_at_20_std": 0.189579, + "nauc_ndcg_at_20_diff1": 0.225449, + "nauc_ndcg_at_100_max": 0.216558, + "nauc_ndcg_at_100_std": 0.170879, + "nauc_ndcg_at_100_diff1": 0.234898, + "nauc_ndcg_at_1000_max": 0.217006, + "nauc_ndcg_at_1000_std": 0.207354, + "nauc_ndcg_at_1000_diff1": 0.196055, + "nauc_map_at_1_max": -0.307373, + "nauc_map_at_1_std": -0.353831, + "nauc_map_at_1_diff1": 0.400461, + "nauc_map_at_3_max": -0.182548, + "nauc_map_at_3_std": -0.30988, + "nauc_map_at_3_diff1": 0.321096, + "nauc_map_at_5_max": -0.103915, + "nauc_map_at_5_std": -0.275059, + "nauc_map_at_5_diff1": 0.280235, + "nauc_map_at_10_max": -0.018571, + "nauc_map_at_10_std": -0.171658, + "nauc_map_at_10_diff1": 0.324817, + "nauc_map_at_20_max": 0.037399, + "nauc_map_at_20_std": -0.109027, + "nauc_map_at_20_diff1": 0.329019, + "nauc_map_at_100_max": 0.146475, + "nauc_map_at_100_std": 0.070246, + "nauc_map_at_100_diff1": 0.249155, + "nauc_map_at_1000_max": 0.174533, + "nauc_map_at_1000_std": 0.120952, + "nauc_map_at_1000_diff1": 0.210812, + "nauc_recall_at_1_max": -0.307373, + "nauc_recall_at_1_std": -0.353831, + "nauc_recall_at_1_diff1": 0.400461, + "nauc_recall_at_3_max": -0.137566, + "nauc_recall_at_3_std": -0.252857, + "nauc_recall_at_3_diff1": 0.222884, + "nauc_recall_at_5_max": -0.07065, + "nauc_recall_at_5_std": -0.23614, + "nauc_recall_at_5_diff1": 0.168984, + "nauc_recall_at_10_max": -0.015767, + "nauc_recall_at_10_std": -0.127322, + "nauc_recall_at_10_diff1": 0.221318, + "nauc_recall_at_20_max": 0.044689, + "nauc_recall_at_20_std": -0.119681, + "nauc_recall_at_20_diff1": 0.248652, + "nauc_recall_at_100_max": 0.128173, + "nauc_recall_at_100_std": -0.014805, + "nauc_recall_at_100_diff1": 0.232354, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.161312, + "nauc_precision_at_1_std": 0.177432, + "nauc_precision_at_1_diff1": 0.257717, + "nauc_precision_at_3_max": 0.432114, + "nauc_precision_at_3_std": 0.339933, + "nauc_precision_at_3_diff1": 0.250125, + "nauc_precision_at_5_max": 0.478469, + "nauc_precision_at_5_std": 0.393013, + "nauc_precision_at_5_diff1": 0.141807, + "nauc_precision_at_10_max": 0.469069, + "nauc_precision_at_10_std": 0.516759, + "nauc_precision_at_10_diff1": 0.080165, + "nauc_precision_at_20_max": 0.409159, + "nauc_precision_at_20_std": 0.518212, + "nauc_precision_at_20_diff1": -0.06158, + "nauc_precision_at_100_max": 0.306425, + "nauc_precision_at_100_std": 0.487945, + "nauc_precision_at_100_diff1": -0.217063, + "nauc_precision_at_1000_max": 0.314884, + "nauc_precision_at_1000_std": 0.494515, + "nauc_precision_at_1000_diff1": -0.236382, + "nauc_cv_recall_at_1_max": 0.161312, + "nauc_cv_recall_at_1_std": 0.177432, + "nauc_cv_recall_at_1_diff1": 0.257717, + "nauc_cv_recall_at_3_max": 0.561903, + "nauc_cv_recall_at_3_std": 0.632453, + "nauc_cv_recall_at_3_diff1": 0.247395, + "nauc_cv_recall_at_5_max": 0.520491, + "nauc_cv_recall_at_5_std": 0.621687, + "nauc_cv_recall_at_5_diff1": 0.270718, + "nauc_cv_recall_at_10_max": 0.233385, + "nauc_cv_recall_at_10_std": 0.424041, + "nauc_cv_recall_at_10_diff1": -0.096254, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.161312, + "nauc_mrr_at_1_std": 0.177432, + "nauc_mrr_at_1_diff1": 0.257717, + "nauc_mrr_at_3_max": 0.265197, + "nauc_mrr_at_3_std": 0.271707, + "nauc_mrr_at_3_diff1": 0.251997, + "nauc_mrr_at_5_max": 0.248012, + "nauc_mrr_at_5_std": 0.254817, + "nauc_mrr_at_5_diff1": 0.254469, + "nauc_mrr_at_10_max": 0.229046, + "nauc_mrr_at_10_std": 0.235728, + "nauc_mrr_at_10_diff1": 0.240886, + "nauc_mrr_at_20_max": 0.229094, + "nauc_mrr_at_20_std": 0.232382, + "nauc_mrr_at_20_diff1": 0.247525, + "nauc_mrr_at_100_max": 0.229094, + "nauc_mrr_at_100_std": 0.232382, + "nauc_mrr_at_100_diff1": 0.247525, + "nauc_mrr_at_1000_max": 0.229094, + "nauc_mrr_at_1000_std": 0.232382, + "nauc_mrr_at_1000_diff1": 0.247525, + "main_score": 0.53126, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.55172, + "ndcg_at_3": 0.52639, + "ndcg_at_5": 0.48979, + "ndcg_at_10": 0.48925, + "ndcg_at_20": 0.50428, + "ndcg_at_100": 0.64229, + "ndcg_at_1000": 0.68579, + "map_at_1": 0.11943, + "map_at_3": 0.17514, + "map_at_5": 0.20714, + "map_at_10": 0.26272, + "map_at_20": 0.30095, + "map_at_100": 0.39206, + "map_at_1000": 0.41, + "recall_at_1": 0.11943, + "recall_at_3": 0.20076, + "recall_at_5": 0.26176, + "recall_at_10": 0.38542, + "recall_at_20": 0.49428, + "recall_at_100": 0.8797, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.55172, + "cv_recall_at_3": 0.82759, + "cv_recall_at_5": 0.89655, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.55172, + "precision_at_3": 0.46552, + "precision_at_5": 0.4, + "precision_at_10": 0.33621, + "precision_at_20": 0.25086, + "precision_at_100": 0.12397, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.551724, + "mrr_at_3": 0.678161, + "mrr_at_5": 0.693678, + "mrr_at_10": 0.700739, + "mrr_at_20": 0.703456, + "mrr_at_100": 0.703857, + "mrr_at_1000": 0.703857, + "nauc_ndcg_at_1_max": 0.101687, + "nauc_ndcg_at_1_std": -0.005752, + "nauc_ndcg_at_1_diff1": 0.077843, + "nauc_ndcg_at_3_max": 0.111211, + "nauc_ndcg_at_3_std": 0.051202, + "nauc_ndcg_at_3_diff1": 0.059425, + "nauc_ndcg_at_5_max": 0.059659, + "nauc_ndcg_at_5_std": 0.005425, + "nauc_ndcg_at_5_diff1": 0.056271, + "nauc_ndcg_at_10_max": 0.140024, + "nauc_ndcg_at_10_std": 0.06321, + "nauc_ndcg_at_10_diff1": 0.176531, + "nauc_ndcg_at_20_max": 0.111559, + "nauc_ndcg_at_20_std": 0.025302, + "nauc_ndcg_at_20_diff1": 0.180463, + "nauc_ndcg_at_100_max": 0.080055, + "nauc_ndcg_at_100_std": -0.021322, + "nauc_ndcg_at_100_diff1": 0.131042, + "nauc_ndcg_at_1000_max": 0.041293, + "nauc_ndcg_at_1000_std": -0.002951, + "nauc_ndcg_at_1000_diff1": 0.077306, + "nauc_map_at_1_max": -0.317519, + "nauc_map_at_1_std": -0.365062, + "nauc_map_at_1_diff1": -0.041705, + "nauc_map_at_3_max": -0.19989, + "nauc_map_at_3_std": -0.252217, + "nauc_map_at_3_diff1": 0.010777, + "nauc_map_at_5_max": -0.181177, + "nauc_map_at_5_std": -0.240893, + "nauc_map_at_5_diff1": 0.03976, + "nauc_map_at_10_max": -0.045329, + "nauc_map_at_10_std": -0.134394, + "nauc_map_at_10_diff1": 0.146943, + "nauc_map_at_20_max": -0.010927, + "nauc_map_at_20_std": -0.118008, + "nauc_map_at_20_diff1": 0.176668, + "nauc_map_at_100_max": 0.009947, + "nauc_map_at_100_std": -0.070996, + "nauc_map_at_100_diff1": 0.136412, + "nauc_map_at_1000_max": 0.006951, + "nauc_map_at_1000_std": -0.051878, + "nauc_map_at_1000_diff1": 0.109172, + "nauc_recall_at_1_max": -0.317519, + "nauc_recall_at_1_std": -0.365062, + "nauc_recall_at_1_diff1": -0.041705, + "nauc_recall_at_3_max": -0.113279, + "nauc_recall_at_3_std": -0.173301, + "nauc_recall_at_3_diff1": 0.056551, + "nauc_recall_at_5_max": -0.107227, + "nauc_recall_at_5_std": -0.180708, + "nauc_recall_at_5_diff1": 0.089696, + "nauc_recall_at_10_max": 0.127225, + "nauc_recall_at_10_std": -0.004606, + "nauc_recall_at_10_diff1": 0.275094, + "nauc_recall_at_20_max": 0.165458, + "nauc_recall_at_20_std": 0.016626, + "nauc_recall_at_20_diff1": 0.326469, + "nauc_recall_at_100_max": 0.250379, + "nauc_recall_at_100_std": -0.044794, + "nauc_recall_at_100_diff1": 0.291412, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.101687, + "nauc_precision_at_1_std": -0.005752, + "nauc_precision_at_1_diff1": 0.077843, + "nauc_precision_at_3_max": 0.250759, + "nauc_precision_at_3_std": 0.21789, + "nauc_precision_at_3_diff1": 0.080716, + "nauc_precision_at_5_max": 0.240689, + "nauc_precision_at_5_std": 0.223207, + "nauc_precision_at_5_diff1": 0.075086, + "nauc_precision_at_10_max": 0.331993, + "nauc_precision_at_10_std": 0.303329, + "nauc_precision_at_10_diff1": 0.139507, + "nauc_precision_at_20_max": 0.25945, + "nauc_precision_at_20_std": 0.240991, + "nauc_precision_at_20_diff1": 0.106828, + "nauc_precision_at_100_max": 0.059489, + "nauc_precision_at_100_std": 0.155431, + "nauc_precision_at_100_diff1": -0.12721, + "nauc_precision_at_1000_max": 0.041906, + "nauc_precision_at_1000_std": 0.157016, + "nauc_precision_at_1000_diff1": -0.152602, + "nauc_cv_recall_at_1_max": 0.101687, + "nauc_cv_recall_at_1_std": -0.005752, + "nauc_cv_recall_at_1_diff1": 0.077843, + "nauc_cv_recall_at_3_max": 0.38013, + "nauc_cv_recall_at_3_std": 0.260141, + "nauc_cv_recall_at_3_diff1": 0.097574, + "nauc_cv_recall_at_5_max": 0.304403, + "nauc_cv_recall_at_5_std": 0.286403, + "nauc_cv_recall_at_5_diff1": -0.223744, + "nauc_cv_recall_at_10_max": 0.658935, + "nauc_cv_recall_at_10_std": 0.544503, + "nauc_cv_recall_at_10_diff1": 0.139414, + "nauc_cv_recall_at_20_max": 0.554013, + "nauc_cv_recall_at_20_std": 0.359055, + "nauc_cv_recall_at_20_diff1": 0.868778, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.101687, + "nauc_mrr_at_1_std": -0.005752, + "nauc_mrr_at_1_diff1": 0.077843, + "nauc_mrr_at_3_max": 0.154636, + "nauc_mrr_at_3_std": 0.051523, + "nauc_mrr_at_3_diff1": 0.086747, + "nauc_mrr_at_5_max": 0.134192, + "nauc_mrr_at_5_std": 0.038151, + "nauc_mrr_at_5_diff1": 0.055868, + "nauc_mrr_at_10_max": 0.134686, + "nauc_mrr_at_10_std": 0.038019, + "nauc_mrr_at_10_diff1": 0.070861, + "nauc_mrr_at_20_max": 0.12825, + "nauc_mrr_at_20_std": 0.031187, + "nauc_mrr_at_20_diff1": 0.074937, + "nauc_mrr_at_100_max": 0.127517, + "nauc_mrr_at_100_std": 0.030623, + "nauc_mrr_at_100_diff1": 0.073569, + "nauc_mrr_at_1000_max": 0.127517, + "nauc_mrr_at_1000_std": 0.030623, + "nauc_mrr_at_1000_diff1": 0.073569, + "main_score": 0.48979, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 241.23444509506226, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/model_meta.json b/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/model_meta.json index 14135e009e..62319ecec7 100644 --- a/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/model_meta.json +++ b/results/royokong__e5-v/0c1f22679417b3ae925d779442221c40cd1861ab/model_meta.json @@ -1 +1 @@ -{"name": "royokong/e5-v", "revision": "0c1f22679417b3ae925d779442221c40cd1861ab", "release_date": "2024-07-17", "languages": ["eng_Latn"], "n_parameters": 8360000000, "memory_usage_mb": 15936.0, "max_tokens": 8192.0, "embed_dim": 4096, "license": null, "open_weights": true, "public_training_code": "https://github.com/kongds/E5-V", "public_training_data": "https://huggingface.co/datasets/princeton-nlp/datasets-for-simcse", "framework": ["PyTorch"], "reference": "https://huggingface.co/royokong/e5-v", "similarity_fn_name": null, "use_instructions": true, "training_datasets": {}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "E5VWrapper"} \ No newline at end of file +{"name": "royokong/e5-v", "revision": "0c1f22679417b3ae925d779442221c40cd1861ab", "release_date": "2024-07-17", "languages": ["eng-Latn"], "n_parameters": 8360000000, "memory_usage_mb": 15936.0, "max_tokens": 8192.0, "embed_dim": 4096, "license": null, "open_weights": true, "public_training_code": "https://github.com/kongds/E5-V", "public_training_data": "https://huggingface.co/datasets/princeton-nlp/datasets-for-simcse", "framework": ["PyTorch"], "reference": "https://huggingface.co/royokong/e5-v", "similarity_fn_name": null, "use_instructions": true, "training_datasets": {}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "E5VWrapper"} \ No newline at end of file