diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..1302c88371 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.6, + "ndcg_at_3": 0.61334, + "ndcg_at_5": 0.62753, + "ndcg_at_10": 0.65339, + "ndcg_at_20": 0.67966, + "ndcg_at_100": 0.71165, + "ndcg_at_1000": 0.72717, + "map_at_1": 0.34903, + "map_at_3": 0.49359, + "map_at_5": 0.53238, + "map_at_10": 0.56365, + "map_at_20": 0.57904, + "map_at_100": 0.58982, + "map_at_1000": 0.5914, + "recall_at_1": 0.34903, + "recall_at_3": 0.58461, + "recall_at_5": 0.6615, + "recall_at_10": 0.74271, + "recall_at_20": 0.8178, + "recall_at_100": 0.91762, + "recall_at_1000": 0.99026, + "cv_recall_at_1": 0.6, + "cv_recall_at_3": 0.81875, + "cv_recall_at_5": 0.875, + "cv_recall_at_10": 0.925, + "cv_recall_at_20": 0.95, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 0.99375, + "precision_at_1": 0.6, + "precision_at_3": 0.38333, + "precision_at_5": 0.2875, + "precision_at_10": 0.17938, + "precision_at_20": 0.105, + "precision_at_100": 0.02719, + "precision_at_1000": 0.0032, + "mrr_at_1": 0.6, + "mrr_at_3": 0.695833, + "mrr_at_5": 0.709271, + "mrr_at_10": 0.715761, + "mrr_at_20": 0.717335, + "mrr_at_100": 0.718162, + "mrr_at_1000": 0.718218, + "nauc_ndcg_at_1_max": 0.154497, + "nauc_ndcg_at_1_std": -0.068251, + "nauc_ndcg_at_1_diff1": 0.417798, + "nauc_ndcg_at_3_max": 0.228794, + "nauc_ndcg_at_3_std": 0.012049, + "nauc_ndcg_at_3_diff1": 0.321429, + "nauc_ndcg_at_5_max": 0.312623, + "nauc_ndcg_at_5_std": 0.082851, + "nauc_ndcg_at_5_diff1": 0.360767, + "nauc_ndcg_at_10_max": 0.309748, + "nauc_ndcg_at_10_std": 0.071029, + "nauc_ndcg_at_10_diff1": 0.36791, + "nauc_ndcg_at_20_max": 0.291857, + "nauc_ndcg_at_20_std": 0.067943, + "nauc_ndcg_at_20_diff1": 0.352894, + "nauc_ndcg_at_100_max": 0.289118, + "nauc_ndcg_at_100_std": 0.091577, + "nauc_ndcg_at_100_diff1": 0.338573, + "nauc_ndcg_at_1000_max": 0.275044, + "nauc_ndcg_at_1000_std": 0.072465, + "nauc_ndcg_at_1000_diff1": 0.340254, + "nauc_map_at_1_max": 0.27885, + "nauc_map_at_1_std": 0.060976, + "nauc_map_at_1_diff1": 0.576173, + "nauc_map_at_3_max": 0.267566, + "nauc_map_at_3_std": 0.039264, + "nauc_map_at_3_diff1": 0.416917, + "nauc_map_at_5_max": 0.295917, + "nauc_map_at_5_std": 0.050516, + "nauc_map_at_5_diff1": 0.408991, + "nauc_map_at_10_max": 0.281736, + "nauc_map_at_10_std": 0.045404, + "nauc_map_at_10_diff1": 0.38718, + "nauc_map_at_20_max": 0.27647, + "nauc_map_at_20_std": 0.048694, + "nauc_map_at_20_diff1": 0.380001, + "nauc_map_at_100_max": 0.281193, + "nauc_map_at_100_std": 0.060898, + "nauc_map_at_100_diff1": 0.375156, + "nauc_map_at_1000_max": 0.279753, + "nauc_map_at_1000_std": 0.059513, + "nauc_map_at_1000_diff1": 0.374714, + "nauc_recall_at_1_max": 0.27885, + "nauc_recall_at_1_std": 0.060976, + "nauc_recall_at_1_diff1": 0.576173, + "nauc_recall_at_3_max": 0.281917, + "nauc_recall_at_3_std": 0.07835, + "nauc_recall_at_3_diff1": 0.301018, + "nauc_recall_at_5_max": 0.370192, + "nauc_recall_at_5_std": 0.140894, + "nauc_recall_at_5_diff1": 0.295357, + "nauc_recall_at_10_max": 0.373375, + "nauc_recall_at_10_std": 0.142083, + "nauc_recall_at_10_diff1": 0.268506, + "nauc_recall_at_20_max": 0.339328, + "nauc_recall_at_20_std": 0.155073, + "nauc_recall_at_20_diff1": 0.177394, + "nauc_recall_at_100_max": 0.377384, + "nauc_recall_at_100_std": 0.376627, + "nauc_recall_at_100_diff1": 0.02438, + "nauc_recall_at_1000_max": 0.703504, + "nauc_recall_at_1000_std": 0.899318, + "nauc_recall_at_1000_diff1": -0.698104, + "nauc_precision_at_1_max": 0.154497, + "nauc_precision_at_1_std": -0.068251, + "nauc_precision_at_1_diff1": 0.417798, + "nauc_precision_at_3_max": 0.009449, + "nauc_precision_at_3_std": -0.053879, + "nauc_precision_at_3_diff1": -0.161895, + "nauc_precision_at_5_max": 0.044978, + "nauc_precision_at_5_std": 0.024785, + "nauc_precision_at_5_diff1": -0.204319, + "nauc_precision_at_10_max": -0.0385, + "nauc_precision_at_10_std": 0.01939, + "nauc_precision_at_10_diff1": -0.245725, + "nauc_precision_at_20_max": -0.087502, + "nauc_precision_at_20_std": 0.037161, + "nauc_precision_at_20_diff1": -0.285149, + "nauc_precision_at_100_max": -0.109539, + "nauc_precision_at_100_std": 0.08413, + "nauc_precision_at_100_diff1": -0.321648, + "nauc_precision_at_1000_max": -0.203447, + "nauc_precision_at_1000_std": -0.02032, + "nauc_precision_at_1000_diff1": -0.345633, + "nauc_cv_recall_at_1_max": 0.154497, + "nauc_cv_recall_at_1_std": -0.068251, + "nauc_cv_recall_at_1_diff1": 0.417798, + "nauc_cv_recall_at_3_max": 0.364701, + "nauc_cv_recall_at_3_std": 0.114316, + "nauc_cv_recall_at_3_diff1": 0.281764, + "nauc_cv_recall_at_5_max": 0.558581, + "nauc_cv_recall_at_5_std": 0.31534, + "nauc_cv_recall_at_5_diff1": 0.310954, + "nauc_cv_recall_at_10_max": 0.613951, + "nauc_cv_recall_at_10_std": 0.451953, + "nauc_cv_recall_at_10_diff1": 0.292873, + "nauc_cv_recall_at_20_max": 0.60796, + "nauc_cv_recall_at_20_std": 0.540558, + "nauc_cv_recall_at_20_diff1": 0.09162, + "nauc_cv_recall_at_100_max": 0.435224, + "nauc_cv_recall_at_100_std": 0.645892, + "nauc_cv_recall_at_100_diff1": -0.389122, + "nauc_cv_recall_at_1000_max": 0.554155, + "nauc_cv_recall_at_1000_std": 0.869281, + "nauc_cv_recall_at_1000_diff1": -0.563492, + "nauc_mrr_at_1_max": 0.154497, + "nauc_mrr_at_1_std": -0.068251, + "nauc_mrr_at_1_diff1": 0.417798, + "nauc_mrr_at_3_max": 0.206663, + "nauc_mrr_at_3_std": -0.026868, + "nauc_mrr_at_3_diff1": 0.375085, + "nauc_mrr_at_5_max": 0.223149, + "nauc_mrr_at_5_std": -0.01031, + "nauc_mrr_at_5_diff1": 0.38221, + "nauc_mrr_at_10_max": 0.215799, + "nauc_mrr_at_10_std": -0.0139, + "nauc_mrr_at_10_diff1": 0.384206, + "nauc_mrr_at_20_max": 0.213175, + "nauc_mrr_at_20_std": -0.015593, + "nauc_mrr_at_20_diff1": 0.382225, + "nauc_mrr_at_100_max": 0.211361, + "nauc_mrr_at_100_std": -0.016911, + "nauc_mrr_at_100_diff1": 0.381847, + "nauc_mrr_at_1000_max": 0.211277, + "nauc_mrr_at_1000_std": -0.017053, + "nauc_mrr_at_1000_diff1": 0.381987, + "main_score": 0.62753, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.58125, + "ndcg_at_3": 0.59874, + "ndcg_at_5": 0.61587, + "ndcg_at_10": 0.65114, + "ndcg_at_20": 0.67327, + "ndcg_at_100": 0.70465, + "ndcg_at_1000": 0.72114, + "map_at_1": 0.34517, + "map_at_3": 0.48252, + "map_at_5": 0.5225, + "map_at_10": 0.55845, + "map_at_20": 0.57106, + "map_at_100": 0.58193, + "map_at_1000": 0.58365, + "recall_at_1": 0.34517, + "recall_at_3": 0.55977, + "recall_at_5": 0.64676, + "recall_at_10": 0.75469, + "recall_at_20": 0.81631, + "recall_at_100": 0.91339, + "recall_at_1000": 0.99026, + "cv_recall_at_1": 0.58125, + "cv_recall_at_3": 0.80625, + "cv_recall_at_5": 0.8625, + "cv_recall_at_10": 0.9125, + "cv_recall_at_20": 0.95, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 0.99375, + "precision_at_1": 0.58125, + "precision_at_3": 0.37708, + "precision_at_5": 0.285, + "precision_at_10": 0.18125, + "precision_at_20": 0.10469, + "precision_at_100": 0.027, + "precision_at_1000": 0.0032, + "mrr_at_1": 0.58125, + "mrr_at_3": 0.684375, + "mrr_at_5": 0.696562, + "mrr_at_10": 0.703209, + "mrr_at_20": 0.705912, + "mrr_at_100": 0.706619, + "mrr_at_1000": 0.70671, + "nauc_ndcg_at_1_max": 0.209674, + "nauc_ndcg_at_1_std": -0.084607, + "nauc_ndcg_at_1_diff1": 0.530338, + "nauc_ndcg_at_3_max": 0.271065, + "nauc_ndcg_at_3_std": 0.00201, + "nauc_ndcg_at_3_diff1": 0.391404, + "nauc_ndcg_at_5_max": 0.331266, + "nauc_ndcg_at_5_std": 0.043248, + "nauc_ndcg_at_5_diff1": 0.420242, + "nauc_ndcg_at_10_max": 0.329112, + "nauc_ndcg_at_10_std": 0.01054, + "nauc_ndcg_at_10_diff1": 0.422176, + "nauc_ndcg_at_20_max": 0.337149, + "nauc_ndcg_at_20_std": 0.029068, + "nauc_ndcg_at_20_diff1": 0.412403, + "nauc_ndcg_at_100_max": 0.314971, + "nauc_ndcg_at_100_std": 0.046079, + "nauc_ndcg_at_100_diff1": 0.404311, + "nauc_ndcg_at_1000_max": 0.302413, + "nauc_ndcg_at_1000_std": 0.034942, + "nauc_ndcg_at_1000_diff1": 0.404864, + "nauc_map_at_1_max": 0.337876, + "nauc_map_at_1_std": 0.074091, + "nauc_map_at_1_diff1": 0.55822, + "nauc_map_at_3_max": 0.320056, + "nauc_map_at_3_std": 0.04348, + "nauc_map_at_3_diff1": 0.449275, + "nauc_map_at_5_max": 0.331657, + "nauc_map_at_5_std": 0.039875, + "nauc_map_at_5_diff1": 0.440973, + "nauc_map_at_10_max": 0.320541, + "nauc_map_at_10_std": 0.019204, + "nauc_map_at_10_diff1": 0.430675, + "nauc_map_at_20_max": 0.323694, + "nauc_map_at_20_std": 0.02885, + "nauc_map_at_20_diff1": 0.423493, + "nauc_map_at_100_max": 0.32082, + "nauc_map_at_100_std": 0.036447, + "nauc_map_at_100_diff1": 0.419517, + "nauc_map_at_1000_max": 0.318948, + "nauc_map_at_1000_std": 0.035423, + "nauc_map_at_1000_diff1": 0.418831, + "nauc_recall_at_1_max": 0.337876, + "nauc_recall_at_1_std": 0.074091, + "nauc_recall_at_1_diff1": 0.55822, + "nauc_recall_at_3_max": 0.31326, + "nauc_recall_at_3_std": 0.052299, + "nauc_recall_at_3_diff1": 0.353859, + "nauc_recall_at_5_max": 0.367351, + "nauc_recall_at_5_std": 0.066672, + "nauc_recall_at_5_diff1": 0.356955, + "nauc_recall_at_10_max": 0.33887, + "nauc_recall_at_10_std": -0.019088, + "nauc_recall_at_10_diff1": 0.292632, + "nauc_recall_at_20_max": 0.408121, + "nauc_recall_at_20_std": 0.073262, + "nauc_recall_at_20_diff1": 0.234038, + "nauc_recall_at_100_max": 0.342323, + "nauc_recall_at_100_std": 0.207946, + "nauc_recall_at_100_diff1": 0.130852, + "nauc_recall_at_1000_max": 0.916143, + "nauc_recall_at_1000_std": 0.869281, + "nauc_recall_at_1000_diff1": -0.452081, + "nauc_precision_at_1_max": 0.209674, + "nauc_precision_at_1_std": -0.084607, + "nauc_precision_at_1_diff1": 0.530338, + "nauc_precision_at_3_max": 0.015027, + "nauc_precision_at_3_std": -0.069673, + "nauc_precision_at_3_diff1": -0.006237, + "nauc_precision_at_5_max": 0.007375, + "nauc_precision_at_5_std": -0.020147, + "nauc_precision_at_5_diff1": -0.092585, + "nauc_precision_at_10_max": -0.072772, + "nauc_precision_at_10_std": -0.059053, + "nauc_precision_at_10_diff1": -0.167993, + "nauc_precision_at_20_max": -0.106829, + "nauc_precision_at_20_std": -0.020383, + "nauc_precision_at_20_diff1": -0.226797, + "nauc_precision_at_100_max": -0.173424, + "nauc_precision_at_100_std": 0.037586, + "nauc_precision_at_100_diff1": -0.269825, + "nauc_precision_at_1000_max": -0.264781, + "nauc_precision_at_1000_std": -0.044858, + "nauc_precision_at_1000_diff1": -0.303533, + "nauc_cv_recall_at_1_max": 0.209674, + "nauc_cv_recall_at_1_std": -0.084607, + "nauc_cv_recall_at_1_diff1": 0.530338, + "nauc_cv_recall_at_3_max": 0.313366, + "nauc_cv_recall_at_3_std": 0.015209, + "nauc_cv_recall_at_3_diff1": 0.421626, + "nauc_cv_recall_at_5_max": 0.496853, + "nauc_cv_recall_at_5_std": 0.141801, + "nauc_cv_recall_at_5_diff1": 0.480239, + "nauc_cv_recall_at_10_max": 0.472089, + "nauc_cv_recall_at_10_std": 0.071362, + "nauc_cv_recall_at_10_diff1": 0.318394, + "nauc_cv_recall_at_20_max": 0.509045, + "nauc_cv_recall_at_20_std": 0.194561, + "nauc_cv_recall_at_20_diff1": 0.135971, + "nauc_cv_recall_at_100_max": 0.152894, + "nauc_cv_recall_at_100_std": 0.2905, + "nauc_cv_recall_at_100_diff1": -0.221639, + "nauc_cv_recall_at_1000_max": 0.869281, + "nauc_cv_recall_at_1000_std": 0.869281, + "nauc_cv_recall_at_1000_diff1": -0.171335, + "nauc_mrr_at_1_max": 0.209674, + "nauc_mrr_at_1_std": -0.084607, + "nauc_mrr_at_1_diff1": 0.530338, + "nauc_mrr_at_3_max": 0.247514, + "nauc_mrr_at_3_std": -0.05047, + "nauc_mrr_at_3_diff1": 0.487293, + "nauc_mrr_at_5_max": 0.26285, + "nauc_mrr_at_5_std": -0.040611, + "nauc_mrr_at_5_diff1": 0.495622, + "nauc_mrr_at_10_max": 0.254666, + "nauc_mrr_at_10_std": -0.04896, + "nauc_mrr_at_10_diff1": 0.487919, + "nauc_mrr_at_20_max": 0.25361, + "nauc_mrr_at_20_std": -0.047539, + "nauc_mrr_at_20_diff1": 0.487827, + "nauc_mrr_at_100_max": 0.251874, + "nauc_mrr_at_100_std": -0.047619, + "nauc_mrr_at_100_diff1": 0.487654, + "nauc_mrr_at_1000_max": 0.252009, + "nauc_mrr_at_1000_std": -0.047493, + "nauc_mrr_at_1000_diff1": 0.487726, + "main_score": 0.61587, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.59375, + "ndcg_at_3": 0.61922, + "ndcg_at_5": 0.63935, + "ndcg_at_10": 0.67255, + "ndcg_at_20": 0.69973, + "ndcg_at_100": 0.72895, + "ndcg_at_1000": 0.74134, + "map_at_1": 0.36528, + "map_at_3": 0.50674, + "map_at_5": 0.54621, + "map_at_10": 0.58223, + "map_at_20": 0.59969, + "map_at_100": 0.60891, + "map_at_1000": 0.61022, + "recall_at_1": 0.36528, + "recall_at_3": 0.5876, + "recall_at_5": 0.6749, + "recall_at_10": 0.76865, + "recall_at_20": 0.84205, + "recall_at_100": 0.93672, + "recall_at_1000": 0.99026, + "cv_recall_at_1": 0.59375, + "cv_recall_at_3": 0.825, + "cv_recall_at_5": 0.89375, + "cv_recall_at_10": 0.925, + "cv_recall_at_20": 0.95, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 0.99375, + "precision_at_1": 0.59375, + "precision_at_3": 0.38333, + "precision_at_5": 0.28875, + "precision_at_10": 0.18625, + "precision_at_20": 0.11031, + "precision_at_100": 0.02769, + "precision_at_1000": 0.0032, + "mrr_at_1": 0.59375, + "mrr_at_3": 0.697917, + "mrr_at_5": 0.714479, + "mrr_at_10": 0.719191, + "mrr_at_20": 0.720896, + "mrr_at_100": 0.721756, + "mrr_at_1000": 0.721799, + "nauc_ndcg_at_1_max": 0.351987, + "nauc_ndcg_at_1_std": -0.054405, + "nauc_ndcg_at_1_diff1": 0.438232, + "nauc_ndcg_at_3_max": 0.312408, + "nauc_ndcg_at_3_std": -0.010081, + "nauc_ndcg_at_3_diff1": 0.338341, + "nauc_ndcg_at_5_max": 0.330371, + "nauc_ndcg_at_5_std": -0.001217, + "nauc_ndcg_at_5_diff1": 0.341077, + "nauc_ndcg_at_10_max": 0.34284, + "nauc_ndcg_at_10_std": -0.027518, + "nauc_ndcg_at_10_diff1": 0.358538, + "nauc_ndcg_at_20_max": 0.355295, + "nauc_ndcg_at_20_std": 0.000337, + "nauc_ndcg_at_20_diff1": 0.340768, + "nauc_ndcg_at_100_max": 0.347257, + "nauc_ndcg_at_100_std": 0.02319, + "nauc_ndcg_at_100_diff1": 0.33534, + "nauc_ndcg_at_1000_max": 0.332773, + "nauc_ndcg_at_1000_std": 0.02377, + "nauc_ndcg_at_1000_diff1": 0.326105, + "nauc_map_at_1_max": 0.419457, + "nauc_map_at_1_std": 0.081611, + "nauc_map_at_1_diff1": 0.532104, + "nauc_map_at_3_max": 0.3677, + "nauc_map_at_3_std": 0.037212, + "nauc_map_at_3_diff1": 0.416624, + "nauc_map_at_5_max": 0.351682, + "nauc_map_at_5_std": 0.01948, + "nauc_map_at_5_diff1": 0.380822, + "nauc_map_at_10_max": 0.339023, + "nauc_map_at_10_std": -0.004854, + "nauc_map_at_10_diff1": 0.366911, + "nauc_map_at_20_max": 0.344315, + "nauc_map_at_20_std": 0.011353, + "nauc_map_at_20_diff1": 0.357049, + "nauc_map_at_100_max": 0.342552, + "nauc_map_at_100_std": 0.015385, + "nauc_map_at_100_diff1": 0.356943, + "nauc_map_at_1000_max": 0.340866, + "nauc_map_at_1000_std": 0.016068, + "nauc_map_at_1000_diff1": 0.355554, + "nauc_recall_at_1_max": 0.419457, + "nauc_recall_at_1_std": 0.081611, + "nauc_recall_at_1_diff1": 0.532104, + "nauc_recall_at_3_max": 0.316431, + "nauc_recall_at_3_std": -0.009, + "nauc_recall_at_3_diff1": 0.347678, + "nauc_recall_at_5_max": 0.276271, + "nauc_recall_at_5_std": -0.048901, + "nauc_recall_at_5_diff1": 0.28011, + "nauc_recall_at_10_max": 0.28889, + "nauc_recall_at_10_std": -0.108, + "nauc_recall_at_10_diff1": 0.269711, + "nauc_recall_at_20_max": 0.365227, + "nauc_recall_at_20_std": -0.023282, + "nauc_recall_at_20_diff1": 0.215147, + "nauc_recall_at_100_max": 0.435367, + "nauc_recall_at_100_std": 0.224339, + "nauc_recall_at_100_diff1": 0.084197, + "nauc_recall_at_1000_max": 0.90566, + "nauc_recall_at_1000_std": 0.941, + "nauc_recall_at_1000_diff1": -0.698104, + "nauc_precision_at_1_max": 0.351987, + "nauc_precision_at_1_std": -0.054405, + "nauc_precision_at_1_diff1": 0.438232, + "nauc_precision_at_3_max": -0.012523, + "nauc_precision_at_3_std": -0.065107, + "nauc_precision_at_3_diff1": -0.107777, + "nauc_precision_at_5_max": -0.112123, + "nauc_precision_at_5_std": -0.07081, + "nauc_precision_at_5_diff1": -0.230431, + "nauc_precision_at_10_max": -0.151535, + "nauc_precision_at_10_std": -0.084969, + "nauc_precision_at_10_diff1": -0.252868, + "nauc_precision_at_20_max": -0.164624, + "nauc_precision_at_20_std": -0.009819, + "nauc_precision_at_20_diff1": -0.293617, + "nauc_precision_at_100_max": -0.240083, + "nauc_precision_at_100_std": 0.033949, + "nauc_precision_at_100_diff1": -0.318366, + "nauc_precision_at_1000_max": -0.311234, + "nauc_precision_at_1000_std": 0.029489, + "nauc_precision_at_1000_diff1": -0.359165, + "nauc_cv_recall_at_1_max": 0.351987, + "nauc_cv_recall_at_1_std": -0.054405, + "nauc_cv_recall_at_1_diff1": 0.438232, + "nauc_cv_recall_at_3_max": 0.336053, + "nauc_cv_recall_at_3_std": -0.037887, + "nauc_cv_recall_at_3_diff1": 0.275329, + "nauc_cv_recall_at_5_max": 0.275475, + "nauc_cv_recall_at_5_std": -0.116428, + "nauc_cv_recall_at_5_diff1": 0.190873, + "nauc_cv_recall_at_10_max": 0.605587, + "nauc_cv_recall_at_10_std": 0.125039, + "nauc_cv_recall_at_10_diff1": 0.250272, + "nauc_cv_recall_at_20_max": 0.512547, + "nauc_cv_recall_at_20_std": 0.287932, + "nauc_cv_recall_at_20_diff1": 0.027719, + "nauc_cv_recall_at_100_max": 0.239185, + "nauc_cv_recall_at_100_std": 0.742453, + "nauc_cv_recall_at_100_diff1": -0.75957, + "nauc_cv_recall_at_1000_max": 0.869281, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_1000_diff1": -0.563492, + "nauc_mrr_at_1_max": 0.351987, + "nauc_mrr_at_1_std": -0.054405, + "nauc_mrr_at_1_diff1": 0.438232, + "nauc_mrr_at_3_max": 0.348409, + "nauc_mrr_at_3_std": -0.042703, + "nauc_mrr_at_3_diff1": 0.378716, + "nauc_mrr_at_5_max": 0.342925, + "nauc_mrr_at_5_std": -0.05023, + "nauc_mrr_at_5_diff1": 0.37675, + "nauc_mrr_at_10_max": 0.360368, + "nauc_mrr_at_10_std": -0.036972, + "nauc_mrr_at_10_diff1": 0.384822, + "nauc_mrr_at_20_max": 0.357094, + "nauc_mrr_at_20_std": -0.036169, + "nauc_mrr_at_20_diff1": 0.382223, + "nauc_mrr_at_100_max": 0.35591, + "nauc_mrr_at_100_std": -0.036365, + "nauc_mrr_at_100_diff1": 0.382325, + "nauc_mrr_at_1000_max": 0.355883, + "nauc_mrr_at_1000_std": -0.036513, + "nauc_mrr_at_1000_diff1": 0.382529, + "main_score": 0.63935, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.55625, + "ndcg_at_3": 0.59202, + "ndcg_at_5": 0.61102, + "ndcg_at_10": 0.64062, + "ndcg_at_20": 0.66768, + "ndcg_at_100": 0.70126, + "ndcg_at_1000": 0.71757, + "map_at_1": 0.34513, + "map_at_3": 0.48418, + "map_at_5": 0.52146, + "map_at_10": 0.55233, + "map_at_20": 0.56724, + "map_at_100": 0.578, + "map_at_1000": 0.57968, + "recall_at_1": 0.34513, + "recall_at_3": 0.56983, + "recall_at_5": 0.64436, + "recall_at_10": 0.73204, + "recall_at_20": 0.80954, + "recall_at_100": 0.91844, + "recall_at_1000": 0.99026, + "cv_recall_at_1": 0.55625, + "cv_recall_at_3": 0.80625, + "cv_recall_at_5": 0.85625, + "cv_recall_at_10": 0.9125, + "cv_recall_at_20": 0.95625, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 0.99375, + "precision_at_1": 0.55625, + "precision_at_3": 0.36458, + "precision_at_5": 0.275, + "precision_at_10": 0.17438, + "precision_at_20": 0.10281, + "precision_at_100": 0.02669, + "precision_at_1000": 0.0032, + "mrr_at_1": 0.55625, + "mrr_at_3": 0.670833, + "mrr_at_5": 0.683021, + "mrr_at_10": 0.690546, + "mrr_at_20": 0.693602, + "mrr_at_100": 0.694467, + "mrr_at_1000": 0.694507, + "nauc_ndcg_at_1_max": 0.3239, + "nauc_ndcg_at_1_std": 0.052891, + "nauc_ndcg_at_1_diff1": 0.555113, + "nauc_ndcg_at_3_max": 0.308389, + "nauc_ndcg_at_3_std": 0.065977, + "nauc_ndcg_at_3_diff1": 0.465493, + "nauc_ndcg_at_5_max": 0.339405, + "nauc_ndcg_at_5_std": 0.078585, + "nauc_ndcg_at_5_diff1": 0.446528, + "nauc_ndcg_at_10_max": 0.344313, + "nauc_ndcg_at_10_std": 0.079479, + "nauc_ndcg_at_10_diff1": 0.425597, + "nauc_ndcg_at_20_max": 0.338742, + "nauc_ndcg_at_20_std": 0.072687, + "nauc_ndcg_at_20_diff1": 0.427008, + "nauc_ndcg_at_100_max": 0.344391, + "nauc_ndcg_at_100_std": 0.10623, + "nauc_ndcg_at_100_diff1": 0.441037, + "nauc_ndcg_at_1000_max": 0.329252, + "nauc_ndcg_at_1000_std": 0.086877, + "nauc_ndcg_at_1000_diff1": 0.43912, + "nauc_map_at_1_max": 0.336691, + "nauc_map_at_1_std": 0.084594, + "nauc_map_at_1_diff1": 0.574735, + "nauc_map_at_3_max": 0.317964, + "nauc_map_at_3_std": 0.050438, + "nauc_map_at_3_diff1": 0.490579, + "nauc_map_at_5_max": 0.326439, + "nauc_map_at_5_std": 0.042715, + "nauc_map_at_5_diff1": 0.463839, + "nauc_map_at_10_max": 0.322931, + "nauc_map_at_10_std": 0.053802, + "nauc_map_at_10_diff1": 0.44688, + "nauc_map_at_20_max": 0.322933, + "nauc_map_at_20_std": 0.055777, + "nauc_map_at_20_diff1": 0.449956, + "nauc_map_at_100_max": 0.329124, + "nauc_map_at_100_std": 0.069, + "nauc_map_at_100_diff1": 0.453002, + "nauc_map_at_1000_max": 0.327509, + "nauc_map_at_1000_std": 0.067456, + "nauc_map_at_1000_diff1": 0.452705, + "nauc_recall_at_1_max": 0.336691, + "nauc_recall_at_1_std": 0.084594, + "nauc_recall_at_1_diff1": 0.574735, + "nauc_recall_at_3_max": 0.273244, + "nauc_recall_at_3_std": 0.023561, + "nauc_recall_at_3_diff1": 0.392585, + "nauc_recall_at_5_max": 0.314399, + "nauc_recall_at_5_std": 0.032292, + "nauc_recall_at_5_diff1": 0.326745, + "nauc_recall_at_10_max": 0.333223, + "nauc_recall_at_10_std": 0.071948, + "nauc_recall_at_10_diff1": 0.243738, + "nauc_recall_at_20_max": 0.334498, + "nauc_recall_at_20_std": 0.06747, + "nauc_recall_at_20_diff1": 0.201312, + "nauc_recall_at_100_max": 0.444262, + "nauc_recall_at_100_std": 0.357126, + "nauc_recall_at_100_diff1": 0.240105, + "nauc_recall_at_1000_max": 0.916143, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": -0.432764, + "nauc_precision_at_1_max": 0.3239, + "nauc_precision_at_1_std": 0.052891, + "nauc_precision_at_1_diff1": 0.555113, + "nauc_precision_at_3_max": 0.103277, + "nauc_precision_at_3_std": 0.047226, + "nauc_precision_at_3_diff1": 0.075166, + "nauc_precision_at_5_max": 0.049229, + "nauc_precision_at_5_std": 0.051225, + "nauc_precision_at_5_diff1": -0.069013, + "nauc_precision_at_10_max": -0.018805, + "nauc_precision_at_10_std": 0.061807, + "nauc_precision_at_10_diff1": -0.162361, + "nauc_precision_at_20_max": -0.075665, + "nauc_precision_at_20_std": 0.045853, + "nauc_precision_at_20_diff1": -0.194453, + "nauc_precision_at_100_max": -0.115563, + "nauc_precision_at_100_std": 0.092271, + "nauc_precision_at_100_diff1": -0.221101, + "nauc_precision_at_1000_max": -0.22995, + "nauc_precision_at_1000_std": -0.041227, + "nauc_precision_at_1000_diff1": -0.268731, + "nauc_cv_recall_at_1_max": 0.3239, + "nauc_cv_recall_at_1_std": 0.052891, + "nauc_cv_recall_at_1_diff1": 0.555113, + "nauc_cv_recall_at_3_max": 0.331935, + "nauc_cv_recall_at_3_std": 0.077554, + "nauc_cv_recall_at_3_diff1": 0.430468, + "nauc_cv_recall_at_5_max": 0.524881, + "nauc_cv_recall_at_5_std": 0.263413, + "nauc_cv_recall_at_5_diff1": 0.408976, + "nauc_cv_recall_at_10_max": 0.592737, + "nauc_cv_recall_at_10_std": 0.330932, + "nauc_cv_recall_at_10_diff1": 0.276177, + "nauc_cv_recall_at_20_max": 0.36588, + "nauc_cv_recall_at_20_std": 0.171269, + "nauc_cv_recall_at_20_diff1": -0.0825, + "nauc_cv_recall_at_100_max": 0.146592, + "nauc_cv_recall_at_100_std": 0.46094, + "nauc_cv_recall_at_100_diff1": -0.82493, + "nauc_cv_recall_at_1000_max": 0.869281, + "nauc_cv_recall_at_1000_std": 1.0, + "nauc_cv_recall_at_1000_diff1": -1.151727, + "nauc_mrr_at_1_max": 0.3239, + "nauc_mrr_at_1_std": 0.052891, + "nauc_mrr_at_1_diff1": 0.555113, + "nauc_mrr_at_3_max": 0.326217, + "nauc_mrr_at_3_std": 0.0618, + "nauc_mrr_at_3_diff1": 0.51271, + "nauc_mrr_at_5_max": 0.353179, + "nauc_mrr_at_5_std": 0.087487, + "nauc_mrr_at_5_diff1": 0.513653, + "nauc_mrr_at_10_max": 0.348862, + "nauc_mrr_at_10_std": 0.082654, + "nauc_mrr_at_10_diff1": 0.509639, + "nauc_mrr_at_20_max": 0.342961, + "nauc_mrr_at_20_std": 0.077382, + "nauc_mrr_at_20_diff1": 0.507892, + "nauc_mrr_at_100_max": 0.341934, + "nauc_mrr_at_100_std": 0.077162, + "nauc_mrr_at_100_diff1": 0.507908, + "nauc_mrr_at_1000_max": 0.341979, + "nauc_mrr_at_1000_std": 0.077124, + "nauc_mrr_at_1000_diff1": 0.508078, + "main_score": 0.61102, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 454.06863832473755, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..89c79ea8dd --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.52564, + "ndcg_at_3": 0.53678, + "ndcg_at_5": 0.56258, + "ndcg_at_10": 0.60728, + "ndcg_at_20": 0.62512, + "ndcg_at_100": 0.6438, + "ndcg_at_1000": 0.64727, + "map_at_1": 0.35705, + "map_at_3": 0.46463, + "map_at_5": 0.50013, + "map_at_10": 0.5298, + "map_at_20": 0.53801, + "map_at_100": 0.54335, + "map_at_1000": 0.54354, + "recall_at_1": 0.35705, + "recall_at_3": 0.5088, + "recall_at_5": 0.60674, + "recall_at_10": 0.74172, + "recall_at_20": 0.79209, + "recall_at_100": 0.85534, + "recall_at_1000": 0.87085, + "cv_recall_at_1": 0.53846, + "cv_recall_at_3": 0.67308, + "cv_recall_at_5": 0.76923, + "cv_recall_at_10": 0.86538, + "cv_recall_at_20": 0.88462, + "cv_recall_at_100": 0.92308, + "cv_recall_at_1000": 0.92308, + "precision_at_1": 0.53846, + "precision_at_3": 0.3141, + "precision_at_5": 0.23462, + "precision_at_10": 0.15, + "precision_at_20": 0.08365, + "precision_at_100": 0.01942, + "precision_at_1000": 0.00206, + "mrr_at_1": 0.538462, + "mrr_at_3": 0.602564, + "mrr_at_5": 0.623718, + "mrr_at_10": 0.637088, + "mrr_at_20": 0.638156, + "mrr_at_100": 0.6398, + "mrr_at_1000": 0.6398, + "nauc_ndcg_at_1_max": -0.006873, + "nauc_ndcg_at_1_std": -0.115874, + "nauc_ndcg_at_1_diff1": 0.653078, + "nauc_ndcg_at_3_max": 0.030291, + "nauc_ndcg_at_3_std": -0.128955, + "nauc_ndcg_at_3_diff1": 0.621676, + "nauc_ndcg_at_5_max": 0.051193, + "nauc_ndcg_at_5_std": -0.11253, + "nauc_ndcg_at_5_diff1": 0.603302, + "nauc_ndcg_at_10_max": 0.004413, + "nauc_ndcg_at_10_std": -0.171094, + "nauc_ndcg_at_10_diff1": 0.585866, + "nauc_ndcg_at_20_max": 0.028621, + "nauc_ndcg_at_20_std": -0.165573, + "nauc_ndcg_at_20_diff1": 0.57246, + "nauc_ndcg_at_100_max": 0.005829, + "nauc_ndcg_at_100_std": -0.171213, + "nauc_ndcg_at_100_diff1": 0.581426, + "nauc_ndcg_at_1000_max": -0.005794, + "nauc_ndcg_at_1000_std": -0.181419, + "nauc_ndcg_at_1000_diff1": 0.578665, + "nauc_map_at_1_max": 0.10908, + "nauc_map_at_1_std": -0.016547, + "nauc_map_at_1_diff1": 0.798757, + "nauc_map_at_3_max": 0.087311, + "nauc_map_at_3_std": -0.102847, + "nauc_map_at_3_diff1": 0.67621, + "nauc_map_at_5_max": 0.090685, + "nauc_map_at_5_std": -0.096318, + "nauc_map_at_5_diff1": 0.651714, + "nauc_map_at_10_max": 0.073197, + "nauc_map_at_10_std": -0.112653, + "nauc_map_at_10_diff1": 0.641118, + "nauc_map_at_20_max": 0.086573, + "nauc_map_at_20_std": -0.103388, + "nauc_map_at_20_diff1": 0.635758, + "nauc_map_at_100_max": 0.081848, + "nauc_map_at_100_std": -0.100852, + "nauc_map_at_100_diff1": 0.636786, + "nauc_map_at_1000_max": 0.081376, + "nauc_map_at_1000_std": -0.101278, + "nauc_map_at_1000_diff1": 0.636743, + "nauc_recall_at_1_max": 0.10908, + "nauc_recall_at_1_std": -0.016547, + "nauc_recall_at_1_diff1": 0.798757, + "nauc_recall_at_3_max": 0.055484, + "nauc_recall_at_3_std": -0.150288, + "nauc_recall_at_3_diff1": 0.607356, + "nauc_recall_at_5_max": 0.089113, + "nauc_recall_at_5_std": -0.146594, + "nauc_recall_at_5_diff1": 0.491119, + "nauc_recall_at_10_max": -0.106264, + "nauc_recall_at_10_std": -0.306044, + "nauc_recall_at_10_diff1": 0.35421, + "nauc_recall_at_20_max": -0.07534, + "nauc_recall_at_20_std": -0.335822, + "nauc_recall_at_20_diff1": 0.270151, + "nauc_recall_at_100_max": -0.217213, + "nauc_recall_at_100_std": -0.451964, + "nauc_recall_at_100_diff1": 0.286193, + "nauc_recall_at_1000_max": -0.348085, + "nauc_recall_at_1000_std": -0.579415, + "nauc_recall_at_1000_diff1": 0.236968, + "nauc_precision_at_1_max": -0.028036, + "nauc_precision_at_1_std": -0.096459, + "nauc_precision_at_1_diff1": 0.657316, + "nauc_precision_at_3_max": -0.137687, + "nauc_precision_at_3_std": -0.174772, + "nauc_precision_at_3_diff1": 0.132177, + "nauc_precision_at_5_max": -0.103355, + "nauc_precision_at_5_std": -0.086118, + "nauc_precision_at_5_diff1": -0.022199, + "nauc_precision_at_10_max": -0.127403, + "nauc_precision_at_10_std": -0.040621, + "nauc_precision_at_10_diff1": -0.175909, + "nauc_precision_at_20_max": -0.066029, + "nauc_precision_at_20_std": 0.006654, + "nauc_precision_at_20_diff1": -0.24687, + "nauc_precision_at_100_max": -0.154761, + "nauc_precision_at_100_std": 0.030852, + "nauc_precision_at_100_diff1": -0.286049, + "nauc_precision_at_1000_max": -0.200742, + "nauc_precision_at_1000_std": -0.02002, + "nauc_precision_at_1000_diff1": -0.297478, + "nauc_cv_recall_at_1_max": -0.028036, + "nauc_cv_recall_at_1_std": -0.096459, + "nauc_cv_recall_at_1_diff1": 0.657316, + "nauc_cv_recall_at_3_max": -0.18862, + "nauc_cv_recall_at_3_std": -0.254613, + "nauc_cv_recall_at_3_diff1": 0.56085, + "nauc_cv_recall_at_5_max": -0.017911, + "nauc_cv_recall_at_5_std": -0.24444, + "nauc_cv_recall_at_5_diff1": 0.437267, + "nauc_cv_recall_at_10_max": -0.369317, + "nauc_cv_recall_at_10_std": -0.568087, + "nauc_cv_recall_at_10_diff1": 0.202074, + "nauc_cv_recall_at_20_max": -0.395133, + "nauc_cv_recall_at_20_std": -0.623358, + "nauc_cv_recall_at_20_diff1": 0.137596, + "nauc_cv_recall_at_100_max": -0.691826, + "nauc_cv_recall_at_100_std": -0.915298, + "nauc_cv_recall_at_100_diff1": 0.122473, + "nauc_cv_recall_at_1000_max": -0.691826, + "nauc_cv_recall_at_1000_std": -0.915298, + "nauc_cv_recall_at_1000_diff1": 0.122473, + "nauc_mrr_at_1_max": -0.028036, + "nauc_mrr_at_1_std": -0.096459, + "nauc_mrr_at_1_diff1": 0.657316, + "nauc_mrr_at_3_max": -0.091894, + "nauc_mrr_at_3_std": -0.16061, + "nauc_mrr_at_3_diff1": 0.614245, + "nauc_mrr_at_5_max": -0.057306, + "nauc_mrr_at_5_std": -0.150925, + "nauc_mrr_at_5_diff1": 0.599501, + "nauc_mrr_at_10_max": -0.079952, + "nauc_mrr_at_10_std": -0.16812, + "nauc_mrr_at_10_diff1": 0.591194, + "nauc_mrr_at_20_max": -0.079571, + "nauc_mrr_at_20_std": -0.168031, + "nauc_mrr_at_20_diff1": 0.591042, + "nauc_mrr_at_100_max": -0.081031, + "nauc_mrr_at_100_std": -0.168749, + "nauc_mrr_at_100_diff1": 0.593608, + "nauc_mrr_at_1000_max": -0.081031, + "nauc_mrr_at_1000_std": -0.168749, + "nauc_mrr_at_1000_diff1": 0.593608, + "main_score": 0.56258, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 287.26457595825195, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..68d6b12c6b --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.49123, + "ndcg_at_3": 0.46743, + "ndcg_at_5": 0.4803, + "ndcg_at_10": 0.52543, + "ndcg_at_20": 0.55093, + "ndcg_at_100": 0.58502, + "ndcg_at_1000": 0.58796, + "map_at_1": 0.23952, + "map_at_3": 0.33183, + "map_at_5": 0.36153, + "map_at_10": 0.40642, + "map_at_20": 0.4224, + "map_at_100": 0.44008, + "map_at_1000": 0.44035, + "recall_at_1": 0.23952, + "recall_at_3": 0.40725, + "recall_at_5": 0.4859, + "recall_at_10": 0.61702, + "recall_at_20": 0.70505, + "recall_at_100": 0.80075, + "recall_at_1000": 0.81976, + "cv_recall_at_1": 0.49123, + "cv_recall_at_3": 0.70175, + "cv_recall_at_5": 0.77193, + "cv_recall_at_10": 0.84211, + "cv_recall_at_20": 0.89474, + "cv_recall_at_100": 0.92982, + "cv_recall_at_1000": 0.94737, + "precision_at_1": 0.49123, + "precision_at_3": 0.30994, + "precision_at_5": 0.2386, + "precision_at_10": 0.17544, + "precision_at_20": 0.10965, + "precision_at_100": 0.03246, + "precision_at_1000": 0.0033, + "mrr_at_1": 0.491228, + "mrr_at_3": 0.587719, + "mrr_at_5": 0.603509, + "mrr_at_10": 0.613499, + "mrr_at_20": 0.61652, + "mrr_at_100": 0.61765, + "mrr_at_1000": 0.617745, + "nauc_ndcg_at_1_max": -0.158453, + "nauc_ndcg_at_1_std": -0.16578, + "nauc_ndcg_at_1_diff1": 0.219851, + "nauc_ndcg_at_3_max": -0.087708, + "nauc_ndcg_at_3_std": -0.162574, + "nauc_ndcg_at_3_diff1": -0.005327, + "nauc_ndcg_at_5_max": -0.02982, + "nauc_ndcg_at_5_std": -0.160219, + "nauc_ndcg_at_5_diff1": -0.025982, + "nauc_ndcg_at_10_max": -0.014615, + "nauc_ndcg_at_10_std": -0.110169, + "nauc_ndcg_at_10_diff1": 0.034133, + "nauc_ndcg_at_20_max": -0.008272, + "nauc_ndcg_at_20_std": -0.097219, + "nauc_ndcg_at_20_diff1": 0.010941, + "nauc_ndcg_at_100_max": -0.05176, + "nauc_ndcg_at_100_std": -0.055711, + "nauc_ndcg_at_100_diff1": -0.041458, + "nauc_ndcg_at_1000_max": -0.04935, + "nauc_ndcg_at_1000_std": -0.066327, + "nauc_ndcg_at_1000_diff1": -0.037212, + "nauc_map_at_1_max": 0.059809, + "nauc_map_at_1_std": -0.097321, + "nauc_map_at_1_diff1": 0.016209, + "nauc_map_at_3_max": 0.05107, + "nauc_map_at_3_std": -0.107651, + "nauc_map_at_3_diff1": -0.036284, + "nauc_map_at_5_max": 0.068563, + "nauc_map_at_5_std": -0.109937, + "nauc_map_at_5_diff1": -0.023415, + "nauc_map_at_10_max": 0.048134, + "nauc_map_at_10_std": -0.105734, + "nauc_map_at_10_diff1": 0.011276, + "nauc_map_at_20_max": 0.034481, + "nauc_map_at_20_std": -0.099003, + "nauc_map_at_20_diff1": -0.001954, + "nauc_map_at_100_max": 0.004804, + "nauc_map_at_100_std": -0.08898, + "nauc_map_at_100_diff1": -0.021978, + "nauc_map_at_1000_max": 0.004336, + "nauc_map_at_1000_std": -0.089985, + "nauc_map_at_1000_diff1": -0.022033, + "nauc_recall_at_1_max": 0.059809, + "nauc_recall_at_1_std": -0.097321, + "nauc_recall_at_1_diff1": 0.016209, + "nauc_recall_at_3_max": 0.056353, + "nauc_recall_at_3_std": -0.078525, + "nauc_recall_at_3_diff1": -0.06985, + "nauc_recall_at_5_max": 0.18035, + "nauc_recall_at_5_std": -0.065609, + "nauc_recall_at_5_diff1": -0.037682, + "nauc_recall_at_10_max": 0.129413, + "nauc_recall_at_10_std": -0.017822, + "nauc_recall_at_10_diff1": 0.032293, + "nauc_recall_at_20_max": 0.187814, + "nauc_recall_at_20_std": 0.050267, + "nauc_recall_at_20_diff1": -0.055222, + "nauc_recall_at_100_max": 0.131225, + "nauc_recall_at_100_std": 0.211417, + "nauc_recall_at_100_diff1": -0.289877, + "nauc_recall_at_1000_max": 0.196881, + "nauc_recall_at_1000_std": 0.142589, + "nauc_recall_at_1000_diff1": -0.259399, + "nauc_precision_at_1_max": -0.158453, + "nauc_precision_at_1_std": -0.16578, + "nauc_precision_at_1_diff1": 0.219851, + "nauc_precision_at_3_max": -0.245123, + "nauc_precision_at_3_std": -0.190767, + "nauc_precision_at_3_diff1": 0.019365, + "nauc_precision_at_5_max": -0.205883, + "nauc_precision_at_5_std": -0.153694, + "nauc_precision_at_5_diff1": 0.025285, + "nauc_precision_at_10_max": -0.236443, + "nauc_precision_at_10_std": -0.058601, + "nauc_precision_at_10_diff1": 0.031466, + "nauc_precision_at_20_max": -0.269961, + "nauc_precision_at_20_std": -0.029786, + "nauc_precision_at_20_diff1": -0.036292, + "nauc_precision_at_100_max": -0.277148, + "nauc_precision_at_100_std": 0.110522, + "nauc_precision_at_100_diff1": -0.108057, + "nauc_precision_at_1000_max": -0.281146, + "nauc_precision_at_1000_std": 0.094774, + "nauc_precision_at_1000_diff1": -0.107483, + "nauc_cv_recall_at_1_max": -0.158453, + "nauc_cv_recall_at_1_std": -0.16578, + "nauc_cv_recall_at_1_diff1": 0.219851, + "nauc_cv_recall_at_3_max": -0.318264, + "nauc_cv_recall_at_3_std": -0.320596, + "nauc_cv_recall_at_3_diff1": 0.005994, + "nauc_cv_recall_at_5_max": -0.041713, + "nauc_cv_recall_at_5_std": -0.235398, + "nauc_cv_recall_at_5_diff1": 0.042987, + "nauc_cv_recall_at_10_max": -0.105386, + "nauc_cv_recall_at_10_std": 0.049689, + "nauc_cv_recall_at_10_diff1": -0.10146, + "nauc_cv_recall_at_20_max": -0.171533, + "nauc_cv_recall_at_20_std": 0.086009, + "nauc_cv_recall_at_20_diff1": -0.116724, + "nauc_cv_recall_at_100_max": -0.182169, + "nauc_cv_recall_at_100_std": 0.310977, + "nauc_cv_recall_at_100_diff1": -0.530985, + "nauc_cv_recall_at_1000_max": -0.056422, + "nauc_cv_recall_at_1000_std": 0.173726, + "nauc_cv_recall_at_1000_diff1": -0.521511, + "nauc_mrr_at_1_max": -0.158453, + "nauc_mrr_at_1_std": -0.16578, + "nauc_mrr_at_1_diff1": 0.219851, + "nauc_mrr_at_3_max": -0.232383, + "nauc_mrr_at_3_std": -0.24422, + "nauc_mrr_at_3_diff1": 0.112262, + "nauc_mrr_at_5_max": -0.186338, + "nauc_mrr_at_5_std": -0.225199, + "nauc_mrr_at_5_diff1": 0.116004, + "nauc_mrr_at_10_max": -0.192177, + "nauc_mrr_at_10_std": -0.198164, + "nauc_mrr_at_10_diff1": 0.108881, + "nauc_mrr_at_20_max": -0.194687, + "nauc_mrr_at_20_std": -0.200567, + "nauc_mrr_at_20_diff1": 0.109198, + "nauc_mrr_at_100_max": -0.193885, + "nauc_mrr_at_100_std": -0.198612, + "nauc_mrr_at_100_diff1": 0.106663, + "nauc_mrr_at_1000_max": -0.193764, + "nauc_mrr_at_1000_std": -0.198918, + "nauc_mrr_at_1000_diff1": 0.106884, + "main_score": 0.4803, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.50877, + "ndcg_at_3": 0.47647, + "ndcg_at_5": 0.49456, + "ndcg_at_10": 0.52704, + "ndcg_at_20": 0.55699, + "ndcg_at_100": 0.58756, + "ndcg_at_1000": 0.5901, + "map_at_1": 0.22153, + "map_at_3": 0.32581, + "map_at_5": 0.36374, + "map_at_10": 0.40195, + "map_at_20": 0.42174, + "map_at_100": 0.43937, + "map_at_1000": 0.43954, + "recall_at_1": 0.22153, + "recall_at_3": 0.40798, + "recall_at_5": 0.50115, + "recall_at_10": 0.62813, + "recall_at_20": 0.7226, + "recall_at_100": 0.80148, + "recall_at_1000": 0.81976, + "cv_recall_at_1": 0.50877, + "cv_recall_at_3": 0.73684, + "cv_recall_at_5": 0.80702, + "cv_recall_at_10": 0.85965, + "cv_recall_at_20": 0.92982, + "cv_recall_at_100": 0.92982, + "cv_recall_at_1000": 0.94737, + "precision_at_1": 0.50877, + "precision_at_3": 0.32749, + "precision_at_5": 0.25965, + "precision_at_10": 0.17193, + "precision_at_20": 0.1114, + "precision_at_100": 0.03263, + "precision_at_1000": 0.0033, + "mrr_at_1": 0.508772, + "mrr_at_3": 0.608187, + "mrr_at_5": 0.623977, + "mrr_at_10": 0.631287, + "mrr_at_20": 0.636132, + "mrr_at_100": 0.636132, + "mrr_at_1000": 0.636208, + "nauc_ndcg_at_1_max": -0.195537, + "nauc_ndcg_at_1_std": -0.213457, + "nauc_ndcg_at_1_diff1": 0.023997, + "nauc_ndcg_at_3_max": -0.106376, + "nauc_ndcg_at_3_std": -0.137185, + "nauc_ndcg_at_3_diff1": -0.075091, + "nauc_ndcg_at_5_max": -0.072566, + "nauc_ndcg_at_5_std": -0.122428, + "nauc_ndcg_at_5_diff1": -0.057007, + "nauc_ndcg_at_10_max": -0.025595, + "nauc_ndcg_at_10_std": -0.154587, + "nauc_ndcg_at_10_diff1": -0.032966, + "nauc_ndcg_at_20_max": -0.02834, + "nauc_ndcg_at_20_std": -0.125457, + "nauc_ndcg_at_20_diff1": 0.00232, + "nauc_ndcg_at_100_max": -0.018808, + "nauc_ndcg_at_100_std": -0.040515, + "nauc_ndcg_at_100_diff1": -0.018039, + "nauc_ndcg_at_1000_max": -0.023973, + "nauc_ndcg_at_1000_std": -0.050816, + "nauc_ndcg_at_1000_diff1": -0.024781, + "nauc_map_at_1_max": 0.020323, + "nauc_map_at_1_std": -0.164888, + "nauc_map_at_1_diff1": -0.055786, + "nauc_map_at_3_max": 0.001008, + "nauc_map_at_3_std": -0.143155, + "nauc_map_at_3_diff1": -0.096099, + "nauc_map_at_5_max": -0.00089, + "nauc_map_at_5_std": -0.154746, + "nauc_map_at_5_diff1": -0.058322, + "nauc_map_at_10_max": 0.011278, + "nauc_map_at_10_std": -0.153372, + "nauc_map_at_10_diff1": -0.041986, + "nauc_map_at_20_max": -0.005966, + "nauc_map_at_20_std": -0.143261, + "nauc_map_at_20_diff1": -0.027712, + "nauc_map_at_100_max": -0.014779, + "nauc_map_at_100_std": -0.116396, + "nauc_map_at_100_diff1": -0.037301, + "nauc_map_at_1000_max": -0.015277, + "nauc_map_at_1000_std": -0.11704, + "nauc_map_at_1000_diff1": -0.037761, + "nauc_recall_at_1_max": 0.020323, + "nauc_recall_at_1_std": -0.164888, + "nauc_recall_at_1_diff1": -0.055786, + "nauc_recall_at_3_max": 0.042953, + "nauc_recall_at_3_std": -0.056066, + "nauc_recall_at_3_diff1": -0.14397, + "nauc_recall_at_5_max": 0.113789, + "nauc_recall_at_5_std": -0.008886, + "nauc_recall_at_5_diff1": -0.02026, + "nauc_recall_at_10_max": 0.104897, + "nauc_recall_at_10_std": -0.125031, + "nauc_recall_at_10_diff1": -0.009852, + "nauc_recall_at_20_max": 0.098106, + "nauc_recall_at_20_std": -0.062273, + "nauc_recall_at_20_diff1": 0.058392, + "nauc_recall_at_100_max": 0.198088, + "nauc_recall_at_100_std": 0.246276, + "nauc_recall_at_100_diff1": 0.000922, + "nauc_recall_at_1000_max": 0.172422, + "nauc_recall_at_1000_std": 0.16908, + "nauc_recall_at_1000_diff1": -0.063947, + "nauc_precision_at_1_max": -0.195537, + "nauc_precision_at_1_std": -0.213457, + "nauc_precision_at_1_diff1": 0.023997, + "nauc_precision_at_3_max": -0.159425, + "nauc_precision_at_3_std": -0.045416, + "nauc_precision_at_3_diff1": 0.031929, + "nauc_precision_at_5_max": -0.158937, + "nauc_precision_at_5_std": -0.025299, + "nauc_precision_at_5_diff1": 0.058088, + "nauc_precision_at_10_max": -0.147745, + "nauc_precision_at_10_std": -0.009763, + "nauc_precision_at_10_diff1": 0.037735, + "nauc_precision_at_20_max": -0.188833, + "nauc_precision_at_20_std": 0.037383, + "nauc_precision_at_20_diff1": 0.024967, + "nauc_precision_at_100_max": -0.120458, + "nauc_precision_at_100_std": 0.202082, + "nauc_precision_at_100_diff1": -0.031123, + "nauc_precision_at_1000_max": -0.127139, + "nauc_precision_at_1000_std": 0.192172, + "nauc_precision_at_1000_diff1": -0.037724, + "nauc_cv_recall_at_1_max": -0.195537, + "nauc_cv_recall_at_1_std": -0.213457, + "nauc_cv_recall_at_1_diff1": 0.023997, + "nauc_cv_recall_at_3_max": -0.13837, + "nauc_cv_recall_at_3_std": -0.103201, + "nauc_cv_recall_at_3_diff1": -0.084696, + "nauc_cv_recall_at_5_max": 0.060109, + "nauc_cv_recall_at_5_std": 0.179943, + "nauc_cv_recall_at_5_diff1": 0.046868, + "nauc_cv_recall_at_10_max": 0.030454, + "nauc_cv_recall_at_10_std": -0.011377, + "nauc_cv_recall_at_10_diff1": 0.016958, + "nauc_cv_recall_at_20_max": 0.045987, + "nauc_cv_recall_at_20_std": 0.424674, + "nauc_cv_recall_at_20_diff1": 0.13749, + "nauc_cv_recall_at_100_max": 0.045987, + "nauc_cv_recall_at_100_std": 0.424674, + "nauc_cv_recall_at_100_diff1": 0.13749, + "nauc_cv_recall_at_1000_max": -0.057969, + "nauc_cv_recall_at_1000_std": 0.276728, + "nauc_cv_recall_at_1000_diff1": -0.001708, + "nauc_mrr_at_1_max": -0.195537, + "nauc_mrr_at_1_std": -0.213457, + "nauc_mrr_at_1_diff1": 0.023997, + "nauc_mrr_at_3_max": -0.187877, + "nauc_mrr_at_3_std": -0.194133, + "nauc_mrr_at_3_diff1": -0.029521, + "nauc_mrr_at_5_max": -0.161981, + "nauc_mrr_at_5_std": -0.156675, + "nauc_mrr_at_5_diff1": -0.006207, + "nauc_mrr_at_10_max": -0.165611, + "nauc_mrr_at_10_std": -0.17635, + "nauc_mrr_at_10_diff1": -0.007651, + "nauc_mrr_at_20_max": -0.168403, + "nauc_mrr_at_20_std": -0.171319, + "nauc_mrr_at_20_diff1": -0.006569, + "nauc_mrr_at_100_max": -0.168403, + "nauc_mrr_at_100_std": -0.171319, + "nauc_mrr_at_100_diff1": -0.006569, + "nauc_mrr_at_1000_max": -0.168547, + "nauc_mrr_at_1000_std": -0.171604, + "nauc_mrr_at_1000_diff1": -0.006723, + "main_score": 0.49456, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.52632, + "ndcg_at_3": 0.48534, + "ndcg_at_5": 0.48091, + "ndcg_at_10": 0.53866, + "ndcg_at_20": 0.55681, + "ndcg_at_100": 0.58929, + "ndcg_at_1000": 0.59127, + "map_at_1": 0.22299, + "map_at_3": 0.3304, + "map_at_5": 0.35207, + "map_at_10": 0.40494, + "map_at_20": 0.42135, + "map_at_100": 0.4389, + "map_at_1000": 0.43894, + "recall_at_1": 0.22299, + "recall_at_3": 0.43367, + "recall_at_5": 0.47421, + "recall_at_10": 0.65777, + "recall_at_20": 0.71378, + "recall_at_100": 0.80221, + "recall_at_1000": 0.81976, + "cv_recall_at_1": 0.52632, + "cv_recall_at_3": 0.77193, + "cv_recall_at_5": 0.78947, + "cv_recall_at_10": 0.87719, + "cv_recall_at_20": 0.91228, + "cv_recall_at_100": 0.92982, + "cv_recall_at_1000": 0.94737, + "precision_at_1": 0.52632, + "precision_at_3": 0.33333, + "precision_at_5": 0.24561, + "precision_at_10": 0.18246, + "precision_at_20": 0.11316, + "precision_at_100": 0.03281, + "precision_at_1000": 0.0033, + "mrr_at_1": 0.526316, + "mrr_at_3": 0.625731, + "mrr_at_5": 0.62924, + "mrr_at_10": 0.641124, + "mrr_at_20": 0.643839, + "mrr_at_100": 0.644355, + "mrr_at_1000": 0.644392, + "nauc_ndcg_at_1_max": -0.065006, + "nauc_ndcg_at_1_std": -0.177376, + "nauc_ndcg_at_1_diff1": 0.259509, + "nauc_ndcg_at_3_max": 0.112058, + "nauc_ndcg_at_3_std": -0.060983, + "nauc_ndcg_at_3_diff1": 0.106015, + "nauc_ndcg_at_5_max": 0.106767, + "nauc_ndcg_at_5_std": -0.067516, + "nauc_ndcg_at_5_diff1": 0.070245, + "nauc_ndcg_at_10_max": 0.125224, + "nauc_ndcg_at_10_std": -0.078424, + "nauc_ndcg_at_10_diff1": 0.134397, + "nauc_ndcg_at_20_max": 0.128095, + "nauc_ndcg_at_20_std": -0.0875, + "nauc_ndcg_at_20_diff1": 0.129946, + "nauc_ndcg_at_100_max": 0.132239, + "nauc_ndcg_at_100_std": 0.024237, + "nauc_ndcg_at_100_diff1": 0.062069, + "nauc_ndcg_at_1000_max": 0.138613, + "nauc_ndcg_at_1000_std": 0.019345, + "nauc_ndcg_at_1000_diff1": 0.053425, + "nauc_map_at_1_max": 0.246159, + "nauc_map_at_1_std": -0.072113, + "nauc_map_at_1_diff1": 0.20381, + "nauc_map_at_3_max": 0.278303, + "nauc_map_at_3_std": -0.010364, + "nauc_map_at_3_diff1": 0.105898, + "nauc_map_at_5_max": 0.250685, + "nauc_map_at_5_std": -0.039576, + "nauc_map_at_5_diff1": 0.120716, + "nauc_map_at_10_max": 0.205391, + "nauc_map_at_10_std": -0.074998, + "nauc_map_at_10_diff1": 0.141737, + "nauc_map_at_20_max": 0.191162, + "nauc_map_at_20_std": -0.074746, + "nauc_map_at_20_diff1": 0.122403, + "nauc_map_at_100_max": 0.176309, + "nauc_map_at_100_std": -0.034253, + "nauc_map_at_100_diff1": 0.077127, + "nauc_map_at_1000_max": 0.176425, + "nauc_map_at_1000_std": -0.034346, + "nauc_map_at_1000_diff1": 0.076981, + "nauc_recall_at_1_max": 0.246159, + "nauc_recall_at_1_std": -0.072113, + "nauc_recall_at_1_diff1": 0.20381, + "nauc_recall_at_3_max": 0.29587, + "nauc_recall_at_3_std": 0.066861, + "nauc_recall_at_3_diff1": -0.002949, + "nauc_recall_at_5_max": 0.242743, + "nauc_recall_at_5_std": 0.015774, + "nauc_recall_at_5_diff1": 0.009652, + "nauc_recall_at_10_max": 0.182879, + "nauc_recall_at_10_std": -0.058818, + "nauc_recall_at_10_diff1": 0.089819, + "nauc_recall_at_20_max": 0.201083, + "nauc_recall_at_20_std": -0.085725, + "nauc_recall_at_20_diff1": 0.040753, + "nauc_recall_at_100_max": 0.333341, + "nauc_recall_at_100_std": 0.317935, + "nauc_recall_at_100_diff1": -0.181442, + "nauc_recall_at_1000_max": 0.425753, + "nauc_recall_at_1000_std": 0.285241, + "nauc_recall_at_1000_diff1": -0.308943, + "nauc_precision_at_1_max": -0.065006, + "nauc_precision_at_1_std": -0.177376, + "nauc_precision_at_1_diff1": 0.259509, + "nauc_precision_at_3_max": -0.105724, + "nauc_precision_at_3_std": -0.056129, + "nauc_precision_at_3_diff1": 0.113783, + "nauc_precision_at_5_max": -0.187473, + "nauc_precision_at_5_std": -0.06122, + "nauc_precision_at_5_diff1": 0.047798, + "nauc_precision_at_10_max": -0.256555, + "nauc_precision_at_10_std": -0.057591, + "nauc_precision_at_10_diff1": -0.017577, + "nauc_precision_at_20_max": -0.266114, + "nauc_precision_at_20_std": -0.028502, + "nauc_precision_at_20_diff1": -0.139428, + "nauc_precision_at_100_max": -0.235423, + "nauc_precision_at_100_std": 0.195526, + "nauc_precision_at_100_diff1": -0.294873, + "nauc_precision_at_1000_max": -0.233719, + "nauc_precision_at_1000_std": 0.193635, + "nauc_precision_at_1000_diff1": -0.301684, + "nauc_cv_recall_at_1_max": -0.065006, + "nauc_cv_recall_at_1_std": -0.177376, + "nauc_cv_recall_at_1_diff1": 0.259509, + "nauc_cv_recall_at_3_max": 0.018455, + "nauc_cv_recall_at_3_std": -0.067012, + "nauc_cv_recall_at_3_diff1": 0.215102, + "nauc_cv_recall_at_5_max": 0.008217, + "nauc_cv_recall_at_5_std": 0.084762, + "nauc_cv_recall_at_5_diff1": 0.150661, + "nauc_cv_recall_at_10_max": -0.044619, + "nauc_cv_recall_at_10_std": 0.191052, + "nauc_cv_recall_at_10_diff1": 0.282973, + "nauc_cv_recall_at_20_max": -0.139321, + "nauc_cv_recall_at_20_std": 0.046332, + "nauc_cv_recall_at_20_diff1": 0.356056, + "nauc_cv_recall_at_100_max": 0.118107, + "nauc_cv_recall_at_100_std": 0.489879, + "nauc_cv_recall_at_100_diff1": 0.306299, + "nauc_cv_recall_at_1000_max": 0.343945, + "nauc_cv_recall_at_1000_std": 0.468144, + "nauc_cv_recall_at_1000_diff1": 0.075065, + "nauc_mrr_at_1_max": -0.065006, + "nauc_mrr_at_1_std": -0.177376, + "nauc_mrr_at_1_diff1": 0.259509, + "nauc_mrr_at_3_max": -0.047434, + "nauc_mrr_at_3_std": -0.154799, + "nauc_mrr_at_3_diff1": 0.232806, + "nauc_mrr_at_5_max": -0.049406, + "nauc_mrr_at_5_std": -0.134997, + "nauc_mrr_at_5_diff1": 0.224225, + "nauc_mrr_at_10_max": -0.056394, + "nauc_mrr_at_10_std": -0.142017, + "nauc_mrr_at_10_diff1": 0.233101, + "nauc_mrr_at_20_max": -0.058572, + "nauc_mrr_at_20_std": -0.148541, + "nauc_mrr_at_20_diff1": 0.233949, + "nauc_mrr_at_100_max": -0.056477, + "nauc_mrr_at_100_std": -0.145561, + "nauc_mrr_at_100_diff1": 0.233343, + "nauc_mrr_at_1000_max": -0.056408, + "nauc_mrr_at_1000_std": -0.145657, + "nauc_mrr_at_1000_diff1": 0.233238, + "main_score": 0.48091, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.52632, + "ndcg_at_3": 0.46294, + "ndcg_at_5": 0.48116, + "ndcg_at_10": 0.51908, + "ndcg_at_20": 0.5602, + "ndcg_at_100": 0.59247, + "ndcg_at_1000": 0.59247, + "map_at_1": 0.24858, + "map_at_3": 0.33364, + "map_at_5": 0.36415, + "map_at_10": 0.40305, + "map_at_20": 0.42686, + "map_at_100": 0.4441, + "map_at_1000": 0.4441, + "recall_at_1": 0.24858, + "recall_at_3": 0.39916, + "recall_at_5": 0.47611, + "recall_at_10": 0.59213, + "recall_at_20": 0.72487, + "recall_at_100": 0.81976, + "recall_at_1000": 0.81976, + "cv_recall_at_1": 0.52632, + "cv_recall_at_3": 0.70175, + "cv_recall_at_5": 0.77193, + "cv_recall_at_10": 0.82456, + "cv_recall_at_20": 0.91228, + "cv_recall_at_100": 0.94737, + "cv_recall_at_1000": 0.94737, + "precision_at_1": 0.52632, + "precision_at_3": 0.29825, + "precision_at_5": 0.23509, + "precision_at_10": 0.16842, + "precision_at_20": 0.11404, + "precision_at_100": 0.03298, + "precision_at_1000": 0.0033, + "mrr_at_1": 0.526316, + "mrr_at_3": 0.605263, + "mrr_at_5": 0.62193, + "mrr_at_10": 0.630284, + "mrr_at_20": 0.636336, + "mrr_at_100": 0.637002, + "mrr_at_1000": 0.637002, + "nauc_ndcg_at_1_max": -0.034512, + "nauc_ndcg_at_1_std": -0.003783, + "nauc_ndcg_at_1_diff1": -0.064451, + "nauc_ndcg_at_3_max": 0.042916, + "nauc_ndcg_at_3_std": 0.013325, + "nauc_ndcg_at_3_diff1": -0.123193, + "nauc_ndcg_at_5_max": 0.132902, + "nauc_ndcg_at_5_std": 0.007523, + "nauc_ndcg_at_5_diff1": -0.07605, + "nauc_ndcg_at_10_max": 0.108219, + "nauc_ndcg_at_10_std": 0.014904, + "nauc_ndcg_at_10_diff1": -0.087151, + "nauc_ndcg_at_20_max": 0.087643, + "nauc_ndcg_at_20_std": 0.027988, + "nauc_ndcg_at_20_diff1": -0.116531, + "nauc_ndcg_at_100_max": 0.08414, + "nauc_ndcg_at_100_std": 0.100039, + "nauc_ndcg_at_100_diff1": -0.143861, + "nauc_ndcg_at_1000_max": 0.08414, + "nauc_ndcg_at_1000_std": 0.100039, + "nauc_ndcg_at_1000_diff1": -0.143861, + "nauc_map_at_1_max": 0.038093, + "nauc_map_at_1_std": 0.004225, + "nauc_map_at_1_diff1": -0.062685, + "nauc_map_at_3_max": 0.058418, + "nauc_map_at_3_std": 0.005264, + "nauc_map_at_3_diff1": -0.087846, + "nauc_map_at_5_max": 0.124865, + "nauc_map_at_5_std": -0.000785, + "nauc_map_at_5_diff1": -0.043921, + "nauc_map_at_10_max": 0.107166, + "nauc_map_at_10_std": 0.007442, + "nauc_map_at_10_diff1": -0.075036, + "nauc_map_at_20_max": 0.110669, + "nauc_map_at_20_std": 0.027352, + "nauc_map_at_20_diff1": -0.099554, + "nauc_map_at_100_max": 0.101811, + "nauc_map_at_100_std": 0.058411, + "nauc_map_at_100_diff1": -0.118707, + "nauc_map_at_1000_max": 0.101811, + "nauc_map_at_1000_std": 0.058411, + "nauc_map_at_1000_diff1": -0.118707, + "nauc_recall_at_1_max": 0.038093, + "nauc_recall_at_1_std": 0.004225, + "nauc_recall_at_1_diff1": -0.062685, + "nauc_recall_at_3_max": 0.062498, + "nauc_recall_at_3_std": -0.031641, + "nauc_recall_at_3_diff1": -0.083004, + "nauc_recall_at_5_max": 0.216632, + "nauc_recall_at_5_std": -0.049714, + "nauc_recall_at_5_diff1": 0.023291, + "nauc_recall_at_10_max": 0.176142, + "nauc_recall_at_10_std": -0.03559, + "nauc_recall_at_10_diff1": -0.006024, + "nauc_recall_at_20_max": 0.080968, + "nauc_recall_at_20_std": -0.02354, + "nauc_recall_at_20_diff1": -0.115838, + "nauc_recall_at_100_max": 0.128412, + "nauc_recall_at_100_std": 0.203062, + "nauc_recall_at_100_diff1": -0.218827, + "nauc_recall_at_1000_max": 0.128412, + "nauc_recall_at_1000_std": 0.203062, + "nauc_recall_at_1000_diff1": -0.218827, + "nauc_precision_at_1_max": -0.034512, + "nauc_precision_at_1_std": -0.003783, + "nauc_precision_at_1_diff1": -0.064451, + "nauc_precision_at_3_max": -0.01252, + "nauc_precision_at_3_std": 0.013137, + "nauc_precision_at_3_diff1": -0.102351, + "nauc_precision_at_5_max": 0.158201, + "nauc_precision_at_5_std": 0.073108, + "nauc_precision_at_5_diff1": -0.023109, + "nauc_precision_at_10_max": 0.050889, + "nauc_precision_at_10_std": 0.123627, + "nauc_precision_at_10_diff1": -0.12239, + "nauc_precision_at_20_max": 0.009538, + "nauc_precision_at_20_std": 0.172143, + "nauc_precision_at_20_diff1": -0.182545, + "nauc_precision_at_100_max": -0.058446, + "nauc_precision_at_100_std": 0.285065, + "nauc_precision_at_100_diff1": -0.1939, + "nauc_precision_at_1000_max": -0.058446, + "nauc_precision_at_1000_std": 0.285065, + "nauc_precision_at_1000_diff1": -0.1939, + "nauc_cv_recall_at_1_max": -0.034512, + "nauc_cv_recall_at_1_std": -0.003783, + "nauc_cv_recall_at_1_diff1": -0.064451, + "nauc_cv_recall_at_3_max": -0.085426, + "nauc_cv_recall_at_3_std": -0.121486, + "nauc_cv_recall_at_3_diff1": -0.177757, + "nauc_cv_recall_at_5_max": 0.11789, + "nauc_cv_recall_at_5_std": 0.03784, + "nauc_cv_recall_at_5_diff1": -0.143345, + "nauc_cv_recall_at_10_max": 0.228897, + "nauc_cv_recall_at_10_std": 0.153236, + "nauc_cv_recall_at_10_diff1": -0.054506, + "nauc_cv_recall_at_20_max": -0.16256, + "nauc_cv_recall_at_20_std": 0.11865, + "nauc_cv_recall_at_20_diff1": -0.251086, + "nauc_cv_recall_at_100_max": -0.029483, + "nauc_cv_recall_at_100_std": 0.4024, + "nauc_cv_recall_at_100_diff1": -0.472918, + "nauc_cv_recall_at_1000_max": -0.029483, + "nauc_cv_recall_at_1000_std": 0.4024, + "nauc_cv_recall_at_1000_diff1": -0.472918, + "nauc_mrr_at_1_max": -0.034512, + "nauc_mrr_at_1_std": -0.003783, + "nauc_mrr_at_1_diff1": -0.064451, + "nauc_mrr_at_3_max": -0.045942, + "nauc_mrr_at_3_std": -0.038575, + "nauc_mrr_at_3_diff1": -0.113057, + "nauc_mrr_at_5_max": -0.007763, + "nauc_mrr_at_5_std": -0.011882, + "nauc_mrr_at_5_diff1": -0.103533, + "nauc_mrr_at_10_max": -0.000207, + "nauc_mrr_at_10_std": -0.002674, + "nauc_mrr_at_10_diff1": -0.094663, + "nauc_mrr_at_20_max": -0.011913, + "nauc_mrr_at_20_std": -0.004748, + "nauc_mrr_at_20_diff1": -0.098033, + "nauc_mrr_at_100_max": -0.010973, + "nauc_mrr_at_100_std": -0.003565, + "nauc_mrr_at_100_diff1": -0.098805, + "nauc_mrr_at_1000_max": -0.010973, + "nauc_mrr_at_1000_std": -0.003565, + "nauc_mrr_at_1000_diff1": -0.098805, + "main_score": 0.48116, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 1149.2485148906708, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..7594cbe92d --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.48276, + "ndcg_at_3": 0.5459, + "ndcg_at_5": 0.53846, + "ndcg_at_10": 0.54304, + "ndcg_at_20": 0.57503, + "ndcg_at_100": 0.6794, + "ndcg_at_1000": 0.71205, + "map_at_1": 0.06668, + "map_at_3": 0.16494, + "map_at_5": 0.21602, + "map_at_10": 0.29118, + "map_at_20": 0.35649, + "map_at_100": 0.4435, + "map_at_1000": 0.45915, + "recall_at_1": 0.06668, + "recall_at_3": 0.21602, + "recall_at_5": 0.29863, + "recall_at_10": 0.45385, + "recall_at_20": 0.61874, + "recall_at_100": 0.91752, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.48276, + "cv_recall_at_3": 0.84483, + "cv_recall_at_5": 0.93103, + "cv_recall_at_10": 0.98276, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.48276, + "precision_at_3": 0.52874, + "precision_at_5": 0.4931, + "precision_at_10": 0.41897, + "precision_at_20": 0.3181, + "precision_at_100": 0.12672, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.482759, + "mrr_at_3": 0.649425, + "mrr_at_5": 0.668391, + "mrr_at_10": 0.675883, + "mrr_at_20": 0.67684, + "mrr_at_100": 0.67684, + "mrr_at_1000": 0.67684, + "nauc_ndcg_at_1_max": -0.164023, + "nauc_ndcg_at_1_std": -0.197064, + "nauc_ndcg_at_1_diff1": 0.065133, + "nauc_ndcg_at_3_max": -0.15619, + "nauc_ndcg_at_3_std": -0.178213, + "nauc_ndcg_at_3_diff1": 0.020173, + "nauc_ndcg_at_5_max": -0.110709, + "nauc_ndcg_at_5_std": -0.18467, + "nauc_ndcg_at_5_diff1": 0.064164, + "nauc_ndcg_at_10_max": -0.129887, + "nauc_ndcg_at_10_std": -0.246535, + "nauc_ndcg_at_10_diff1": 0.153377, + "nauc_ndcg_at_20_max": 0.030269, + "nauc_ndcg_at_20_std": -0.105881, + "nauc_ndcg_at_20_diff1": 0.244831, + "nauc_ndcg_at_100_max": -0.029294, + "nauc_ndcg_at_100_std": -0.109041, + "nauc_ndcg_at_100_diff1": 0.099962, + "nauc_ndcg_at_1000_max": -0.083933, + "nauc_ndcg_at_1000_std": -0.155375, + "nauc_ndcg_at_1000_diff1": 0.103705, + "nauc_map_at_1_max": -0.211999, + "nauc_map_at_1_std": -0.293916, + "nauc_map_at_1_diff1": 0.288295, + "nauc_map_at_3_max": -0.154807, + "nauc_map_at_3_std": -0.244992, + "nauc_map_at_3_diff1": 0.229693, + "nauc_map_at_5_max": -0.038125, + "nauc_map_at_5_std": -0.18643, + "nauc_map_at_5_diff1": 0.275589, + "nauc_map_at_10_max": -0.008461, + "nauc_map_at_10_std": -0.199407, + "nauc_map_at_10_diff1": 0.299108, + "nauc_map_at_20_max": 0.050656, + "nauc_map_at_20_std": -0.135966, + "nauc_map_at_20_diff1": 0.262618, + "nauc_map_at_100_max": -0.006404, + "nauc_map_at_100_std": -0.143344, + "nauc_map_at_100_diff1": 0.177201, + "nauc_map_at_1000_max": -0.039895, + "nauc_map_at_1000_std": -0.159712, + "nauc_map_at_1000_diff1": 0.171729, + "nauc_recall_at_1_max": -0.211999, + "nauc_recall_at_1_std": -0.293916, + "nauc_recall_at_1_diff1": 0.288295, + "nauc_recall_at_3_max": -0.208977, + "nauc_recall_at_3_std": -0.26687, + "nauc_recall_at_3_diff1": 0.129564, + "nauc_recall_at_5_max": -0.028582, + "nauc_recall_at_5_std": -0.207918, + "nauc_recall_at_5_diff1": 0.260198, + "nauc_recall_at_10_max": -0.001707, + "nauc_recall_at_10_std": -0.208495, + "nauc_recall_at_10_diff1": 0.273625, + "nauc_recall_at_20_max": 0.193955, + "nauc_recall_at_20_std": -0.027578, + "nauc_recall_at_20_diff1": 0.24001, + "nauc_recall_at_100_max": 0.199383, + "nauc_recall_at_100_std": 0.184218, + "nauc_recall_at_100_diff1": -0.03715, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.164023, + "nauc_precision_at_1_std": -0.197064, + "nauc_precision_at_1_diff1": 0.065133, + "nauc_precision_at_3_max": -0.098943, + "nauc_precision_at_3_std": -0.101315, + "nauc_precision_at_3_diff1": -0.037272, + "nauc_precision_at_5_max": 0.016685, + "nauc_precision_at_5_std": -0.035078, + "nauc_precision_at_5_diff1": -0.006056, + "nauc_precision_at_10_max": -0.021316, + "nauc_precision_at_10_std": -0.053394, + "nauc_precision_at_10_diff1": -0.056085, + "nauc_precision_at_20_max": 0.036146, + "nauc_precision_at_20_std": 0.059076, + "nauc_precision_at_20_diff1": -0.078792, + "nauc_precision_at_100_max": -0.163228, + "nauc_precision_at_100_std": -0.04183, + "nauc_precision_at_100_diff1": -0.137353, + "nauc_precision_at_1000_max": -0.201499, + "nauc_precision_at_1000_std": -0.074602, + "nauc_precision_at_1000_diff1": -0.107645, + "nauc_cv_recall_at_1_max": -0.164023, + "nauc_cv_recall_at_1_std": -0.197064, + "nauc_cv_recall_at_1_diff1": 0.065133, + "nauc_cv_recall_at_3_max": -0.271597, + "nauc_cv_recall_at_3_std": -0.108603, + "nauc_cv_recall_at_3_diff1": 0.067132, + "nauc_cv_recall_at_5_max": 0.094137, + "nauc_cv_recall_at_5_std": -0.059086, + "nauc_cv_recall_at_5_diff1": 0.422101, + "nauc_cv_recall_at_10_max": -0.574298, + "nauc_cv_recall_at_10_std": -1.142926, + "nauc_cv_recall_at_10_diff1": -0.172914, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.164023, + "nauc_mrr_at_1_std": -0.197064, + "nauc_mrr_at_1_diff1": 0.065133, + "nauc_mrr_at_3_max": -0.177857, + "nauc_mrr_at_3_std": -0.167982, + "nauc_mrr_at_3_diff1": 0.073515, + "nauc_mrr_at_5_max": -0.148169, + "nauc_mrr_at_5_std": -0.168857, + "nauc_mrr_at_5_diff1": 0.095998, + "nauc_mrr_at_10_max": -0.162276, + "nauc_mrr_at_10_std": -0.184465, + "nauc_mrr_at_10_diff1": 0.078341, + "nauc_mrr_at_20_max": -0.160604, + "nauc_mrr_at_20_std": -0.180575, + "nauc_mrr_at_20_diff1": 0.079361, + "nauc_mrr_at_100_max": -0.160604, + "nauc_mrr_at_100_std": -0.180575, + "nauc_mrr_at_100_diff1": 0.079361, + "nauc_mrr_at_1000_max": -0.160604, + "nauc_mrr_at_1000_std": -0.180575, + "nauc_mrr_at_1000_diff1": 0.079361, + "main_score": 0.53846, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.48276, + "ndcg_at_3": 0.54663, + "ndcg_at_5": 0.52907, + "ndcg_at_10": 0.5287, + "ndcg_at_20": 0.56503, + "ndcg_at_100": 0.67519, + "ndcg_at_1000": 0.70783, + "map_at_1": 0.06842, + "map_at_3": 0.16625, + "map_at_5": 0.21264, + "map_at_10": 0.28116, + "map_at_20": 0.3445, + "map_at_100": 0.43451, + "map_at_1000": 0.44997, + "recall_at_1": 0.06842, + "recall_at_3": 0.22292, + "recall_at_5": 0.31293, + "recall_at_10": 0.43624, + "recall_at_20": 0.59943, + "recall_at_100": 0.91951, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.48276, + "cv_recall_at_3": 0.84483, + "cv_recall_at_5": 0.94828, + "cv_recall_at_10": 0.98276, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.48276, + "precision_at_3": 0.53448, + "precision_at_5": 0.47586, + "precision_at_10": 0.39828, + "precision_at_20": 0.31121, + "precision_at_100": 0.12603, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.482759, + "mrr_at_3": 0.643678, + "mrr_at_5": 0.666954, + "mrr_at_10": 0.672291, + "mrr_at_20": 0.672291, + "mrr_at_100": 0.67304, + "mrr_at_1000": 0.67304, + "nauc_ndcg_at_1_max": 0.032064, + "nauc_ndcg_at_1_std": -0.078134, + "nauc_ndcg_at_1_diff1": 0.277559, + "nauc_ndcg_at_3_max": 0.119698, + "nauc_ndcg_at_3_std": -0.023395, + "nauc_ndcg_at_3_diff1": 0.296957, + "nauc_ndcg_at_5_max": 0.050011, + "nauc_ndcg_at_5_std": -0.07786, + "nauc_ndcg_at_5_diff1": 0.192623, + "nauc_ndcg_at_10_max": 0.097221, + "nauc_ndcg_at_10_std": -0.058895, + "nauc_ndcg_at_10_diff1": 0.228143, + "nauc_ndcg_at_20_max": 0.117739, + "nauc_ndcg_at_20_std": -0.017638, + "nauc_ndcg_at_20_diff1": 0.247709, + "nauc_ndcg_at_100_max": 0.080938, + "nauc_ndcg_at_100_std": -0.034855, + "nauc_ndcg_at_100_diff1": 0.26258, + "nauc_ndcg_at_1000_max": 0.068048, + "nauc_ndcg_at_1000_std": -0.024069, + "nauc_ndcg_at_1000_diff1": 0.189664, + "nauc_map_at_1_max": -0.201402, + "nauc_map_at_1_std": -0.325881, + "nauc_map_at_1_diff1": 0.449412, + "nauc_map_at_3_max": -0.018735, + "nauc_map_at_3_std": -0.215036, + "nauc_map_at_3_diff1": 0.522783, + "nauc_map_at_5_max": -0.003184, + "nauc_map_at_5_std": -0.202538, + "nauc_map_at_5_diff1": 0.512801, + "nauc_map_at_10_max": 0.101204, + "nauc_map_at_10_std": -0.132524, + "nauc_map_at_10_diff1": 0.548699, + "nauc_map_at_20_max": 0.098005, + "nauc_map_at_20_std": -0.10573, + "nauc_map_at_20_diff1": 0.473484, + "nauc_map_at_100_max": 0.081112, + "nauc_map_at_100_std": -0.069589, + "nauc_map_at_100_diff1": 0.333783, + "nauc_map_at_1000_max": 0.062598, + "nauc_map_at_1000_std": -0.061947, + "nauc_map_at_1000_diff1": 0.273463, + "nauc_recall_at_1_max": -0.201402, + "nauc_recall_at_1_std": -0.325881, + "nauc_recall_at_1_diff1": 0.449412, + "nauc_recall_at_3_max": -0.025677, + "nauc_recall_at_3_std": -0.216777, + "nauc_recall_at_3_diff1": 0.392769, + "nauc_recall_at_5_max": -0.06125, + "nauc_recall_at_5_std": -0.250119, + "nauc_recall_at_5_diff1": 0.353248, + "nauc_recall_at_10_max": 0.0749, + "nauc_recall_at_10_std": -0.153184, + "nauc_recall_at_10_diff1": 0.441913, + "nauc_recall_at_20_max": 0.075454, + "nauc_recall_at_20_std": -0.100806, + "nauc_recall_at_20_diff1": 0.302434, + "nauc_recall_at_100_max": 0.079461, + "nauc_recall_at_100_std": -0.001376, + "nauc_recall_at_100_diff1": 0.208227, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.032064, + "nauc_precision_at_1_std": -0.078134, + "nauc_precision_at_1_diff1": 0.277559, + "nauc_precision_at_3_max": 0.192852, + "nauc_precision_at_3_std": 0.083013, + "nauc_precision_at_3_diff1": 0.206405, + "nauc_precision_at_5_max": 0.124893, + "nauc_precision_at_5_std": 0.068264, + "nauc_precision_at_5_diff1": 0.003739, + "nauc_precision_at_10_max": 0.163464, + "nauc_precision_at_10_std": 0.148537, + "nauc_precision_at_10_diff1": -0.087117, + "nauc_precision_at_20_max": 0.066017, + "nauc_precision_at_20_std": 0.149838, + "nauc_precision_at_20_diff1": -0.291128, + "nauc_precision_at_100_max": -0.060446, + "nauc_precision_at_100_std": 0.09483, + "nauc_precision_at_100_diff1": -0.420127, + "nauc_precision_at_1000_max": -0.088331, + "nauc_precision_at_1000_std": 0.074724, + "nauc_precision_at_1000_diff1": -0.421785, + "nauc_cv_recall_at_1_max": 0.032064, + "nauc_cv_recall_at_1_std": -0.078134, + "nauc_cv_recall_at_1_diff1": 0.277559, + "nauc_cv_recall_at_3_max": -0.047179, + "nauc_cv_recall_at_3_std": -0.239144, + "nauc_cv_recall_at_3_diff1": 0.361088, + "nauc_cv_recall_at_5_max": -0.642265, + "nauc_cv_recall_at_5_std": -1.198673, + "nauc_cv_recall_at_5_diff1": 0.338307, + "nauc_cv_recall_at_10_max": 0.359055, + "nauc_cv_recall_at_10_std": -0.172914, + "nauc_cv_recall_at_10_diff1": -0.574298, + "nauc_cv_recall_at_20_max": 0.359055, + "nauc_cv_recall_at_20_std": -0.172914, + "nauc_cv_recall_at_20_diff1": -0.574298, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.032064, + "nauc_mrr_at_1_std": -0.078134, + "nauc_mrr_at_1_diff1": 0.277559, + "nauc_mrr_at_3_max": 0.032313, + "nauc_mrr_at_3_std": -0.092553, + "nauc_mrr_at_3_diff1": 0.307214, + "nauc_mrr_at_5_max": 0.012817, + "nauc_mrr_at_5_std": -0.122853, + "nauc_mrr_at_5_diff1": 0.299354, + "nauc_mrr_at_10_max": 0.037641, + "nauc_mrr_at_10_std": -0.087387, + "nauc_mrr_at_10_diff1": 0.288171, + "nauc_mrr_at_20_max": 0.037641, + "nauc_mrr_at_20_std": -0.087387, + "nauc_mrr_at_20_diff1": 0.288171, + "nauc_mrr_at_100_max": 0.03663, + "nauc_mrr_at_100_std": -0.087118, + "nauc_mrr_at_100_diff1": 0.290884, + "nauc_mrr_at_1000_max": 0.03663, + "nauc_mrr_at_1000_std": -0.087118, + "nauc_mrr_at_1000_diff1": 0.290884, + "main_score": 0.52907, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.55172, + "ndcg_at_3": 0.62151, + "ndcg_at_5": 0.58687, + "ndcg_at_10": 0.57082, + "ndcg_at_20": 0.61434, + "ndcg_at_100": 0.70485, + "ndcg_at_1000": 0.73822, + "map_at_1": 0.07112, + "map_at_3": 0.19733, + "map_at_5": 0.25431, + "map_at_10": 0.32406, + "map_at_20": 0.39665, + "map_at_100": 0.48211, + "map_at_1000": 0.49771, + "recall_at_1": 0.07112, + "recall_at_3": 0.26535, + "recall_at_5": 0.33848, + "recall_at_10": 0.46094, + "recall_at_20": 0.64141, + "recall_at_100": 0.91051, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.55172, + "cv_recall_at_3": 0.93103, + "cv_recall_at_5": 0.96552, + "cv_recall_at_10": 0.96552, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.55172, + "precision_at_3": 0.60345, + "precision_at_5": 0.52759, + "precision_at_10": 0.42414, + "precision_at_20": 0.33276, + "precision_at_100": 0.12776, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.551724, + "mrr_at_3": 0.727011, + "mrr_at_5": 0.73477, + "mrr_at_10": 0.73477, + "mrr_at_20": 0.736635, + "mrr_at_100": 0.736635, + "mrr_at_1000": 0.736635, + "nauc_ndcg_at_1_max": -0.122728, + "nauc_ndcg_at_1_std": -0.057983, + "nauc_ndcg_at_1_diff1": 0.11415, + "nauc_ndcg_at_3_max": -0.080998, + "nauc_ndcg_at_3_std": -0.100744, + "nauc_ndcg_at_3_diff1": 0.206151, + "nauc_ndcg_at_5_max": -0.086084, + "nauc_ndcg_at_5_std": -0.117262, + "nauc_ndcg_at_5_diff1": 0.138825, + "nauc_ndcg_at_10_max": -0.084388, + "nauc_ndcg_at_10_std": -0.136244, + "nauc_ndcg_at_10_diff1": 0.139606, + "nauc_ndcg_at_20_max": 0.005252, + "nauc_ndcg_at_20_std": -0.068528, + "nauc_ndcg_at_20_diff1": 0.109389, + "nauc_ndcg_at_100_max": 0.015659, + "nauc_ndcg_at_100_std": -0.041036, + "nauc_ndcg_at_100_diff1": 0.122504, + "nauc_ndcg_at_1000_max": -0.00382, + "nauc_ndcg_at_1000_std": -0.027903, + "nauc_ndcg_at_1000_diff1": 0.085611, + "nauc_map_at_1_max": -0.239831, + "nauc_map_at_1_std": -0.245649, + "nauc_map_at_1_diff1": 0.015674, + "nauc_map_at_3_max": -0.219724, + "nauc_map_at_3_std": -0.310526, + "nauc_map_at_3_diff1": 0.135423, + "nauc_map_at_5_max": -0.128976, + "nauc_map_at_5_std": -0.232814, + "nauc_map_at_5_diff1": 0.134075, + "nauc_map_at_10_max": -0.103632, + "nauc_map_at_10_std": -0.242533, + "nauc_map_at_10_diff1": 0.122178, + "nauc_map_at_20_max": -0.040159, + "nauc_map_at_20_std": -0.193863, + "nauc_map_at_20_diff1": 0.09788, + "nauc_map_at_100_max": -0.021931, + "nauc_map_at_100_std": -0.101686, + "nauc_map_at_100_diff1": 0.120097, + "nauc_map_at_1000_max": -0.030655, + "nauc_map_at_1000_std": -0.075812, + "nauc_map_at_1000_diff1": 0.115699, + "nauc_recall_at_1_max": -0.239831, + "nauc_recall_at_1_std": -0.245649, + "nauc_recall_at_1_diff1": 0.015674, + "nauc_recall_at_3_max": -0.269262, + "nauc_recall_at_3_std": -0.388279, + "nauc_recall_at_3_diff1": 0.149139, + "nauc_recall_at_5_max": -0.203515, + "nauc_recall_at_5_std": -0.33209, + "nauc_recall_at_5_diff1": 0.134636, + "nauc_recall_at_10_max": -0.119942, + "nauc_recall_at_10_std": -0.314851, + "nauc_recall_at_10_diff1": 0.152686, + "nauc_recall_at_20_max": 0.010146, + "nauc_recall_at_20_std": -0.205011, + "nauc_recall_at_20_diff1": 0.10301, + "nauc_recall_at_100_max": 0.189632, + "nauc_recall_at_100_std": 0.05643, + "nauc_recall_at_100_diff1": 0.264096, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.122728, + "nauc_precision_at_1_std": -0.057983, + "nauc_precision_at_1_diff1": 0.11415, + "nauc_precision_at_3_max": 0.046567, + "nauc_precision_at_3_std": 0.019628, + "nauc_precision_at_3_diff1": 0.205067, + "nauc_precision_at_5_max": 0.080403, + "nauc_precision_at_5_std": 0.060872, + "nauc_precision_at_5_diff1": 0.070424, + "nauc_precision_at_10_max": 0.115501, + "nauc_precision_at_10_std": 0.108532, + "nauc_precision_at_10_diff1": 0.039686, + "nauc_precision_at_20_max": 0.154313, + "nauc_precision_at_20_std": 0.219811, + "nauc_precision_at_20_diff1": -0.050532, + "nauc_precision_at_100_max": 0.017395, + "nauc_precision_at_100_std": 0.22957, + "nauc_precision_at_100_diff1": -0.02712, + "nauc_precision_at_1000_max": -0.001999, + "nauc_precision_at_1000_std": 0.228928, + "nauc_precision_at_1000_diff1": -0.031667, + "nauc_cv_recall_at_1_max": -0.122728, + "nauc_cv_recall_at_1_std": -0.057983, + "nauc_cv_recall_at_1_diff1": 0.11415, + "nauc_cv_recall_at_3_max": -0.080825, + "nauc_cv_recall_at_3_std": -0.357922, + "nauc_cv_recall_at_3_diff1": 0.494929, + "nauc_cv_recall_at_5_max": -0.137074, + "nauc_cv_recall_at_5_std": -0.57877, + "nauc_cv_recall_at_5_diff1": -0.010143, + "nauc_cv_recall_at_10_max": -0.137074, + "nauc_cv_recall_at_10_std": -0.57877, + "nauc_cv_recall_at_10_diff1": -0.010143, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.122728, + "nauc_mrr_at_1_std": -0.057983, + "nauc_mrr_at_1_diff1": 0.11415, + "nauc_mrr_at_3_max": -0.098852, + "nauc_mrr_at_3_std": -0.078052, + "nauc_mrr_at_3_diff1": 0.151784, + "nauc_mrr_at_5_max": -0.101032, + "nauc_mrr_at_5_std": -0.071607, + "nauc_mrr_at_5_diff1": 0.119778, + "nauc_mrr_at_10_max": -0.101032, + "nauc_mrr_at_10_std": -0.071607, + "nauc_mrr_at_10_diff1": 0.119778, + "nauc_mrr_at_20_max": -0.100951, + "nauc_mrr_at_20_std": -0.067245, + "nauc_mrr_at_20_diff1": 0.120828, + "nauc_mrr_at_100_max": -0.100951, + "nauc_mrr_at_100_std": -0.067245, + "nauc_mrr_at_100_diff1": 0.120828, + "nauc_mrr_at_1000_max": -0.100951, + "nauc_mrr_at_1000_std": -0.067245, + "nauc_mrr_at_1000_diff1": 0.120828, + "main_score": 0.58687, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.60345, + "ndcg_at_3": 0.58318, + "ndcg_at_5": 0.53321, + "ndcg_at_10": 0.53815, + "ndcg_at_20": 0.58039, + "ndcg_at_100": 0.68539, + "ndcg_at_1000": 0.71906, + "map_at_1": 0.07658, + "map_at_3": 0.17802, + "map_at_5": 0.22151, + "map_at_10": 0.29558, + "map_at_20": 0.36209, + "map_at_100": 0.44537, + "map_at_1000": 0.46165, + "recall_at_1": 0.07658, + "recall_at_3": 0.22328, + "recall_at_5": 0.29682, + "recall_at_10": 0.44098, + "recall_at_20": 0.61778, + "recall_at_100": 0.91685, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.60345, + "cv_recall_at_3": 0.86207, + "cv_recall_at_5": 0.93103, + "cv_recall_at_10": 0.96552, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.60345, + "precision_at_3": 0.54598, + "precision_at_5": 0.45862, + "precision_at_10": 0.39655, + "precision_at_20": 0.31034, + "precision_at_100": 0.12534, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.603448, + "mrr_at_3": 0.721264, + "mrr_at_5": 0.73592, + "mrr_at_10": 0.740948, + "mrr_at_20": 0.743424, + "mrr_at_100": 0.743424, + "mrr_at_1000": 0.743424, + "nauc_ndcg_at_1_max": 0.089958, + "nauc_ndcg_at_1_std": 0.187945, + "nauc_ndcg_at_1_diff1": 0.265959, + "nauc_ndcg_at_3_max": 0.19124, + "nauc_ndcg_at_3_std": 0.150828, + "nauc_ndcg_at_3_diff1": 0.25673, + "nauc_ndcg_at_5_max": 0.125684, + "nauc_ndcg_at_5_std": 0.074126, + "nauc_ndcg_at_5_diff1": 0.236721, + "nauc_ndcg_at_10_max": 0.048994, + "nauc_ndcg_at_10_std": -0.01254, + "nauc_ndcg_at_10_diff1": 0.203744, + "nauc_ndcg_at_20_max": 0.028294, + "nauc_ndcg_at_20_std": 0.006345, + "nauc_ndcg_at_20_diff1": 0.243382, + "nauc_ndcg_at_100_max": 0.04124, + "nauc_ndcg_at_100_std": 0.021862, + "nauc_ndcg_at_100_diff1": 0.306575, + "nauc_ndcg_at_1000_max": 0.124743, + "nauc_ndcg_at_1000_std": 0.125411, + "nauc_ndcg_at_1000_diff1": 0.281235, + "nauc_map_at_1_max": -0.197825, + "nauc_map_at_1_std": -0.207494, + "nauc_map_at_1_diff1": 0.174039, + "nauc_map_at_3_max": -0.058125, + "nauc_map_at_3_std": -0.158061, + "nauc_map_at_3_diff1": 0.345508, + "nauc_map_at_5_max": -0.069049, + "nauc_map_at_5_std": -0.162946, + "nauc_map_at_5_diff1": 0.377842, + "nauc_map_at_10_max": -0.080914, + "nauc_map_at_10_std": -0.190684, + "nauc_map_at_10_diff1": 0.345399, + "nauc_map_at_20_max": -0.102562, + "nauc_map_at_20_std": -0.184397, + "nauc_map_at_20_diff1": 0.318583, + "nauc_map_at_100_max": -0.012521, + "nauc_map_at_100_std": -0.056814, + "nauc_map_at_100_diff1": 0.296176, + "nauc_map_at_1000_max": 0.031921, + "nauc_map_at_1000_std": 0.013423, + "nauc_map_at_1000_diff1": 0.264406, + "nauc_recall_at_1_max": -0.197825, + "nauc_recall_at_1_std": -0.207494, + "nauc_recall_at_1_diff1": 0.174039, + "nauc_recall_at_3_max": -0.042852, + "nauc_recall_at_3_std": -0.211979, + "nauc_recall_at_3_diff1": 0.26287, + "nauc_recall_at_5_max": -0.123622, + "nauc_recall_at_5_std": -0.274047, + "nauc_recall_at_5_diff1": 0.244901, + "nauc_recall_at_10_max": -0.179538, + "nauc_recall_at_10_std": -0.344286, + "nauc_recall_at_10_diff1": 0.195243, + "nauc_recall_at_20_max": -0.239043, + "nauc_recall_at_20_std": -0.329613, + "nauc_recall_at_20_diff1": 0.135592, + "nauc_recall_at_100_max": -0.296766, + "nauc_recall_at_100_std": -0.367872, + "nauc_recall_at_100_diff1": 0.169308, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.089958, + "nauc_precision_at_1_std": 0.187945, + "nauc_precision_at_1_diff1": 0.265959, + "nauc_precision_at_3_max": 0.243889, + "nauc_precision_at_3_std": 0.207602, + "nauc_precision_at_3_diff1": 0.236694, + "nauc_precision_at_5_max": 0.196995, + "nauc_precision_at_5_std": 0.174788, + "nauc_precision_at_5_diff1": 0.21801, + "nauc_precision_at_10_max": 0.155704, + "nauc_precision_at_10_std": 0.149662, + "nauc_precision_at_10_diff1": 0.031391, + "nauc_precision_at_20_max": 0.151235, + "nauc_precision_at_20_std": 0.240842, + "nauc_precision_at_20_diff1": -0.084885, + "nauc_precision_at_100_max": 0.271878, + "nauc_precision_at_100_std": 0.445338, + "nauc_precision_at_100_diff1": -0.188951, + "nauc_precision_at_1000_max": 0.314926, + "nauc_precision_at_1000_std": 0.497819, + "nauc_precision_at_1000_diff1": -0.207574, + "nauc_cv_recall_at_1_max": 0.089958, + "nauc_cv_recall_at_1_std": 0.187945, + "nauc_cv_recall_at_1_diff1": 0.265959, + "nauc_cv_recall_at_3_max": 0.485179, + "nauc_cv_recall_at_3_std": 0.386062, + "nauc_cv_recall_at_3_diff1": 0.358422, + "nauc_cv_recall_at_5_max": 0.243064, + "nauc_cv_recall_at_5_std": -0.039014, + "nauc_cv_recall_at_5_diff1": 0.324489, + "nauc_cv_recall_at_10_max": 0.338887, + "nauc_cv_recall_at_10_std": -0.225269, + "nauc_cv_recall_at_10_diff1": 0.720441, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.089958, + "nauc_mrr_at_1_std": 0.187945, + "nauc_mrr_at_1_diff1": 0.265959, + "nauc_mrr_at_3_max": 0.212824, + "nauc_mrr_at_3_std": 0.264335, + "nauc_mrr_at_3_diff1": 0.297534, + "nauc_mrr_at_5_max": 0.180157, + "nauc_mrr_at_5_std": 0.22919, + "nauc_mrr_at_5_diff1": 0.291429, + "nauc_mrr_at_10_max": 0.178511, + "nauc_mrr_at_10_std": 0.228694, + "nauc_mrr_at_10_diff1": 0.296351, + "nauc_mrr_at_20_max": 0.176817, + "nauc_mrr_at_20_std": 0.234289, + "nauc_mrr_at_20_diff1": 0.291397, + "nauc_mrr_at_100_max": 0.176817, + "nauc_mrr_at_100_std": 0.234289, + "nauc_mrr_at_100_diff1": 0.291397, + "nauc_mrr_at_1000_max": 0.176817, + "nauc_mrr_at_1000_std": 0.234289, + "nauc_mrr_at_1000_diff1": 0.291397, + "main_score": 0.53321, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 205.11101078987122, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..987ffdb65d --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", + "task_name": "VidoreArxivQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.834, + "ndcg_at_3": 0.87276, + "ndcg_at_5": 0.88171, + "ndcg_at_10": 0.89144, + "ndcg_at_20": 0.8966, + "ndcg_at_100": 0.90092, + "ndcg_at_1000": 0.90168, + "map_at_1": 0.834, + "map_at_3": 0.86333, + "map_at_5": 0.86823, + "map_at_10": 0.87227, + "map_at_20": 0.87373, + "map_at_100": 0.87443, + "map_at_1000": 0.87446, + "recall_at_1": 0.834, + "recall_at_3": 0.9, + "recall_at_5": 0.922, + "recall_at_10": 0.952, + "recall_at_20": 0.972, + "recall_at_100": 0.994, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.834, + "cv_recall_at_3": 0.9, + "cv_recall_at_5": 0.922, + "cv_recall_at_10": 0.952, + "cv_recall_at_20": 0.972, + "cv_recall_at_100": 0.994, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.834, + "precision_at_3": 0.3, + "precision_at_5": 0.1844, + "precision_at_10": 0.0952, + "precision_at_20": 0.0486, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "mrr_at_1": 0.834, + "mrr_at_3": 0.863333, + "mrr_at_5": 0.868233, + "mrr_at_10": 0.872268, + "mrr_at_20": 0.873734, + "mrr_at_100": 0.874427, + "mrr_at_1000": 0.874457, + "nauc_ndcg_at_1_max": 0.814448, + "nauc_ndcg_at_1_std": 0.005019, + "nauc_ndcg_at_1_diff1": 0.951082, + "nauc_ndcg_at_3_max": 0.812501, + "nauc_ndcg_at_3_std": 0.010721, + "nauc_ndcg_at_3_diff1": 0.931237, + "nauc_ndcg_at_5_max": 0.828884, + "nauc_ndcg_at_5_std": 0.032585, + "nauc_ndcg_at_5_diff1": 0.935934, + "nauc_ndcg_at_10_max": 0.827961, + "nauc_ndcg_at_10_std": 0.033449, + "nauc_ndcg_at_10_diff1": 0.935054, + "nauc_ndcg_at_20_max": 0.82545, + "nauc_ndcg_at_20_std": 0.033909, + "nauc_ndcg_at_20_diff1": 0.937897, + "nauc_ndcg_at_100_max": 0.823959, + "nauc_ndcg_at_100_std": 0.030812, + "nauc_ndcg_at_100_diff1": 0.937906, + "nauc_ndcg_at_1000_max": 0.822948, + "nauc_ndcg_at_1000_std": 0.024384, + "nauc_ndcg_at_1000_diff1": 0.938812, + "nauc_map_at_1_max": 0.814448, + "nauc_map_at_1_std": 0.005019, + "nauc_map_at_1_diff1": 0.951082, + "nauc_map_at_3_max": 0.81415, + "nauc_map_at_3_std": 0.00793, + "nauc_map_at_3_diff1": 0.936997, + "nauc_map_at_5_max": 0.822983, + "nauc_map_at_5_std": 0.019757, + "nauc_map_at_5_diff1": 0.939577, + "nauc_map_at_10_max": 0.82202, + "nauc_map_at_10_std": 0.018636, + "nauc_map_at_10_diff1": 0.939437, + "nauc_map_at_20_max": 0.821261, + "nauc_map_at_20_std": 0.017829, + "nauc_map_at_20_diff1": 0.940131, + "nauc_map_at_100_max": 0.821061, + "nauc_map_at_100_std": 0.017618, + "nauc_map_at_100_diff1": 0.940154, + "nauc_map_at_1000_max": 0.821037, + "nauc_map_at_1000_std": 0.017445, + "nauc_map_at_1000_diff1": 0.940211, + "nauc_recall_at_1_max": 0.814448, + "nauc_recall_at_1_std": 0.005019, + "nauc_recall_at_1_diff1": 0.951082, + "nauc_recall_at_3_max": 0.80563, + "nauc_recall_at_3_std": 0.02197, + "nauc_recall_at_3_diff1": 0.90929, + "nauc_recall_at_5_max": 0.85846, + "nauc_recall_at_5_std": 0.096663, + "nauc_recall_at_5_diff1": 0.919857, + "nauc_recall_at_10_max": 0.876867, + "nauc_recall_at_10_std": 0.156415, + "nauc_recall_at_10_diff1": 0.903128, + "nauc_recall_at_20_max": 0.880285, + "nauc_recall_at_20_std": 0.274176, + "nauc_recall_at_20_diff1": 0.920301, + "nauc_recall_at_100_max": 0.956427, + "nauc_recall_at_100_std": 0.851385, + "nauc_recall_at_100_diff1": 0.851385, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.814448, + "nauc_precision_at_1_std": 0.005019, + "nauc_precision_at_1_diff1": 0.951082, + "nauc_precision_at_3_max": 0.80563, + "nauc_precision_at_3_std": 0.02197, + "nauc_precision_at_3_diff1": 0.90929, + "nauc_precision_at_5_max": 0.85846, + "nauc_precision_at_5_std": 0.096663, + "nauc_precision_at_5_diff1": 0.919857, + "nauc_precision_at_10_max": 0.876867, + "nauc_precision_at_10_std": 0.156415, + "nauc_precision_at_10_diff1": 0.903128, + "nauc_precision_at_20_max": 0.880285, + "nauc_precision_at_20_std": 0.274176, + "nauc_precision_at_20_diff1": 0.920301, + "nauc_precision_at_100_max": 0.956427, + "nauc_precision_at_100_std": 0.851385, + "nauc_precision_at_100_diff1": 0.851385, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_cv_recall_at_1_max": 0.814448, + "nauc_cv_recall_at_1_std": 0.005019, + "nauc_cv_recall_at_1_diff1": 0.951082, + "nauc_cv_recall_at_3_max": 0.80563, + "nauc_cv_recall_at_3_std": 0.02197, + "nauc_cv_recall_at_3_diff1": 0.90929, + "nauc_cv_recall_at_5_max": 0.85846, + "nauc_cv_recall_at_5_std": 0.096663, + "nauc_cv_recall_at_5_diff1": 0.919857, + "nauc_cv_recall_at_10_max": 0.876867, + "nauc_cv_recall_at_10_std": 0.156415, + "nauc_cv_recall_at_10_diff1": 0.903128, + "nauc_cv_recall_at_20_max": 0.880285, + "nauc_cv_recall_at_20_std": 0.274176, + "nauc_cv_recall_at_20_diff1": 0.920301, + "nauc_cv_recall_at_100_max": 0.956427, + "nauc_cv_recall_at_100_std": 0.851385, + "nauc_cv_recall_at_100_diff1": 0.851385, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.814448, + "nauc_mrr_at_1_std": 0.005019, + "nauc_mrr_at_1_diff1": 0.951082, + "nauc_mrr_at_3_max": 0.81415, + "nauc_mrr_at_3_std": 0.00793, + "nauc_mrr_at_3_diff1": 0.936997, + "nauc_mrr_at_5_max": 0.822983, + "nauc_mrr_at_5_std": 0.019757, + "nauc_mrr_at_5_diff1": 0.939577, + "nauc_mrr_at_10_max": 0.82202, + "nauc_mrr_at_10_std": 0.018636, + "nauc_mrr_at_10_diff1": 0.939437, + "nauc_mrr_at_20_max": 0.821261, + "nauc_mrr_at_20_std": 0.017829, + "nauc_mrr_at_20_diff1": 0.940131, + "nauc_mrr_at_100_max": 0.821061, + "nauc_mrr_at_100_std": 0.017618, + "nauc_mrr_at_100_diff1": 0.940154, + "nauc_mrr_at_1000_max": 0.821037, + "nauc_mrr_at_1000_std": 0.017445, + "nauc_mrr_at_1000_diff1": 0.940211, + "main_score": 0.88171, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 79.5927848815918, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..dc73dc896c --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", + "task_name": "VidoreDocVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.52772, + "ndcg_at_3": 0.59288, + "ndcg_at_5": 0.61339, + "ndcg_at_10": 0.62909, + "ndcg_at_20": 0.644, + "ndcg_at_100": 0.66382, + "ndcg_at_1000": 0.6784, + "map_at_1": 0.52661, + "map_at_3": 0.57512, + "map_at_5": 0.58695, + "map_at_10": 0.594, + "map_at_20": 0.59857, + "map_at_100": 0.60124, + "map_at_1000": 0.60186, + "recall_at_1": 0.52661, + "recall_at_3": 0.63976, + "recall_at_5": 0.68758, + "recall_at_10": 0.73422, + "recall_at_20": 0.78991, + "recall_at_100": 0.89521, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.52772, + "cv_recall_at_3": 0.64745, + "cv_recall_at_5": 0.69623, + "cv_recall_at_10": 0.73836, + "cv_recall_at_20": 0.79379, + "cv_recall_at_100": 0.90022, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.52772, + "precision_at_3": 0.21582, + "precision_at_5": 0.14146, + "precision_at_10": 0.07605, + "precision_at_20": 0.04124, + "precision_at_100": 0.00947, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.527716, + "mrr_at_3": 0.579084, + "mrr_at_5": 0.590281, + "mrr_at_10": 0.596165, + "mrr_at_20": 0.600321, + "mrr_at_100": 0.602842, + "mrr_at_1000": 0.603297, + "nauc_ndcg_at_1_max": 0.416512, + "nauc_ndcg_at_1_std": 0.518642, + "nauc_ndcg_at_1_diff1": 0.919367, + "nauc_ndcg_at_3_max": 0.344455, + "nauc_ndcg_at_3_std": 0.575548, + "nauc_ndcg_at_3_diff1": 0.896824, + "nauc_ndcg_at_5_max": 0.321415, + "nauc_ndcg_at_5_std": 0.570746, + "nauc_ndcg_at_5_diff1": 0.894288, + "nauc_ndcg_at_10_max": 0.304045, + "nauc_ndcg_at_10_std": 0.595289, + "nauc_ndcg_at_10_diff1": 0.900125, + "nauc_ndcg_at_20_max": 0.293037, + "nauc_ndcg_at_20_std": 0.59112, + "nauc_ndcg_at_20_diff1": 0.894422, + "nauc_ndcg_at_100_max": 0.287987, + "nauc_ndcg_at_100_std": 0.598571, + "nauc_ndcg_at_100_diff1": 0.89333, + "nauc_ndcg_at_1000_max": 0.313267, + "nauc_ndcg_at_1000_std": 0.578482, + "nauc_ndcg_at_1000_diff1": 0.896733, + "nauc_map_at_1_max": 0.419281, + "nauc_map_at_1_std": 0.518436, + "nauc_map_at_1_diff1": 0.921123, + "nauc_map_at_3_max": 0.36697, + "nauc_map_at_3_std": 0.560935, + "nauc_map_at_3_diff1": 0.904545, + "nauc_map_at_5_max": 0.354634, + "nauc_map_at_5_std": 0.558955, + "nauc_map_at_5_diff1": 0.902653, + "nauc_map_at_10_max": 0.347498, + "nauc_map_at_10_std": 0.568551, + "nauc_map_at_10_diff1": 0.904523, + "nauc_map_at_20_max": 0.344475, + "nauc_map_at_20_std": 0.566203, + "nauc_map_at_20_diff1": 0.902634, + "nauc_map_at_100_max": 0.343979, + "nauc_map_at_100_std": 0.567367, + "nauc_map_at_100_diff1": 0.902519, + "nauc_map_at_1000_max": 0.344688, + "nauc_map_at_1000_std": 0.566609, + "nauc_map_at_1000_diff1": 0.902658, + "nauc_recall_at_1_max": 0.419281, + "nauc_recall_at_1_std": 0.518436, + "nauc_recall_at_1_diff1": 0.921123, + "nauc_recall_at_3_max": 0.279882, + "nauc_recall_at_3_std": 0.625045, + "nauc_recall_at_3_diff1": 0.876623, + "nauc_recall_at_5_max": 0.215041, + "nauc_recall_at_5_std": 0.6174, + "nauc_recall_at_5_diff1": 0.871493, + "nauc_recall_at_10_max": 0.135973, + "nauc_recall_at_10_std": 0.714509, + "nauc_recall_at_10_diff1": 0.892374, + "nauc_recall_at_20_max": 0.046279, + "nauc_recall_at_20_std": 0.727715, + "nauc_recall_at_20_diff1": 0.865246, + "nauc_recall_at_100_max": -0.218591, + "nauc_recall_at_100_std": 0.936633, + "nauc_recall_at_100_diff1": 0.824649, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.416512, + "nauc_precision_at_1_std": 0.518642, + "nauc_precision_at_1_diff1": 0.919367, + "nauc_precision_at_3_max": 0.26782, + "nauc_precision_at_3_std": 0.621758, + "nauc_precision_at_3_diff1": 0.868764, + "nauc_precision_at_5_max": 0.160118, + "nauc_precision_at_5_std": 0.545643, + "nauc_precision_at_5_diff1": 0.771549, + "nauc_precision_at_10_max": 0.057408, + "nauc_precision_at_10_std": 0.571767, + "nauc_precision_at_10_diff1": 0.71186, + "nauc_precision_at_20_max": -0.053749, + "nauc_precision_at_20_std": 0.488928, + "nauc_precision_at_20_diff1": 0.584024, + "nauc_precision_at_100_max": -0.33402, + "nauc_precision_at_100_std": 0.356298, + "nauc_precision_at_100_diff1": 0.271823, + "nauc_precision_at_1000_max": -0.31274, + "nauc_precision_at_1000_std": -0.498851, + "nauc_precision_at_1000_diff1": -0.462367, + "nauc_cv_recall_at_1_max": 0.416512, + "nauc_cv_recall_at_1_std": 0.518642, + "nauc_cv_recall_at_1_diff1": 0.919367, + "nauc_cv_recall_at_3_max": 0.26782, + "nauc_cv_recall_at_3_std": 0.621758, + "nauc_cv_recall_at_3_diff1": 0.868764, + "nauc_cv_recall_at_5_max": 0.19471, + "nauc_cv_recall_at_5_std": 0.603113, + "nauc_cv_recall_at_5_diff1": 0.861001, + "nauc_cv_recall_at_10_max": 0.123876, + "nauc_cv_recall_at_10_std": 0.708817, + "nauc_cv_recall_at_10_diff1": 0.890201, + "nauc_cv_recall_at_20_max": 0.030318, + "nauc_cv_recall_at_20_std": 0.720114, + "nauc_cv_recall_at_20_diff1": 0.861616, + "nauc_cv_recall_at_100_max": -0.314658, + "nauc_cv_recall_at_100_std": 0.929481, + "nauc_cv_recall_at_100_diff1": 0.833837, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.416512, + "nauc_mrr_at_1_std": 0.518642, + "nauc_mrr_at_1_diff1": 0.919367, + "nauc_mrr_at_3_max": 0.360288, + "nauc_mrr_at_3_std": 0.558166, + "nauc_mrr_at_3_diff1": 0.899767, + "nauc_mrr_at_5_max": 0.34743, + "nauc_mrr_at_5_std": 0.553564, + "nauc_mrr_at_5_diff1": 0.898324, + "nauc_mrr_at_10_max": 0.342519, + "nauc_mrr_at_10_std": 0.564786, + "nauc_mrr_at_10_diff1": 0.901759, + "nauc_mrr_at_20_max": 0.340094, + "nauc_mrr_at_20_std": 0.563187, + "nauc_mrr_at_20_diff1": 0.900374, + "nauc_mrr_at_100_max": 0.339484, + "nauc_mrr_at_100_std": 0.564548, + "nauc_mrr_at_100_diff1": 0.900426, + "nauc_mrr_at_1000_max": 0.340377, + "nauc_mrr_at_1000_std": 0.564042, + "nauc_mrr_at_1000_diff1": 0.900544, + "main_score": 0.61339, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 84.03595614433289, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..2973689e96 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", + "task_name": "VidoreInfoVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.88259, + "ndcg_at_3": 0.9198, + "ndcg_at_5": 0.9283, + "ndcg_at_10": 0.93003, + "ndcg_at_20": 0.93191, + "ndcg_at_100": 0.93452, + "ndcg_at_1000": 0.93588, + "map_at_1": 0.88259, + "map_at_3": 0.91093, + "map_at_5": 0.91554, + "map_at_10": 0.91637, + "map_at_20": 0.91695, + "map_at_100": 0.91736, + "map_at_1000": 0.91741, + "recall_at_1": 0.88259, + "recall_at_3": 0.94534, + "recall_at_5": 0.96559, + "recall_at_10": 0.97065, + "recall_at_20": 0.97713, + "recall_at_100": 0.99008, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.88259, + "cv_recall_at_3": 0.94534, + "cv_recall_at_5": 0.96761, + "cv_recall_at_10": 0.97166, + "cv_recall_at_20": 0.97976, + "cv_recall_at_100": 0.9919, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.88259, + "precision_at_3": 0.31511, + "precision_at_5": 0.19352, + "precision_at_10": 0.09737, + "precision_at_20": 0.04909, + "precision_at_100": 0.00998, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.882591, + "mrr_at_3": 0.910931, + "mrr_at_5": 0.915992, + "mrr_at_10": 0.916534, + "mrr_at_20": 0.917242, + "mrr_at_100": 0.917619, + "mrr_at_1000": 0.917655, + "nauc_ndcg_at_1_max": 0.73895, + "nauc_ndcg_at_1_std": -0.00672, + "nauc_ndcg_at_1_diff1": 0.966584, + "nauc_ndcg_at_3_max": 0.747682, + "nauc_ndcg_at_3_std": 0.03609, + "nauc_ndcg_at_3_diff1": 0.96442, + "nauc_ndcg_at_5_max": 0.777568, + "nauc_ndcg_at_5_std": 0.121931, + "nauc_ndcg_at_5_diff1": 0.961755, + "nauc_ndcg_at_10_max": 0.780947, + "nauc_ndcg_at_10_std": 0.133863, + "nauc_ndcg_at_10_diff1": 0.962074, + "nauc_ndcg_at_20_max": 0.777172, + "nauc_ndcg_at_20_std": 0.122846, + "nauc_ndcg_at_20_diff1": 0.962114, + "nauc_ndcg_at_100_max": 0.770523, + "nauc_ndcg_at_100_std": 0.100743, + "nauc_ndcg_at_100_diff1": 0.963728, + "nauc_ndcg_at_1000_max": 0.765585, + "nauc_ndcg_at_1000_std": 0.085763, + "nauc_ndcg_at_1000_diff1": 0.963468, + "nauc_map_at_1_max": 0.73895, + "nauc_map_at_1_std": -0.00672, + "nauc_map_at_1_diff1": 0.966584, + "nauc_map_at_3_max": 0.745554, + "nauc_map_at_3_std": 0.02515, + "nauc_map_at_3_diff1": 0.965094, + "nauc_map_at_5_max": 0.759873, + "nauc_map_at_5_std": 0.065773, + "nauc_map_at_5_diff1": 0.963957, + "nauc_map_at_10_max": 0.761267, + "nauc_map_at_10_std": 0.071439, + "nauc_map_at_10_diff1": 0.964052, + "nauc_map_at_20_max": 0.760191, + "nauc_map_at_20_std": 0.068338, + "nauc_map_at_20_diff1": 0.964089, + "nauc_map_at_100_max": 0.759273, + "nauc_map_at_100_std": 0.065379, + "nauc_map_at_100_diff1": 0.964284, + "nauc_map_at_1000_max": 0.759099, + "nauc_map_at_1000_std": 0.064836, + "nauc_map_at_1000_diff1": 0.96427, + "nauc_recall_at_1_max": 0.73895, + "nauc_recall_at_1_std": -0.00672, + "nauc_recall_at_1_diff1": 0.966584, + "nauc_recall_at_3_max": 0.757137, + "nauc_recall_at_3_std": 0.08514, + "nauc_recall_at_3_diff1": 0.961303, + "nauc_recall_at_5_max": 0.909739, + "nauc_recall_at_5_std": 0.53052, + "nauc_recall_at_5_diff1": 0.946223, + "nauc_recall_at_10_max": 0.953825, + "nauc_recall_at_10_std": 0.67205, + "nauc_recall_at_10_diff1": 0.945958, + "nauc_recall_at_20_max": 0.965327, + "nauc_recall_at_20_std": 0.717547, + "nauc_recall_at_20_diff1": 0.942212, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.789739, + "nauc_recall_at_100_diff1": 0.973347, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.73895, + "nauc_precision_at_1_std": -0.00672, + "nauc_precision_at_1_diff1": 0.966584, + "nauc_precision_at_3_max": 0.757137, + "nauc_precision_at_3_std": 0.08514, + "nauc_precision_at_3_diff1": 0.961303, + "nauc_precision_at_5_max": 0.918026, + "nauc_precision_at_5_std": 0.577062, + "nauc_precision_at_5_diff1": 0.942862, + "nauc_precision_at_10_max": 0.811513, + "nauc_precision_at_10_std": 0.669108, + "nauc_precision_at_10_diff1": 0.775957, + "nauc_precision_at_20_max": 0.770962, + "nauc_precision_at_20_std": 0.689544, + "nauc_precision_at_20_diff1": 0.710767, + "nauc_precision_at_100_max": 0.156616, + "nauc_precision_at_100_std": 0.244742, + "nauc_precision_at_100_diff1": 0.094183, + "nauc_precision_at_1000_max": -0.530911, + "nauc_precision_at_1000_std": -0.3414, + "nauc_precision_at_1000_diff1": -0.573514, + "nauc_cv_recall_at_1_max": 0.73895, + "nauc_cv_recall_at_1_std": -0.00672, + "nauc_cv_recall_at_1_diff1": 0.966584, + "nauc_cv_recall_at_3_max": 0.757137, + "nauc_cv_recall_at_3_std": 0.08514, + "nauc_cv_recall_at_3_diff1": 0.961303, + "nauc_cv_recall_at_5_max": 0.918026, + "nauc_cv_recall_at_5_std": 0.577062, + "nauc_cv_recall_at_5_diff1": 0.942862, + "nauc_cv_recall_at_10_max": 0.952176, + "nauc_cv_recall_at_10_std": 0.670257, + "nauc_cv_recall_at_10_diff1": 0.944028, + "nauc_cv_recall_at_20_max": 0.96082, + "nauc_cv_recall_at_20_std": 0.694715, + "nauc_cv_recall_at_20_diff1": 0.934699, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 0.777148, + "nauc_cv_recall_at_100_diff1": 0.96735, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.73895, + "nauc_mrr_at_1_std": -0.00672, + "nauc_mrr_at_1_diff1": 0.966584, + "nauc_mrr_at_3_max": 0.745554, + "nauc_mrr_at_3_std": 0.02515, + "nauc_mrr_at_3_diff1": 0.965094, + "nauc_mrr_at_5_max": 0.759642, + "nauc_mrr_at_5_std": 0.06674, + "nauc_mrr_at_5_diff1": 0.963759, + "nauc_mrr_at_10_max": 0.760315, + "nauc_mrr_at_10_std": 0.068149, + "nauc_mrr_at_10_diff1": 0.963979, + "nauc_mrr_at_20_max": 0.758861, + "nauc_mrr_at_20_std": 0.063591, + "nauc_mrr_at_20_diff1": 0.963961, + "nauc_mrr_at_100_max": 0.758014, + "nauc_mrr_at_100_std": 0.060906, + "nauc_mrr_at_100_diff1": 0.964168, + "nauc_mrr_at_1000_max": 0.757908, + "nauc_mrr_at_1000_std": 0.060601, + "nauc_mrr_at_1000_diff1": 0.964164, + "main_score": 0.9283, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 80.11045241355896, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..21d9c5be9d --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "84a382e05c4473fed9cff2bbae95fe2379416117", + "task_name": "VidoreShiftProjectRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.81, + "ndcg_at_3": 0.8894, + "ndcg_at_5": 0.90188, + "ndcg_at_10": 0.90545, + "ndcg_at_20": 0.91086, + "ndcg_at_100": 0.91086, + "ndcg_at_1000": 0.91086, + "map_at_1": 0.81, + "map_at_3": 0.87167, + "map_at_5": 0.87867, + "map_at_10": 0.88033, + "map_at_20": 0.88201, + "map_at_100": 0.88201, + "map_at_1000": 0.88201, + "recall_at_1": 0.81, + "recall_at_3": 0.94, + "recall_at_5": 0.97, + "recall_at_10": 0.98, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.81, + "cv_recall_at_3": 0.94, + "cv_recall_at_5": 0.97, + "cv_recall_at_10": 0.98, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.81, + "precision_at_3": 0.31333, + "precision_at_5": 0.194, + "precision_at_10": 0.098, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.81, + "mrr_at_3": 0.871667, + "mrr_at_5": 0.878667, + "mrr_at_10": 0.880333, + "mrr_at_20": 0.882012, + "mrr_at_100": 0.882012, + "mrr_at_1000": 0.882012, + "nauc_ndcg_at_1_max": 0.0966, + "nauc_ndcg_at_1_std": -0.547174, + "nauc_ndcg_at_1_diff1": 0.729347, + "nauc_ndcg_at_3_max": 0.162522, + "nauc_ndcg_at_3_std": -0.534647, + "nauc_ndcg_at_3_diff1": 0.714061, + "nauc_ndcg_at_5_max": 0.155835, + "nauc_ndcg_at_5_std": -0.503839, + "nauc_ndcg_at_5_diff1": 0.700154, + "nauc_ndcg_at_10_max": 0.13347, + "nauc_ndcg_at_10_std": -0.537873, + "nauc_ndcg_at_10_diff1": 0.722952, + "nauc_ndcg_at_20_max": 0.114434, + "nauc_ndcg_at_20_std": -0.554782, + "nauc_ndcg_at_20_diff1": 0.709557, + "nauc_ndcg_at_100_max": 0.114434, + "nauc_ndcg_at_100_std": -0.554782, + "nauc_ndcg_at_100_diff1": 0.709557, + "nauc_ndcg_at_1000_max": 0.114434, + "nauc_ndcg_at_1000_std": -0.554782, + "nauc_ndcg_at_1000_diff1": 0.709557, + "nauc_map_at_1_max": 0.0966, + "nauc_map_at_1_std": -0.547174, + "nauc_map_at_1_diff1": 0.729347, + "nauc_map_at_3_max": 0.131121, + "nauc_map_at_3_std": -0.550815, + "nauc_map_at_3_diff1": 0.710562, + "nauc_map_at_5_max": 0.124519, + "nauc_map_at_5_std": -0.538583, + "nauc_map_at_5_diff1": 0.704123, + "nauc_map_at_10_max": 0.115765, + "nauc_map_at_10_std": -0.551717, + "nauc_map_at_10_diff1": 0.712638, + "nauc_map_at_20_max": 0.110799, + "nauc_map_at_20_std": -0.556567, + "nauc_map_at_20_diff1": 0.709396, + "nauc_map_at_100_max": 0.110799, + "nauc_map_at_100_std": -0.556567, + "nauc_map_at_100_diff1": 0.709396, + "nauc_map_at_1000_max": 0.110799, + "nauc_map_at_1000_std": -0.556567, + "nauc_map_at_1000_diff1": 0.709396, + "nauc_recall_at_1_max": 0.0966, + "nauc_recall_at_1_std": -0.547174, + "nauc_recall_at_1_diff1": 0.729347, + "nauc_recall_at_3_max": 0.35442, + "nauc_recall_at_3_std": -0.429505, + "nauc_recall_at_3_diff1": 0.742141, + "nauc_recall_at_5_max": 0.522409, + "nauc_recall_at_5_std": -0.079832, + "nauc_recall_at_5_diff1": 0.664021, + "nauc_recall_at_10_max": 0.422502, + "nauc_recall_at_10_std": -0.298786, + "nauc_recall_at_10_diff1": 0.934641, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.0966, + "nauc_precision_at_1_std": -0.547174, + "nauc_precision_at_1_diff1": 0.729347, + "nauc_precision_at_3_max": 0.35442, + "nauc_precision_at_3_std": -0.429505, + "nauc_precision_at_3_diff1": 0.742141, + "nauc_precision_at_5_max": 0.522409, + "nauc_precision_at_5_std": -0.079832, + "nauc_precision_at_5_diff1": 0.664021, + "nauc_precision_at_10_max": 0.422502, + "nauc_precision_at_10_std": -0.298786, + "nauc_precision_at_10_diff1": 0.934641, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.0966, + "nauc_cv_recall_at_1_std": -0.547174, + "nauc_cv_recall_at_1_diff1": 0.729347, + "nauc_cv_recall_at_3_max": 0.35442, + "nauc_cv_recall_at_3_std": -0.429505, + "nauc_cv_recall_at_3_diff1": 0.742141, + "nauc_cv_recall_at_5_max": 0.522409, + "nauc_cv_recall_at_5_std": -0.079832, + "nauc_cv_recall_at_5_diff1": 0.664021, + "nauc_cv_recall_at_10_max": 0.422502, + "nauc_cv_recall_at_10_std": -0.298786, + "nauc_cv_recall_at_10_diff1": 0.934641, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.0966, + "nauc_mrr_at_1_std": -0.547174, + "nauc_mrr_at_1_diff1": 0.729347, + "nauc_mrr_at_3_max": 0.131121, + "nauc_mrr_at_3_std": -0.550815, + "nauc_mrr_at_3_diff1": 0.710562, + "nauc_mrr_at_5_max": 0.124519, + "nauc_mrr_at_5_std": -0.538583, + "nauc_mrr_at_5_diff1": 0.704123, + "nauc_mrr_at_10_max": 0.115765, + "nauc_mrr_at_10_std": -0.551717, + "nauc_mrr_at_10_diff1": 0.712638, + "nauc_mrr_at_20_max": 0.110799, + "nauc_mrr_at_20_std": -0.556567, + "nauc_mrr_at_20_diff1": 0.709396, + "nauc_mrr_at_100_max": 0.110799, + "nauc_mrr_at_100_std": -0.556567, + "nauc_mrr_at_100_diff1": 0.709396, + "nauc_mrr_at_1000_max": 0.110799, + "nauc_mrr_at_1000_std": -0.556567, + "nauc_mrr_at_1000_diff1": 0.709396, + "main_score": 0.90188, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 159.50661301612854, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..7473f94f8a --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", + "task_name": "VidoreSyntheticDocQAAIRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.95, + "ndcg_at_3": 0.96262, + "ndcg_at_5": 0.96262, + "ndcg_at_10": 0.96618, + "ndcg_at_20": 0.96618, + "ndcg_at_100": 0.96618, + "ndcg_at_1000": 0.9682, + "map_at_1": 0.95, + "map_at_3": 0.96, + "map_at_5": 0.96, + "map_at_10": 0.96167, + "map_at_20": 0.96167, + "map_at_100": 0.96167, + "map_at_1000": 0.96169, + "recall_at_1": 0.95, + "recall_at_3": 0.97, + "recall_at_5": 0.97, + "recall_at_10": 0.98, + "recall_at_20": 0.98, + "recall_at_100": 0.98, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.95, + "cv_recall_at_3": 0.97, + "cv_recall_at_5": 0.97, + "cv_recall_at_10": 0.98, + "cv_recall_at_20": 0.98, + "cv_recall_at_100": 0.98, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.95, + "precision_at_3": 0.32333, + "precision_at_5": 0.194, + "precision_at_10": 0.098, + "precision_at_20": 0.049, + "precision_at_100": 0.0098, + "precision_at_1000": 0.001, + "mrr_at_1": 0.95, + "mrr_at_3": 0.96, + "mrr_at_5": 0.96, + "mrr_at_10": 0.961667, + "mrr_at_20": 0.961667, + "mrr_at_100": 0.961667, + "mrr_at_1000": 0.961687, + "nauc_ndcg_at_1_max": 0.598133, + "nauc_ndcg_at_1_std": 0.283567, + "nauc_ndcg_at_1_diff1": 0.855275, + "nauc_ndcg_at_3_max": 0.735427, + "nauc_ndcg_at_3_std": 0.314676, + "nauc_ndcg_at_3_diff1": 0.806422, + "nauc_ndcg_at_5_max": 0.735427, + "nauc_ndcg_at_5_std": 0.314676, + "nauc_ndcg_at_5_diff1": 0.806422, + "nauc_ndcg_at_10_max": 0.721329, + "nauc_ndcg_at_10_std": 0.310104, + "nauc_ndcg_at_10_diff1": 0.786033, + "nauc_ndcg_at_20_max": 0.721329, + "nauc_ndcg_at_20_std": 0.310104, + "nauc_ndcg_at_20_diff1": 0.786033, + "nauc_ndcg_at_100_max": 0.721329, + "nauc_ndcg_at_100_std": 0.310104, + "nauc_ndcg_at_100_diff1": 0.786033, + "nauc_ndcg_at_1000_max": 0.711947, + "nauc_ndcg_at_1000_std": 0.308306, + "nauc_ndcg_at_1000_diff1": 0.795408, + "nauc_map_at_1_max": 0.598133, + "nauc_map_at_1_std": 0.283567, + "nauc_map_at_1_diff1": 0.855275, + "nauc_map_at_3_max": 0.699813, + "nauc_map_at_3_std": 0.306606, + "nauc_map_at_3_diff1": 0.819094, + "nauc_map_at_5_max": 0.699813, + "nauc_map_at_5_std": 0.306606, + "nauc_map_at_5_diff1": 0.819094, + "nauc_map_at_10_max": 0.692445, + "nauc_map_at_10_std": 0.304368, + "nauc_map_at_10_diff1": 0.811229, + "nauc_map_at_20_max": 0.692445, + "nauc_map_at_20_std": 0.304368, + "nauc_map_at_20_diff1": 0.811229, + "nauc_map_at_100_max": 0.692445, + "nauc_map_at_100_std": 0.304368, + "nauc_map_at_100_diff1": 0.811229, + "nauc_map_at_1000_max": 0.69235, + "nauc_map_at_1000_std": 0.30435, + "nauc_map_at_1000_diff1": 0.811322, + "nauc_recall_at_1_max": 0.598133, + "nauc_recall_at_1_std": 0.283567, + "nauc_recall_at_1_diff1": 0.855275, + "nauc_recall_at_3_max": 0.869281, + "nauc_recall_at_3_std": 0.345005, + "nauc_recall_at_3_diff1": 0.758792, + "nauc_recall_at_5_max": 0.869281, + "nauc_recall_at_5_std": 0.345005, + "nauc_recall_at_5_diff1": 0.758792, + "nauc_recall_at_10_max": 0.869281, + "nauc_recall_at_10_std": 0.338469, + "nauc_recall_at_10_diff1": 0.638189, + "nauc_recall_at_20_max": 0.869281, + "nauc_recall_at_20_std": 0.338469, + "nauc_recall_at_20_diff1": 0.638189, + "nauc_recall_at_100_max": 0.869281, + "nauc_recall_at_100_std": 0.338469, + "nauc_recall_at_100_diff1": 0.638189, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.598133, + "nauc_precision_at_1_std": 0.283567, + "nauc_precision_at_1_diff1": 0.855275, + "nauc_precision_at_3_max": 0.869281, + "nauc_precision_at_3_std": 0.345005, + "nauc_precision_at_3_diff1": 0.758792, + "nauc_precision_at_5_max": 0.869281, + "nauc_precision_at_5_std": 0.345005, + "nauc_precision_at_5_diff1": 0.758792, + "nauc_precision_at_10_max": 0.869281, + "nauc_precision_at_10_std": 0.338469, + "nauc_precision_at_10_diff1": 0.638189, + "nauc_precision_at_20_max": 0.869281, + "nauc_precision_at_20_std": 0.338469, + "nauc_precision_at_20_diff1": 0.638189, + "nauc_precision_at_100_max": 0.869281, + "nauc_precision_at_100_std": 0.338469, + "nauc_precision_at_100_diff1": 0.638189, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.598133, + "nauc_cv_recall_at_1_std": 0.283567, + "nauc_cv_recall_at_1_diff1": 0.855275, + "nauc_cv_recall_at_3_max": 0.869281, + "nauc_cv_recall_at_3_std": 0.345005, + "nauc_cv_recall_at_3_diff1": 0.758792, + "nauc_cv_recall_at_5_max": 0.869281, + "nauc_cv_recall_at_5_std": 0.345005, + "nauc_cv_recall_at_5_diff1": 0.758792, + "nauc_cv_recall_at_10_max": 0.869281, + "nauc_cv_recall_at_10_std": 0.338469, + "nauc_cv_recall_at_10_diff1": 0.638189, + "nauc_cv_recall_at_20_max": 0.869281, + "nauc_cv_recall_at_20_std": 0.338469, + "nauc_cv_recall_at_20_diff1": 0.638189, + "nauc_cv_recall_at_100_max": 0.869281, + "nauc_cv_recall_at_100_std": 0.338469, + "nauc_cv_recall_at_100_diff1": 0.638189, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.598133, + "nauc_mrr_at_1_std": 0.283567, + "nauc_mrr_at_1_diff1": 0.855275, + "nauc_mrr_at_3_max": 0.699813, + "nauc_mrr_at_3_std": 0.306606, + "nauc_mrr_at_3_diff1": 0.819094, + "nauc_mrr_at_5_max": 0.699813, + "nauc_mrr_at_5_std": 0.306606, + "nauc_mrr_at_5_diff1": 0.819094, + "nauc_mrr_at_10_max": 0.692445, + "nauc_mrr_at_10_std": 0.304368, + "nauc_mrr_at_10_diff1": 0.811229, + "nauc_mrr_at_20_max": 0.692445, + "nauc_mrr_at_20_std": 0.304368, + "nauc_mrr_at_20_diff1": 0.811229, + "nauc_mrr_at_100_max": 0.692445, + "nauc_mrr_at_100_std": 0.304368, + "nauc_mrr_at_100_diff1": 0.811229, + "nauc_mrr_at_1000_max": 0.692349, + "nauc_mrr_at_1000_std": 0.30435, + "nauc_mrr_at_1000_diff1": 0.811323, + "main_score": 0.96262, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 155.90147137641907, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..b02c0d2e53 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", + "task_name": "VidoreSyntheticDocQAEnergyRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.95, + "ndcg_at_3": 0.96893, + "ndcg_at_5": 0.97323, + "ndcg_at_10": 0.97323, + "ndcg_at_20": 0.97602, + "ndcg_at_100": 0.97602, + "ndcg_at_1000": 0.97602, + "map_at_1": 0.95, + "map_at_3": 0.965, + "map_at_5": 0.9675, + "map_at_10": 0.9675, + "map_at_20": 0.96841, + "map_at_100": 0.96841, + "map_at_1000": 0.96841, + "recall_at_1": 0.95, + "recall_at_3": 0.98, + "recall_at_5": 0.99, + "recall_at_10": 0.99, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.95, + "cv_recall_at_3": 0.98, + "cv_recall_at_5": 0.99, + "cv_recall_at_10": 0.99, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.95, + "precision_at_3": 0.32667, + "precision_at_5": 0.198, + "precision_at_10": 0.099, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.95, + "mrr_at_3": 0.965, + "mrr_at_5": 0.9675, + "mrr_at_10": 0.9675, + "mrr_at_20": 0.968409, + "mrr_at_100": 0.968409, + "mrr_at_1000": 0.968409, + "nauc_ndcg_at_1_max": 0.65042, + "nauc_ndcg_at_1_std": -0.536041, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.701857, + "nauc_ndcg_at_3_std": -0.559133, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.653883, + "nauc_ndcg_at_5_std": -0.738271, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.653883, + "nauc_ndcg_at_10_std": -0.738271, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.665485, + "nauc_ndcg_at_20_std": -0.621731, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.665485, + "nauc_ndcg_at_100_std": -0.621731, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.665485, + "nauc_ndcg_at_1000_std": -0.621731, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.65042, + "nauc_map_at_1_std": -0.536041, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.686608, + "nauc_map_at_3_std": -0.552288, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.662501, + "nauc_map_at_5_std": -0.637399, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.662501, + "nauc_map_at_10_std": -0.637399, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.665619, + "nauc_map_at_20_std": -0.60567, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.665619, + "nauc_map_at_100_std": -0.60567, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.665619, + "nauc_map_at_1000_std": -0.60567, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.65042, + "nauc_recall_at_1_std": -0.536041, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 0.777077, + "nauc_recall_at_3_std": -0.592904, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 0.554155, + "nauc_recall_at_5_std": -1.739963, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 0.554155, + "nauc_recall_at_10_std": -1.739963, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.65042, + "nauc_precision_at_1_std": -0.536041, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 0.777077, + "nauc_precision_at_3_std": -0.592904, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 0.554155, + "nauc_precision_at_5_std": -1.739963, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 0.554155, + "nauc_precision_at_10_std": -1.739963, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.65042, + "nauc_cv_recall_at_1_std": -0.536041, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": 0.777077, + "nauc_cv_recall_at_3_std": -0.592904, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 0.554155, + "nauc_cv_recall_at_5_std": -1.739963, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": 0.554155, + "nauc_cv_recall_at_10_std": -1.739963, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.65042, + "nauc_mrr_at_1_std": -0.536041, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.686608, + "nauc_mrr_at_3_std": -0.552288, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.662501, + "nauc_mrr_at_5_std": -0.637399, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.662501, + "nauc_mrr_at_10_std": -0.637399, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.665619, + "nauc_mrr_at_20_std": -0.60567, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.665619, + "nauc_mrr_at_100_std": -0.60567, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.665619, + "nauc_mrr_at_1000_std": -0.60567, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.97323, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 149.383154630661, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..cc95b9dae5 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b4909afa930f81282fd20601e860668073ad02aa", + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.92, + "ndcg_at_3": 0.95786, + "ndcg_at_5": 0.96603, + "ndcg_at_10": 0.96603, + "ndcg_at_20": 0.96603, + "ndcg_at_100": 0.96603, + "ndcg_at_1000": 0.96603, + "map_at_1": 0.92, + "map_at_3": 0.95, + "map_at_5": 0.9545, + "map_at_10": 0.9545, + "map_at_20": 0.9545, + "map_at_100": 0.9545, + "map_at_1000": 0.9545, + "recall_at_1": 0.92, + "recall_at_3": 0.98, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.92, + "cv_recall_at_3": 0.98, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.92, + "precision_at_3": 0.32667, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.92, + "mrr_at_3": 0.95, + "mrr_at_5": 0.9545, + "mrr_at_10": 0.9545, + "mrr_at_20": 0.9545, + "mrr_at_100": 0.9545, + "mrr_at_1000": 0.9545, + "nauc_ndcg_at_1_max": 0.692285, + "nauc_ndcg_at_1_std": 0.619981, + "nauc_ndcg_at_1_diff1": 0.934641, + "nauc_ndcg_at_3_max": 0.717673, + "nauc_ndcg_at_3_std": 0.602439, + "nauc_ndcg_at_3_diff1": 0.934641, + "nauc_ndcg_at_5_max": 0.706252, + "nauc_ndcg_at_5_std": 0.617976, + "nauc_ndcg_at_5_diff1": 0.933797, + "nauc_ndcg_at_10_max": 0.706252, + "nauc_ndcg_at_10_std": 0.617976, + "nauc_ndcg_at_10_diff1": 0.933797, + "nauc_ndcg_at_20_max": 0.706252, + "nauc_ndcg_at_20_std": 0.617976, + "nauc_ndcg_at_20_diff1": 0.933797, + "nauc_ndcg_at_100_max": 0.706252, + "nauc_ndcg_at_100_std": 0.617976, + "nauc_ndcg_at_100_diff1": 0.933797, + "nauc_ndcg_at_1000_max": 0.706252, + "nauc_ndcg_at_1000_std": 0.617976, + "nauc_ndcg_at_1000_diff1": 0.933797, + "nauc_map_at_1_max": 0.692285, + "nauc_map_at_1_std": 0.619981, + "nauc_map_at_1_diff1": 0.934641, + "nauc_map_at_3_max": 0.709244, + "nauc_map_at_3_std": 0.608263, + "nauc_map_at_3_diff1": 0.934641, + "nauc_map_at_5_max": 0.704985, + "nauc_map_at_5_std": 0.617719, + "nauc_map_at_5_diff1": 0.933922, + "nauc_map_at_10_max": 0.704985, + "nauc_map_at_10_std": 0.617719, + "nauc_map_at_10_diff1": 0.933922, + "nauc_map_at_20_max": 0.704985, + "nauc_map_at_20_std": 0.617719, + "nauc_map_at_20_diff1": 0.933922, + "nauc_map_at_100_max": 0.704985, + "nauc_map_at_100_std": 0.617719, + "nauc_map_at_100_diff1": 0.933922, + "nauc_map_at_1000_max": 0.704985, + "nauc_map_at_1000_std": 0.617719, + "nauc_map_at_1000_diff1": 0.933922, + "nauc_recall_at_1_max": 0.692285, + "nauc_recall_at_1_std": 0.619981, + "nauc_recall_at_1_diff1": 0.934641, + "nauc_recall_at_3_max": 0.777077, + "nauc_recall_at_3_std": 0.561391, + "nauc_recall_at_3_diff1": 0.934641, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.692285, + "nauc_precision_at_1_std": 0.619981, + "nauc_precision_at_1_diff1": 0.934641, + "nauc_precision_at_3_max": 0.777077, + "nauc_precision_at_3_std": 0.561391, + "nauc_precision_at_3_diff1": 0.934641, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.692285, + "nauc_cv_recall_at_1_std": 0.619981, + "nauc_cv_recall_at_1_diff1": 0.934641, + "nauc_cv_recall_at_3_max": 0.777077, + "nauc_cv_recall_at_3_std": 0.561391, + "nauc_cv_recall_at_3_diff1": 0.934641, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.692285, + "nauc_mrr_at_1_std": 0.619981, + "nauc_mrr_at_1_diff1": 0.934641, + "nauc_mrr_at_3_max": 0.709244, + "nauc_mrr_at_3_std": 0.608263, + "nauc_mrr_at_3_diff1": 0.934641, + "nauc_mrr_at_5_max": 0.704985, + "nauc_mrr_at_5_std": 0.617719, + "nauc_mrr_at_5_diff1": 0.933922, + "nauc_mrr_at_10_max": 0.704985, + "nauc_mrr_at_10_std": 0.617719, + "nauc_mrr_at_10_diff1": 0.933922, + "nauc_mrr_at_20_max": 0.704985, + "nauc_mrr_at_20_std": 0.617719, + "nauc_mrr_at_20_diff1": 0.933922, + "nauc_mrr_at_100_max": 0.704985, + "nauc_mrr_at_100_std": 0.617719, + "nauc_mrr_at_100_diff1": 0.933922, + "nauc_mrr_at_1000_max": 0.704985, + "nauc_mrr_at_1000_std": 0.617719, + "nauc_mrr_at_1000_diff1": 0.933922, + "main_score": 0.96603, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 153.23047542572021, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..152ee44eb6 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.96, + "ndcg_at_3": 0.97893, + "ndcg_at_5": 0.98323, + "ndcg_at_10": 0.98323, + "ndcg_at_20": 0.98323, + "ndcg_at_100": 0.98323, + "ndcg_at_1000": 0.98323, + "map_at_1": 0.96, + "map_at_3": 0.975, + "map_at_5": 0.9775, + "map_at_10": 0.9775, + "map_at_20": 0.9775, + "map_at_100": 0.9775, + "map_at_1000": 0.9775, + "recall_at_1": 0.96, + "recall_at_3": 0.99, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.96, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.96, + "precision_at_3": 0.33, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.96, + "mrr_at_3": 0.975, + "mrr_at_5": 0.9775, + "mrr_at_10": 0.9775, + "mrr_at_20": 0.9775, + "mrr_at_100": 0.9775, + "mrr_at_1000": 0.9775, + "nauc_ndcg_at_1_max": 0.711718, + "nauc_ndcg_at_1_std": -0.070495, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.758895, + "nauc_ndcg_at_3_std": -0.218105, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.730538, + "nauc_ndcg_at_5_std": -0.129381, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.730538, + "nauc_ndcg_at_10_std": -0.129381, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.730538, + "nauc_ndcg_at_20_std": -0.129381, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.730538, + "nauc_ndcg_at_100_std": -0.129381, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.730538, + "nauc_ndcg_at_1000_std": -0.129381, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.711718, + "nauc_map_at_1_std": -0.070495, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.743231, + "nauc_map_at_3_std": -0.169094, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.729225, + "nauc_map_at_5_std": -0.125272, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.729225, + "nauc_map_at_10_std": -0.125272, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.729225, + "nauc_map_at_20_std": -0.125272, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.729225, + "nauc_map_at_100_std": -0.125272, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.729225, + "nauc_map_at_1000_std": -0.125272, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.711718, + "nauc_recall_at_1_std": -0.070495, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 0.869281, + "nauc_recall_at_3_std": -0.563492, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.711718, + "nauc_precision_at_1_std": -0.070495, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 0.869281, + "nauc_precision_at_3_std": -0.563492, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.711718, + "nauc_cv_recall_at_1_std": -0.070495, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": 0.869281, + "nauc_cv_recall_at_3_std": -0.563492, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.711718, + "nauc_mrr_at_1_std": -0.070495, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.743231, + "nauc_mrr_at_3_std": -0.169094, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.729225, + "nauc_mrr_at_5_std": -0.125272, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.729225, + "nauc_mrr_at_10_std": -0.125272, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.729225, + "nauc_mrr_at_20_std": -0.125272, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.729225, + "nauc_mrr_at_100_std": -0.125272, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.729225, + "nauc_mrr_at_1000_std": -0.125272, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.98323, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 152.20528554916382, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..79e7d9a1ae --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", + "task_name": "VidoreTabfquadRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.89643, + "ndcg_at_3": 0.94451, + "ndcg_at_5": 0.94451, + "ndcg_at_10": 0.9514, + "ndcg_at_20": 0.9514, + "ndcg_at_100": 0.9514, + "ndcg_at_1000": 0.9514, + "map_at_1": 0.89643, + "map_at_3": 0.93274, + "map_at_5": 0.93274, + "map_at_10": 0.93555, + "map_at_20": 0.93555, + "map_at_100": 0.93555, + "map_at_1000": 0.93555, + "recall_at_1": 0.89643, + "recall_at_3": 0.97857, + "recall_at_5": 0.97857, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.89643, + "cv_recall_at_3": 0.97857, + "cv_recall_at_5": 0.97857, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.89643, + "precision_at_3": 0.32619, + "precision_at_5": 0.19571, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.896429, + "mrr_at_3": 0.932738, + "mrr_at_5": 0.932738, + "mrr_at_10": 0.935554, + "mrr_at_20": 0.935554, + "mrr_at_100": 0.935554, + "mrr_at_1000": 0.935554, + "nauc_ndcg_at_1_max": 0.571623, + "nauc_ndcg_at_1_std": 0.14255, + "nauc_ndcg_at_1_diff1": 0.923429, + "nauc_ndcg_at_3_max": 0.650894, + "nauc_ndcg_at_3_std": 0.267349, + "nauc_ndcg_at_3_diff1": 0.933358, + "nauc_ndcg_at_5_max": 0.650894, + "nauc_ndcg_at_5_std": 0.267349, + "nauc_ndcg_at_5_diff1": 0.933358, + "nauc_ndcg_at_10_max": 0.612205, + "nauc_ndcg_at_10_std": 0.200033, + "nauc_ndcg_at_10_diff1": 0.92988, + "nauc_ndcg_at_20_max": 0.612205, + "nauc_ndcg_at_20_std": 0.200033, + "nauc_ndcg_at_20_diff1": 0.92988, + "nauc_ndcg_at_100_max": 0.612205, + "nauc_ndcg_at_100_std": 0.200033, + "nauc_ndcg_at_100_diff1": 0.92988, + "nauc_ndcg_at_1000_max": 0.612205, + "nauc_ndcg_at_1000_std": 0.200033, + "nauc_ndcg_at_1000_diff1": 0.92988, + "nauc_map_at_1_max": 0.571623, + "nauc_map_at_1_std": 0.14255, + "nauc_map_at_1_diff1": 0.923429, + "nauc_map_at_3_max": 0.620582, + "nauc_map_at_3_std": 0.215774, + "nauc_map_at_3_diff1": 0.930661, + "nauc_map_at_5_max": 0.620582, + "nauc_map_at_5_std": 0.215774, + "nauc_map_at_5_diff1": 0.930661, + "nauc_map_at_10_max": 0.607485, + "nauc_map_at_10_std": 0.192662, + "nauc_map_at_10_diff1": 0.929386, + "nauc_map_at_20_max": 0.607485, + "nauc_map_at_20_std": 0.192662, + "nauc_map_at_20_diff1": 0.929386, + "nauc_map_at_100_max": 0.607485, + "nauc_map_at_100_std": 0.192662, + "nauc_map_at_100_diff1": 0.929386, + "nauc_map_at_1000_max": 0.607485, + "nauc_map_at_1000_std": 0.192662, + "nauc_map_at_1000_diff1": 0.929386, + "nauc_recall_at_1_max": 0.571623, + "nauc_recall_at_1_std": 0.14255, + "nauc_recall_at_1_diff1": 0.923429, + "nauc_recall_at_3_max": 0.925692, + "nauc_recall_at_3_std": 0.739729, + "nauc_recall_at_3_diff1": 0.956427, + "nauc_recall_at_5_max": 0.925692, + "nauc_recall_at_5_std": 0.739729, + "nauc_recall_at_5_diff1": 0.956427, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.571623, + "nauc_precision_at_1_std": 0.14255, + "nauc_precision_at_1_diff1": 0.923429, + "nauc_precision_at_3_max": 0.925692, + "nauc_precision_at_3_std": 0.739729, + "nauc_precision_at_3_diff1": 0.956427, + "nauc_precision_at_5_max": 0.925692, + "nauc_precision_at_5_std": 0.739729, + "nauc_precision_at_5_diff1": 0.956427, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.571623, + "nauc_cv_recall_at_1_std": 0.14255, + "nauc_cv_recall_at_1_diff1": 0.923429, + "nauc_cv_recall_at_3_max": 0.925692, + "nauc_cv_recall_at_3_std": 0.739729, + "nauc_cv_recall_at_3_diff1": 0.956427, + "nauc_cv_recall_at_5_max": 0.925692, + "nauc_cv_recall_at_5_std": 0.739729, + "nauc_cv_recall_at_5_diff1": 0.956427, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.571623, + "nauc_mrr_at_1_std": 0.14255, + "nauc_mrr_at_1_diff1": 0.923429, + "nauc_mrr_at_3_max": 0.620582, + "nauc_mrr_at_3_std": 0.215774, + "nauc_mrr_at_3_diff1": 0.930661, + "nauc_mrr_at_5_max": 0.620582, + "nauc_mrr_at_5_std": 0.215774, + "nauc_mrr_at_5_diff1": 0.930661, + "nauc_mrr_at_10_max": 0.607485, + "nauc_mrr_at_10_std": 0.192662, + "nauc_mrr_at_10_diff1": 0.929386, + "nauc_mrr_at_20_max": 0.607485, + "nauc_mrr_at_20_std": 0.192662, + "nauc_mrr_at_20_diff1": 0.929386, + "nauc_mrr_at_100_max": 0.607485, + "nauc_mrr_at_100_std": 0.192662, + "nauc_mrr_at_100_diff1": 0.929386, + "nauc_mrr_at_1000_max": 0.607485, + "nauc_mrr_at_1000_std": 0.192662, + "nauc_mrr_at_1000_diff1": 0.929386, + "main_score": 0.94451, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.10863184928894, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..3e178e5d09 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", + "task_name": "VidoreTatdqaRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.7339, + "ndcg_at_3": 0.81303, + "ndcg_at_5": 0.83143, + "ndcg_at_10": 0.84065, + "ndcg_at_20": 0.84423, + "ndcg_at_100": 0.84959, + "ndcg_at_1000": 0.85187, + "map_at_1": 0.73269, + "map_at_3": 0.79344, + "map_at_5": 0.80371, + "map_at_10": 0.80764, + "map_at_20": 0.80867, + "map_at_100": 0.80942, + "map_at_1000": 0.80954, + "recall_at_1": 0.73269, + "recall_at_3": 0.86908, + "recall_at_5": 0.91373, + "recall_at_10": 0.94168, + "recall_at_20": 0.95522, + "recall_at_100": 0.9836, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.7339, + "cv_recall_at_3": 0.86999, + "cv_recall_at_5": 0.91434, + "cv_recall_at_10": 0.94168, + "cv_recall_at_20": 0.95565, + "cv_recall_at_100": 0.9842, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.7339, + "precision_at_3": 0.2906, + "precision_at_5": 0.18335, + "precision_at_10": 0.09453, + "precision_at_20": 0.04803, + "precision_at_100": 0.00991, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.7339, + "mrr_at_3": 0.794046, + "mrr_at_5": 0.804162, + "mrr_at_10": 0.807955, + "mrr_at_20": 0.809015, + "mrr_at_100": 0.809763, + "mrr_at_1000": 0.809862, + "nauc_ndcg_at_1_max": 0.248726, + "nauc_ndcg_at_1_std": 0.084058, + "nauc_ndcg_at_1_diff1": 0.853962, + "nauc_ndcg_at_3_max": 0.231481, + "nauc_ndcg_at_3_std": 0.102509, + "nauc_ndcg_at_3_diff1": 0.822225, + "nauc_ndcg_at_5_max": 0.24045, + "nauc_ndcg_at_5_std": 0.108251, + "nauc_ndcg_at_5_diff1": 0.821873, + "nauc_ndcg_at_10_max": 0.254456, + "nauc_ndcg_at_10_std": 0.120258, + "nauc_ndcg_at_10_diff1": 0.823623, + "nauc_ndcg_at_20_max": 0.254772, + "nauc_ndcg_at_20_std": 0.120888, + "nauc_ndcg_at_20_diff1": 0.824252, + "nauc_ndcg_at_100_max": 0.250758, + "nauc_ndcg_at_100_std": 0.11799, + "nauc_ndcg_at_100_diff1": 0.827993, + "nauc_ndcg_at_1000_max": 0.247529, + "nauc_ndcg_at_1000_std": 0.109845, + "nauc_ndcg_at_1000_diff1": 0.829085, + "nauc_map_at_1_max": 0.248351, + "nauc_map_at_1_std": 0.085432, + "nauc_map_at_1_diff1": 0.854838, + "nauc_map_at_3_max": 0.236389, + "nauc_map_at_3_std": 0.097123, + "nauc_map_at_3_diff1": 0.831556, + "nauc_map_at_5_max": 0.241021, + "nauc_map_at_5_std": 0.099301, + "nauc_map_at_5_diff1": 0.831667, + "nauc_map_at_10_max": 0.246028, + "nauc_map_at_10_std": 0.103558, + "nauc_map_at_10_diff1": 0.832484, + "nauc_map_at_20_max": 0.246144, + "nauc_map_at_20_std": 0.103729, + "nauc_map_at_20_diff1": 0.832694, + "nauc_map_at_100_max": 0.245673, + "nauc_map_at_100_std": 0.103393, + "nauc_map_at_100_diff1": 0.833144, + "nauc_map_at_1000_max": 0.245508, + "nauc_map_at_1000_std": 0.103033, + "nauc_map_at_1000_diff1": 0.833172, + "nauc_recall_at_1_max": 0.248351, + "nauc_recall_at_1_std": 0.085432, + "nauc_recall_at_1_diff1": 0.854838, + "nauc_recall_at_3_max": 0.210593, + "nauc_recall_at_3_std": 0.124789, + "nauc_recall_at_3_diff1": 0.78316, + "nauc_recall_at_5_max": 0.238658, + "nauc_recall_at_5_std": 0.162915, + "nauc_recall_at_5_diff1": 0.763725, + "nauc_recall_at_10_max": 0.345844, + "nauc_recall_at_10_std": 0.279626, + "nauc_recall_at_10_diff1": 0.751075, + "nauc_recall_at_20_max": 0.378969, + "nauc_recall_at_20_std": 0.337124, + "nauc_recall_at_20_diff1": 0.737553, + "nauc_recall_at_100_max": 0.426705, + "nauc_recall_at_100_std": 0.586312, + "nauc_recall_at_100_diff1": 0.75622, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.248726, + "nauc_precision_at_1_std": 0.084058, + "nauc_precision_at_1_diff1": 0.853962, + "nauc_precision_at_3_max": 0.207813, + "nauc_precision_at_3_std": 0.122184, + "nauc_precision_at_3_diff1": 0.75237, + "nauc_precision_at_5_max": 0.228749, + "nauc_precision_at_5_std": 0.155496, + "nauc_precision_at_5_diff1": 0.70603, + "nauc_precision_at_10_max": 0.313541, + "nauc_precision_at_10_std": 0.255103, + "nauc_precision_at_10_diff1": 0.633294, + "nauc_precision_at_20_max": 0.292255, + "nauc_precision_at_20_std": 0.25835, + "nauc_precision_at_20_diff1": 0.5403, + "nauc_precision_at_100_max": 0.126114, + "nauc_precision_at_100_std": 0.224648, + "nauc_precision_at_100_diff1": 0.21535, + "nauc_precision_at_1000_max": -0.366355, + "nauc_precision_at_1000_std": -0.361713, + "nauc_precision_at_1000_diff1": -0.531096, + "nauc_cv_recall_at_1_max": 0.248726, + "nauc_cv_recall_at_1_std": 0.084058, + "nauc_cv_recall_at_1_diff1": 0.853962, + "nauc_cv_recall_at_3_max": 0.211617, + "nauc_cv_recall_at_3_std": 0.127371, + "nauc_cv_recall_at_3_diff1": 0.782242, + "nauc_cv_recall_at_5_max": 0.241875, + "nauc_cv_recall_at_5_std": 0.167154, + "nauc_cv_recall_at_5_diff1": 0.764094, + "nauc_cv_recall_at_10_max": 0.345844, + "nauc_cv_recall_at_10_std": 0.279626, + "nauc_cv_recall_at_10_diff1": 0.751075, + "nauc_cv_recall_at_20_max": 0.373013, + "nauc_cv_recall_at_20_std": 0.330767, + "nauc_cv_recall_at_20_diff1": 0.735037, + "nauc_cv_recall_at_100_max": 0.404655, + "nauc_cv_recall_at_100_std": 0.5704, + "nauc_cv_recall_at_100_diff1": 0.746844, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.248726, + "nauc_mrr_at_1_std": 0.084058, + "nauc_mrr_at_1_diff1": 0.853962, + "nauc_mrr_at_3_max": 0.236313, + "nauc_mrr_at_3_std": 0.097274, + "nauc_mrr_at_3_diff1": 0.831032, + "nauc_mrr_at_5_max": 0.241247, + "nauc_mrr_at_5_std": 0.099497, + "nauc_mrr_at_5_diff1": 0.831352, + "nauc_mrr_at_10_max": 0.245885, + "nauc_mrr_at_10_std": 0.103278, + "nauc_mrr_at_10_diff1": 0.832195, + "nauc_mrr_at_20_max": 0.245853, + "nauc_mrr_at_20_std": 0.103273, + "nauc_mrr_at_20_diff1": 0.83237, + "nauc_mrr_at_100_max": 0.245398, + "nauc_mrr_at_100_std": 0.102955, + "nauc_mrr_at_100_diff1": 0.832824, + "nauc_mrr_at_1000_max": 0.245303, + "nauc_mrr_at_1000_std": 0.102678, + "nauc_mrr_at_1000_diff1": 0.832867, + "main_score": 0.83143, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 55.76857042312622, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json new file mode 100644 index 0000000000..eefbd48b9c --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-3b/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json @@ -0,0 +1 @@ +{"name": "nomic-ai/colnomic-embed-multimodal-3b", "revision": "530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f", "release_date": "2025-03-31", "languages": ["deu-Latn", "spa-Latn", "eng-Latn", "fra-Latn", "ita-Latn"], "n_parameters": 3000000000, "memory_usage_mb": 7200.0, "max_tokens": 128000.0, "embed_dim": 128, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/nomic-ai/colpali", "public_training_data": "https://huggingface.co/datasets/vidore/colpali_train_set", "framework": ["ColPali"], "reference": "https://huggingface.co/nomic-ai/colnomic-embed-multimodal-3b", "similarity_fn_name": "max_sim", "use_instructions": true, "training_datasets": {"VDRMultilingual": ["Train"], "DocVQA": ["train"], "InfoVQA": ["train"], "TATDQA": ["train"], "arXivQA": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "ColQwen2_5Wrapper"} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/Vidore2BioMedicalLecturesRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..981066ff67 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.59375, + "ndcg_at_3": 0.60429, + "ndcg_at_5": 0.63352, + "ndcg_at_10": 0.66101, + "ndcg_at_20": 0.68998, + "ndcg_at_100": 0.7234, + "ndcg_at_1000": 0.73762, + "map_at_1": 0.36429, + "map_at_3": 0.49515, + "map_at_5": 0.54277, + "map_at_10": 0.5741, + "map_at_20": 0.59142, + "map_at_100": 0.60255, + "map_at_1000": 0.60411, + "recall_at_1": 0.36429, + "recall_at_3": 0.56175, + "recall_at_5": 0.66574, + "recall_at_10": 0.74914, + "recall_at_20": 0.82394, + "recall_at_100": 0.93173, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.59375, + "cv_recall_at_3": 0.8125, + "cv_recall_at_5": 0.88125, + "cv_recall_at_10": 0.925, + "cv_recall_at_20": 0.95625, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.59375, + "precision_at_3": 0.37292, + "precision_at_5": 0.2875, + "precision_at_10": 0.18063, + "precision_at_20": 0.10906, + "precision_at_100": 0.02794, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.59375, + "mrr_at_3": 0.694792, + "mrr_at_5": 0.711042, + "mrr_at_10": 0.716979, + "mrr_at_20": 0.719252, + "mrr_at_100": 0.720087, + "mrr_at_1000": 0.720166, + "nauc_ndcg_at_1_max": 0.285508, + "nauc_ndcg_at_1_std": -0.110661, + "nauc_ndcg_at_1_diff1": 0.509556, + "nauc_ndcg_at_3_max": 0.3371, + "nauc_ndcg_at_3_std": -0.059034, + "nauc_ndcg_at_3_diff1": 0.385384, + "nauc_ndcg_at_5_max": 0.398273, + "nauc_ndcg_at_5_std": 0.006369, + "nauc_ndcg_at_5_diff1": 0.372803, + "nauc_ndcg_at_10_max": 0.368394, + "nauc_ndcg_at_10_std": -0.030717, + "nauc_ndcg_at_10_diff1": 0.385292, + "nauc_ndcg_at_20_max": 0.369969, + "nauc_ndcg_at_20_std": -0.025682, + "nauc_ndcg_at_20_diff1": 0.389531, + "nauc_ndcg_at_100_max": 0.366572, + "nauc_ndcg_at_100_std": -0.009177, + "nauc_ndcg_at_100_diff1": 0.360413, + "nauc_ndcg_at_1000_max": 0.353859, + "nauc_ndcg_at_1000_std": -0.027034, + "nauc_ndcg_at_1000_diff1": 0.370963, + "nauc_map_at_1_max": 0.355755, + "nauc_map_at_1_std": 0.023722, + "nauc_map_at_1_diff1": 0.557791, + "nauc_map_at_3_max": 0.353489, + "nauc_map_at_3_std": -0.01299, + "nauc_map_at_3_diff1": 0.435358, + "nauc_map_at_5_max": 0.382809, + "nauc_map_at_5_std": -0.004712, + "nauc_map_at_5_diff1": 0.40255, + "nauc_map_at_10_max": 0.360365, + "nauc_map_at_10_std": -0.032698, + "nauc_map_at_10_diff1": 0.39501, + "nauc_map_at_20_max": 0.364167, + "nauc_map_at_20_std": -0.029072, + "nauc_map_at_20_diff1": 0.396801, + "nauc_map_at_100_max": 0.366935, + "nauc_map_at_100_std": -0.022796, + "nauc_map_at_100_diff1": 0.388295, + "nauc_map_at_1000_max": 0.365621, + "nauc_map_at_1000_std": -0.024212, + "nauc_map_at_1000_diff1": 0.388183, + "nauc_recall_at_1_max": 0.355755, + "nauc_recall_at_1_std": 0.023722, + "nauc_recall_at_1_diff1": 0.557791, + "nauc_recall_at_3_max": 0.330629, + "nauc_recall_at_3_std": -0.025845, + "nauc_recall_at_3_diff1": 0.373661, + "nauc_recall_at_5_max": 0.420323, + "nauc_recall_at_5_std": 0.067326, + "nauc_recall_at_5_diff1": 0.272756, + "nauc_recall_at_10_max": 0.351534, + "nauc_recall_at_10_std": -0.020258, + "nauc_recall_at_10_diff1": 0.305145, + "nauc_recall_at_20_max": 0.362325, + "nauc_recall_at_20_std": -0.013261, + "nauc_recall_at_20_diff1": 0.290742, + "nauc_recall_at_100_max": 0.390277, + "nauc_recall_at_100_std": 0.195869, + "nauc_recall_at_100_diff1": -0.021621, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.285508, + "nauc_precision_at_1_std": -0.110661, + "nauc_precision_at_1_diff1": 0.509556, + "nauc_precision_at_3_max": 0.092382, + "nauc_precision_at_3_std": -0.096821, + "nauc_precision_at_3_diff1": -0.076792, + "nauc_precision_at_5_max": 0.06012, + "nauc_precision_at_5_std": -0.052176, + "nauc_precision_at_5_diff1": -0.199501, + "nauc_precision_at_10_max": -0.053141, + "nauc_precision_at_10_std": -0.094155, + "nauc_precision_at_10_diff1": -0.216114, + "nauc_precision_at_20_max": -0.092178, + "nauc_precision_at_20_std": -0.036998, + "nauc_precision_at_20_diff1": -0.246616, + "nauc_precision_at_100_max": -0.160906, + "nauc_precision_at_100_std": -0.016439, + "nauc_precision_at_100_diff1": -0.339039, + "nauc_precision_at_1000_max": -0.231167, + "nauc_precision_at_1000_std": -0.075931, + "nauc_precision_at_1000_diff1": -0.351832, + "nauc_cv_recall_at_1_max": 0.285508, + "nauc_cv_recall_at_1_std": -0.110661, + "nauc_cv_recall_at_1_diff1": 0.509556, + "nauc_cv_recall_at_3_max": 0.360872, + "nauc_cv_recall_at_3_std": -0.055834, + "nauc_cv_recall_at_3_diff1": 0.381737, + "nauc_cv_recall_at_5_max": 0.537609, + "nauc_cv_recall_at_5_std": 0.243765, + "nauc_cv_recall_at_5_diff1": 0.215891, + "nauc_cv_recall_at_10_max": 0.568705, + "nauc_cv_recall_at_10_std": 0.322207, + "nauc_cv_recall_at_10_diff1": 0.334111, + "nauc_cv_recall_at_20_max": 0.52601, + "nauc_cv_recall_at_20_std": 0.372216, + "nauc_cv_recall_at_20_diff1": 0.168868, + "nauc_cv_recall_at_100_max": 0.18643, + "nauc_cv_recall_at_100_std": 0.068783, + "nauc_cv_recall_at_100_diff1": -0.330999, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.285508, + "nauc_mrr_at_1_std": -0.110661, + "nauc_mrr_at_1_diff1": 0.509556, + "nauc_mrr_at_3_max": 0.314699, + "nauc_mrr_at_3_std": -0.081256, + "nauc_mrr_at_3_diff1": 0.45486, + "nauc_mrr_at_5_max": 0.329789, + "nauc_mrr_at_5_std": -0.049679, + "nauc_mrr_at_5_diff1": 0.439923, + "nauc_mrr_at_10_max": 0.326172, + "nauc_mrr_at_10_std": -0.052663, + "nauc_mrr_at_10_diff1": 0.448274, + "nauc_mrr_at_20_max": 0.323152, + "nauc_mrr_at_20_std": -0.055851, + "nauc_mrr_at_20_diff1": 0.446965, + "nauc_mrr_at_100_max": 0.321547, + "nauc_mrr_at_100_std": -0.058184, + "nauc_mrr_at_100_diff1": 0.4467, + "nauc_mrr_at_1000_max": 0.321558, + "nauc_mrr_at_1000_std": -0.058229, + "nauc_mrr_at_1000_diff1": 0.446941, + "main_score": 0.63352, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.58125, + "ndcg_at_3": 0.59462, + "ndcg_at_5": 0.61687, + "ndcg_at_10": 0.65484, + "ndcg_at_20": 0.68634, + "ndcg_at_100": 0.71171, + "ndcg_at_1000": 0.72838, + "map_at_1": 0.35361, + "map_at_3": 0.48232, + "map_at_5": 0.52433, + "map_at_10": 0.56285, + "map_at_20": 0.5804, + "map_at_100": 0.58961, + "map_at_1000": 0.59145, + "recall_at_1": 0.35361, + "recall_at_3": 0.55481, + "recall_at_5": 0.64693, + "recall_at_10": 0.75485, + "recall_at_20": 0.84331, + "recall_at_100": 0.9229, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.58125, + "cv_recall_at_3": 0.81875, + "cv_recall_at_5": 0.8625, + "cv_recall_at_10": 0.91875, + "cv_recall_at_20": 0.96875, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.58125, + "precision_at_3": 0.37083, + "precision_at_5": 0.28125, + "precision_at_10": 0.18438, + "precision_at_20": 0.11125, + "precision_at_100": 0.02706, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.58125, + "mrr_at_3": 0.689583, + "mrr_at_5": 0.700208, + "mrr_at_10": 0.708609, + "mrr_at_20": 0.711892, + "mrr_at_100": 0.71213, + "mrr_at_1000": 0.712209, + "nauc_ndcg_at_1_max": 0.224823, + "nauc_ndcg_at_1_std": -0.061748, + "nauc_ndcg_at_1_diff1": 0.49293, + "nauc_ndcg_at_3_max": 0.274806, + "nauc_ndcg_at_3_std": -0.037644, + "nauc_ndcg_at_3_diff1": 0.385702, + "nauc_ndcg_at_5_max": 0.304227, + "nauc_ndcg_at_5_std": -0.043146, + "nauc_ndcg_at_5_diff1": 0.404386, + "nauc_ndcg_at_10_max": 0.305067, + "nauc_ndcg_at_10_std": -0.047125, + "nauc_ndcg_at_10_diff1": 0.402024, + "nauc_ndcg_at_20_max": 0.294784, + "nauc_ndcg_at_20_std": -0.04368, + "nauc_ndcg_at_20_diff1": 0.379749, + "nauc_ndcg_at_100_max": 0.28017, + "nauc_ndcg_at_100_std": -0.042373, + "nauc_ndcg_at_100_diff1": 0.374529, + "nauc_ndcg_at_1000_max": 0.266453, + "nauc_ndcg_at_1000_std": -0.045699, + "nauc_ndcg_at_1000_diff1": 0.377555, + "nauc_map_at_1_max": 0.310675, + "nauc_map_at_1_std": 0.05387, + "nauc_map_at_1_diff1": 0.53568, + "nauc_map_at_3_max": 0.306115, + "nauc_map_at_3_std": 0.007725, + "nauc_map_at_3_diff1": 0.434651, + "nauc_map_at_5_max": 0.314116, + "nauc_map_at_5_std": -0.015505, + "nauc_map_at_5_diff1": 0.418721, + "nauc_map_at_10_max": 0.303144, + "nauc_map_at_10_std": -0.03638, + "nauc_map_at_10_diff1": 0.406053, + "nauc_map_at_20_max": 0.300597, + "nauc_map_at_20_std": -0.031618, + "nauc_map_at_20_diff1": 0.394468, + "nauc_map_at_100_max": 0.297808, + "nauc_map_at_100_std": -0.033996, + "nauc_map_at_100_diff1": 0.390629, + "nauc_map_at_1000_max": 0.296123, + "nauc_map_at_1000_std": -0.034298, + "nauc_map_at_1000_diff1": 0.389855, + "nauc_recall_at_1_max": 0.310675, + "nauc_recall_at_1_std": 0.05387, + "nauc_recall_at_1_diff1": 0.53568, + "nauc_recall_at_3_max": 0.29076, + "nauc_recall_at_3_std": -0.003302, + "nauc_recall_at_3_diff1": 0.382719, + "nauc_recall_at_5_max": 0.30257, + "nauc_recall_at_5_std": -0.049839, + "nauc_recall_at_5_diff1": 0.345866, + "nauc_recall_at_10_max": 0.321985, + "nauc_recall_at_10_std": -0.065189, + "nauc_recall_at_10_diff1": 0.314947, + "nauc_recall_at_20_max": 0.303579, + "nauc_recall_at_20_std": -0.06345, + "nauc_recall_at_20_diff1": 0.179608, + "nauc_recall_at_100_max": 0.265254, + "nauc_recall_at_100_std": -0.009276, + "nauc_recall_at_100_diff1": 0.080256, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.224823, + "nauc_precision_at_1_std": -0.061748, + "nauc_precision_at_1_diff1": 0.49293, + "nauc_precision_at_3_max": 0.076148, + "nauc_precision_at_3_std": -0.085875, + "nauc_precision_at_3_diff1": -0.013427, + "nauc_precision_at_5_max": -0.000244, + "nauc_precision_at_5_std": -0.128723, + "nauc_precision_at_5_diff1": -0.125278, + "nauc_precision_at_10_max": -0.095177, + "nauc_precision_at_10_std": -0.124122, + "nauc_precision_at_10_diff1": -0.206377, + "nauc_precision_at_20_max": -0.143306, + "nauc_precision_at_20_std": -0.086565, + "nauc_precision_at_20_diff1": -0.285766, + "nauc_precision_at_100_max": -0.197516, + "nauc_precision_at_100_std": -0.072638, + "nauc_precision_at_100_diff1": -0.328231, + "nauc_precision_at_1000_max": -0.269047, + "nauc_precision_at_1000_std": -0.080577, + "nauc_precision_at_1000_diff1": -0.361402, + "nauc_cv_recall_at_1_max": 0.224823, + "nauc_cv_recall_at_1_std": -0.061748, + "nauc_cv_recall_at_1_diff1": 0.49293, + "nauc_cv_recall_at_3_max": 0.32472, + "nauc_cv_recall_at_3_std": 0.011526, + "nauc_cv_recall_at_3_diff1": 0.462482, + "nauc_cv_recall_at_5_max": 0.354876, + "nauc_cv_recall_at_5_std": -0.048609, + "nauc_cv_recall_at_5_diff1": 0.475948, + "nauc_cv_recall_at_10_max": 0.409251, + "nauc_cv_recall_at_10_std": 0.07247, + "nauc_cv_recall_at_10_diff1": 0.437908, + "nauc_cv_recall_at_20_max": 0.348459, + "nauc_cv_recall_at_20_std": 0.309617, + "nauc_cv_recall_at_20_diff1": 0.000187, + "nauc_cv_recall_at_100_max": -0.042328, + "nauc_cv_recall_at_100_std": 0.09057, + "nauc_cv_recall_at_100_diff1": -0.452381, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.224823, + "nauc_mrr_at_1_std": -0.061748, + "nauc_mrr_at_1_diff1": 0.49293, + "nauc_mrr_at_3_max": 0.238349, + "nauc_mrr_at_3_std": -0.053829, + "nauc_mrr_at_3_diff1": 0.462911, + "nauc_mrr_at_5_max": 0.238595, + "nauc_mrr_at_5_std": -0.065174, + "nauc_mrr_at_5_diff1": 0.46516, + "nauc_mrr_at_10_max": 0.238593, + "nauc_mrr_at_10_std": -0.058282, + "nauc_mrr_at_10_diff1": 0.462788, + "nauc_mrr_at_20_max": 0.235898, + "nauc_mrr_at_20_std": -0.057664, + "nauc_mrr_at_20_diff1": 0.458922, + "nauc_mrr_at_100_max": 0.235204, + "nauc_mrr_at_100_std": -0.058397, + "nauc_mrr_at_100_diff1": 0.458789, + "nauc_mrr_at_1000_max": 0.235268, + "nauc_mrr_at_1000_std": -0.058423, + "nauc_mrr_at_1000_diff1": 0.459059, + "main_score": 0.61687, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.6375, + "ndcg_at_3": 0.62364, + "ndcg_at_5": 0.64683, + "ndcg_at_10": 0.68715, + "ndcg_at_20": 0.70996, + "ndcg_at_100": 0.74101, + "ndcg_at_1000": 0.75449, + "map_at_1": 0.38924, + "map_at_3": 0.51414, + "map_at_5": 0.55813, + "map_at_10": 0.5977, + "map_at_20": 0.61322, + "map_at_100": 0.62363, + "map_at_1000": 0.62515, + "recall_at_1": 0.38924, + "recall_at_3": 0.5668, + "recall_at_5": 0.66811, + "recall_at_10": 0.78194, + "recall_at_20": 0.8405, + "recall_at_100": 0.93855, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.6375, + "cv_recall_at_3": 0.81875, + "cv_recall_at_5": 0.8875, + "cv_recall_at_10": 0.94375, + "cv_recall_at_20": 0.95625, + "cv_recall_at_100": 0.9875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.6375, + "precision_at_3": 0.38333, + "precision_at_5": 0.29, + "precision_at_10": 0.18938, + "precision_at_20": 0.11063, + "precision_at_100": 0.028, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.6375, + "mrr_at_3": 0.71875, + "mrr_at_5": 0.734063, + "mrr_at_10": 0.742376, + "mrr_at_20": 0.743056, + "mrr_at_100": 0.744111, + "mrr_at_1000": 0.744174, + "nauc_ndcg_at_1_max": 0.21572, + "nauc_ndcg_at_1_std": -0.181545, + "nauc_ndcg_at_1_diff1": 0.434231, + "nauc_ndcg_at_3_max": 0.324839, + "nauc_ndcg_at_3_std": -0.117042, + "nauc_ndcg_at_3_diff1": 0.341487, + "nauc_ndcg_at_5_max": 0.324855, + "nauc_ndcg_at_5_std": -0.128888, + "nauc_ndcg_at_5_diff1": 0.36501, + "nauc_ndcg_at_10_max": 0.304189, + "nauc_ndcg_at_10_std": -0.168759, + "nauc_ndcg_at_10_diff1": 0.383355, + "nauc_ndcg_at_20_max": 0.312577, + "nauc_ndcg_at_20_std": -0.159314, + "nauc_ndcg_at_20_diff1": 0.359227, + "nauc_ndcg_at_100_max": 0.309793, + "nauc_ndcg_at_100_std": -0.124935, + "nauc_ndcg_at_100_diff1": 0.355825, + "nauc_ndcg_at_1000_max": 0.287952, + "nauc_ndcg_at_1000_std": -0.145102, + "nauc_ndcg_at_1000_diff1": 0.353841, + "nauc_map_at_1_max": 0.325199, + "nauc_map_at_1_std": -0.089064, + "nauc_map_at_1_diff1": 0.499677, + "nauc_map_at_3_max": 0.348089, + "nauc_map_at_3_std": -0.0986, + "nauc_map_at_3_diff1": 0.404687, + "nauc_map_at_5_max": 0.320855, + "nauc_map_at_5_std": -0.131165, + "nauc_map_at_5_diff1": 0.375853, + "nauc_map_at_10_max": 0.301388, + "nauc_map_at_10_std": -0.162168, + "nauc_map_at_10_diff1": 0.376903, + "nauc_map_at_20_max": 0.308314, + "nauc_map_at_20_std": -0.150038, + "nauc_map_at_20_diff1": 0.365313, + "nauc_map_at_100_max": 0.306604, + "nauc_map_at_100_std": -0.146749, + "nauc_map_at_100_diff1": 0.366246, + "nauc_map_at_1000_max": 0.304281, + "nauc_map_at_1000_std": -0.148288, + "nauc_map_at_1000_diff1": 0.365239, + "nauc_recall_at_1_max": 0.325199, + "nauc_recall_at_1_std": -0.089064, + "nauc_recall_at_1_diff1": 0.499677, + "nauc_recall_at_3_max": 0.3888, + "nauc_recall_at_3_std": -0.074864, + "nauc_recall_at_3_diff1": 0.366595, + "nauc_recall_at_5_max": 0.322674, + "nauc_recall_at_5_std": -0.116744, + "nauc_recall_at_5_diff1": 0.329388, + "nauc_recall_at_10_max": 0.257046, + "nauc_recall_at_10_std": -0.239396, + "nauc_recall_at_10_diff1": 0.32237, + "nauc_recall_at_20_max": 0.282944, + "nauc_recall_at_20_std": -0.2499, + "nauc_recall_at_20_diff1": 0.235681, + "nauc_recall_at_100_max": 0.426998, + "nauc_recall_at_100_std": 0.147252, + "nauc_recall_at_100_diff1": 0.127155, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.21572, + "nauc_precision_at_1_std": -0.181545, + "nauc_precision_at_1_diff1": 0.434231, + "nauc_precision_at_3_max": 0.054558, + "nauc_precision_at_3_std": -0.055327, + "nauc_precision_at_3_diff1": -0.101731, + "nauc_precision_at_5_max": -0.062751, + "nauc_precision_at_5_std": -0.090542, + "nauc_precision_at_5_diff1": -0.184519, + "nauc_precision_at_10_max": -0.162807, + "nauc_precision_at_10_std": -0.083555, + "nauc_precision_at_10_diff1": -0.207658, + "nauc_precision_at_20_max": -0.170609, + "nauc_precision_at_20_std": -0.026241, + "nauc_precision_at_20_diff1": -0.249603, + "nauc_precision_at_100_max": -0.25269, + "nauc_precision_at_100_std": 0.01882, + "nauc_precision_at_100_diff1": -0.291999, + "nauc_precision_at_1000_max": -0.318836, + "nauc_precision_at_1000_std": -0.028154, + "nauc_precision_at_1000_diff1": -0.333272, + "nauc_cv_recall_at_1_max": 0.21572, + "nauc_cv_recall_at_1_std": -0.181545, + "nauc_cv_recall_at_1_diff1": 0.434231, + "nauc_cv_recall_at_3_max": 0.500274, + "nauc_cv_recall_at_3_std": -0.008841, + "nauc_cv_recall_at_3_diff1": 0.398817, + "nauc_cv_recall_at_5_max": 0.579772, + "nauc_cv_recall_at_5_std": 0.060243, + "nauc_cv_recall_at_5_diff1": 0.535951, + "nauc_cv_recall_at_10_max": 0.563803, + "nauc_cv_recall_at_10_std": 0.013072, + "nauc_cv_recall_at_10_diff1": 0.379759, + "nauc_cv_recall_at_20_max": 0.521542, + "nauc_cv_recall_at_20_std": -0.113512, + "nauc_cv_recall_at_20_diff1": 0.367547, + "nauc_cv_recall_at_100_max": -0.075864, + "nauc_cv_recall_at_100_std": 0.348973, + "nauc_cv_recall_at_100_diff1": -0.563492, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.21572, + "nauc_mrr_at_1_std": -0.181545, + "nauc_mrr_at_1_diff1": 0.434231, + "nauc_mrr_at_3_max": 0.311321, + "nauc_mrr_at_3_std": -0.117369, + "nauc_mrr_at_3_diff1": 0.415544, + "nauc_mrr_at_5_max": 0.311677, + "nauc_mrr_at_5_std": -0.113468, + "nauc_mrr_at_5_diff1": 0.429125, + "nauc_mrr_at_10_max": 0.301657, + "nauc_mrr_at_10_std": -0.120166, + "nauc_mrr_at_10_diff1": 0.4196, + "nauc_mrr_at_20_max": 0.300334, + "nauc_mrr_at_20_std": -0.121994, + "nauc_mrr_at_20_diff1": 0.419667, + "nauc_mrr_at_100_max": 0.297888, + "nauc_mrr_at_100_std": -0.121103, + "nauc_mrr_at_100_diff1": 0.418316, + "nauc_mrr_at_1000_max": 0.297852, + "nauc_mrr_at_1000_std": -0.121311, + "nauc_mrr_at_1000_diff1": 0.418604, + "main_score": 0.64683, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.5875, + "ndcg_at_3": 0.60085, + "ndcg_at_5": 0.61678, + "ndcg_at_10": 0.65947, + "ndcg_at_20": 0.68829, + "ndcg_at_100": 0.71532, + "ndcg_at_1000": 0.73126, + "map_at_1": 0.35371, + "map_at_3": 0.49482, + "map_at_5": 0.53, + "map_at_10": 0.57087, + "map_at_20": 0.58715, + "map_at_100": 0.59606, + "map_at_1000": 0.5978, + "recall_at_1": 0.35371, + "recall_at_3": 0.56331, + "recall_at_5": 0.64754, + "recall_at_10": 0.75773, + "recall_at_20": 0.84079, + "recall_at_100": 0.92527, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.5875, + "cv_recall_at_3": 0.79375, + "cv_recall_at_5": 0.85625, + "cv_recall_at_10": 0.9125, + "cv_recall_at_20": 0.95625, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.5875, + "precision_at_3": 0.37708, + "precision_at_5": 0.275, + "precision_at_10": 0.18625, + "precision_at_20": 0.11063, + "precision_at_100": 0.02744, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.5875, + "mrr_at_3": 0.678125, + "mrr_at_5": 0.693438, + "mrr_at_10": 0.7013, + "mrr_at_20": 0.70411, + "mrr_at_100": 0.704794, + "mrr_at_1000": 0.704882, + "nauc_ndcg_at_1_max": 0.377751, + "nauc_ndcg_at_1_std": 0.018191, + "nauc_ndcg_at_1_diff1": 0.46296, + "nauc_ndcg_at_3_max": 0.336251, + "nauc_ndcg_at_3_std": -0.03368, + "nauc_ndcg_at_3_diff1": 0.366469, + "nauc_ndcg_at_5_max": 0.349671, + "nauc_ndcg_at_5_std": -0.015968, + "nauc_ndcg_at_5_diff1": 0.364423, + "nauc_ndcg_at_10_max": 0.354664, + "nauc_ndcg_at_10_std": -0.029559, + "nauc_ndcg_at_10_diff1": 0.362231, + "nauc_ndcg_at_20_max": 0.373808, + "nauc_ndcg_at_20_std": 1.6e-05, + "nauc_ndcg_at_20_diff1": 0.374684, + "nauc_ndcg_at_100_max": 0.364805, + "nauc_ndcg_at_100_std": -0.007448, + "nauc_ndcg_at_100_diff1": 0.37791, + "nauc_ndcg_at_1000_max": 0.353859, + "nauc_ndcg_at_1000_std": -0.022384, + "nauc_ndcg_at_1000_diff1": 0.379866, + "nauc_map_at_1_max": 0.375084, + "nauc_map_at_1_std": 0.021123, + "nauc_map_at_1_diff1": 0.547027, + "nauc_map_at_3_max": 0.357146, + "nauc_map_at_3_std": -0.01633, + "nauc_map_at_3_diff1": 0.412406, + "nauc_map_at_5_max": 0.350693, + "nauc_map_at_5_std": -0.026152, + "nauc_map_at_5_diff1": 0.377711, + "nauc_map_at_10_max": 0.344609, + "nauc_map_at_10_std": -0.045215, + "nauc_map_at_10_diff1": 0.366972, + "nauc_map_at_20_max": 0.358549, + "nauc_map_at_20_std": -0.026605, + "nauc_map_at_20_diff1": 0.372332, + "nauc_map_at_100_max": 0.358615, + "nauc_map_at_100_std": -0.02831, + "nauc_map_at_100_diff1": 0.374024, + "nauc_map_at_1000_max": 0.357122, + "nauc_map_at_1000_std": -0.030312, + "nauc_map_at_1000_diff1": 0.373938, + "nauc_recall_at_1_max": 0.375084, + "nauc_recall_at_1_std": 0.021123, + "nauc_recall_at_1_diff1": 0.547027, + "nauc_recall_at_3_max": 0.318584, + "nauc_recall_at_3_std": -0.019404, + "nauc_recall_at_3_diff1": 0.346219, + "nauc_recall_at_5_max": 0.292771, + "nauc_recall_at_5_std": -0.006082, + "nauc_recall_at_5_diff1": 0.250897, + "nauc_recall_at_10_max": 0.28799, + "nauc_recall_at_10_std": -0.054592, + "nauc_recall_at_10_diff1": 0.214356, + "nauc_recall_at_20_max": 0.341931, + "nauc_recall_at_20_std": 0.036302, + "nauc_recall_at_20_diff1": 0.244772, + "nauc_recall_at_100_max": 0.336216, + "nauc_recall_at_100_std": 0.07459, + "nauc_recall_at_100_diff1": 0.221441, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.377751, + "nauc_precision_at_1_std": 0.018191, + "nauc_precision_at_1_diff1": 0.46296, + "nauc_precision_at_3_max": 0.101246, + "nauc_precision_at_3_std": -0.04491, + "nauc_precision_at_3_diff1": -0.041673, + "nauc_precision_at_5_max": 0.038837, + "nauc_precision_at_5_std": -0.047151, + "nauc_precision_at_5_diff1": -0.150026, + "nauc_precision_at_10_max": -0.051723, + "nauc_precision_at_10_std": -0.059767, + "nauc_precision_at_10_diff1": -0.182612, + "nauc_precision_at_20_max": -0.053655, + "nauc_precision_at_20_std": 0.004044, + "nauc_precision_at_20_diff1": -0.171325, + "nauc_precision_at_100_max": -0.146394, + "nauc_precision_at_100_std": -0.038322, + "nauc_precision_at_100_diff1": -0.190814, + "nauc_precision_at_1000_max": -0.230559, + "nauc_precision_at_1000_std": -0.107428, + "nauc_precision_at_1000_diff1": -0.22561, + "nauc_cv_recall_at_1_max": 0.377751, + "nauc_cv_recall_at_1_std": 0.018191, + "nauc_cv_recall_at_1_diff1": 0.46296, + "nauc_cv_recall_at_3_max": 0.453445, + "nauc_cv_recall_at_3_std": 0.086342, + "nauc_cv_recall_at_3_diff1": 0.456981, + "nauc_cv_recall_at_5_max": 0.557878, + "nauc_cv_recall_at_5_std": 0.235002, + "nauc_cv_recall_at_5_diff1": 0.453663, + "nauc_cv_recall_at_10_max": 0.513405, + "nauc_cv_recall_at_10_std": 0.186074, + "nauc_cv_recall_at_10_diff1": 0.254702, + "nauc_cv_recall_at_20_max": 0.377551, + "nauc_cv_recall_at_20_std": 0.17427, + "nauc_cv_recall_at_20_diff1": 0.151861, + "nauc_cv_recall_at_100_max": -0.151261, + "nauc_cv_recall_at_100_std": -0.151261, + "nauc_cv_recall_at_100_diff1": -0.238406, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.377751, + "nauc_mrr_at_1_std": 0.018191, + "nauc_mrr_at_1_diff1": 0.46296, + "nauc_mrr_at_3_max": 0.384652, + "nauc_mrr_at_3_std": 0.030122, + "nauc_mrr_at_3_diff1": 0.447887, + "nauc_mrr_at_5_max": 0.394225, + "nauc_mrr_at_5_std": 0.046822, + "nauc_mrr_at_5_diff1": 0.446909, + "nauc_mrr_at_10_max": 0.386043, + "nauc_mrr_at_10_std": 0.036904, + "nauc_mrr_at_10_diff1": 0.436161, + "nauc_mrr_at_20_max": 0.382654, + "nauc_mrr_at_20_std": 0.034982, + "nauc_mrr_at_20_diff1": 0.436215, + "nauc_mrr_at_100_max": 0.38132, + "nauc_mrr_at_100_std": 0.033553, + "nauc_mrr_at_100_diff1": 0.435877, + "nauc_mrr_at_1000_max": 0.381489, + "nauc_mrr_at_1000_std": 0.033676, + "nauc_mrr_at_1000_diff1": 0.436029, + "main_score": 0.61678, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 380.4892044067383, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/Vidore2ESGReportsHLRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..0ee4409bb6 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.62179, + "ndcg_at_3": 0.63397, + "ndcg_at_5": 0.68024, + "ndcg_at_10": 0.71594, + "ndcg_at_20": 0.72467, + "ndcg_at_100": 0.75305, + "ndcg_at_1000": 0.75968, + "map_at_1": 0.44103, + "map_at_3": 0.54477, + "map_at_5": 0.60244, + "map_at_10": 0.63478, + "map_at_20": 0.63963, + "map_at_100": 0.64876, + "map_at_1000": 0.64968, + "recall_at_1": 0.44103, + "recall_at_3": 0.61781, + "recall_at_5": 0.75838, + "recall_at_10": 0.8409, + "recall_at_20": 0.86609, + "recall_at_100": 0.9717, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.63462, + "cv_recall_at_3": 0.80769, + "cv_recall_at_5": 0.90385, + "cv_recall_at_10": 0.94231, + "cv_recall_at_20": 0.96154, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.63462, + "precision_at_3": 0.35256, + "precision_at_5": 0.27692, + "precision_at_10": 0.175, + "precision_at_20": 0.09327, + "precision_at_100": 0.0225, + "precision_at_1000": 0.00246, + "mrr_at_1": 0.634615, + "mrr_at_3": 0.711538, + "mrr_at_5": 0.735577, + "mrr_at_10": 0.741529, + "mrr_at_20": 0.743009, + "mrr_at_100": 0.744257, + "mrr_at_1000": 0.744257, + "nauc_ndcg_at_1_max": 0.08559, + "nauc_ndcg_at_1_std": 0.205165, + "nauc_ndcg_at_1_diff1": 0.536563, + "nauc_ndcg_at_3_max": 0.094804, + "nauc_ndcg_at_3_std": 0.114469, + "nauc_ndcg_at_3_diff1": 0.48254, + "nauc_ndcg_at_5_max": 0.127991, + "nauc_ndcg_at_5_std": 0.190158, + "nauc_ndcg_at_5_diff1": 0.482276, + "nauc_ndcg_at_10_max": 0.143887, + "nauc_ndcg_at_10_std": 0.196865, + "nauc_ndcg_at_10_diff1": 0.428075, + "nauc_ndcg_at_20_max": 0.134433, + "nauc_ndcg_at_20_std": 0.18403, + "nauc_ndcg_at_20_diff1": 0.447985, + "nauc_ndcg_at_100_max": 0.138984, + "nauc_ndcg_at_100_std": 0.176803, + "nauc_ndcg_at_100_diff1": 0.476474, + "nauc_ndcg_at_1000_max": 0.122445, + "nauc_ndcg_at_1000_std": 0.157882, + "nauc_ndcg_at_1000_diff1": 0.464841, + "nauc_map_at_1_max": 0.18431, + "nauc_map_at_1_std": 0.086564, + "nauc_map_at_1_diff1": 0.680994, + "nauc_map_at_3_max": 0.149924, + "nauc_map_at_3_std": 0.079076, + "nauc_map_at_3_diff1": 0.5467, + "nauc_map_at_5_max": 0.165075, + "nauc_map_at_5_std": 0.146107, + "nauc_map_at_5_diff1": 0.526571, + "nauc_map_at_10_max": 0.177927, + "nauc_map_at_10_std": 0.170709, + "nauc_map_at_10_diff1": 0.496667, + "nauc_map_at_20_max": 0.174799, + "nauc_map_at_20_std": 0.170916, + "nauc_map_at_20_diff1": 0.502532, + "nauc_map_at_100_max": 0.17528, + "nauc_map_at_100_std": 0.171292, + "nauc_map_at_100_diff1": 0.507306, + "nauc_map_at_1000_max": 0.174533, + "nauc_map_at_1000_std": 0.169748, + "nauc_map_at_1000_diff1": 0.506075, + "nauc_recall_at_1_max": 0.18431, + "nauc_recall_at_1_std": 0.086564, + "nauc_recall_at_1_diff1": 0.680994, + "nauc_recall_at_3_max": 0.07724, + "nauc_recall_at_3_std": 0.007555, + "nauc_recall_at_3_diff1": 0.414346, + "nauc_recall_at_5_max": 0.157083, + "nauc_recall_at_5_std": 0.238552, + "nauc_recall_at_5_diff1": 0.330822, + "nauc_recall_at_10_max": 0.240977, + "nauc_recall_at_10_std": 0.319587, + "nauc_recall_at_10_diff1": 0.132316, + "nauc_recall_at_20_max": 0.225337, + "nauc_recall_at_20_std": 0.282092, + "nauc_recall_at_20_diff1": 0.227253, + "nauc_recall_at_100_max": 0.522108, + "nauc_recall_at_100_std": 0.641073, + "nauc_recall_at_100_diff1": 0.67342, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.072156, + "nauc_precision_at_1_std": 0.217469, + "nauc_precision_at_1_diff1": 0.599997, + "nauc_precision_at_3_max": -0.039178, + "nauc_precision_at_3_std": 0.123226, + "nauc_precision_at_3_diff1": -0.101261, + "nauc_precision_at_5_max": -0.035867, + "nauc_precision_at_5_std": 0.203108, + "nauc_precision_at_5_diff1": -0.224684, + "nauc_precision_at_10_max": -0.053609, + "nauc_precision_at_10_std": 0.148578, + "nauc_precision_at_10_diff1": -0.345519, + "nauc_precision_at_20_max": -0.073042, + "nauc_precision_at_20_std": 0.122911, + "nauc_precision_at_20_diff1": -0.335639, + "nauc_precision_at_100_max": -0.122331, + "nauc_precision_at_100_std": 0.047214, + "nauc_precision_at_100_diff1": -0.345229, + "nauc_precision_at_1000_max": -0.150556, + "nauc_precision_at_1000_std": -0.031002, + "nauc_precision_at_1000_diff1": -0.363507, + "nauc_cv_recall_at_1_max": 0.072156, + "nauc_cv_recall_at_1_std": 0.217469, + "nauc_cv_recall_at_1_diff1": 0.599997, + "nauc_cv_recall_at_3_max": -0.095713, + "nauc_cv_recall_at_3_std": 0.030134, + "nauc_cv_recall_at_3_diff1": 0.314645, + "nauc_cv_recall_at_5_max": -0.082595, + "nauc_cv_recall_at_5_std": 0.327501, + "nauc_cv_recall_at_5_diff1": 0.207772, + "nauc_cv_recall_at_10_max": -0.200055, + "nauc_cv_recall_at_10_std": 0.176339, + "nauc_cv_recall_at_10_diff1": -0.22844, + "nauc_cv_recall_at_20_max": -0.215228, + "nauc_cv_recall_at_20_std": -0.097581, + "nauc_cv_recall_at_20_diff1": -0.52111, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.072156, + "nauc_mrr_at_1_std": 0.217469, + "nauc_mrr_at_1_diff1": 0.599997, + "nauc_mrr_at_3_max": 0.007301, + "nauc_mrr_at_3_std": 0.149039, + "nauc_mrr_at_3_diff1": 0.507869, + "nauc_mrr_at_5_max": 0.018344, + "nauc_mrr_at_5_std": 0.192124, + "nauc_mrr_at_5_diff1": 0.514448, + "nauc_mrr_at_10_max": 0.015732, + "nauc_mrr_at_10_std": 0.182221, + "nauc_mrr_at_10_diff1": 0.505231, + "nauc_mrr_at_20_max": 0.017001, + "nauc_mrr_at_20_std": 0.178513, + "nauc_mrr_at_20_diff1": 0.506246, + "nauc_mrr_at_100_max": 0.018905, + "nauc_mrr_at_100_std": 0.180865, + "nauc_mrr_at_100_diff1": 0.513273, + "nauc_mrr_at_1000_max": 0.018905, + "nauc_mrr_at_1000_std": 0.180865, + "nauc_mrr_at_1000_diff1": 0.513273, + "main_score": 0.68024, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 243.98739051818848, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/Vidore2ESGReportsRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..5c2b437b0c --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.52632, + "ndcg_at_3": 0.47962, + "ndcg_at_5": 0.5441, + "ndcg_at_10": 0.60126, + "ndcg_at_20": 0.63759, + "ndcg_at_100": 0.69103, + "ndcg_at_1000": 0.69103, + "map_at_1": 0.29737, + "map_at_3": 0.37161, + "map_at_5": 0.43866, + "map_at_10": 0.4869, + "map_at_20": 0.50648, + "map_at_100": 0.53268, + "map_at_1000": 0.53268, + "recall_at_1": 0.29737, + "recall_at_3": 0.41886, + "recall_at_5": 0.56938, + "recall_at_10": 0.73559, + "recall_at_20": 0.8424, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.52632, + "cv_recall_at_3": 0.70175, + "cv_recall_at_5": 0.80702, + "cv_recall_at_10": 0.87719, + "cv_recall_at_20": 0.94737, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.52632, + "precision_at_3": 0.29825, + "precision_at_5": 0.27368, + "precision_at_10": 0.18947, + "precision_at_20": 0.12105, + "precision_at_100": 0.03895, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.526316, + "mrr_at_3": 0.611111, + "mrr_at_5": 0.634795, + "mrr_at_10": 0.643442, + "mrr_at_20": 0.648825, + "mrr_at_100": 0.649958, + "mrr_at_1000": 0.649958, + "nauc_ndcg_at_1_max": 0.083087, + "nauc_ndcg_at_1_std": -0.004367, + "nauc_ndcg_at_1_diff1": 0.237421, + "nauc_ndcg_at_3_max": 0.073434, + "nauc_ndcg_at_3_std": -0.03705, + "nauc_ndcg_at_3_diff1": 0.146406, + "nauc_ndcg_at_5_max": 0.09407, + "nauc_ndcg_at_5_std": 0.012895, + "nauc_ndcg_at_5_diff1": 0.157845, + "nauc_ndcg_at_10_max": 0.149019, + "nauc_ndcg_at_10_std": 0.013565, + "nauc_ndcg_at_10_diff1": 0.25983, + "nauc_ndcg_at_20_max": 0.15156, + "nauc_ndcg_at_20_std": 0.039924, + "nauc_ndcg_at_20_diff1": 0.242984, + "nauc_ndcg_at_100_max": 0.025725, + "nauc_ndcg_at_100_std": -0.006154, + "nauc_ndcg_at_100_diff1": 0.142564, + "nauc_ndcg_at_1000_max": 0.025725, + "nauc_ndcg_at_1000_std": -0.006154, + "nauc_ndcg_at_1000_diff1": 0.142564, + "nauc_map_at_1_max": 0.157817, + "nauc_map_at_1_std": -0.113462, + "nauc_map_at_1_diff1": 0.211464, + "nauc_map_at_3_max": 0.144657, + "nauc_map_at_3_std": -0.070371, + "nauc_map_at_3_diff1": 0.177603, + "nauc_map_at_5_max": 0.160078, + "nauc_map_at_5_std": -0.019671, + "nauc_map_at_5_diff1": 0.189386, + "nauc_map_at_10_max": 0.167535, + "nauc_map_at_10_std": -0.010633, + "nauc_map_at_10_diff1": 0.226498, + "nauc_map_at_20_max": 0.167653, + "nauc_map_at_20_std": 0.001434, + "nauc_map_at_20_diff1": 0.225561, + "nauc_map_at_100_max": 0.11193, + "nauc_map_at_100_std": -0.017699, + "nauc_map_at_100_diff1": 0.171593, + "nauc_map_at_1000_max": 0.11193, + "nauc_map_at_1000_std": -0.017699, + "nauc_map_at_1000_diff1": 0.171593, + "nauc_recall_at_1_max": 0.157817, + "nauc_recall_at_1_std": -0.113462, + "nauc_recall_at_1_diff1": 0.211464, + "nauc_recall_at_3_max": 0.124397, + "nauc_recall_at_3_std": -0.036733, + "nauc_recall_at_3_diff1": 0.138421, + "nauc_recall_at_5_max": 0.205306, + "nauc_recall_at_5_std": 0.102805, + "nauc_recall_at_5_diff1": 0.183219, + "nauc_recall_at_10_max": 0.339027, + "nauc_recall_at_10_std": 0.098319, + "nauc_recall_at_10_diff1": 0.410625, + "nauc_recall_at_20_max": 0.495088, + "nauc_recall_at_20_std": 0.296223, + "nauc_recall_at_20_diff1": 0.404822, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.083087, + "nauc_precision_at_1_std": -0.004367, + "nauc_precision_at_1_diff1": 0.237421, + "nauc_precision_at_3_max": -0.096564, + "nauc_precision_at_3_std": 0.016016, + "nauc_precision_at_3_diff1": 0.059439, + "nauc_precision_at_5_max": -0.122906, + "nauc_precision_at_5_std": 0.075368, + "nauc_precision_at_5_diff1": -0.029648, + "nauc_precision_at_10_max": -0.147245, + "nauc_precision_at_10_std": 0.079008, + "nauc_precision_at_10_diff1": -0.003074, + "nauc_precision_at_20_max": -0.23942, + "nauc_precision_at_20_std": 0.075737, + "nauc_precision_at_20_diff1": -0.151465, + "nauc_precision_at_100_max": -0.365036, + "nauc_precision_at_100_std": -0.005155, + "nauc_precision_at_100_diff1": -0.353244, + "nauc_precision_at_1000_max": -0.365036, + "nauc_precision_at_1000_std": -0.005155, + "nauc_precision_at_1000_diff1": -0.353244, + "nauc_cv_recall_at_1_max": 0.083087, + "nauc_cv_recall_at_1_std": -0.004367, + "nauc_cv_recall_at_1_diff1": 0.237421, + "nauc_cv_recall_at_3_max": -0.178147, + "nauc_cv_recall_at_3_std": -0.098181, + "nauc_cv_recall_at_3_diff1": 0.131699, + "nauc_cv_recall_at_5_max": -0.161164, + "nauc_cv_recall_at_5_std": 0.000528, + "nauc_cv_recall_at_5_diff1": 0.1628, + "nauc_cv_recall_at_10_max": 0.057507, + "nauc_cv_recall_at_10_std": -0.054701, + "nauc_cv_recall_at_10_diff1": 0.392111, + "nauc_cv_recall_at_20_max": 0.040764, + "nauc_cv_recall_at_20_std": 0.184566, + "nauc_cv_recall_at_20_diff1": 0.22337, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.083087, + "nauc_mrr_at_1_std": -0.004367, + "nauc_mrr_at_1_diff1": 0.237421, + "nauc_mrr_at_3_max": -0.020703, + "nauc_mrr_at_3_std": -0.034769, + "nauc_mrr_at_3_diff1": 0.200619, + "nauc_mrr_at_5_max": -0.007212, + "nauc_mrr_at_5_std": -0.013522, + "nauc_mrr_at_5_diff1": 0.209201, + "nauc_mrr_at_10_max": 0.010654, + "nauc_mrr_at_10_std": -0.018914, + "nauc_mrr_at_10_diff1": 0.224414, + "nauc_mrr_at_20_max": 0.007097, + "nauc_mrr_at_20_std": -0.015145, + "nauc_mrr_at_20_diff1": 0.218836, + "nauc_mrr_at_100_max": 0.006796, + "nauc_mrr_at_100_std": -0.013756, + "nauc_mrr_at_100_diff1": 0.217536, + "nauc_mrr_at_1000_max": 0.006796, + "nauc_mrr_at_1000_std": -0.013756, + "nauc_mrr_at_1000_diff1": 0.217536, + "main_score": 0.5441, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.45614, + "ndcg_at_3": 0.46325, + "ndcg_at_5": 0.5188, + "ndcg_at_10": 0.57772, + "ndcg_at_20": 0.61448, + "ndcg_at_100": 0.66705, + "ndcg_at_1000": 0.66705, + "map_at_1": 0.25088, + "map_at_3": 0.34382, + "map_at_5": 0.40931, + "map_at_10": 0.4547, + "map_at_20": 0.47492, + "map_at_100": 0.50036, + "map_at_1000": 0.50036, + "recall_at_1": 0.25088, + "recall_at_3": 0.43191, + "recall_at_5": 0.56543, + "recall_at_10": 0.72755, + "recall_at_20": 0.84467, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.45614, + "cv_recall_at_3": 0.75439, + "cv_recall_at_5": 0.80702, + "cv_recall_at_10": 0.89474, + "cv_recall_at_20": 0.96491, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.45614, + "precision_at_3": 0.29825, + "precision_at_5": 0.26667, + "precision_at_10": 0.19123, + "precision_at_20": 0.12018, + "precision_at_100": 0.03895, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.45614, + "mrr_at_3": 0.587719, + "mrr_at_5": 0.6, + "mrr_at_10": 0.612058, + "mrr_at_20": 0.616916, + "mrr_at_100": 0.61739, + "mrr_at_1000": 0.61739, + "nauc_ndcg_at_1_max": -0.273361, + "nauc_ndcg_at_1_std": -0.106313, + "nauc_ndcg_at_1_diff1": 0.328974, + "nauc_ndcg_at_3_max": 0.013079, + "nauc_ndcg_at_3_std": 0.011205, + "nauc_ndcg_at_3_diff1": 0.089752, + "nauc_ndcg_at_5_max": 0.001769, + "nauc_ndcg_at_5_std": -0.008604, + "nauc_ndcg_at_5_diff1": 0.134232, + "nauc_ndcg_at_10_max": 0.009337, + "nauc_ndcg_at_10_std": 0.000991, + "nauc_ndcg_at_10_diff1": 0.245484, + "nauc_ndcg_at_20_max": 0.004006, + "nauc_ndcg_at_20_std": 0.025724, + "nauc_ndcg_at_20_diff1": 0.208364, + "nauc_ndcg_at_100_max": -0.099282, + "nauc_ndcg_at_100_std": 0.02182, + "nauc_ndcg_at_100_diff1": 0.145389, + "nauc_ndcg_at_1000_max": -0.099282, + "nauc_ndcg_at_1000_std": 0.02182, + "nauc_ndcg_at_1000_diff1": 0.145389, + "nauc_map_at_1_max": -0.177631, + "nauc_map_at_1_std": -0.15885, + "nauc_map_at_1_diff1": 0.220965, + "nauc_map_at_3_max": 0.042119, + "nauc_map_at_3_std": -0.027739, + "nauc_map_at_3_diff1": 0.131222, + "nauc_map_at_5_max": 0.026536, + "nauc_map_at_5_std": -0.0489, + "nauc_map_at_5_diff1": 0.158367, + "nauc_map_at_10_max": 0.018289, + "nauc_map_at_10_std": -0.030621, + "nauc_map_at_10_diff1": 0.216527, + "nauc_map_at_20_max": 0.00493, + "nauc_map_at_20_std": -0.020374, + "nauc_map_at_20_diff1": 0.200623, + "nauc_map_at_100_max": -0.037606, + "nauc_map_at_100_std": -0.011917, + "nauc_map_at_100_diff1": 0.16046, + "nauc_map_at_1000_max": -0.037606, + "nauc_map_at_1000_std": -0.011917, + "nauc_map_at_1000_diff1": 0.16046, + "nauc_recall_at_1_max": -0.177631, + "nauc_recall_at_1_std": -0.15885, + "nauc_recall_at_1_diff1": 0.220965, + "nauc_recall_at_3_max": 0.244599, + "nauc_recall_at_3_std": 0.095356, + "nauc_recall_at_3_diff1": 0.021355, + "nauc_recall_at_5_max": 0.239983, + "nauc_recall_at_5_std": 0.075885, + "nauc_recall_at_5_diff1": 0.113146, + "nauc_recall_at_10_max": 0.270142, + "nauc_recall_at_10_std": 0.042901, + "nauc_recall_at_10_diff1": 0.363377, + "nauc_recall_at_20_max": 0.383566, + "nauc_recall_at_20_std": 0.196769, + "nauc_recall_at_20_diff1": 0.240569, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.273361, + "nauc_precision_at_1_std": -0.106313, + "nauc_precision_at_1_diff1": 0.328974, + "nauc_precision_at_3_max": 0.03009, + "nauc_precision_at_3_std": 0.141282, + "nauc_precision_at_3_diff1": -0.038332, + "nauc_precision_at_5_max": -0.056173, + "nauc_precision_at_5_std": 0.063172, + "nauc_precision_at_5_diff1": -0.016879, + "nauc_precision_at_10_max": -0.117056, + "nauc_precision_at_10_std": 0.129754, + "nauc_precision_at_10_diff1": 0.049998, + "nauc_precision_at_20_max": -0.142297, + "nauc_precision_at_20_std": 0.194779, + "nauc_precision_at_20_diff1": -0.109284, + "nauc_precision_at_100_max": -0.228066, + "nauc_precision_at_100_std": 0.142314, + "nauc_precision_at_100_diff1": -0.264354, + "nauc_precision_at_1000_max": -0.228066, + "nauc_precision_at_1000_std": 0.142314, + "nauc_precision_at_1000_diff1": -0.264354, + "nauc_cv_recall_at_1_max": -0.273361, + "nauc_cv_recall_at_1_std": -0.106313, + "nauc_cv_recall_at_1_diff1": 0.328974, + "nauc_cv_recall_at_3_max": -0.110383, + "nauc_cv_recall_at_3_std": -0.006633, + "nauc_cv_recall_at_3_diff1": -0.04039, + "nauc_cv_recall_at_5_max": -0.07573, + "nauc_cv_recall_at_5_std": 0.105426, + "nauc_cv_recall_at_5_diff1": 0.088319, + "nauc_cv_recall_at_10_max": -0.041285, + "nauc_cv_recall_at_10_std": 0.115726, + "nauc_cv_recall_at_10_diff1": 0.28604, + "nauc_cv_recall_at_20_max": 0.456469, + "nauc_cv_recall_at_20_std": 1.0, + "nauc_cv_recall_at_20_diff1": -0.362176, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.273361, + "nauc_mrr_at_1_std": -0.106313, + "nauc_mrr_at_1_diff1": 0.328974, + "nauc_mrr_at_3_max": -0.208664, + "nauc_mrr_at_3_std": -0.075096, + "nauc_mrr_at_3_diff1": 0.197222, + "nauc_mrr_at_5_max": -0.207699, + "nauc_mrr_at_5_std": -0.063147, + "nauc_mrr_at_5_diff1": 0.227438, + "nauc_mrr_at_10_max": -0.208199, + "nauc_mrr_at_10_std": -0.064854, + "nauc_mrr_at_10_diff1": 0.244815, + "nauc_mrr_at_20_max": -0.208684, + "nauc_mrr_at_20_std": -0.063078, + "nauc_mrr_at_20_diff1": 0.238493, + "nauc_mrr_at_100_max": -0.20985, + "nauc_mrr_at_100_std": -0.064921, + "nauc_mrr_at_100_diff1": 0.239509, + "nauc_mrr_at_1000_max": -0.20985, + "nauc_mrr_at_1000_std": -0.064921, + "nauc_mrr_at_1000_diff1": 0.239509, + "main_score": 0.5188, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.54386, + "ndcg_at_3": 0.48398, + "ndcg_at_5": 0.54073, + "ndcg_at_10": 0.61778, + "ndcg_at_20": 0.65583, + "ndcg_at_100": 0.69263, + "ndcg_at_1000": 0.69506, + "map_at_1": 0.29288, + "map_at_3": 0.36256, + "map_at_5": 0.42967, + "map_at_10": 0.48881, + "map_at_20": 0.51757, + "map_at_100": 0.53807, + "map_at_1000": 0.53819, + "recall_at_1": 0.29288, + "recall_at_3": 0.40827, + "recall_at_5": 0.54837, + "recall_at_10": 0.76702, + "recall_at_20": 0.88064, + "recall_at_100": 0.98246, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.54386, + "cv_recall_at_3": 0.70175, + "cv_recall_at_5": 0.77193, + "cv_recall_at_10": 0.91228, + "cv_recall_at_20": 0.96491, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.54386, + "precision_at_3": 0.30994, + "precision_at_5": 0.28421, + "precision_at_10": 0.20877, + "precision_at_20": 0.13509, + "precision_at_100": 0.03877, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.54386, + "mrr_at_3": 0.614035, + "mrr_at_5": 0.629825, + "mrr_at_10": 0.651824, + "mrr_at_20": 0.655051, + "mrr_at_100": 0.655469, + "mrr_at_1000": 0.655588, + "nauc_ndcg_at_1_max": 0.095647, + "nauc_ndcg_at_1_std": -0.006758, + "nauc_ndcg_at_1_diff1": 0.35544, + "nauc_ndcg_at_3_max": 0.131238, + "nauc_ndcg_at_3_std": 0.071137, + "nauc_ndcg_at_3_diff1": 0.14043, + "nauc_ndcg_at_5_max": 0.019782, + "nauc_ndcg_at_5_std": -0.059501, + "nauc_ndcg_at_5_diff1": 0.103515, + "nauc_ndcg_at_10_max": 0.051553, + "nauc_ndcg_at_10_std": -0.088303, + "nauc_ndcg_at_10_diff1": 0.267062, + "nauc_ndcg_at_20_max": 0.05854, + "nauc_ndcg_at_20_std": -0.071097, + "nauc_ndcg_at_20_diff1": 0.222619, + "nauc_ndcg_at_100_max": -0.027174, + "nauc_ndcg_at_100_std": -0.052367, + "nauc_ndcg_at_100_diff1": 0.168197, + "nauc_ndcg_at_1000_max": -0.012535, + "nauc_ndcg_at_1000_std": -0.054662, + "nauc_ndcg_at_1000_diff1": 0.163237, + "nauc_map_at_1_max": 0.197311, + "nauc_map_at_1_std": -0.122021, + "nauc_map_at_1_diff1": 0.270686, + "nauc_map_at_3_max": 0.216454, + "nauc_map_at_3_std": 0.002949, + "nauc_map_at_3_diff1": 0.21145, + "nauc_map_at_5_max": 0.121563, + "nauc_map_at_5_std": -0.077198, + "nauc_map_at_5_diff1": 0.171149, + "nauc_map_at_10_max": 0.127104, + "nauc_map_at_10_std": -0.078888, + "nauc_map_at_10_diff1": 0.229081, + "nauc_map_at_20_max": 0.109249, + "nauc_map_at_20_std": -0.077698, + "nauc_map_at_20_diff1": 0.194559, + "nauc_map_at_100_max": 0.064569, + "nauc_map_at_100_std": -0.078438, + "nauc_map_at_100_diff1": 0.15677, + "nauc_map_at_1000_max": 0.065162, + "nauc_map_at_1000_std": -0.078536, + "nauc_map_at_1000_diff1": 0.156579, + "nauc_recall_at_1_max": 0.197311, + "nauc_recall_at_1_std": -0.122021, + "nauc_recall_at_1_diff1": 0.270686, + "nauc_recall_at_3_max": 0.198568, + "nauc_recall_at_3_std": 0.114377, + "nauc_recall_at_3_diff1": 0.142944, + "nauc_recall_at_5_max": 0.085279, + "nauc_recall_at_5_std": -0.051471, + "nauc_recall_at_5_diff1": 0.126925, + "nauc_recall_at_10_max": 0.10525, + "nauc_recall_at_10_std": -0.197542, + "nauc_recall_at_10_diff1": 0.488327, + "nauc_recall_at_20_max": 0.188912, + "nauc_recall_at_20_std": -0.15988, + "nauc_recall_at_20_diff1": 0.386484, + "nauc_recall_at_100_max": -1.169034, + "nauc_recall_at_100_std": 0.126616, + "nauc_recall_at_100_diff1": 0.555085, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.095647, + "nauc_precision_at_1_std": -0.006758, + "nauc_precision_at_1_diff1": 0.35544, + "nauc_precision_at_3_max": -0.062185, + "nauc_precision_at_3_std": 0.125116, + "nauc_precision_at_3_diff1": -0.042368, + "nauc_precision_at_5_max": -0.25502, + "nauc_precision_at_5_std": -0.072795, + "nauc_precision_at_5_diff1": -0.163402, + "nauc_precision_at_10_max": -0.281549, + "nauc_precision_at_10_std": -0.040217, + "nauc_precision_at_10_diff1": -0.112386, + "nauc_precision_at_20_max": -0.297882, + "nauc_precision_at_20_std": -0.011326, + "nauc_precision_at_20_diff1": -0.233254, + "nauc_precision_at_100_max": -0.343027, + "nauc_precision_at_100_std": 0.041168, + "nauc_precision_at_100_diff1": -0.310494, + "nauc_precision_at_1000_max": -0.339058, + "nauc_precision_at_1000_std": 0.040758, + "nauc_precision_at_1000_diff1": -0.314653, + "nauc_cv_recall_at_1_max": 0.095647, + "nauc_cv_recall_at_1_std": -0.006758, + "nauc_cv_recall_at_1_diff1": 0.35544, + "nauc_cv_recall_at_3_max": -0.119241, + "nauc_cv_recall_at_3_std": 0.103208, + "nauc_cv_recall_at_3_diff1": 0.06837, + "nauc_cv_recall_at_5_max": -0.112767, + "nauc_cv_recall_at_5_std": 0.011004, + "nauc_cv_recall_at_5_diff1": 0.183597, + "nauc_cv_recall_at_10_max": -0.40654, + "nauc_cv_recall_at_10_std": -0.376727, + "nauc_cv_recall_at_10_diff1": 0.744656, + "nauc_cv_recall_at_20_max": -0.864221, + "nauc_cv_recall_at_20_std": -0.800621, + "nauc_cv_recall_at_20_diff1": 0.638909, + "nauc_cv_recall_at_100_max": -1.169034, + "nauc_cv_recall_at_100_std": 0.126616, + "nauc_cv_recall_at_100_diff1": 0.555085, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.095647, + "nauc_mrr_at_1_std": -0.006758, + "nauc_mrr_at_1_diff1": 0.35544, + "nauc_mrr_at_3_max": 0.005617, + "nauc_mrr_at_3_std": 0.032823, + "nauc_mrr_at_3_diff1": 0.233546, + "nauc_mrr_at_5_max": 0.008174, + "nauc_mrr_at_5_std": 0.015202, + "nauc_mrr_at_5_diff1": 0.253397, + "nauc_mrr_at_10_max": 0.004463, + "nauc_mrr_at_10_std": -0.005273, + "nauc_mrr_at_10_diff1": 0.291226, + "nauc_mrr_at_20_max": 0.005709, + "nauc_mrr_at_20_std": -0.004012, + "nauc_mrr_at_20_diff1": 0.284731, + "nauc_mrr_at_100_max": 0.006591, + "nauc_mrr_at_100_std": -0.00132, + "nauc_mrr_at_100_diff1": 0.284047, + "nauc_mrr_at_1000_max": 0.007116, + "nauc_mrr_at_1000_std": -0.001377, + "nauc_mrr_at_1000_diff1": 0.283926, + "main_score": 0.54073, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.49123, + "ndcg_at_3": 0.46031, + "ndcg_at_5": 0.5103, + "ndcg_at_10": 0.57504, + "ndcg_at_20": 0.61486, + "ndcg_at_100": 0.66343, + "ndcg_at_1000": 0.66579, + "map_at_1": 0.25706, + "map_at_3": 0.33618, + "map_at_5": 0.39106, + "map_at_10": 0.44623, + "map_at_20": 0.4713, + "map_at_100": 0.49679, + "map_at_1000": 0.49689, + "recall_at_1": 0.25706, + "recall_at_3": 0.4056, + "recall_at_5": 0.53858, + "recall_at_10": 0.72575, + "recall_at_20": 0.84979, + "recall_at_100": 0.98246, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.49123, + "cv_recall_at_3": 0.73684, + "cv_recall_at_5": 0.80702, + "cv_recall_at_10": 0.89474, + "cv_recall_at_20": 0.94737, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.49123, + "precision_at_3": 0.30409, + "precision_at_5": 0.26316, + "precision_at_10": 0.18947, + "precision_at_20": 0.12368, + "precision_at_100": 0.03877, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.491228, + "mrr_at_3": 0.596491, + "mrr_at_5": 0.613158, + "mrr_at_10": 0.626072, + "mrr_at_20": 0.629527, + "mrr_at_100": 0.630667, + "mrr_at_1000": 0.630769, + "nauc_ndcg_at_1_max": -0.028234, + "nauc_ndcg_at_1_std": -0.056423, + "nauc_ndcg_at_1_diff1": 0.213023, + "nauc_ndcg_at_3_max": 0.135788, + "nauc_ndcg_at_3_std": 0.010893, + "nauc_ndcg_at_3_diff1": 0.272581, + "nauc_ndcg_at_5_max": 0.075805, + "nauc_ndcg_at_5_std": 0.025937, + "nauc_ndcg_at_5_diff1": 0.252901, + "nauc_ndcg_at_10_max": 0.163113, + "nauc_ndcg_at_10_std": 0.056698, + "nauc_ndcg_at_10_diff1": 0.304772, + "nauc_ndcg_at_20_max": 0.199105, + "nauc_ndcg_at_20_std": 0.090735, + "nauc_ndcg_at_20_diff1": 0.3161, + "nauc_ndcg_at_100_max": 0.112218, + "nauc_ndcg_at_100_std": 0.086207, + "nauc_ndcg_at_100_diff1": 0.244414, + "nauc_ndcg_at_1000_max": 0.103026, + "nauc_ndcg_at_1000_std": 0.075101, + "nauc_ndcg_at_1000_diff1": 0.245846, + "nauc_map_at_1_max": 0.013519, + "nauc_map_at_1_std": -0.149942, + "nauc_map_at_1_diff1": 0.400395, + "nauc_map_at_3_max": 0.1144, + "nauc_map_at_3_std": -0.078641, + "nauc_map_at_3_diff1": 0.372403, + "nauc_map_at_5_max": 0.080863, + "nauc_map_at_5_std": -0.049021, + "nauc_map_at_5_diff1": 0.346506, + "nauc_map_at_10_max": 0.160585, + "nauc_map_at_10_std": 0.008763, + "nauc_map_at_10_diff1": 0.359807, + "nauc_map_at_20_max": 0.182923, + "nauc_map_at_20_std": 0.038795, + "nauc_map_at_20_diff1": 0.355208, + "nauc_map_at_100_max": 0.144703, + "nauc_map_at_100_std": 0.039897, + "nauc_map_at_100_diff1": 0.328532, + "nauc_map_at_1000_max": 0.144409, + "nauc_map_at_1000_std": 0.039506, + "nauc_map_at_1000_diff1": 0.328614, + "nauc_recall_at_1_max": 0.013519, + "nauc_recall_at_1_std": -0.149942, + "nauc_recall_at_1_diff1": 0.400395, + "nauc_recall_at_3_max": 0.191153, + "nauc_recall_at_3_std": -0.06163, + "nauc_recall_at_3_diff1": 0.298315, + "nauc_recall_at_5_max": 0.100614, + "nauc_recall_at_5_std": 0.018453, + "nauc_recall_at_5_diff1": 0.193038, + "nauc_recall_at_10_max": 0.251376, + "nauc_recall_at_10_std": 0.052611, + "nauc_recall_at_10_diff1": 0.284885, + "nauc_recall_at_20_max": 0.538092, + "nauc_recall_at_20_std": 0.252153, + "nauc_recall_at_20_diff1": 0.446636, + "nauc_recall_at_100_max": 0.868512, + "nauc_recall_at_100_std": 1.0, + "nauc_recall_at_100_diff1": 0.126616, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.028234, + "nauc_precision_at_1_std": -0.056423, + "nauc_precision_at_1_diff1": 0.213023, + "nauc_precision_at_3_max": 0.126507, + "nauc_precision_at_3_std": 0.149606, + "nauc_precision_at_3_diff1": 0.024835, + "nauc_precision_at_5_max": 0.01514, + "nauc_precision_at_5_std": 0.162948, + "nauc_precision_at_5_diff1": -0.088192, + "nauc_precision_at_10_max": 0.184919, + "nauc_precision_at_10_std": 0.319647, + "nauc_precision_at_10_diff1": -0.140589, + "nauc_precision_at_20_max": 0.116359, + "nauc_precision_at_20_std": 0.305567, + "nauc_precision_at_20_diff1": -0.203763, + "nauc_precision_at_100_max": -0.117973, + "nauc_precision_at_100_std": 0.194117, + "nauc_precision_at_100_diff1": -0.275319, + "nauc_precision_at_1000_max": -0.122713, + "nauc_precision_at_1000_std": 0.190244, + "nauc_precision_at_1000_diff1": -0.27725, + "nauc_cv_recall_at_1_max": -0.028234, + "nauc_cv_recall_at_1_std": -0.056423, + "nauc_cv_recall_at_1_diff1": 0.213023, + "nauc_cv_recall_at_3_max": -0.00538, + "nauc_cv_recall_at_3_std": 0.014912, + "nauc_cv_recall_at_3_diff1": 0.127047, + "nauc_cv_recall_at_5_max": -0.05002, + "nauc_cv_recall_at_5_std": 0.135752, + "nauc_cv_recall_at_5_diff1": -0.006451, + "nauc_cv_recall_at_10_max": 0.071766, + "nauc_cv_recall_at_10_std": 0.208921, + "nauc_cv_recall_at_10_diff1": 0.089677, + "nauc_cv_recall_at_20_max": 0.57262, + "nauc_cv_recall_at_20_std": 0.567856, + "nauc_cv_recall_at_20_diff1": 0.450994, + "nauc_cv_recall_at_100_max": 0.868512, + "nauc_cv_recall_at_100_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.126616, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.028234, + "nauc_mrr_at_1_std": -0.056423, + "nauc_mrr_at_1_diff1": 0.213023, + "nauc_mrr_at_3_max": -0.009063, + "nauc_mrr_at_3_std": -0.005287, + "nauc_mrr_at_3_diff1": 0.197123, + "nauc_mrr_at_5_max": -0.015114, + "nauc_mrr_at_5_std": 0.014961, + "nauc_mrr_at_5_diff1": 0.178345, + "nauc_mrr_at_10_max": -0.001194, + "nauc_mrr_at_10_std": 0.018425, + "nauc_mrr_at_10_diff1": 0.190073, + "nauc_mrr_at_20_max": 0.003573, + "nauc_mrr_at_20_std": 0.019538, + "nauc_mrr_at_20_diff1": 0.196634, + "nauc_mrr_at_100_max": 0.001532, + "nauc_mrr_at_100_std": 0.017657, + "nauc_mrr_at_100_diff1": 0.195144, + "nauc_mrr_at_1000_max": 0.001209, + "nauc_mrr_at_1000_std": 0.01729, + "nauc_mrr_at_1000_diff1": 0.19517, + "main_score": 0.5103, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 959.0791382789612, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/Vidore2EconomicsReportsRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..66d9a87da7 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.60345, + "ndcg_at_3": 0.57435, + "ndcg_at_5": 0.56952, + "ndcg_at_10": 0.54464, + "ndcg_at_20": 0.5709, + "ndcg_at_100": 0.68515, + "ndcg_at_1000": 0.72016, + "map_at_1": 0.08059, + "map_at_3": 0.17827, + "map_at_5": 0.23961, + "map_at_10": 0.30382, + "map_at_20": 0.35936, + "map_at_100": 0.44658, + "map_at_1000": 0.46317, + "recall_at_1": 0.08059, + "recall_at_3": 0.2228, + "recall_at_5": 0.32239, + "recall_at_10": 0.43412, + "recall_at_20": 0.58404, + "recall_at_100": 0.91353, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.60345, + "cv_recall_at_3": 0.87931, + "cv_recall_at_5": 0.94828, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.60345, + "precision_at_3": 0.53448, + "precision_at_5": 0.5069, + "precision_at_10": 0.39655, + "precision_at_20": 0.29741, + "precision_at_100": 0.12379, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.603448, + "mrr_at_3": 0.721264, + "mrr_at_5": 0.736782, + "mrr_at_10": 0.736782, + "mrr_at_20": 0.739545, + "mrr_at_100": 0.740366, + "mrr_at_1000": 0.740366, + "nauc_ndcg_at_1_max": 0.078714, + "nauc_ndcg_at_1_std": 0.041758, + "nauc_ndcg_at_1_diff1": 0.248964, + "nauc_ndcg_at_3_max": 0.014291, + "nauc_ndcg_at_3_std": -0.021754, + "nauc_ndcg_at_3_diff1": 0.207147, + "nauc_ndcg_at_5_max": 0.002786, + "nauc_ndcg_at_5_std": -0.082915, + "nauc_ndcg_at_5_diff1": 0.186967, + "nauc_ndcg_at_10_max": -0.028274, + "nauc_ndcg_at_10_std": -0.18415, + "nauc_ndcg_at_10_diff1": 0.298051, + "nauc_ndcg_at_20_max": 0.090959, + "nauc_ndcg_at_20_std": -0.037329, + "nauc_ndcg_at_20_diff1": 0.302758, + "nauc_ndcg_at_100_max": 0.103993, + "nauc_ndcg_at_100_std": 0.028405, + "nauc_ndcg_at_100_diff1": 0.209579, + "nauc_ndcg_at_1000_max": 0.040336, + "nauc_ndcg_at_1000_std": -0.039036, + "nauc_ndcg_at_1000_diff1": 0.196397, + "nauc_map_at_1_max": -0.03924, + "nauc_map_at_1_std": -0.064525, + "nauc_map_at_1_diff1": 0.074203, + "nauc_map_at_3_max": 0.005164, + "nauc_map_at_3_std": -0.09034, + "nauc_map_at_3_diff1": 0.23947, + "nauc_map_at_5_max": 0.061865, + "nauc_map_at_5_std": -0.073117, + "nauc_map_at_5_diff1": 0.253882, + "nauc_map_at_10_max": 0.096407, + "nauc_map_at_10_std": -0.057269, + "nauc_map_at_10_diff1": 0.32412, + "nauc_map_at_20_max": 0.134668, + "nauc_map_at_20_std": -0.004051, + "nauc_map_at_20_diff1": 0.315508, + "nauc_map_at_100_max": 0.086596, + "nauc_map_at_100_std": -0.013602, + "nauc_map_at_100_diff1": 0.257042, + "nauc_map_at_1000_max": 0.046829, + "nauc_map_at_1000_std": -0.039162, + "nauc_map_at_1000_diff1": 0.239751, + "nauc_recall_at_1_max": -0.03924, + "nauc_recall_at_1_std": -0.064525, + "nauc_recall_at_1_diff1": 0.074203, + "nauc_recall_at_3_max": -0.005787, + "nauc_recall_at_3_std": -0.118323, + "nauc_recall_at_3_diff1": 0.247537, + "nauc_recall_at_5_max": 0.021994, + "nauc_recall_at_5_std": -0.137563, + "nauc_recall_at_5_diff1": 0.242697, + "nauc_recall_at_10_max": 0.046501, + "nauc_recall_at_10_std": -0.142647, + "nauc_recall_at_10_diff1": 0.345127, + "nauc_recall_at_20_max": 0.144583, + "nauc_recall_at_20_std": 0.017082, + "nauc_recall_at_20_diff1": 0.349933, + "nauc_recall_at_100_max": 0.235687, + "nauc_recall_at_100_std": 0.261675, + "nauc_recall_at_100_diff1": 0.050988, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.078714, + "nauc_precision_at_1_std": 0.041758, + "nauc_precision_at_1_diff1": 0.248964, + "nauc_precision_at_3_max": 0.030814, + "nauc_precision_at_3_std": 0.003777, + "nauc_precision_at_3_diff1": 0.179251, + "nauc_precision_at_5_max": 0.073867, + "nauc_precision_at_5_std": -0.000824, + "nauc_precision_at_5_diff1": 0.145703, + "nauc_precision_at_10_max": -0.008117, + "nauc_precision_at_10_std": -0.061435, + "nauc_precision_at_10_diff1": 0.143271, + "nauc_precision_at_20_max": -0.020208, + "nauc_precision_at_20_std": -0.007721, + "nauc_precision_at_20_diff1": 0.017379, + "nauc_precision_at_100_max": -0.215899, + "nauc_precision_at_100_std": -0.115027, + "nauc_precision_at_100_diff1": -0.165629, + "nauc_precision_at_1000_max": -0.245321, + "nauc_precision_at_1000_std": -0.140177, + "nauc_precision_at_1000_diff1": -0.152822, + "nauc_cv_recall_at_1_max": 0.078714, + "nauc_cv_recall_at_1_std": 0.041758, + "nauc_cv_recall_at_1_diff1": 0.248964, + "nauc_cv_recall_at_3_max": 0.381997, + "nauc_cv_recall_at_3_std": 0.175646, + "nauc_cv_recall_at_3_diff1": 0.314282, + "nauc_cv_recall_at_5_max": -0.129386, + "nauc_cv_recall_at_5_std": -0.630046, + "nauc_cv_recall_at_5_diff1": 0.431493, + "nauc_cv_recall_at_10_max": -0.129386, + "nauc_cv_recall_at_10_std": -0.630046, + "nauc_cv_recall_at_10_diff1": 0.431493, + "nauc_cv_recall_at_20_max": -0.574298, + "nauc_cv_recall_at_20_std": -1.142926, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.078714, + "nauc_mrr_at_1_std": 0.041758, + "nauc_mrr_at_1_diff1": 0.248964, + "nauc_mrr_at_3_max": 0.111486, + "nauc_mrr_at_3_std": 0.042617, + "nauc_mrr_at_3_diff1": 0.235445, + "nauc_mrr_at_5_max": 0.065046, + "nauc_mrr_at_5_std": -0.010494, + "nauc_mrr_at_5_diff1": 0.238785, + "nauc_mrr_at_10_max": 0.065046, + "nauc_mrr_at_10_std": -0.010494, + "nauc_mrr_at_10_diff1": 0.238785, + "nauc_mrr_at_20_max": 0.064822, + "nauc_mrr_at_20_std": -0.00592, + "nauc_mrr_at_20_diff1": 0.239593, + "nauc_mrr_at_100_max": 0.067278, + "nauc_mrr_at_100_std": -0.001551, + "nauc_mrr_at_100_diff1": 0.236671, + "nauc_mrr_at_1000_max": 0.067278, + "nauc_mrr_at_1000_std": -0.001551, + "nauc_mrr_at_1000_diff1": 0.236671, + "main_score": 0.56952, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.53448, + "ndcg_at_3": 0.54184, + "ndcg_at_5": 0.53493, + "ndcg_at_10": 0.54034, + "ndcg_at_20": 0.55948, + "ndcg_at_100": 0.67423, + "ndcg_at_1000": 0.70945, + "map_at_1": 0.07931, + "map_at_3": 0.1678, + "map_at_5": 0.22657, + "map_at_10": 0.29859, + "map_at_20": 0.3502, + "map_at_100": 0.43729, + "map_at_1000": 0.45366, + "recall_at_1": 0.07931, + "recall_at_3": 0.21256, + "recall_at_5": 0.30517, + "recall_at_10": 0.43707, + "recall_at_20": 0.57822, + "recall_at_100": 0.9085, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.53448, + "cv_recall_at_3": 0.86207, + "cv_recall_at_5": 0.89655, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.53448, + "precision_at_3": 0.51149, + "precision_at_5": 0.47586, + "precision_at_10": 0.40862, + "precision_at_20": 0.29741, + "precision_at_100": 0.125, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.534483, + "mrr_at_3": 0.681034, + "mrr_at_5": 0.688793, + "mrr_at_10": 0.697414, + "mrr_at_20": 0.70112, + "mrr_at_100": 0.70112, + "mrr_at_1000": 0.70112, + "nauc_ndcg_at_1_max": -0.148543, + "nauc_ndcg_at_1_std": -0.137914, + "nauc_ndcg_at_1_diff1": 0.259317, + "nauc_ndcg_at_3_max": -0.029597, + "nauc_ndcg_at_3_std": -0.108538, + "nauc_ndcg_at_3_diff1": 0.31691, + "nauc_ndcg_at_5_max": -0.042362, + "nauc_ndcg_at_5_std": -0.160608, + "nauc_ndcg_at_5_diff1": 0.256992, + "nauc_ndcg_at_10_max": -0.043993, + "nauc_ndcg_at_10_std": -0.188999, + "nauc_ndcg_at_10_diff1": 0.217212, + "nauc_ndcg_at_20_max": 0.069834, + "nauc_ndcg_at_20_std": -0.052704, + "nauc_ndcg_at_20_diff1": 0.198472, + "nauc_ndcg_at_100_max": 0.014298, + "nauc_ndcg_at_100_std": -0.10412, + "nauc_ndcg_at_100_diff1": 0.173122, + "nauc_ndcg_at_1000_max": -0.019807, + "nauc_ndcg_at_1000_std": -0.117013, + "nauc_ndcg_at_1000_diff1": 0.159373, + "nauc_map_at_1_max": -0.21161, + "nauc_map_at_1_std": -0.289253, + "nauc_map_at_1_diff1": 0.410709, + "nauc_map_at_3_max": -0.080462, + "nauc_map_at_3_std": -0.253934, + "nauc_map_at_3_diff1": 0.513247, + "nauc_map_at_5_max": -0.007695, + "nauc_map_at_5_std": -0.21509, + "nauc_map_at_5_diff1": 0.472354, + "nauc_map_at_10_max": 0.065978, + "nauc_map_at_10_std": -0.161407, + "nauc_map_at_10_diff1": 0.440954, + "nauc_map_at_20_max": 0.096012, + "nauc_map_at_20_std": -0.114445, + "nauc_map_at_20_diff1": 0.36987, + "nauc_map_at_100_max": 0.059341, + "nauc_map_at_100_std": -0.115118, + "nauc_map_at_100_diff1": 0.254462, + "nauc_map_at_1000_max": 0.03495, + "nauc_map_at_1000_std": -0.110054, + "nauc_map_at_1000_diff1": 0.21024, + "nauc_recall_at_1_max": -0.21161, + "nauc_recall_at_1_std": -0.289253, + "nauc_recall_at_1_diff1": 0.410709, + "nauc_recall_at_3_max": -0.031353, + "nauc_recall_at_3_std": -0.230575, + "nauc_recall_at_3_diff1": 0.495393, + "nauc_recall_at_5_max": -0.011349, + "nauc_recall_at_5_std": -0.228394, + "nauc_recall_at_5_diff1": 0.357846, + "nauc_recall_at_10_max": 0.075008, + "nauc_recall_at_10_std": -0.148268, + "nauc_recall_at_10_diff1": 0.310152, + "nauc_recall_at_20_max": 0.198697, + "nauc_recall_at_20_std": 0.028873, + "nauc_recall_at_20_diff1": 0.197032, + "nauc_recall_at_100_max": 0.149168, + "nauc_recall_at_100_std": 0.103672, + "nauc_recall_at_100_diff1": -0.010746, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.148543, + "nauc_precision_at_1_std": -0.137914, + "nauc_precision_at_1_diff1": 0.259317, + "nauc_precision_at_3_max": 0.056194, + "nauc_precision_at_3_std": -0.008265, + "nauc_precision_at_3_diff1": 0.225153, + "nauc_precision_at_5_max": 0.084059, + "nauc_precision_at_5_std": -0.002011, + "nauc_precision_at_5_diff1": 0.093548, + "nauc_precision_at_10_max": 0.04909, + "nauc_precision_at_10_std": 0.026559, + "nauc_precision_at_10_diff1": -0.098686, + "nauc_precision_at_20_max": 0.030583, + "nauc_precision_at_20_std": 0.096013, + "nauc_precision_at_20_diff1": -0.294498, + "nauc_precision_at_100_max": -0.123799, + "nauc_precision_at_100_std": 0.038194, + "nauc_precision_at_100_diff1": -0.372749, + "nauc_precision_at_1000_max": -0.150352, + "nauc_precision_at_1000_std": 0.026714, + "nauc_precision_at_1000_diff1": -0.343633, + "nauc_cv_recall_at_1_max": -0.148543, + "nauc_cv_recall_at_1_std": -0.137914, + "nauc_cv_recall_at_1_diff1": 0.259317, + "nauc_cv_recall_at_3_max": 0.0612, + "nauc_cv_recall_at_3_std": -0.015508, + "nauc_cv_recall_at_3_diff1": 0.441436, + "nauc_cv_recall_at_5_max": -0.061777, + "nauc_cv_recall_at_5_std": -0.353333, + "nauc_cv_recall_at_5_diff1": 0.304403, + "nauc_cv_recall_at_10_max": -0.074022, + "nauc_cv_recall_at_10_std": -0.642265, + "nauc_cv_recall_at_10_diff1": 0.093289, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.148543, + "nauc_mrr_at_1_std": -0.137914, + "nauc_mrr_at_1_diff1": 0.259317, + "nauc_mrr_at_3_max": -0.113864, + "nauc_mrr_at_3_std": -0.125266, + "nauc_mrr_at_3_diff1": 0.307855, + "nauc_mrr_at_5_max": -0.13157, + "nauc_mrr_at_5_std": -0.160692, + "nauc_mrr_at_5_diff1": 0.29075, + "nauc_mrr_at_10_max": -0.134589, + "nauc_mrr_at_10_std": -0.164237, + "nauc_mrr_at_10_diff1": 0.282477, + "nauc_mrr_at_20_max": -0.135141, + "nauc_mrr_at_20_std": -0.153585, + "nauc_mrr_at_20_diff1": 0.285037, + "nauc_mrr_at_100_max": -0.135141, + "nauc_mrr_at_100_std": -0.153585, + "nauc_mrr_at_100_diff1": 0.285037, + "nauc_mrr_at_1000_max": -0.135141, + "nauc_mrr_at_1000_std": -0.153585, + "nauc_mrr_at_1000_diff1": 0.285037, + "main_score": 0.53493, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.62069, + "ndcg_at_3": 0.59706, + "ndcg_at_5": 0.60015, + "ndcg_at_10": 0.58159, + "ndcg_at_20": 0.59699, + "ndcg_at_100": 0.70767, + "ndcg_at_1000": 0.73888, + "map_at_1": 0.0891, + "map_at_3": 0.18827, + "map_at_5": 0.25898, + "map_at_10": 0.33057, + "map_at_20": 0.3873, + "map_at_100": 0.47558, + "map_at_1000": 0.4919, + "recall_at_1": 0.0891, + "recall_at_3": 0.22936, + "recall_at_5": 0.33731, + "recall_at_10": 0.46139, + "recall_at_20": 0.59975, + "recall_at_100": 0.92406, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.62069, + "cv_recall_at_3": 0.87931, + "cv_recall_at_5": 0.93103, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.62069, + "precision_at_3": 0.56322, + "precision_at_5": 0.54138, + "precision_at_10": 0.42759, + "precision_at_20": 0.30948, + "precision_at_100": 0.12638, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.62069, + "mrr_at_3": 0.744253, + "mrr_at_5": 0.756322, + "mrr_at_10": 0.758785, + "mrr_at_20": 0.762229, + "mrr_at_100": 0.762229, + "mrr_at_1000": 0.762229, + "nauc_ndcg_at_1_max": -0.177803, + "nauc_ndcg_at_1_std": 0.067159, + "nauc_ndcg_at_1_diff1": -0.126386, + "nauc_ndcg_at_3_max": 0.008073, + "nauc_ndcg_at_3_std": 0.006912, + "nauc_ndcg_at_3_diff1": -0.071436, + "nauc_ndcg_at_5_max": -0.034585, + "nauc_ndcg_at_5_std": -0.0748, + "nauc_ndcg_at_5_diff1": -0.030183, + "nauc_ndcg_at_10_max": -0.036281, + "nauc_ndcg_at_10_std": -0.11918, + "nauc_ndcg_at_10_diff1": -0.060398, + "nauc_ndcg_at_20_max": 0.052619, + "nauc_ndcg_at_20_std": -0.011875, + "nauc_ndcg_at_20_diff1": -0.107596, + "nauc_ndcg_at_100_max": 0.055373, + "nauc_ndcg_at_100_std": 0.050662, + "nauc_ndcg_at_100_diff1": -0.211406, + "nauc_ndcg_at_1000_max": 0.01514, + "nauc_ndcg_at_1000_std": 0.037151, + "nauc_ndcg_at_1000_diff1": -0.165476, + "nauc_map_at_1_max": -0.230844, + "nauc_map_at_1_std": -0.187715, + "nauc_map_at_1_diff1": 0.035627, + "nauc_map_at_3_max": -0.064836, + "nauc_map_at_3_std": -0.214895, + "nauc_map_at_3_diff1": 0.135979, + "nauc_map_at_5_max": -0.044388, + "nauc_map_at_5_std": -0.195828, + "nauc_map_at_5_diff1": 0.099602, + "nauc_map_at_10_max": 0.014236, + "nauc_map_at_10_std": -0.132786, + "nauc_map_at_10_diff1": 0.022959, + "nauc_map_at_20_max": 0.052114, + "nauc_map_at_20_std": -0.077159, + "nauc_map_at_20_diff1": -0.038691, + "nauc_map_at_100_max": 0.034569, + "nauc_map_at_100_std": -0.031766, + "nauc_map_at_100_diff1": -0.127525, + "nauc_map_at_1000_max": 0.005182, + "nauc_map_at_1000_std": -0.030131, + "nauc_map_at_1000_diff1": -0.128971, + "nauc_recall_at_1_max": -0.230844, + "nauc_recall_at_1_std": -0.187715, + "nauc_recall_at_1_diff1": 0.035627, + "nauc_recall_at_3_max": -0.001497, + "nauc_recall_at_3_std": -0.244329, + "nauc_recall_at_3_diff1": 0.268573, + "nauc_recall_at_5_max": -0.018821, + "nauc_recall_at_5_std": -0.257332, + "nauc_recall_at_5_diff1": 0.189909, + "nauc_recall_at_10_max": 0.027561, + "nauc_recall_at_10_std": -0.213087, + "nauc_recall_at_10_diff1": 0.097809, + "nauc_recall_at_20_max": 0.152418, + "nauc_recall_at_20_std": -0.037495, + "nauc_recall_at_20_diff1": 0.027775, + "nauc_recall_at_100_max": 0.214756, + "nauc_recall_at_100_std": 0.099827, + "nauc_recall_at_100_diff1": -0.281124, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.177803, + "nauc_precision_at_1_std": 0.067159, + "nauc_precision_at_1_diff1": -0.126386, + "nauc_precision_at_3_max": 0.090093, + "nauc_precision_at_3_std": 0.067675, + "nauc_precision_at_3_diff1": -0.124611, + "nauc_precision_at_5_max": 0.061982, + "nauc_precision_at_5_std": 0.048207, + "nauc_precision_at_5_diff1": -0.139699, + "nauc_precision_at_10_max": 0.045263, + "nauc_precision_at_10_std": 0.094111, + "nauc_precision_at_10_diff1": -0.223516, + "nauc_precision_at_20_max": 0.016125, + "nauc_precision_at_20_std": 0.144724, + "nauc_precision_at_20_diff1": -0.240905, + "nauc_precision_at_100_max": -0.119911, + "nauc_precision_at_100_std": 0.079422, + "nauc_precision_at_100_diff1": -0.186177, + "nauc_precision_at_1000_max": -0.144728, + "nauc_precision_at_1000_std": 0.058614, + "nauc_precision_at_1000_diff1": -0.118498, + "nauc_cv_recall_at_1_max": -0.177803, + "nauc_cv_recall_at_1_std": 0.067159, + "nauc_cv_recall_at_1_diff1": -0.126386, + "nauc_cv_recall_at_3_max": 0.435717, + "nauc_cv_recall_at_3_std": 0.387189, + "nauc_cv_recall_at_3_diff1": 0.255176, + "nauc_cv_recall_at_5_max": 0.282882, + "nauc_cv_recall_at_5_std": 0.017573, + "nauc_cv_recall_at_5_diff1": 0.120155, + "nauc_cv_recall_at_10_max": 0.043843, + "nauc_cv_recall_at_10_std": -0.266162, + "nauc_cv_recall_at_10_diff1": 0.217845, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.177803, + "nauc_mrr_at_1_std": 0.067159, + "nauc_mrr_at_1_diff1": -0.126386, + "nauc_mrr_at_3_max": -0.01482, + "nauc_mrr_at_3_std": 0.151515, + "nauc_mrr_at_3_diff1": -0.041632, + "nauc_mrr_at_5_max": -0.054403, + "nauc_mrr_at_5_std": 0.109084, + "nauc_mrr_at_5_diff1": -0.0693, + "nauc_mrr_at_10_max": -0.067271, + "nauc_mrr_at_10_std": 0.099813, + "nauc_mrr_at_10_diff1": -0.068036, + "nauc_mrr_at_20_max": -0.070695, + "nauc_mrr_at_20_std": 0.104358, + "nauc_mrr_at_20_diff1": -0.074705, + "nauc_mrr_at_100_max": -0.070695, + "nauc_mrr_at_100_std": 0.104358, + "nauc_mrr_at_100_diff1": -0.074705, + "nauc_mrr_at_1000_max": -0.070695, + "nauc_mrr_at_1000_std": 0.104358, + "nauc_mrr_at_1000_diff1": -0.074705, + "main_score": 0.60015, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.53448, + "ndcg_at_3": 0.57895, + "ndcg_at_5": 0.55368, + "ndcg_at_10": 0.55441, + "ndcg_at_20": 0.57555, + "ndcg_at_100": 0.68951, + "ndcg_at_1000": 0.72271, + "map_at_1": 0.09089, + "map_at_3": 0.19444, + "map_at_5": 0.24687, + "map_at_10": 0.32179, + "map_at_20": 0.37235, + "map_at_100": 0.45806, + "map_at_1000": 0.47412, + "recall_at_1": 0.09089, + "recall_at_3": 0.23684, + "recall_at_5": 0.31637, + "recall_at_10": 0.44856, + "recall_at_20": 0.58304, + "recall_at_100": 0.91715, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.53448, + "cv_recall_at_3": 0.89655, + "cv_recall_at_5": 0.93103, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.53448, + "precision_at_3": 0.55172, + "precision_at_5": 0.48966, + "precision_at_10": 0.40862, + "precision_at_20": 0.29914, + "precision_at_100": 0.12517, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.534483, + "mrr_at_3": 0.689655, + "mrr_at_5": 0.697414, + "mrr_at_10": 0.700287, + "mrr_at_20": 0.7042, + "mrr_at_100": 0.7042, + "mrr_at_1000": 0.7042, + "nauc_ndcg_at_1_max": -0.183958, + "nauc_ndcg_at_1_std": -0.137081, + "nauc_ndcg_at_1_diff1": 0.135484, + "nauc_ndcg_at_3_max": -0.146831, + "nauc_ndcg_at_3_std": -0.123378, + "nauc_ndcg_at_3_diff1": 0.100851, + "nauc_ndcg_at_5_max": -0.076588, + "nauc_ndcg_at_5_std": -0.047331, + "nauc_ndcg_at_5_diff1": -0.00145, + "nauc_ndcg_at_10_max": -0.132226, + "nauc_ndcg_at_10_std": -0.116158, + "nauc_ndcg_at_10_diff1": -0.058247, + "nauc_ndcg_at_20_max": -0.068214, + "nauc_ndcg_at_20_std": -0.073887, + "nauc_ndcg_at_20_diff1": -0.042294, + "nauc_ndcg_at_100_max": -0.218715, + "nauc_ndcg_at_100_std": -0.150776, + "nauc_ndcg_at_100_diff1": -0.102716, + "nauc_ndcg_at_1000_max": -0.149121, + "nauc_ndcg_at_1000_std": -0.080733, + "nauc_ndcg_at_1000_diff1": -0.059075, + "nauc_map_at_1_max": -0.396443, + "nauc_map_at_1_std": -0.426522, + "nauc_map_at_1_diff1": -0.178434, + "nauc_map_at_3_max": -0.303775, + "nauc_map_at_3_std": -0.416407, + "nauc_map_at_3_diff1": -0.112556, + "nauc_map_at_5_max": -0.259977, + "nauc_map_at_5_std": -0.36125, + "nauc_map_at_5_diff1": -0.124554, + "nauc_map_at_10_max": -0.219415, + "nauc_map_at_10_std": -0.287969, + "nauc_map_at_10_diff1": -0.132849, + "nauc_map_at_20_max": -0.212324, + "nauc_map_at_20_std": -0.25768, + "nauc_map_at_20_diff1": -0.141349, + "nauc_map_at_100_max": -0.216997, + "nauc_map_at_100_std": -0.202514, + "nauc_map_at_100_diff1": -0.137882, + "nauc_map_at_1000_max": -0.175643, + "nauc_map_at_1000_std": -0.152124, + "nauc_map_at_1000_diff1": -0.116669, + "nauc_recall_at_1_max": -0.396443, + "nauc_recall_at_1_std": -0.426522, + "nauc_recall_at_1_diff1": -0.178434, + "nauc_recall_at_3_max": -0.202436, + "nauc_recall_at_3_std": -0.392459, + "nauc_recall_at_3_diff1": -0.132247, + "nauc_recall_at_5_max": -0.148518, + "nauc_recall_at_5_std": -0.330647, + "nauc_recall_at_5_diff1": -0.171646, + "nauc_recall_at_10_max": -0.111937, + "nauc_recall_at_10_std": -0.261986, + "nauc_recall_at_10_diff1": -0.140084, + "nauc_recall_at_20_max": -0.066086, + "nauc_recall_at_20_std": -0.172244, + "nauc_recall_at_20_diff1": -0.111778, + "nauc_recall_at_100_max": -0.30535, + "nauc_recall_at_100_std": -0.269829, + "nauc_recall_at_100_diff1": -0.192782, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.183958, + "nauc_precision_at_1_std": -0.137081, + "nauc_precision_at_1_diff1": 0.135484, + "nauc_precision_at_3_max": -0.09261, + "nauc_precision_at_3_std": 0.010255, + "nauc_precision_at_3_diff1": 0.178191, + "nauc_precision_at_5_max": 0.070788, + "nauc_precision_at_5_std": 0.221971, + "nauc_precision_at_5_diff1": 0.062472, + "nauc_precision_at_10_max": 0.056106, + "nauc_precision_at_10_std": 0.236612, + "nauc_precision_at_10_diff1": 0.043702, + "nauc_precision_at_20_max": 0.131755, + "nauc_precision_at_20_std": 0.302995, + "nauc_precision_at_20_diff1": 0.070359, + "nauc_precision_at_100_max": 0.171945, + "nauc_precision_at_100_std": 0.335173, + "nauc_precision_at_100_diff1": 0.100086, + "nauc_precision_at_1000_max": 0.236316, + "nauc_precision_at_1000_std": 0.368346, + "nauc_precision_at_1000_diff1": 0.129531, + "nauc_cv_recall_at_1_max": -0.183958, + "nauc_cv_recall_at_1_std": -0.137081, + "nauc_cv_recall_at_1_diff1": 0.135484, + "nauc_cv_recall_at_3_max": 0.007973, + "nauc_cv_recall_at_3_std": 0.178735, + "nauc_cv_recall_at_3_diff1": -0.035394, + "nauc_cv_recall_at_5_max": -0.124923, + "nauc_cv_recall_at_5_std": -0.199091, + "nauc_cv_recall_at_5_diff1": -0.441594, + "nauc_cv_recall_at_10_max": -0.207817, + "nauc_cv_recall_at_10_std": -0.306709, + "nauc_cv_recall_at_10_diff1": -0.39736, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.183958, + "nauc_mrr_at_1_std": -0.137081, + "nauc_mrr_at_1_diff1": 0.135484, + "nauc_mrr_at_3_max": -0.131498, + "nauc_mrr_at_3_std": -0.076248, + "nauc_mrr_at_3_diff1": 0.087816, + "nauc_mrr_at_5_max": -0.146587, + "nauc_mrr_at_5_std": -0.110012, + "nauc_mrr_at_5_diff1": 0.064134, + "nauc_mrr_at_10_max": -0.149949, + "nauc_mrr_at_10_std": -0.11292, + "nauc_mrr_at_10_diff1": 0.072075, + "nauc_mrr_at_20_max": -0.148711, + "nauc_mrr_at_20_std": -0.109855, + "nauc_mrr_at_20_diff1": 0.08093, + "nauc_mrr_at_100_max": -0.148711, + "nauc_mrr_at_100_std": -0.109855, + "nauc_mrr_at_100_diff1": 0.08093, + "nauc_mrr_at_1000_max": -0.148711, + "nauc_mrr_at_1000_std": -0.109855, + "nauc_mrr_at_1000_diff1": 0.08093, + "main_score": 0.55368, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 167.03795647621155, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreArxivQARetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..e082aa1819 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", + "task_name": "VidoreArxivQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.828, + "ndcg_at_3": 0.87459, + "ndcg_at_5": 0.88432, + "ndcg_at_10": 0.89275, + "ndcg_at_20": 0.89727, + "ndcg_at_100": 0.90108, + "ndcg_at_1000": 0.9019, + "map_at_1": 0.828, + "map_at_3": 0.86367, + "map_at_5": 0.86897, + "map_at_10": 0.87247, + "map_at_20": 0.87369, + "map_at_100": 0.87425, + "map_at_1000": 0.87429, + "recall_at_1": 0.828, + "recall_at_3": 0.906, + "recall_at_5": 0.93, + "recall_at_10": 0.956, + "recall_at_20": 0.974, + "recall_at_100": 0.994, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.828, + "cv_recall_at_3": 0.906, + "cv_recall_at_5": 0.93, + "cv_recall_at_10": 0.956, + "cv_recall_at_20": 0.974, + "cv_recall_at_100": 0.994, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.828, + "precision_at_3": 0.302, + "precision_at_5": 0.186, + "precision_at_10": 0.0956, + "precision_at_20": 0.0487, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "mrr_at_1": 0.828, + "mrr_at_3": 0.863667, + "mrr_at_5": 0.868967, + "mrr_at_10": 0.872466, + "mrr_at_20": 0.873686, + "mrr_at_100": 0.87425, + "mrr_at_1000": 0.874291, + "nauc_ndcg_at_1_max": 0.831834, + "nauc_ndcg_at_1_std": -0.005013, + "nauc_ndcg_at_1_diff1": 0.970195, + "nauc_ndcg_at_3_max": 0.815044, + "nauc_ndcg_at_3_std": -0.010792, + "nauc_ndcg_at_3_diff1": 0.952475, + "nauc_ndcg_at_5_max": 0.808369, + "nauc_ndcg_at_5_std": -0.0605, + "nauc_ndcg_at_5_diff1": 0.953983, + "nauc_ndcg_at_10_max": 0.816188, + "nauc_ndcg_at_10_std": -0.010803, + "nauc_ndcg_at_10_diff1": 0.953559, + "nauc_ndcg_at_20_max": 0.823249, + "nauc_ndcg_at_20_std": -0.006587, + "nauc_ndcg_at_20_diff1": 0.957499, + "nauc_ndcg_at_100_max": 0.821813, + "nauc_ndcg_at_100_std": -0.01818, + "nauc_ndcg_at_100_diff1": 0.958605, + "nauc_ndcg_at_1000_max": 0.821138, + "nauc_ndcg_at_1000_std": -0.016418, + "nauc_ndcg_at_1000_diff1": 0.958246, + "nauc_map_at_1_max": 0.831834, + "nauc_map_at_1_std": -0.005013, + "nauc_map_at_1_diff1": 0.970195, + "nauc_map_at_3_max": 0.820827, + "nauc_map_at_3_std": -0.010741, + "nauc_map_at_3_diff1": 0.957535, + "nauc_map_at_5_max": 0.81778, + "nauc_map_at_5_std": -0.034626, + "nauc_map_at_5_diff1": 0.958394, + "nauc_map_at_10_max": 0.820649, + "nauc_map_at_10_std": -0.015874, + "nauc_map_at_10_diff1": 0.958398, + "nauc_map_at_20_max": 0.822326, + "nauc_map_at_20_std": -0.014783, + "nauc_map_at_20_diff1": 0.959284, + "nauc_map_at_100_max": 0.822118, + "nauc_map_at_100_std": -0.016587, + "nauc_map_at_100_diff1": 0.959478, + "nauc_map_at_1000_max": 0.822099, + "nauc_map_at_1000_std": -0.016418, + "nauc_map_at_1000_diff1": 0.959464, + "nauc_recall_at_1_max": 0.831834, + "nauc_recall_at_1_std": -0.005013, + "nauc_recall_at_1_diff1": 0.970195, + "nauc_recall_at_3_max": 0.791287, + "nauc_recall_at_3_std": -0.010032, + "nauc_recall_at_3_diff1": 0.932207, + "nauc_recall_at_5_max": 0.75721, + "nauc_recall_at_5_std": -0.207443, + "nauc_recall_at_5_diff1": 0.931839, + "nauc_recall_at_10_max": 0.785481, + "nauc_recall_at_10_std": 0.057041, + "nauc_recall_at_10_diff1": 0.91533, + "nauc_recall_at_20_max": 0.868563, + "nauc_recall_at_20_std": 0.159987, + "nauc_recall_at_20_diff1": 0.949723, + "nauc_recall_at_100_max": 0.907407, + "nauc_recall_at_100_std": -0.127295, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.831834, + "nauc_precision_at_1_std": -0.005013, + "nauc_precision_at_1_diff1": 0.970195, + "nauc_precision_at_3_max": 0.791287, + "nauc_precision_at_3_std": -0.010032, + "nauc_precision_at_3_diff1": 0.932207, + "nauc_precision_at_5_max": 0.75721, + "nauc_precision_at_5_std": -0.207443, + "nauc_precision_at_5_diff1": 0.931839, + "nauc_precision_at_10_max": 0.785481, + "nauc_precision_at_10_std": 0.057041, + "nauc_precision_at_10_diff1": 0.91533, + "nauc_precision_at_20_max": 0.868563, + "nauc_precision_at_20_std": 0.159987, + "nauc_precision_at_20_diff1": 0.949723, + "nauc_precision_at_100_max": 0.907407, + "nauc_precision_at_100_std": -0.127295, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_cv_recall_at_1_max": 0.831834, + "nauc_cv_recall_at_1_std": -0.005013, + "nauc_cv_recall_at_1_diff1": 0.970195, + "nauc_cv_recall_at_3_max": 0.791287, + "nauc_cv_recall_at_3_std": -0.010032, + "nauc_cv_recall_at_3_diff1": 0.932207, + "nauc_cv_recall_at_5_max": 0.75721, + "nauc_cv_recall_at_5_std": -0.207443, + "nauc_cv_recall_at_5_diff1": 0.931839, + "nauc_cv_recall_at_10_max": 0.785481, + "nauc_cv_recall_at_10_std": 0.057041, + "nauc_cv_recall_at_10_diff1": 0.91533, + "nauc_cv_recall_at_20_max": 0.868563, + "nauc_cv_recall_at_20_std": 0.159987, + "nauc_cv_recall_at_20_diff1": 0.949723, + "nauc_cv_recall_at_100_max": 0.907407, + "nauc_cv_recall_at_100_std": -0.127295, + "nauc_cv_recall_at_100_diff1": 1.0, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.831834, + "nauc_mrr_at_1_std": -0.005013, + "nauc_mrr_at_1_diff1": 0.970195, + "nauc_mrr_at_3_max": 0.820827, + "nauc_mrr_at_3_std": -0.010741, + "nauc_mrr_at_3_diff1": 0.957535, + "nauc_mrr_at_5_max": 0.81778, + "nauc_mrr_at_5_std": -0.034626, + "nauc_mrr_at_5_diff1": 0.958394, + "nauc_mrr_at_10_max": 0.820649, + "nauc_mrr_at_10_std": -0.015874, + "nauc_mrr_at_10_diff1": 0.958398, + "nauc_mrr_at_20_max": 0.822326, + "nauc_mrr_at_20_std": -0.014783, + "nauc_mrr_at_20_diff1": 0.959284, + "nauc_mrr_at_100_max": 0.822118, + "nauc_mrr_at_100_std": -0.016587, + "nauc_mrr_at_100_diff1": 0.959478, + "nauc_mrr_at_1000_max": 0.822099, + "nauc_mrr_at_1000_std": -0.016418, + "nauc_mrr_at_1000_diff1": 0.959464, + "main_score": 0.88432, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 61.08024048805237, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreDocVQARetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..d2098ed165 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", + "task_name": "VidoreDocVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.52993, + "ndcg_at_3": 0.585, + "ndcg_at_5": 0.60106, + "ndcg_at_10": 0.62045, + "ndcg_at_20": 0.63154, + "ndcg_at_100": 0.64873, + "ndcg_at_1000": 0.66822, + "map_at_1": 0.52679, + "map_at_3": 0.56954, + "map_at_5": 0.57854, + "map_at_10": 0.58681, + "map_at_20": 0.58984, + "map_at_100": 0.59225, + "map_at_1000": 0.59305, + "recall_at_1": 0.52679, + "recall_at_3": 0.62509, + "recall_at_5": 0.66353, + "recall_at_10": 0.72339, + "recall_at_20": 0.76615, + "recall_at_100": 0.85671, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.52993, + "cv_recall_at_3": 0.62971, + "cv_recall_at_5": 0.67184, + "cv_recall_at_10": 0.73171, + "cv_recall_at_20": 0.77827, + "cv_recall_at_100": 0.86475, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.52993, + "precision_at_3": 0.21064, + "precision_at_5": 0.1357, + "precision_at_10": 0.07406, + "precision_at_20": 0.03947, + "precision_at_100": 0.00907, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.529933, + "mrr_at_3": 0.572801, + "mrr_at_5": 0.582225, + "mrr_at_10": 0.590303, + "mrr_at_20": 0.593597, + "mrr_at_100": 0.595551, + "mrr_at_1000": 0.596197, + "nauc_ndcg_at_1_max": 0.81942, + "nauc_ndcg_at_1_std": 0.638484, + "nauc_ndcg_at_1_diff1": 0.899958, + "nauc_ndcg_at_3_max": 0.842884, + "nauc_ndcg_at_3_std": 0.694045, + "nauc_ndcg_at_3_diff1": 0.873958, + "nauc_ndcg_at_5_max": 0.845252, + "nauc_ndcg_at_5_std": 0.703463, + "nauc_ndcg_at_5_diff1": 0.872746, + "nauc_ndcg_at_10_max": 0.842831, + "nauc_ndcg_at_10_std": 0.720536, + "nauc_ndcg_at_10_diff1": 0.868091, + "nauc_ndcg_at_20_max": 0.83934, + "nauc_ndcg_at_20_std": 0.718157, + "nauc_ndcg_at_20_diff1": 0.865105, + "nauc_ndcg_at_100_max": 0.837377, + "nauc_ndcg_at_100_std": 0.715192, + "nauc_ndcg_at_100_diff1": 0.867866, + "nauc_ndcg_at_1000_max": 0.835618, + "nauc_ndcg_at_1000_std": 0.700757, + "nauc_ndcg_at_1000_diff1": 0.870364, + "nauc_map_at_1_max": 0.82503, + "nauc_map_at_1_std": 0.643366, + "nauc_map_at_1_diff1": 0.903644, + "nauc_map_at_3_max": 0.839672, + "nauc_map_at_3_std": 0.680821, + "nauc_map_at_3_diff1": 0.88225, + "nauc_map_at_5_max": 0.841227, + "nauc_map_at_5_std": 0.686805, + "nauc_map_at_5_diff1": 0.881616, + "nauc_map_at_10_max": 0.839734, + "nauc_map_at_10_std": 0.693015, + "nauc_map_at_10_diff1": 0.879449, + "nauc_map_at_20_max": 0.839057, + "nauc_map_at_20_std": 0.69233, + "nauc_map_at_20_diff1": 0.878877, + "nauc_map_at_100_max": 0.838349, + "nauc_map_at_100_std": 0.691321, + "nauc_map_at_100_diff1": 0.878831, + "nauc_map_at_1000_max": 0.83821, + "nauc_map_at_1000_std": 0.690729, + "nauc_map_at_1000_diff1": 0.878889, + "nauc_recall_at_1_max": 0.82503, + "nauc_recall_at_1_std": 0.643366, + "nauc_recall_at_1_diff1": 0.903644, + "nauc_recall_at_3_max": 0.859165, + "nauc_recall_at_3_std": 0.738755, + "nauc_recall_at_3_diff1": 0.852705, + "nauc_recall_at_5_max": 0.866494, + "nauc_recall_at_5_std": 0.766496, + "nauc_recall_at_5_diff1": 0.849232, + "nauc_recall_at_10_max": 0.858669, + "nauc_recall_at_10_std": 0.837023, + "nauc_recall_at_10_diff1": 0.827859, + "nauc_recall_at_20_max": 0.841558, + "nauc_recall_at_20_std": 0.842672, + "nauc_recall_at_20_diff1": 0.806323, + "nauc_recall_at_100_max": 0.837026, + "nauc_recall_at_100_std": 0.894331, + "nauc_recall_at_100_diff1": 0.818823, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.81942, + "nauc_precision_at_1_std": 0.638484, + "nauc_precision_at_1_diff1": 0.899958, + "nauc_precision_at_3_max": 0.839139, + "nauc_precision_at_3_std": 0.727779, + "nauc_precision_at_3_diff1": 0.831292, + "nauc_precision_at_5_max": 0.810156, + "nauc_precision_at_5_std": 0.728003, + "nauc_precision_at_5_diff1": 0.785874, + "nauc_precision_at_10_max": 0.778149, + "nauc_precision_at_10_std": 0.776703, + "nauc_precision_at_10_diff1": 0.740034, + "nauc_precision_at_20_max": 0.72062, + "nauc_precision_at_20_std": 0.743821, + "nauc_precision_at_20_diff1": 0.680353, + "nauc_precision_at_100_max": 0.359596, + "nauc_precision_at_100_std": 0.421899, + "nauc_precision_at_100_diff1": 0.347157, + "nauc_precision_at_1000_max": -0.500356, + "nauc_precision_at_1000_std": -0.472539, + "nauc_precision_at_1000_diff1": -0.435916, + "nauc_cv_recall_at_1_max": 0.81942, + "nauc_cv_recall_at_1_std": 0.638484, + "nauc_cv_recall_at_1_diff1": 0.899958, + "nauc_cv_recall_at_3_max": 0.855407, + "nauc_cv_recall_at_3_std": 0.741352, + "nauc_cv_recall_at_3_diff1": 0.847458, + "nauc_cv_recall_at_5_max": 0.853309, + "nauc_cv_recall_at_5_std": 0.753634, + "nauc_cv_recall_at_5_diff1": 0.838821, + "nauc_cv_recall_at_10_max": 0.84765, + "nauc_cv_recall_at_10_std": 0.828394, + "nauc_cv_recall_at_10_diff1": 0.819618, + "nauc_cv_recall_at_20_max": 0.821977, + "nauc_cv_recall_at_20_std": 0.825303, + "nauc_cv_recall_at_20_diff1": 0.789494, + "nauc_cv_recall_at_100_max": 0.824783, + "nauc_cv_recall_at_100_std": 0.888157, + "nauc_cv_recall_at_100_diff1": 0.812055, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.81942, + "nauc_mrr_at_1_std": 0.638484, + "nauc_mrr_at_1_diff1": 0.899958, + "nauc_mrr_at_3_max": 0.834975, + "nauc_mrr_at_3_std": 0.678431, + "nauc_mrr_at_3_diff1": 0.878471, + "nauc_mrr_at_5_max": 0.834416, + "nauc_mrr_at_5_std": 0.680059, + "nauc_mrr_at_5_diff1": 0.877385, + "nauc_mrr_at_10_max": 0.833206, + "nauc_mrr_at_10_std": 0.686571, + "nauc_mrr_at_10_diff1": 0.875536, + "nauc_mrr_at_20_max": 0.832202, + "nauc_mrr_at_20_std": 0.685394, + "nauc_mrr_at_20_diff1": 0.874668, + "nauc_mrr_at_100_max": 0.832002, + "nauc_mrr_at_100_std": 0.685058, + "nauc_mrr_at_100_diff1": 0.875078, + "nauc_mrr_at_1000_max": 0.831989, + "nauc_mrr_at_1000_std": 0.684617, + "nauc_mrr_at_1000_diff1": 0.875171, + "main_score": 0.60106, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 65.4463279247284, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreInfoVQARetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..cd0622a9df --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", + "task_name": "VidoreInfoVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.88462, + "ndcg_at_3": 0.91456, + "ndcg_at_5": 0.92259, + "ndcg_at_10": 0.92801, + "ndcg_at_20": 0.932, + "ndcg_at_100": 0.93369, + "ndcg_at_1000": 0.93445, + "map_at_1": 0.88462, + "map_at_3": 0.90756, + "map_at_5": 0.91221, + "map_at_10": 0.91455, + "map_at_20": 0.9156, + "map_at_100": 0.91578, + "map_at_1000": 0.91582, + "recall_at_1": 0.88462, + "recall_at_3": 0.93421, + "recall_at_5": 0.95344, + "recall_at_10": 0.96964, + "recall_at_20": 0.98583, + "recall_at_100": 0.99453, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.88462, + "cv_recall_at_3": 0.93522, + "cv_recall_at_5": 0.95344, + "cv_recall_at_10": 0.96964, + "cv_recall_at_20": 0.98583, + "cv_recall_at_100": 0.99595, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.88462, + "precision_at_3": 0.31174, + "precision_at_5": 0.19109, + "precision_at_10": 0.09717, + "precision_at_20": 0.04939, + "precision_at_100": 0.01004, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.884615, + "mrr_at_3": 0.907895, + "mrr_at_5": 0.912045, + "mrr_at_10": 0.914386, + "mrr_at_20": 0.915432, + "mrr_at_100": 0.915659, + "mrr_at_1000": 0.915677, + "nauc_ndcg_at_1_max": 0.63825, + "nauc_ndcg_at_1_std": 0.085138, + "nauc_ndcg_at_1_diff1": 0.96236, + "nauc_ndcg_at_3_max": 0.706977, + "nauc_ndcg_at_3_std": 0.149421, + "nauc_ndcg_at_3_diff1": 0.955389, + "nauc_ndcg_at_5_max": 0.721119, + "nauc_ndcg_at_5_std": 0.167059, + "nauc_ndcg_at_5_diff1": 0.959178, + "nauc_ndcg_at_10_max": 0.717238, + "nauc_ndcg_at_10_std": 0.189043, + "nauc_ndcg_at_10_diff1": 0.958634, + "nauc_ndcg_at_20_max": 0.711436, + "nauc_ndcg_at_20_std": 0.190496, + "nauc_ndcg_at_20_diff1": 0.961082, + "nauc_ndcg_at_100_max": 0.704642, + "nauc_ndcg_at_100_std": 0.172072, + "nauc_ndcg_at_100_diff1": 0.960079, + "nauc_ndcg_at_1000_max": 0.701707, + "nauc_ndcg_at_1000_std": 0.164486, + "nauc_ndcg_at_1000_diff1": 0.959611, + "nauc_map_at_1_max": 0.63825, + "nauc_map_at_1_std": 0.085138, + "nauc_map_at_1_diff1": 0.96236, + "nauc_map_at_3_max": 0.690696, + "nauc_map_at_3_std": 0.134266, + "nauc_map_at_3_diff1": 0.957387, + "nauc_map_at_5_max": 0.697468, + "nauc_map_at_5_std": 0.143765, + "nauc_map_at_5_diff1": 0.959214, + "nauc_map_at_10_max": 0.695388, + "nauc_map_at_10_std": 0.151709, + "nauc_map_at_10_diff1": 0.959052, + "nauc_map_at_20_max": 0.694051, + "nauc_map_at_20_std": 0.151773, + "nauc_map_at_20_diff1": 0.959599, + "nauc_map_at_100_max": 0.693426, + "nauc_map_at_100_std": 0.150182, + "nauc_map_at_100_diff1": 0.95951, + "nauc_map_at_1000_max": 0.693308, + "nauc_map_at_1000_std": 0.149873, + "nauc_map_at_1000_diff1": 0.959493, + "nauc_recall_at_1_max": 0.63825, + "nauc_recall_at_1_std": 0.085138, + "nauc_recall_at_1_diff1": 0.96236, + "nauc_recall_at_3_max": 0.76868, + "nauc_recall_at_3_std": 0.205498, + "nauc_recall_at_3_diff1": 0.947759, + "nauc_recall_at_5_max": 0.847085, + "nauc_recall_at_5_std": 0.290298, + "nauc_recall_at_5_diff1": 0.960252, + "nauc_recall_at_10_max": 0.887066, + "nauc_recall_at_10_std": 0.501692, + "nauc_recall_at_10_diff1": 0.956466, + "nauc_recall_at_20_max": 0.962685, + "nauc_recall_at_20_std": 0.877994, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": 0.951629, + "nauc_recall_at_100_std": 0.814597, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.63825, + "nauc_precision_at_1_std": 0.085138, + "nauc_precision_at_1_diff1": 0.96236, + "nauc_precision_at_3_max": 0.772029, + "nauc_precision_at_3_std": 0.211378, + "nauc_precision_at_3_diff1": 0.946943, + "nauc_precision_at_5_max": 0.764991, + "nauc_precision_at_5_std": 0.276747, + "nauc_precision_at_5_diff1": 0.85217, + "nauc_precision_at_10_max": 0.761584, + "nauc_precision_at_10_std": 0.459652, + "nauc_precision_at_10_diff1": 0.797138, + "nauc_precision_at_20_max": 0.707315, + "nauc_precision_at_20_std": 0.722448, + "nauc_precision_at_20_diff1": 0.686213, + "nauc_precision_at_100_max": -0.187742, + "nauc_precision_at_100_std": -0.123334, + "nauc_precision_at_100_diff1": -0.222852, + "nauc_precision_at_1000_max": -0.530911, + "nauc_precision_at_1000_std": -0.418993, + "nauc_precision_at_1000_diff1": -0.573514, + "nauc_cv_recall_at_1_max": 0.63825, + "nauc_cv_recall_at_1_std": 0.085138, + "nauc_cv_recall_at_1_diff1": 0.96236, + "nauc_cv_recall_at_3_max": 0.772029, + "nauc_cv_recall_at_3_std": 0.211378, + "nauc_cv_recall_at_3_diff1": 0.946943, + "nauc_cv_recall_at_5_max": 0.847085, + "nauc_cv_recall_at_5_std": 0.290298, + "nauc_cv_recall_at_5_diff1": 0.960252, + "nauc_cv_recall_at_10_max": 0.887066, + "nauc_cv_recall_at_10_std": 0.501692, + "nauc_cv_recall_at_10_diff1": 0.956466, + "nauc_cv_recall_at_20_max": 0.962685, + "nauc_cv_recall_at_20_std": 0.877994, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": 0.934699, + "nauc_cv_recall_at_100_std": 0.861133, + "nauc_cv_recall_at_100_diff1": 1.0, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.63825, + "nauc_mrr_at_1_std": 0.085138, + "nauc_mrr_at_1_diff1": 0.96236, + "nauc_mrr_at_3_max": 0.691202, + "nauc_mrr_at_3_std": 0.135397, + "nauc_mrr_at_3_diff1": 0.957229, + "nauc_mrr_at_5_max": 0.69719, + "nauc_mrr_at_5_std": 0.143154, + "nauc_mrr_at_5_diff1": 0.959293, + "nauc_mrr_at_10_max": 0.695106, + "nauc_mrr_at_10_std": 0.151065, + "nauc_mrr_at_10_diff1": 0.959134, + "nauc_mrr_at_20_max": 0.693769, + "nauc_mrr_at_20_std": 0.151122, + "nauc_mrr_at_20_diff1": 0.95968, + "nauc_mrr_at_100_max": 0.692979, + "nauc_mrr_at_100_std": 0.14929, + "nauc_mrr_at_100_diff1": 0.959571, + "nauc_mrr_at_1000_max": 0.692924, + "nauc_mrr_at_1000_std": 0.14913, + "nauc_mrr_at_1000_diff1": 0.959561, + "main_score": 0.92259, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 66.753089427948, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreShiftProjectRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..077c093db6 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "84a382e05c4473fed9cff2bbae95fe2379416117", + "task_name": "VidoreShiftProjectRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.8, + "ndcg_at_3": 0.8844, + "ndcg_at_5": 0.89302, + "ndcg_at_10": 0.89936, + "ndcg_at_20": 0.90199, + "ndcg_at_100": 0.90358, + "ndcg_at_1000": 0.90358, + "map_at_1": 0.8, + "map_at_3": 0.865, + "map_at_5": 0.87, + "map_at_10": 0.87254, + "map_at_20": 0.87331, + "map_at_100": 0.87344, + "map_at_1000": 0.87344, + "recall_at_1": 0.8, + "recall_at_3": 0.94, + "recall_at_5": 0.96, + "recall_at_10": 0.98, + "recall_at_20": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.8, + "cv_recall_at_3": 0.94, + "cv_recall_at_5": 0.96, + "cv_recall_at_10": 0.98, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.8, + "precision_at_3": 0.31333, + "precision_at_5": 0.192, + "precision_at_10": 0.098, + "precision_at_20": 0.0495, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.8, + "mrr_at_3": 0.865, + "mrr_at_5": 0.87, + "mrr_at_10": 0.87254, + "mrr_at_20": 0.873309, + "mrr_at_100": 0.873439, + "mrr_at_1000": 0.873439, + "nauc_ndcg_at_1_max": -0.032967, + "nauc_ndcg_at_1_std": -0.600649, + "nauc_ndcg_at_1_diff1": 0.832817, + "nauc_ndcg_at_3_max": 0.028443, + "nauc_ndcg_at_3_std": -0.571303, + "nauc_ndcg_at_3_diff1": 0.846062, + "nauc_ndcg_at_5_max": 0.048948, + "nauc_ndcg_at_5_std": -0.522364, + "nauc_ndcg_at_5_diff1": 0.838593, + "nauc_ndcg_at_10_max": 0.013542, + "nauc_ndcg_at_10_std": -0.576691, + "nauc_ndcg_at_10_diff1": 0.836634, + "nauc_ndcg_at_20_max": 0.018753, + "nauc_ndcg_at_20_std": -0.543905, + "nauc_ndcg_at_20_diff1": 0.839859, + "nauc_ndcg_at_100_max": 0.009453, + "nauc_ndcg_at_100_std": -0.559571, + "nauc_ndcg_at_100_diff1": 0.839348, + "nauc_ndcg_at_1000_max": 0.009453, + "nauc_ndcg_at_1000_std": -0.559571, + "nauc_ndcg_at_1000_diff1": 0.839348, + "nauc_map_at_1_max": -0.032967, + "nauc_map_at_1_std": -0.600649, + "nauc_map_at_1_diff1": 0.832817, + "nauc_map_at_3_max": 0.004379, + "nauc_map_at_3_std": -0.579514, + "nauc_map_at_3_diff1": 0.842964, + "nauc_map_at_5_max": 0.013253, + "nauc_map_at_5_std": -0.556341, + "nauc_map_at_5_diff1": 0.839251, + "nauc_map_at_10_max": 0.00059, + "nauc_map_at_10_std": -0.574983, + "nauc_map_at_10_diff1": 0.838427, + "nauc_map_at_20_max": 0.00169, + "nauc_map_at_20_std": -0.567525, + "nauc_map_at_20_diff1": 0.839171, + "nauc_map_at_100_max": 0.001093, + "nauc_map_at_100_std": -0.568527, + "nauc_map_at_100_diff1": 0.839139, + "nauc_map_at_1000_max": 0.001093, + "nauc_map_at_1000_std": -0.568527, + "nauc_map_at_1000_diff1": 0.839139, + "nauc_recall_at_1_max": -0.032967, + "nauc_recall_at_1_std": -0.600649, + "nauc_recall_at_1_diff1": 0.832817, + "nauc_recall_at_3_max": 0.17935, + "nauc_recall_at_3_std": -0.522409, + "nauc_recall_at_3_diff1": 0.863834, + "nauc_recall_at_5_max": 0.376401, + "nauc_recall_at_5_std": -0.20775, + "nauc_recall_at_5_diff1": 0.828431, + "nauc_recall_at_10_max": 0.19141, + "nauc_recall_at_10_std": -0.690943, + "nauc_recall_at_10_diff1": 0.795752, + "nauc_recall_at_20_max": 0.554155, + "nauc_recall_at_20_std": 0.358077, + "nauc_recall_at_20_diff1": 0.869281, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.032967, + "nauc_precision_at_1_std": -0.600649, + "nauc_precision_at_1_diff1": 0.832817, + "nauc_precision_at_3_max": 0.17935, + "nauc_precision_at_3_std": -0.522409, + "nauc_precision_at_3_diff1": 0.863834, + "nauc_precision_at_5_max": 0.376401, + "nauc_precision_at_5_std": -0.20775, + "nauc_precision_at_5_diff1": 0.828431, + "nauc_precision_at_10_max": 0.19141, + "nauc_precision_at_10_std": -0.690943, + "nauc_precision_at_10_diff1": 0.795752, + "nauc_precision_at_20_max": 0.554155, + "nauc_precision_at_20_std": 0.358077, + "nauc_precision_at_20_diff1": 0.869281, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": -0.032967, + "nauc_cv_recall_at_1_std": -0.600649, + "nauc_cv_recall_at_1_diff1": 0.832817, + "nauc_cv_recall_at_3_max": 0.17935, + "nauc_cv_recall_at_3_std": -0.522409, + "nauc_cv_recall_at_3_diff1": 0.863834, + "nauc_cv_recall_at_5_max": 0.376401, + "nauc_cv_recall_at_5_std": -0.20775, + "nauc_cv_recall_at_5_diff1": 0.828431, + "nauc_cv_recall_at_10_max": 0.19141, + "nauc_cv_recall_at_10_std": -0.690943, + "nauc_cv_recall_at_10_diff1": 0.795752, + "nauc_cv_recall_at_20_max": 0.554155, + "nauc_cv_recall_at_20_std": 0.358077, + "nauc_cv_recall_at_20_diff1": 0.869281, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.032967, + "nauc_mrr_at_1_std": -0.600649, + "nauc_mrr_at_1_diff1": 0.832817, + "nauc_mrr_at_3_max": 0.004379, + "nauc_mrr_at_3_std": -0.579514, + "nauc_mrr_at_3_diff1": 0.842964, + "nauc_mrr_at_5_max": 0.013253, + "nauc_mrr_at_5_std": -0.556341, + "nauc_mrr_at_5_diff1": 0.839251, + "nauc_mrr_at_10_max": 0.00059, + "nauc_mrr_at_10_std": -0.574983, + "nauc_mrr_at_10_diff1": 0.838427, + "nauc_mrr_at_20_max": 0.00169, + "nauc_mrr_at_20_std": -0.567525, + "nauc_mrr_at_20_diff1": 0.839171, + "nauc_mrr_at_100_max": 0.001093, + "nauc_mrr_at_100_std": -0.568527, + "nauc_mrr_at_100_diff1": 0.839139, + "nauc_mrr_at_1000_max": 0.001093, + "nauc_mrr_at_1000_std": -0.568527, + "nauc_mrr_at_1000_diff1": 0.839139, + "main_score": 0.89302, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 128.75925064086914, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreSyntheticDocQAAIRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..e4171715c7 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", + "task_name": "VidoreSyntheticDocQAAIRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.98, + "ndcg_at_3": 0.99262, + "ndcg_at_5": 0.99262, + "ndcg_at_10": 0.99262, + "ndcg_at_20": 0.99262, + "ndcg_at_100": 0.99262, + "ndcg_at_1000": 0.99262, + "map_at_1": 0.98, + "map_at_3": 0.99, + "map_at_5": 0.99, + "map_at_10": 0.99, + "map_at_20": 0.99, + "map_at_100": 0.99, + "map_at_1000": 0.99, + "recall_at_1": 0.98, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.98, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.98, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.98, + "mrr_at_3": 0.99, + "mrr_at_5": 0.99, + "mrr_at_10": 0.99, + "mrr_at_20": 0.99, + "mrr_at_100": 0.99, + "mrr_at_1000": 0.99, + "nauc_ndcg_at_1_max": 0.540149, + "nauc_ndcg_at_1_std": 0.24043, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.540149, + "nauc_ndcg_at_3_std": 0.24043, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.540149, + "nauc_ndcg_at_5_std": 0.24043, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.540149, + "nauc_ndcg_at_10_std": 0.24043, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.540149, + "nauc_ndcg_at_20_std": 0.24043, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.540149, + "nauc_ndcg_at_100_std": 0.24043, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.540149, + "nauc_ndcg_at_1000_std": 0.24043, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.540149, + "nauc_map_at_1_std": 0.24043, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.540149, + "nauc_map_at_3_std": 0.24043, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.540149, + "nauc_map_at_5_std": 0.24043, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.540149, + "nauc_map_at_10_std": 0.24043, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.540149, + "nauc_map_at_20_std": 0.24043, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.540149, + "nauc_map_at_100_std": 0.24043, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.540149, + "nauc_map_at_1000_std": 0.24043, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.540149, + "nauc_recall_at_1_std": 0.24043, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.540149, + "nauc_precision_at_1_std": 0.24043, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.540149, + "nauc_cv_recall_at_1_std": 0.24043, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.540149, + "nauc_mrr_at_1_std": 0.24043, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.540149, + "nauc_mrr_at_3_std": 0.24043, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.540149, + "nauc_mrr_at_5_std": 0.24043, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.540149, + "nauc_mrr_at_10_std": 0.24043, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.540149, + "nauc_mrr_at_20_std": 0.24043, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.540149, + "nauc_mrr_at_100_std": 0.24043, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.540149, + "nauc_mrr_at_1000_std": 0.24043, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.99262, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 123.2853786945343, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreSyntheticDocQAEnergyRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..536c655bcf --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", + "task_name": "VidoreSyntheticDocQAEnergyRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.95, + "ndcg_at_3": 0.96262, + "ndcg_at_5": 0.96649, + "ndcg_at_10": 0.97298, + "ndcg_at_20": 0.97298, + "ndcg_at_100": 0.97298, + "ndcg_at_1000": 0.97298, + "map_at_1": 0.95, + "map_at_3": 0.96, + "map_at_5": 0.962, + "map_at_10": 0.96468, + "map_at_20": 0.96468, + "map_at_100": 0.96468, + "map_at_1000": 0.96468, + "recall_at_1": 0.95, + "recall_at_3": 0.97, + "recall_at_5": 0.98, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.95, + "cv_recall_at_3": 0.97, + "cv_recall_at_5": 0.98, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.95, + "precision_at_3": 0.32333, + "precision_at_5": 0.196, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.95, + "mrr_at_3": 0.96, + "mrr_at_5": 0.962, + "mrr_at_10": 0.964679, + "mrr_at_20": 0.964679, + "mrr_at_100": 0.964679, + "mrr_at_1000": 0.964679, + "nauc_ndcg_at_1_max": 0.50859, + "nauc_ndcg_at_1_std": -0.816153, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.587292, + "nauc_ndcg_at_3_std": -0.769064, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.554741, + "nauc_ndcg_at_5_std": -0.724892, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.556044, + "nauc_ndcg_at_10_std": -0.768614, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.556044, + "nauc_ndcg_at_20_std": -0.768614, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.556044, + "nauc_ndcg_at_100_std": -0.768614, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.556044, + "nauc_ndcg_at_1000_std": -0.768614, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.50859, + "nauc_map_at_1_std": -0.816153, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.566877, + "nauc_map_at_3_std": -0.781279, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.550961, + "nauc_map_at_5_std": -0.761782, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.552387, + "nauc_map_at_10_std": -0.774737, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.552387, + "nauc_map_at_20_std": -0.774737, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.552387, + "nauc_map_at_100_std": -0.774737, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.552387, + "nauc_map_at_1000_std": -0.774737, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.50859, + "nauc_recall_at_1_std": -0.816153, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 0.664021, + "nauc_recall_at_3_std": -0.723156, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 0.561391, + "nauc_recall_at_5_std": -0.50887, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.50859, + "nauc_precision_at_1_std": -0.816153, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 0.664021, + "nauc_precision_at_3_std": -0.723156, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 0.561391, + "nauc_precision_at_5_std": -0.50887, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.50859, + "nauc_cv_recall_at_1_std": -0.816153, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": 0.664021, + "nauc_cv_recall_at_3_std": -0.723156, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 0.561391, + "nauc_cv_recall_at_5_std": -0.50887, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.50859, + "nauc_mrr_at_1_std": -0.816153, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.566877, + "nauc_mrr_at_3_std": -0.781279, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.550961, + "nauc_mrr_at_5_std": -0.761782, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.552387, + "nauc_mrr_at_10_std": -0.774737, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.552387, + "nauc_mrr_at_20_std": -0.774737, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.552387, + "nauc_mrr_at_100_std": -0.774737, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.552387, + "nauc_mrr_at_1000_std": -0.774737, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.96649, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 126.9228093624115, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..2b9e388cfb --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b4909afa930f81282fd20601e860668073ad02aa", + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.9, + "ndcg_at_3": 0.95047, + "ndcg_at_5": 0.95434, + "ndcg_at_10": 0.9579, + "ndcg_at_20": 0.9579, + "ndcg_at_100": 0.9579, + "ndcg_at_1000": 0.9579, + "map_at_1": 0.9, + "map_at_3": 0.94, + "map_at_5": 0.942, + "map_at_10": 0.94367, + "map_at_20": 0.94367, + "map_at_100": 0.94367, + "map_at_1000": 0.94367, + "recall_at_1": 0.9, + "recall_at_3": 0.98, + "recall_at_5": 0.99, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.9, + "cv_recall_at_3": 0.98, + "cv_recall_at_5": 0.99, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.9, + "precision_at_3": 0.32667, + "precision_at_5": 0.198, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.9, + "mrr_at_3": 0.94, + "mrr_at_5": 0.942, + "mrr_at_10": 0.943667, + "mrr_at_20": 0.943667, + "mrr_at_100": 0.943667, + "mrr_at_1000": 0.943667, + "nauc_ndcg_at_1_max": 0.721802, + "nauc_ndcg_at_1_std": 0.199533, + "nauc_ndcg_at_1_diff1": 0.919935, + "nauc_ndcg_at_3_max": 0.792684, + "nauc_ndcg_at_3_std": 0.093978, + "nauc_ndcg_at_3_diff1": 0.923682, + "nauc_ndcg_at_5_max": 0.775118, + "nauc_ndcg_at_5_std": 0.040746, + "nauc_ndcg_at_5_diff1": 0.928291, + "nauc_ndcg_at_10_max": 0.756089, + "nauc_ndcg_at_10_std": 0.141653, + "nauc_ndcg_at_10_diff1": 0.922223, + "nauc_ndcg_at_20_max": 0.756089, + "nauc_ndcg_at_20_std": 0.141653, + "nauc_ndcg_at_20_diff1": 0.922223, + "nauc_ndcg_at_100_max": 0.756089, + "nauc_ndcg_at_100_std": 0.141653, + "nauc_ndcg_at_100_diff1": 0.922223, + "nauc_ndcg_at_1000_max": 0.756089, + "nauc_ndcg_at_1000_std": 0.141653, + "nauc_ndcg_at_1000_diff1": 0.922223, + "nauc_map_at_1_max": 0.721802, + "nauc_map_at_1_std": 0.199533, + "nauc_map_at_1_diff1": 0.919935, + "nauc_map_at_3_max": 0.768168, + "nauc_map_at_3_std": 0.130486, + "nauc_map_at_3_diff1": 0.922386, + "nauc_map_at_5_max": 0.760174, + "nauc_map_at_5_std": 0.110081, + "nauc_map_at_5_diff1": 0.924217, + "nauc_map_at_10_max": 0.753079, + "nauc_map_at_10_std": 0.147412, + "nauc_map_at_10_diff1": 0.921975, + "nauc_map_at_20_max": 0.753079, + "nauc_map_at_20_std": 0.147412, + "nauc_map_at_20_diff1": 0.921975, + "nauc_map_at_100_max": 0.753079, + "nauc_map_at_100_std": 0.147412, + "nauc_map_at_100_diff1": 0.921975, + "nauc_map_at_1000_max": 0.753079, + "nauc_map_at_1000_std": 0.147412, + "nauc_map_at_1000_diff1": 0.921975, + "nauc_recall_at_1_max": 0.721802, + "nauc_recall_at_1_std": 0.199533, + "nauc_recall_at_1_diff1": 0.919935, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": -0.214753, + "nauc_recall_at_3_diff1": 0.934641, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": -1.151727, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.721802, + "nauc_precision_at_1_std": 0.199533, + "nauc_precision_at_1_diff1": 0.919935, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": -0.214753, + "nauc_precision_at_3_diff1": 0.934641, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": -1.151727, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.721802, + "nauc_cv_recall_at_1_std": 0.199533, + "nauc_cv_recall_at_1_diff1": 0.919935, + "nauc_cv_recall_at_3_max": 1.0, + "nauc_cv_recall_at_3_std": -0.214753, + "nauc_cv_recall_at_3_diff1": 0.934641, + "nauc_cv_recall_at_5_max": 1.0, + "nauc_cv_recall_at_5_std": -1.151727, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.721802, + "nauc_mrr_at_1_std": 0.199533, + "nauc_mrr_at_1_diff1": 0.919935, + "nauc_mrr_at_3_max": 0.768168, + "nauc_mrr_at_3_std": 0.130486, + "nauc_mrr_at_3_diff1": 0.922386, + "nauc_mrr_at_5_max": 0.760174, + "nauc_mrr_at_5_std": 0.110081, + "nauc_mrr_at_5_diff1": 0.924217, + "nauc_mrr_at_10_max": 0.753079, + "nauc_mrr_at_10_std": 0.147412, + "nauc_mrr_at_10_diff1": 0.921975, + "nauc_mrr_at_20_max": 0.753079, + "nauc_mrr_at_20_std": 0.147412, + "nauc_mrr_at_20_diff1": 0.921975, + "nauc_mrr_at_100_max": 0.753079, + "nauc_mrr_at_100_std": 0.147412, + "nauc_mrr_at_100_diff1": 0.921975, + "nauc_mrr_at_1000_max": 0.753079, + "nauc_mrr_at_1000_std": 0.147412, + "nauc_mrr_at_1000_diff1": 0.921975, + "main_score": 0.95434, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 127.21970534324646, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..fa255926fb --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.98, + "ndcg_at_3": 0.99262, + "ndcg_at_5": 0.99262, + "ndcg_at_10": 0.99262, + "ndcg_at_20": 0.99262, + "ndcg_at_100": 0.99262, + "ndcg_at_1000": 0.99262, + "map_at_1": 0.98, + "map_at_3": 0.99, + "map_at_5": 0.99, + "map_at_10": 0.99, + "map_at_20": 0.99, + "map_at_100": 0.99, + "map_at_1000": 0.99, + "recall_at_1": 0.98, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.98, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.98, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.98, + "mrr_at_3": 0.99, + "mrr_at_5": 0.99, + "mrr_at_10": 0.99, + "mrr_at_20": 0.99, + "mrr_at_100": 0.99, + "mrr_at_1000": 0.99, + "nauc_ndcg_at_1_max": 0.869281, + "nauc_ndcg_at_1_std": 0.561391, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.869281, + "nauc_ndcg_at_3_std": 0.561391, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.869281, + "nauc_ndcg_at_5_std": 0.561391, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.869281, + "nauc_ndcg_at_10_std": 0.561391, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.869281, + "nauc_ndcg_at_20_std": 0.561391, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.869281, + "nauc_ndcg_at_100_std": 0.561391, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.869281, + "nauc_ndcg_at_1000_std": 0.561391, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.869281, + "nauc_map_at_1_std": 0.561391, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.869281, + "nauc_map_at_3_std": 0.561391, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.869281, + "nauc_map_at_5_std": 0.561391, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.869281, + "nauc_map_at_10_std": 0.561391, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.869281, + "nauc_map_at_20_std": 0.561391, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.869281, + "nauc_map_at_100_std": 0.561391, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.869281, + "nauc_map_at_1000_std": 0.561391, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.869281, + "nauc_recall_at_1_std": 0.561391, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.869281, + "nauc_precision_at_1_std": 0.561391, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.869281, + "nauc_cv_recall_at_1_std": 0.561391, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.869281, + "nauc_mrr_at_1_std": 0.561391, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.869281, + "nauc_mrr_at_3_std": 0.561391, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.869281, + "nauc_mrr_at_5_std": 0.561391, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.869281, + "nauc_mrr_at_10_std": 0.561391, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.869281, + "nauc_mrr_at_20_std": 0.561391, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.869281, + "nauc_mrr_at_100_std": 0.561391, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.869281, + "nauc_mrr_at_1000_std": 0.561391, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.99262, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 120.10931038856506, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreTabfquadRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..1acd0064eb --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", + "task_name": "VidoreTabfquadRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.92143, + "ndcg_at_3": 0.95786, + "ndcg_at_5": 0.96094, + "ndcg_at_10": 0.9631, + "ndcg_at_20": 0.9631, + "ndcg_at_100": 0.9639, + "ndcg_at_1000": 0.9639, + "map_at_1": 0.92143, + "map_at_3": 0.9494, + "map_at_5": 0.95119, + "map_at_10": 0.95199, + "map_at_20": 0.95199, + "map_at_100": 0.95216, + "map_at_1000": 0.95216, + "recall_at_1": 0.92143, + "recall_at_3": 0.98214, + "recall_at_5": 0.98929, + "recall_at_10": 0.99643, + "recall_at_20": 0.99643, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.92143, + "cv_recall_at_3": 0.98214, + "cv_recall_at_5": 0.98929, + "cv_recall_at_10": 0.99643, + "cv_recall_at_20": 0.99643, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.92143, + "precision_at_3": 0.32738, + "precision_at_5": 0.19786, + "precision_at_10": 0.09964, + "precision_at_20": 0.04982, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.921429, + "mrr_at_3": 0.949405, + "mrr_at_5": 0.95119, + "mrr_at_10": 0.951994, + "mrr_at_20": 0.951994, + "mrr_at_100": 0.952164, + "mrr_at_1000": 0.952164, + "nauc_ndcg_at_1_max": 0.528818, + "nauc_ndcg_at_1_std": 0.153383, + "nauc_ndcg_at_1_diff1": 0.934428, + "nauc_ndcg_at_3_max": 0.661455, + "nauc_ndcg_at_3_std": 0.317631, + "nauc_ndcg_at_3_diff1": 0.953422, + "nauc_ndcg_at_5_max": 0.634791, + "nauc_ndcg_at_5_std": 0.294315, + "nauc_ndcg_at_5_diff1": 0.949753, + "nauc_ndcg_at_10_max": 0.613423, + "nauc_ndcg_at_10_std": 0.26949, + "nauc_ndcg_at_10_diff1": 0.946813, + "nauc_ndcg_at_20_max": 0.613423, + "nauc_ndcg_at_20_std": 0.26949, + "nauc_ndcg_at_20_diff1": 0.946813, + "nauc_ndcg_at_100_max": 0.604847, + "nauc_ndcg_at_100_std": 0.253283, + "nauc_ndcg_at_100_diff1": 0.945633, + "nauc_ndcg_at_1000_max": 0.604847, + "nauc_ndcg_at_1000_std": 0.253283, + "nauc_ndcg_at_1000_diff1": 0.945633, + "nauc_map_at_1_max": 0.528818, + "nauc_map_at_1_std": 0.153383, + "nauc_map_at_1_diff1": 0.934428, + "nauc_map_at_3_max": 0.619015, + "nauc_map_at_3_std": 0.265793, + "nauc_map_at_3_diff1": 0.947548, + "nauc_map_at_5_max": 0.605076, + "nauc_map_at_5_std": 0.253066, + "nauc_map_at_5_diff1": 0.945629, + "nauc_map_at_10_max": 0.598466, + "nauc_map_at_10_std": 0.245095, + "nauc_map_at_10_diff1": 0.944719, + "nauc_map_at_20_max": 0.598466, + "nauc_map_at_20_std": 0.245095, + "nauc_map_at_20_diff1": 0.944719, + "nauc_map_at_100_max": 0.597038, + "nauc_map_at_100_std": 0.242412, + "nauc_map_at_100_diff1": 0.944522, + "nauc_map_at_1000_max": 0.597038, + "nauc_map_at_1000_std": 0.242412, + "nauc_map_at_1000_diff1": 0.944522, + "nauc_recall_at_1_max": 0.528818, + "nauc_recall_at_1_std": 0.153383, + "nauc_recall_at_1_diff1": 0.934428, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": 0.730159, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": 0.807812, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": 1.0, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 1.0, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.528818, + "nauc_precision_at_1_std": 0.153383, + "nauc_precision_at_1_diff1": 0.934428, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 0.730159, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 0.807812, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.528818, + "nauc_cv_recall_at_1_std": 0.153383, + "nauc_cv_recall_at_1_diff1": 0.934428, + "nauc_cv_recall_at_3_max": 1.0, + "nauc_cv_recall_at_3_std": 0.730159, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 1.0, + "nauc_cv_recall_at_5_std": 0.807812, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": 1.0, + "nauc_cv_recall_at_10_std": 1.0, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": 1.0, + "nauc_cv_recall_at_20_std": 1.0, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.528818, + "nauc_mrr_at_1_std": 0.153383, + "nauc_mrr_at_1_diff1": 0.934428, + "nauc_mrr_at_3_max": 0.619015, + "nauc_mrr_at_3_std": 0.265793, + "nauc_mrr_at_3_diff1": 0.947548, + "nauc_mrr_at_5_max": 0.605076, + "nauc_mrr_at_5_std": 0.253066, + "nauc_mrr_at_5_diff1": 0.945629, + "nauc_mrr_at_10_max": 0.598466, + "nauc_mrr_at_10_std": 0.245095, + "nauc_mrr_at_10_diff1": 0.944719, + "nauc_mrr_at_20_max": 0.598466, + "nauc_mrr_at_20_std": 0.245095, + "nauc_mrr_at_20_diff1": 0.944719, + "nauc_mrr_at_100_max": 0.597038, + "nauc_mrr_at_100_std": 0.242412, + "nauc_mrr_at_100_diff1": 0.944522, + "nauc_mrr_at_1000_max": 0.597038, + "nauc_mrr_at_1000_std": 0.242412, + "nauc_mrr_at_1000_diff1": 0.944522, + "main_score": 0.96094, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 8.678192138671875, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreTatdqaRetrieval.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..f58bc840e0 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", + "task_name": "VidoreTatdqaRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.70656, + "ndcg_at_3": 0.79446, + "ndcg_at_5": 0.81248, + "ndcg_at_10": 0.82344, + "ndcg_at_20": 0.82794, + "ndcg_at_100": 0.83441, + "ndcg_at_1000": 0.83651, + "map_at_1": 0.70504, + "map_at_3": 0.77319, + "map_at_5": 0.7833, + "map_at_10": 0.78788, + "map_at_20": 0.78914, + "map_at_100": 0.78999, + "map_at_1000": 0.79011, + "recall_at_1": 0.70504, + "recall_at_3": 0.85541, + "recall_at_5": 0.89885, + "recall_at_10": 0.93256, + "recall_at_20": 0.95018, + "recall_at_100": 0.98512, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.70656, + "cv_recall_at_3": 0.85601, + "cv_recall_at_5": 0.89915, + "cv_recall_at_10": 0.93256, + "cv_recall_at_20": 0.95018, + "cv_recall_at_100": 0.98603, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.70656, + "precision_at_3": 0.28615, + "precision_at_5": 0.18044, + "precision_at_10": 0.09362, + "precision_at_20": 0.04769, + "precision_at_100": 0.00992, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.706561, + "mrr_at_3": 0.773593, + "mrr_at_5": 0.783556, + "mrr_at_10": 0.788049, + "mrr_at_20": 0.789304, + "mrr_at_100": 0.790173, + "mrr_at_1000": 0.790265, + "nauc_ndcg_at_1_max": 0.255851, + "nauc_ndcg_at_1_std": 0.073965, + "nauc_ndcg_at_1_diff1": 0.865883, + "nauc_ndcg_at_3_max": 0.275974, + "nauc_ndcg_at_3_std": 0.132506, + "nauc_ndcg_at_3_diff1": 0.837361, + "nauc_ndcg_at_5_max": 0.282959, + "nauc_ndcg_at_5_std": 0.132395, + "nauc_ndcg_at_5_diff1": 0.831167, + "nauc_ndcg_at_10_max": 0.285889, + "nauc_ndcg_at_10_std": 0.134425, + "nauc_ndcg_at_10_diff1": 0.833628, + "nauc_ndcg_at_20_max": 0.281353, + "nauc_ndcg_at_20_std": 0.127559, + "nauc_ndcg_at_20_diff1": 0.833074, + "nauc_ndcg_at_100_max": 0.278715, + "nauc_ndcg_at_100_std": 0.126047, + "nauc_ndcg_at_100_diff1": 0.83869, + "nauc_ndcg_at_1000_max": 0.275851, + "nauc_ndcg_at_1000_std": 0.119098, + "nauc_ndcg_at_1000_diff1": 0.839309, + "nauc_map_at_1_max": 0.253867, + "nauc_map_at_1_std": 0.071398, + "nauc_map_at_1_diff1": 0.866331, + "nauc_map_at_3_max": 0.269953, + "nauc_map_at_3_std": 0.114465, + "nauc_map_at_3_diff1": 0.844854, + "nauc_map_at_5_max": 0.27278, + "nauc_map_at_5_std": 0.113403, + "nauc_map_at_5_diff1": 0.842212, + "nauc_map_at_10_max": 0.273505, + "nauc_map_at_10_std": 0.113516, + "nauc_map_at_10_diff1": 0.843313, + "nauc_map_at_20_max": 0.272344, + "nauc_map_at_20_std": 0.111734, + "nauc_map_at_20_diff1": 0.84322, + "nauc_map_at_100_max": 0.271995, + "nauc_map_at_100_std": 0.111453, + "nauc_map_at_100_diff1": 0.843787, + "nauc_map_at_1000_max": 0.271847, + "nauc_map_at_1000_std": 0.111122, + "nauc_map_at_1000_diff1": 0.843801, + "nauc_recall_at_1_max": 0.253867, + "nauc_recall_at_1_std": 0.071398, + "nauc_recall_at_1_diff1": 0.866331, + "nauc_recall_at_3_max": 0.297198, + "nauc_recall_at_3_std": 0.20501, + "nauc_recall_at_3_diff1": 0.805555, + "nauc_recall_at_5_max": 0.335745, + "nauc_recall_at_5_std": 0.232762, + "nauc_recall_at_5_diff1": 0.767901, + "nauc_recall_at_10_max": 0.387725, + "nauc_recall_at_10_std": 0.302863, + "nauc_recall_at_10_diff1": 0.75555, + "nauc_recall_at_20_max": 0.369347, + "nauc_recall_at_20_std": 0.28029, + "nauc_recall_at_20_diff1": 0.722024, + "nauc_recall_at_100_max": 0.464719, + "nauc_recall_at_100_std": 0.593342, + "nauc_recall_at_100_diff1": 0.787545, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.255851, + "nauc_precision_at_1_std": 0.073965, + "nauc_precision_at_1_diff1": 0.865883, + "nauc_precision_at_3_max": 0.287775, + "nauc_precision_at_3_std": 0.203287, + "nauc_precision_at_3_diff1": 0.768936, + "nauc_precision_at_5_max": 0.315974, + "nauc_precision_at_5_std": 0.225556, + "nauc_precision_at_5_diff1": 0.708384, + "nauc_precision_at_10_max": 0.355987, + "nauc_precision_at_10_std": 0.289475, + "nauc_precision_at_10_diff1": 0.655835, + "nauc_precision_at_20_max": 0.329727, + "nauc_precision_at_20_std": 0.26525, + "nauc_precision_at_20_diff1": 0.594808, + "nauc_precision_at_100_max": 0.150459, + "nauc_precision_at_100_std": 0.261732, + "nauc_precision_at_100_diff1": 0.236921, + "nauc_precision_at_1000_max": -0.365643, + "nauc_precision_at_1000_std": -0.320215, + "nauc_precision_at_1000_diff1": -0.515068, + "nauc_cv_recall_at_1_max": 0.255851, + "nauc_cv_recall_at_1_std": 0.073965, + "nauc_cv_recall_at_1_diff1": 0.865883, + "nauc_cv_recall_at_3_max": 0.300936, + "nauc_cv_recall_at_3_std": 0.208826, + "nauc_cv_recall_at_3_diff1": 0.806258, + "nauc_cv_recall_at_5_max": 0.341867, + "nauc_cv_recall_at_5_std": 0.238613, + "nauc_cv_recall_at_5_diff1": 0.768834, + "nauc_cv_recall_at_10_max": 0.387725, + "nauc_cv_recall_at_10_std": 0.302863, + "nauc_cv_recall_at_10_diff1": 0.75555, + "nauc_cv_recall_at_20_max": 0.369347, + "nauc_cv_recall_at_20_std": 0.28029, + "nauc_cv_recall_at_20_diff1": 0.722024, + "nauc_cv_recall_at_100_max": 0.429809, + "nauc_cv_recall_at_100_std": 0.566821, + "nauc_cv_recall_at_100_diff1": 0.773689, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.255851, + "nauc_mrr_at_1_std": 0.073965, + "nauc_mrr_at_1_diff1": 0.865883, + "nauc_mrr_at_3_max": 0.272687, + "nauc_mrr_at_3_std": 0.117061, + "nauc_mrr_at_3_diff1": 0.845237, + "nauc_mrr_at_5_max": 0.275888, + "nauc_mrr_at_5_std": 0.116316, + "nauc_mrr_at_5_diff1": 0.8426, + "nauc_mrr_at_10_max": 0.275802, + "nauc_mrr_at_10_std": 0.115666, + "nauc_mrr_at_10_diff1": 0.843597, + "nauc_mrr_at_20_max": 0.274655, + "nauc_mrr_at_20_std": 0.113897, + "nauc_mrr_at_20_diff1": 0.843506, + "nauc_mrr_at_100_max": 0.27427, + "nauc_mrr_at_100_std": 0.113569, + "nauc_mrr_at_100_diff1": 0.844065, + "nauc_mrr_at_1000_max": 0.2742, + "nauc_mrr_at_1000_std": 0.113333, + "nauc_mrr_at_1000_diff1": 0.844095, + "main_score": 0.81248, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 46.28056049346924, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/model_meta.json b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/model_meta.json new file mode 100644 index 0000000000..8facbce883 --- /dev/null +++ b/results/nomic-ai__colnomic-embed-multimodal-7b/09dbc9502b66605d5be56d2226019b49c9fd3293/model_meta.json @@ -0,0 +1 @@ +{"name": "nomic-ai/colnomic-embed-multimodal-7b", "revision": "09dbc9502b66605d5be56d2226019b49c9fd3293", "release_date": "2025-03-31", "languages": ["deu-Latn", "spa-Latn", "eng-Latn", "fra-Latn", "ita-Latn"], "n_parameters": 7000000000, "memory_usage_mb": 14400.0, "max_tokens": 128000.0, "embed_dim": 128, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/nomic-ai/colpali", "public_training_data": "https://huggingface.co/datasets/vidore/colpali_train_set", "framework": ["ColPali"], "reference": "https://huggingface.co/nomic-ai/colnomic-embed-multimodal-7b", "similarity_fn_name": "max_sim", "use_instructions": true, "training_datasets": {"VDRMultilingual": ["Train"], "DocVQA": ["train"], "InfoVQA": ["train"], "TATDQA": ["train"], "arXivQA": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "ColQwen2_5Wrapper"} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..697c2e4126 --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.30625, + "ndcg_at_3": 0.32251, + "ndcg_at_5": 0.35041, + "ndcg_at_10": 0.38297, + "ndcg_at_20": 0.40822, + "ndcg_at_100": 0.4617, + "ndcg_at_1000": 0.50544, + "map_at_1": 0.2022, + "map_at_3": 0.25546, + "map_at_5": 0.28373, + "map_at_10": 0.30635, + "map_at_20": 0.31823, + "map_at_100": 0.32848, + "map_at_1000": 0.33167, + "recall_at_1": 0.2022, + "recall_at_3": 0.3082, + "recall_at_5": 0.38882, + "recall_at_10": 0.48522, + "recall_at_20": 0.56033, + "recall_at_100": 0.77469, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.30625, + "cv_recall_at_3": 0.49375, + "cv_recall_at_5": 0.58125, + "cv_recall_at_10": 0.66875, + "cv_recall_at_20": 0.7125, + "cv_recall_at_100": 0.91875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.30625, + "precision_at_3": 0.19167, + "precision_at_5": 0.1575, + "precision_at_10": 0.105, + "precision_at_20": 0.065, + "precision_at_100": 0.0205, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.30625, + "mrr_at_3": 0.384375, + "mrr_at_5": 0.40375, + "mrr_at_10": 0.415952, + "mrr_at_20": 0.418791, + "mrr_at_100": 0.42395, + "mrr_at_1000": 0.424307, + "nauc_ndcg_at_1_max": 0.258311, + "nauc_ndcg_at_1_std": 0.100359, + "nauc_ndcg_at_1_diff1": 0.507699, + "nauc_ndcg_at_3_max": 0.229435, + "nauc_ndcg_at_3_std": 0.09537, + "nauc_ndcg_at_3_diff1": 0.463202, + "nauc_ndcg_at_5_max": 0.287991, + "nauc_ndcg_at_5_std": 0.159244, + "nauc_ndcg_at_5_diff1": 0.479507, + "nauc_ndcg_at_10_max": 0.277587, + "nauc_ndcg_at_10_std": 0.156431, + "nauc_ndcg_at_10_diff1": 0.458003, + "nauc_ndcg_at_20_max": 0.293384, + "nauc_ndcg_at_20_std": 0.182005, + "nauc_ndcg_at_20_diff1": 0.428804, + "nauc_ndcg_at_100_max": 0.285246, + "nauc_ndcg_at_100_std": 0.168172, + "nauc_ndcg_at_100_diff1": 0.454083, + "nauc_ndcg_at_1000_max": 0.270173, + "nauc_ndcg_at_1000_std": 0.145882, + "nauc_ndcg_at_1000_diff1": 0.460152, + "nauc_map_at_1_max": 0.311676, + "nauc_map_at_1_std": 0.148324, + "nauc_map_at_1_diff1": 0.56245, + "nauc_map_at_3_max": 0.265807, + "nauc_map_at_3_std": 0.122712, + "nauc_map_at_3_diff1": 0.501112, + "nauc_map_at_5_max": 0.283674, + "nauc_map_at_5_std": 0.144738, + "nauc_map_at_5_diff1": 0.497223, + "nauc_map_at_10_max": 0.268573, + "nauc_map_at_10_std": 0.132693, + "nauc_map_at_10_diff1": 0.482586, + "nauc_map_at_20_max": 0.273959, + "nauc_map_at_20_std": 0.143528, + "nauc_map_at_20_diff1": 0.468543, + "nauc_map_at_100_max": 0.272396, + "nauc_map_at_100_std": 0.142446, + "nauc_map_at_100_diff1": 0.472983, + "nauc_map_at_1000_max": 0.271174, + "nauc_map_at_1000_std": 0.140547, + "nauc_map_at_1000_diff1": 0.474096, + "nauc_recall_at_1_max": 0.311676, + "nauc_recall_at_1_std": 0.148324, + "nauc_recall_at_1_diff1": 0.56245, + "nauc_recall_at_3_max": 0.245943, + "nauc_recall_at_3_std": 0.136475, + "nauc_recall_at_3_diff1": 0.424846, + "nauc_recall_at_5_max": 0.328582, + "nauc_recall_at_5_std": 0.229549, + "nauc_recall_at_5_diff1": 0.415732, + "nauc_recall_at_10_max": 0.271842, + "nauc_recall_at_10_std": 0.196591, + "nauc_recall_at_10_diff1": 0.341367, + "nauc_recall_at_20_max": 0.321614, + "nauc_recall_at_20_std": 0.268468, + "nauc_recall_at_20_diff1": 0.276662, + "nauc_recall_at_100_max": 0.300216, + "nauc_recall_at_100_std": 0.247725, + "nauc_recall_at_100_diff1": 0.321861, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.258311, + "nauc_precision_at_1_std": 0.100359, + "nauc_precision_at_1_diff1": 0.507699, + "nauc_precision_at_3_max": 0.102296, + "nauc_precision_at_3_std": 0.02695, + "nauc_precision_at_3_diff1": 0.24553, + "nauc_precision_at_5_max": 0.128248, + "nauc_precision_at_5_std": 0.085435, + "nauc_precision_at_5_diff1": 0.198781, + "nauc_precision_at_10_max": 0.038495, + "nauc_precision_at_10_std": 0.021486, + "nauc_precision_at_10_diff1": 0.11364, + "nauc_precision_at_20_max": 0.03471, + "nauc_precision_at_20_std": 0.051146, + "nauc_precision_at_20_diff1": 0.012379, + "nauc_precision_at_100_max": -0.078627, + "nauc_precision_at_100_std": -0.069139, + "nauc_precision_at_100_diff1": -8.7e-05, + "nauc_precision_at_1000_max": -0.217462, + "nauc_precision_at_1000_std": -0.210444, + "nauc_precision_at_1000_diff1": -0.114205, + "nauc_cv_recall_at_1_max": 0.258311, + "nauc_cv_recall_at_1_std": 0.100359, + "nauc_cv_recall_at_1_diff1": 0.507699, + "nauc_cv_recall_at_3_max": 0.23094, + "nauc_cv_recall_at_3_std": 0.147911, + "nauc_cv_recall_at_3_diff1": 0.35408, + "nauc_cv_recall_at_5_max": 0.386036, + "nauc_cv_recall_at_5_std": 0.283465, + "nauc_cv_recall_at_5_diff1": 0.412182, + "nauc_cv_recall_at_10_max": 0.361495, + "nauc_cv_recall_at_10_std": 0.307522, + "nauc_cv_recall_at_10_diff1": 0.398529, + "nauc_cv_recall_at_20_max": 0.419379, + "nauc_cv_recall_at_20_std": 0.346422, + "nauc_cv_recall_at_20_diff1": 0.39172, + "nauc_cv_recall_at_100_max": 0.501113, + "nauc_cv_recall_at_100_std": 0.404187, + "nauc_cv_recall_at_100_diff1": 0.548517, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.258311, + "nauc_mrr_at_1_std": 0.100359, + "nauc_mrr_at_1_diff1": 0.507699, + "nauc_mrr_at_3_max": 0.235186, + "nauc_mrr_at_3_std": 0.103676, + "nauc_mrr_at_3_diff1": 0.444178, + "nauc_mrr_at_5_max": 0.266052, + "nauc_mrr_at_5_std": 0.129789, + "nauc_mrr_at_5_diff1": 0.458416, + "nauc_mrr_at_10_max": 0.26087, + "nauc_mrr_at_10_std": 0.129576, + "nauc_mrr_at_10_diff1": 0.458283, + "nauc_mrr_at_20_max": 0.263216, + "nauc_mrr_at_20_std": 0.130746, + "nauc_mrr_at_20_diff1": 0.458307, + "nauc_mrr_at_100_max": 0.261851, + "nauc_mrr_at_100_std": 0.128177, + "nauc_mrr_at_100_diff1": 0.460072, + "nauc_mrr_at_1000_max": 0.26169, + "nauc_mrr_at_1000_std": 0.12808, + "nauc_mrr_at_1000_diff1": 0.459978, + "main_score": 0.35041, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.20625, + "ndcg_at_3": 0.23851, + "ndcg_at_5": 0.27063, + "ndcg_at_10": 0.30438, + "ndcg_at_20": 0.32509, + "ndcg_at_100": 0.382, + "ndcg_at_1000": 0.43595, + "map_at_1": 0.13479, + "map_at_3": 0.18359, + "map_at_5": 0.20958, + "map_at_10": 0.22874, + "map_at_20": 0.23841, + "map_at_100": 0.24964, + "map_at_1000": 0.25308, + "recall_at_1": 0.13479, + "recall_at_3": 0.23813, + "recall_at_5": 0.32693, + "recall_at_10": 0.42086, + "recall_at_20": 0.47866, + "recall_at_100": 0.71224, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.20625, + "cv_recall_at_3": 0.4, + "cv_recall_at_5": 0.50625, + "cv_recall_at_10": 0.60625, + "cv_recall_at_20": 0.6625, + "cv_recall_at_100": 0.84375, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.20625, + "precision_at_3": 0.14792, + "precision_at_5": 0.125, + "precision_at_10": 0.0875, + "precision_at_20": 0.05594, + "precision_at_100": 0.01837, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.20625, + "mrr_at_3": 0.284375, + "mrr_at_5": 0.30875, + "mrr_at_10": 0.321627, + "mrr_at_20": 0.32572, + "mrr_at_100": 0.329969, + "mrr_at_1000": 0.330756, + "nauc_ndcg_at_1_max": 0.194655, + "nauc_ndcg_at_1_std": 0.05793, + "nauc_ndcg_at_1_diff1": 0.447578, + "nauc_ndcg_at_3_max": 0.188069, + "nauc_ndcg_at_3_std": 0.045661, + "nauc_ndcg_at_3_diff1": 0.354102, + "nauc_ndcg_at_5_max": 0.298719, + "nauc_ndcg_at_5_std": 0.171868, + "nauc_ndcg_at_5_diff1": 0.350169, + "nauc_ndcg_at_10_max": 0.296644, + "nauc_ndcg_at_10_std": 0.177233, + "nauc_ndcg_at_10_diff1": 0.337124, + "nauc_ndcg_at_20_max": 0.295932, + "nauc_ndcg_at_20_std": 0.178377, + "nauc_ndcg_at_20_diff1": 0.329074, + "nauc_ndcg_at_100_max": 0.289941, + "nauc_ndcg_at_100_std": 0.18352, + "nauc_ndcg_at_100_diff1": 0.349178, + "nauc_ndcg_at_1000_max": 0.263326, + "nauc_ndcg_at_1000_std": 0.145905, + "nauc_ndcg_at_1000_diff1": 0.357913, + "nauc_map_at_1_max": 0.256693, + "nauc_map_at_1_std": 0.089117, + "nauc_map_at_1_diff1": 0.513333, + "nauc_map_at_3_max": 0.234227, + "nauc_map_at_3_std": 0.072683, + "nauc_map_at_3_diff1": 0.424433, + "nauc_map_at_5_max": 0.290682, + "nauc_map_at_5_std": 0.144155, + "nauc_map_at_5_diff1": 0.409941, + "nauc_map_at_10_max": 0.277222, + "nauc_map_at_10_std": 0.139446, + "nauc_map_at_10_diff1": 0.393507, + "nauc_map_at_20_max": 0.270339, + "nauc_map_at_20_std": 0.136402, + "nauc_map_at_20_diff1": 0.38507, + "nauc_map_at_100_max": 0.270385, + "nauc_map_at_100_std": 0.139318, + "nauc_map_at_100_diff1": 0.389671, + "nauc_map_at_1000_max": 0.268534, + "nauc_map_at_1000_std": 0.137093, + "nauc_map_at_1000_diff1": 0.389682, + "nauc_recall_at_1_max": 0.256693, + "nauc_recall_at_1_std": 0.089117, + "nauc_recall_at_1_diff1": 0.513333, + "nauc_recall_at_3_max": 0.222893, + "nauc_recall_at_3_std": 0.072884, + "nauc_recall_at_3_diff1": 0.320987, + "nauc_recall_at_5_max": 0.386047, + "nauc_recall_at_5_std": 0.293303, + "nauc_recall_at_5_diff1": 0.269764, + "nauc_recall_at_10_max": 0.341858, + "nauc_recall_at_10_std": 0.275513, + "nauc_recall_at_10_diff1": 0.214943, + "nauc_recall_at_20_max": 0.344975, + "nauc_recall_at_20_std": 0.275522, + "nauc_recall_at_20_diff1": 0.187027, + "nauc_recall_at_100_max": 0.324328, + "nauc_recall_at_100_std": 0.320756, + "nauc_recall_at_100_diff1": 0.251877, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.194655, + "nauc_precision_at_1_std": 0.05793, + "nauc_precision_at_1_diff1": 0.447578, + "nauc_precision_at_3_max": 0.10096, + "nauc_precision_at_3_std": -0.024246, + "nauc_precision_at_3_diff1": 0.212433, + "nauc_precision_at_5_max": 0.202208, + "nauc_precision_at_5_std": 0.143626, + "nauc_precision_at_5_diff1": 0.152672, + "nauc_precision_at_10_max": 0.111971, + "nauc_precision_at_10_std": 0.110194, + "nauc_precision_at_10_diff1": 0.039432, + "nauc_precision_at_20_max": 0.05228, + "nauc_precision_at_20_std": 0.076392, + "nauc_precision_at_20_diff1": 0.000793, + "nauc_precision_at_100_max": -0.052336, + "nauc_precision_at_100_std": 0.011571, + "nauc_precision_at_100_diff1": -0.051591, + "nauc_precision_at_1000_max": -0.24765, + "nauc_precision_at_1000_std": -0.185056, + "nauc_precision_at_1000_diff1": -0.086736, + "nauc_cv_recall_at_1_max": 0.194655, + "nauc_cv_recall_at_1_std": 0.05793, + "nauc_cv_recall_at_1_diff1": 0.447578, + "nauc_cv_recall_at_3_max": 0.185875, + "nauc_cv_recall_at_3_std": 0.03652, + "nauc_cv_recall_at_3_diff1": 0.246741, + "nauc_cv_recall_at_5_max": 0.377494, + "nauc_cv_recall_at_5_std": 0.264154, + "nauc_cv_recall_at_5_diff1": 0.229179, + "nauc_cv_recall_at_10_max": 0.393356, + "nauc_cv_recall_at_10_std": 0.319506, + "nauc_cv_recall_at_10_diff1": 0.171261, + "nauc_cv_recall_at_20_max": 0.449749, + "nauc_cv_recall_at_20_std": 0.375847, + "nauc_cv_recall_at_20_diff1": 0.222873, + "nauc_cv_recall_at_100_max": 0.488928, + "nauc_cv_recall_at_100_std": 0.53837, + "nauc_cv_recall_at_100_diff1": 0.2806, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.194655, + "nauc_mrr_at_1_std": 0.05793, + "nauc_mrr_at_1_diff1": 0.447578, + "nauc_mrr_at_3_max": 0.1934, + "nauc_mrr_at_3_std": 0.053928, + "nauc_mrr_at_3_diff1": 0.352843, + "nauc_mrr_at_5_max": 0.244133, + "nauc_mrr_at_5_std": 0.11308, + "nauc_mrr_at_5_diff1": 0.348477, + "nauc_mrr_at_10_max": 0.244414, + "nauc_mrr_at_10_std": 0.116676, + "nauc_mrr_at_10_diff1": 0.342759, + "nauc_mrr_at_20_max": 0.247779, + "nauc_mrr_at_20_std": 0.119605, + "nauc_mrr_at_20_diff1": 0.348759, + "nauc_mrr_at_100_max": 0.247463, + "nauc_mrr_at_100_std": 0.120212, + "nauc_mrr_at_100_diff1": 0.350376, + "nauc_mrr_at_1000_max": 0.24673, + "nauc_mrr_at_1000_std": 0.119168, + "nauc_mrr_at_1000_diff1": 0.350436, + "main_score": 0.27063, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.48125, + "ndcg_at_3": 0.50119, + "ndcg_at_5": 0.53142, + "ndcg_at_10": 0.56019, + "ndcg_at_20": 0.58861, + "ndcg_at_100": 0.62654, + "ndcg_at_1000": 0.65097, + "map_at_1": 0.29491, + "map_at_3": 0.39754, + "map_at_5": 0.43503, + "map_at_10": 0.46033, + "map_at_20": 0.47486, + "map_at_100": 0.48617, + "map_at_1000": 0.48834, + "recall_at_1": 0.29491, + "recall_at_3": 0.48038, + "recall_at_5": 0.57681, + "recall_at_10": 0.66948, + "recall_at_20": 0.74961, + "recall_at_100": 0.8751, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.48125, + "cv_recall_at_3": 0.73125, + "cv_recall_at_5": 0.825, + "cv_recall_at_10": 0.8875, + "cv_recall_at_20": 0.925, + "cv_recall_at_100": 0.96875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.48125, + "precision_at_3": 0.30833, + "precision_at_5": 0.24125, + "precision_at_10": 0.15313, + "precision_at_20": 0.09406, + "precision_at_100": 0.02525, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.48125, + "mrr_at_3": 0.589583, + "mrr_at_5": 0.611146, + "mrr_at_10": 0.619035, + "mrr_at_20": 0.621664, + "mrr_at_100": 0.623113, + "mrr_at_1000": 0.623209, + "nauc_ndcg_at_1_max": 0.290083, + "nauc_ndcg_at_1_std": -0.0265, + "nauc_ndcg_at_1_diff1": 0.524237, + "nauc_ndcg_at_3_max": 0.291543, + "nauc_ndcg_at_3_std": 0.022051, + "nauc_ndcg_at_3_diff1": 0.442844, + "nauc_ndcg_at_5_max": 0.265473, + "nauc_ndcg_at_5_std": 0.012543, + "nauc_ndcg_at_5_diff1": 0.402903, + "nauc_ndcg_at_10_max": 0.256099, + "nauc_ndcg_at_10_std": -0.007155, + "nauc_ndcg_at_10_diff1": 0.402625, + "nauc_ndcg_at_20_max": 0.25325, + "nauc_ndcg_at_20_std": 0.008494, + "nauc_ndcg_at_20_diff1": 0.389055, + "nauc_ndcg_at_100_max": 0.250964, + "nauc_ndcg_at_100_std": 0.031174, + "nauc_ndcg_at_100_diff1": 0.385172, + "nauc_ndcg_at_1000_max": 0.249372, + "nauc_ndcg_at_1000_std": 0.021744, + "nauc_ndcg_at_1000_diff1": 0.38706, + "nauc_map_at_1_max": 0.306721, + "nauc_map_at_1_std": 0.053663, + "nauc_map_at_1_diff1": 0.54933, + "nauc_map_at_3_max": 0.289254, + "nauc_map_at_3_std": 0.03815, + "nauc_map_at_3_diff1": 0.482681, + "nauc_map_at_5_max": 0.264912, + "nauc_map_at_5_std": 0.011716, + "nauc_map_at_5_diff1": 0.445606, + "nauc_map_at_10_max": 0.257783, + "nauc_map_at_10_std": -0.005388, + "nauc_map_at_10_diff1": 0.431751, + "nauc_map_at_20_max": 0.259085, + "nauc_map_at_20_std": 0.000752, + "nauc_map_at_20_diff1": 0.424103, + "nauc_map_at_100_max": 0.259211, + "nauc_map_at_100_std": 0.008916, + "nauc_map_at_100_diff1": 0.421275, + "nauc_map_at_1000_max": 0.258952, + "nauc_map_at_1000_std": 0.008749, + "nauc_map_at_1000_diff1": 0.420764, + "nauc_recall_at_1_max": 0.306721, + "nauc_recall_at_1_std": 0.053663, + "nauc_recall_at_1_diff1": 0.54933, + "nauc_recall_at_3_max": 0.256002, + "nauc_recall_at_3_std": 0.047114, + "nauc_recall_at_3_diff1": 0.412597, + "nauc_recall_at_5_max": 0.214277, + "nauc_recall_at_5_std": 0.020108, + "nauc_recall_at_5_diff1": 0.30819, + "nauc_recall_at_10_max": 0.193624, + "nauc_recall_at_10_std": -0.019757, + "nauc_recall_at_10_diff1": 0.250678, + "nauc_recall_at_20_max": 0.154, + "nauc_recall_at_20_std": 0.005899, + "nauc_recall_at_20_diff1": 0.186846, + "nauc_recall_at_100_max": 0.127117, + "nauc_recall_at_100_std": 0.128925, + "nauc_recall_at_100_diff1": 0.143173, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.290083, + "nauc_precision_at_1_std": -0.0265, + "nauc_precision_at_1_diff1": 0.524237, + "nauc_precision_at_3_max": 0.161082, + "nauc_precision_at_3_std": -0.002801, + "nauc_precision_at_3_diff1": 0.074326, + "nauc_precision_at_5_max": 0.012606, + "nauc_precision_at_5_std": -0.05404, + "nauc_precision_at_5_diff1": -0.139099, + "nauc_precision_at_10_max": -0.059461, + "nauc_precision_at_10_std": -0.090366, + "nauc_precision_at_10_diff1": -0.217199, + "nauc_precision_at_20_max": -0.073322, + "nauc_precision_at_20_std": -0.004065, + "nauc_precision_at_20_diff1": -0.271495, + "nauc_precision_at_100_max": -0.129169, + "nauc_precision_at_100_std": 0.039389, + "nauc_precision_at_100_diff1": -0.322071, + "nauc_precision_at_1000_max": -0.177876, + "nauc_precision_at_1000_std": 0.005739, + "nauc_precision_at_1000_diff1": -0.35195, + "nauc_cv_recall_at_1_max": 0.290083, + "nauc_cv_recall_at_1_std": -0.0265, + "nauc_cv_recall_at_1_diff1": 0.524237, + "nauc_cv_recall_at_3_max": 0.330879, + "nauc_cv_recall_at_3_std": 0.088146, + "nauc_cv_recall_at_3_diff1": 0.43073, + "nauc_cv_recall_at_5_max": 0.353716, + "nauc_cv_recall_at_5_std": 0.189293, + "nauc_cv_recall_at_5_diff1": 0.212961, + "nauc_cv_recall_at_10_max": 0.319165, + "nauc_cv_recall_at_10_std": 0.169149, + "nauc_cv_recall_at_10_diff1": 0.123145, + "nauc_cv_recall_at_20_max": 0.219538, + "nauc_cv_recall_at_20_std": 0.310963, + "nauc_cv_recall_at_20_diff1": 0.069055, + "nauc_cv_recall_at_100_max": -0.039029, + "nauc_cv_recall_at_100_std": 0.253688, + "nauc_cv_recall_at_100_diff1": -0.406349, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.290083, + "nauc_mrr_at_1_std": -0.0265, + "nauc_mrr_at_1_diff1": 0.524237, + "nauc_mrr_at_3_max": 0.293589, + "nauc_mrr_at_3_std": 0.006265, + "nauc_mrr_at_3_diff1": 0.494616, + "nauc_mrr_at_5_max": 0.295256, + "nauc_mrr_at_5_std": 0.014323, + "nauc_mrr_at_5_diff1": 0.46856, + "nauc_mrr_at_10_max": 0.29114, + "nauc_mrr_at_10_std": 0.00975, + "nauc_mrr_at_10_diff1": 0.469914, + "nauc_mrr_at_20_max": 0.289026, + "nauc_mrr_at_20_std": 0.01102, + "nauc_mrr_at_20_diff1": 0.472056, + "nauc_mrr_at_100_max": 0.288461, + "nauc_mrr_at_100_std": 0.009696, + "nauc_mrr_at_100_diff1": 0.472524, + "nauc_mrr_at_1000_max": 0.288422, + "nauc_mrr_at_1000_std": 0.00948, + "nauc_mrr_at_1000_diff1": 0.472932, + "main_score": 0.53142, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.1875, + "ndcg_at_3": 0.20565, + "ndcg_at_5": 0.21771, + "ndcg_at_10": 0.24921, + "ndcg_at_20": 0.27784, + "ndcg_at_100": 0.3355, + "ndcg_at_1000": 0.39718, + "map_at_1": 0.13125, + "map_at_3": 0.16637, + "map_at_5": 0.1779, + "map_at_10": 0.1946, + "map_at_20": 0.20545, + "map_at_100": 0.21621, + "map_at_1000": 0.21966, + "recall_at_1": 0.13125, + "recall_at_3": 0.20359, + "recall_at_5": 0.24276, + "recall_at_10": 0.3289, + "recall_at_20": 0.41627, + "recall_at_100": 0.64991, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.1875, + "cv_recall_at_3": 0.3, + "cv_recall_at_5": 0.3625, + "cv_recall_at_10": 0.48125, + "cv_recall_at_20": 0.5875, + "cv_recall_at_100": 0.8, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.1875, + "precision_at_3": 0.11458, + "precision_at_5": 0.09, + "precision_at_10": 0.0675, + "precision_at_20": 0.04781, + "precision_at_100": 0.01706, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.1875, + "mrr_at_3": 0.235417, + "mrr_at_5": 0.249167, + "mrr_at_10": 0.264385, + "mrr_at_20": 0.271431, + "mrr_at_100": 0.276664, + "mrr_at_1000": 0.277423, + "nauc_ndcg_at_1_max": 0.306117, + "nauc_ndcg_at_1_std": 0.251938, + "nauc_ndcg_at_1_diff1": 0.516247, + "nauc_ndcg_at_3_max": 0.298627, + "nauc_ndcg_at_3_std": 0.173765, + "nauc_ndcg_at_3_diff1": 0.395628, + "nauc_ndcg_at_5_max": 0.277756, + "nauc_ndcg_at_5_std": 0.177093, + "nauc_ndcg_at_5_diff1": 0.391897, + "nauc_ndcg_at_10_max": 0.272553, + "nauc_ndcg_at_10_std": 0.176536, + "nauc_ndcg_at_10_diff1": 0.374225, + "nauc_ndcg_at_20_max": 0.266686, + "nauc_ndcg_at_20_std": 0.177587, + "nauc_ndcg_at_20_diff1": 0.363442, + "nauc_ndcg_at_100_max": 0.293459, + "nauc_ndcg_at_100_std": 0.202169, + "nauc_ndcg_at_100_diff1": 0.346464, + "nauc_ndcg_at_1000_max": 0.278664, + "nauc_ndcg_at_1000_std": 0.194219, + "nauc_ndcg_at_1000_diff1": 0.377004, + "nauc_map_at_1_max": 0.282434, + "nauc_map_at_1_std": 0.167386, + "nauc_map_at_1_diff1": 0.541031, + "nauc_map_at_3_max": 0.310044, + "nauc_map_at_3_std": 0.167662, + "nauc_map_at_3_diff1": 0.438083, + "nauc_map_at_5_max": 0.296365, + "nauc_map_at_5_std": 0.170626, + "nauc_map_at_5_diff1": 0.426868, + "nauc_map_at_10_max": 0.286299, + "nauc_map_at_10_std": 0.167351, + "nauc_map_at_10_diff1": 0.413176, + "nauc_map_at_20_max": 0.277367, + "nauc_map_at_20_std": 0.162795, + "nauc_map_at_20_diff1": 0.405761, + "nauc_map_at_100_max": 0.28069, + "nauc_map_at_100_std": 0.167185, + "nauc_map_at_100_diff1": 0.400214, + "nauc_map_at_1000_max": 0.280341, + "nauc_map_at_1000_std": 0.167499, + "nauc_map_at_1000_diff1": 0.401622, + "nauc_recall_at_1_max": 0.282434, + "nauc_recall_at_1_std": 0.167386, + "nauc_recall_at_1_diff1": 0.541031, + "nauc_recall_at_3_max": 0.322116, + "nauc_recall_at_3_std": 0.159747, + "nauc_recall_at_3_diff1": 0.34438, + "nauc_recall_at_5_max": 0.271217, + "nauc_recall_at_5_std": 0.177096, + "nauc_recall_at_5_diff1": 0.324269, + "nauc_recall_at_10_max": 0.228637, + "nauc_recall_at_10_std": 0.15408, + "nauc_recall_at_10_diff1": 0.249338, + "nauc_recall_at_20_max": 0.208898, + "nauc_recall_at_20_std": 0.153403, + "nauc_recall_at_20_diff1": 0.217717, + "nauc_recall_at_100_max": 0.312371, + "nauc_recall_at_100_std": 0.24918, + "nauc_recall_at_100_diff1": 0.130065, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.306117, + "nauc_precision_at_1_std": 0.251938, + "nauc_precision_at_1_diff1": 0.516247, + "nauc_precision_at_3_max": 0.266856, + "nauc_precision_at_3_std": 0.165579, + "nauc_precision_at_3_diff1": 0.284627, + "nauc_precision_at_5_max": 0.187233, + "nauc_precision_at_5_std": 0.161646, + "nauc_precision_at_5_diff1": 0.224699, + "nauc_precision_at_10_max": 0.10804, + "nauc_precision_at_10_std": 0.111913, + "nauc_precision_at_10_diff1": 0.155854, + "nauc_precision_at_20_max": 0.054067, + "nauc_precision_at_20_std": 0.083342, + "nauc_precision_at_20_diff1": 0.077368, + "nauc_precision_at_100_max": 0.053117, + "nauc_precision_at_100_std": 0.086422, + "nauc_precision_at_100_diff1": 0.002856, + "nauc_precision_at_1000_max": -0.108015, + "nauc_precision_at_1000_std": -0.005718, + "nauc_precision_at_1000_diff1": 0.002142, + "nauc_cv_recall_at_1_max": 0.306117, + "nauc_cv_recall_at_1_std": 0.251938, + "nauc_cv_recall_at_1_diff1": 0.516247, + "nauc_cv_recall_at_3_max": 0.288601, + "nauc_cv_recall_at_3_std": 0.170627, + "nauc_cv_recall_at_3_diff1": 0.360399, + "nauc_cv_recall_at_5_max": 0.253731, + "nauc_cv_recall_at_5_std": 0.208378, + "nauc_cv_recall_at_5_diff1": 0.344978, + "nauc_cv_recall_at_10_max": 0.242941, + "nauc_cv_recall_at_10_std": 0.206397, + "nauc_cv_recall_at_10_diff1": 0.325609, + "nauc_cv_recall_at_20_max": 0.31589, + "nauc_cv_recall_at_20_std": 0.288425, + "nauc_cv_recall_at_20_diff1": 0.277752, + "nauc_cv_recall_at_100_max": 0.494989, + "nauc_cv_recall_at_100_std": 0.49858, + "nauc_cv_recall_at_100_diff1": 0.183957, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.306117, + "nauc_mrr_at_1_std": 0.251938, + "nauc_mrr_at_1_diff1": 0.516247, + "nauc_mrr_at_3_max": 0.298832, + "nauc_mrr_at_3_std": 0.210857, + "nauc_mrr_at_3_diff1": 0.435345, + "nauc_mrr_at_5_max": 0.288743, + "nauc_mrr_at_5_std": 0.217799, + "nauc_mrr_at_5_diff1": 0.429316, + "nauc_mrr_at_10_max": 0.286525, + "nauc_mrr_at_10_std": 0.217295, + "nauc_mrr_at_10_diff1": 0.422582, + "nauc_mrr_at_20_max": 0.29103, + "nauc_mrr_at_20_std": 0.222047, + "nauc_mrr_at_20_diff1": 0.41883, + "nauc_mrr_at_100_max": 0.293754, + "nauc_mrr_at_100_std": 0.225195, + "nauc_mrr_at_100_diff1": 0.4168, + "nauc_mrr_at_1000_max": 0.293141, + "nauc_mrr_at_1000_std": 0.224348, + "nauc_mrr_at_1000_diff1": 0.4173, + "main_score": 0.21771, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 1158.5815997123718, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..5b34a89a5f --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.44872, + "ndcg_at_3": 0.42511, + "ndcg_at_5": 0.47898, + "ndcg_at_10": 0.51072, + "ndcg_at_20": 0.5707, + "ndcg_at_100": 0.6101, + "ndcg_at_1000": 0.61372, + "map_at_1": 0.3, + "map_at_3": 0.35849, + "map_at_5": 0.40708, + "map_at_10": 0.4231, + "map_at_20": 0.44803, + "map_at_100": 0.46002, + "map_at_1000": 0.46025, + "recall_at_1": 0.3, + "recall_at_3": 0.39712, + "recall_at_5": 0.55353, + "recall_at_10": 0.64406, + "recall_at_20": 0.84744, + "recall_at_100": 0.97896, + "recall_at_1000": 0.99505, + "cv_recall_at_1": 0.46154, + "cv_recall_at_3": 0.59615, + "cv_recall_at_5": 0.75, + "cv_recall_at_10": 0.84615, + "cv_recall_at_20": 0.96154, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.46154, + "precision_at_3": 0.24359, + "precision_at_5": 0.20769, + "precision_at_10": 0.12308, + "precision_at_20": 0.08365, + "precision_at_100": 0.02269, + "precision_at_1000": 0.00242, + "mrr_at_1": 0.461538, + "mrr_at_3": 0.522436, + "mrr_at_5": 0.555128, + "mrr_at_10": 0.5664, + "mrr_at_20": 0.573788, + "mrr_at_100": 0.575416, + "mrr_at_1000": 0.575416, + "nauc_ndcg_at_1_max": 0.156868, + "nauc_ndcg_at_1_std": 0.014879, + "nauc_ndcg_at_1_diff1": 0.574738, + "nauc_ndcg_at_3_max": 0.198849, + "nauc_ndcg_at_3_std": 0.0521, + "nauc_ndcg_at_3_diff1": 0.547911, + "nauc_ndcg_at_5_max": 0.126229, + "nauc_ndcg_at_5_std": 0.033295, + "nauc_ndcg_at_5_diff1": 0.499753, + "nauc_ndcg_at_10_max": 0.175534, + "nauc_ndcg_at_10_std": 0.056816, + "nauc_ndcg_at_10_diff1": 0.521702, + "nauc_ndcg_at_20_max": 0.17815, + "nauc_ndcg_at_20_std": 0.056946, + "nauc_ndcg_at_20_diff1": 0.537958, + "nauc_ndcg_at_100_max": 0.151049, + "nauc_ndcg_at_100_std": 0.03075, + "nauc_ndcg_at_100_diff1": 0.530403, + "nauc_ndcg_at_1000_max": 0.143691, + "nauc_ndcg_at_1000_std": 0.018192, + "nauc_ndcg_at_1000_diff1": 0.527221, + "nauc_map_at_1_max": 0.245524, + "nauc_map_at_1_std": 0.115891, + "nauc_map_at_1_diff1": 0.604003, + "nauc_map_at_3_max": 0.229268, + "nauc_map_at_3_std": 0.097372, + "nauc_map_at_3_diff1": 0.569605, + "nauc_map_at_5_max": 0.188615, + "nauc_map_at_5_std": 0.087676, + "nauc_map_at_5_diff1": 0.526955, + "nauc_map_at_10_max": 0.204728, + "nauc_map_at_10_std": 0.090975, + "nauc_map_at_10_diff1": 0.526236, + "nauc_map_at_20_max": 0.201047, + "nauc_map_at_20_std": 0.088705, + "nauc_map_at_20_diff1": 0.528213, + "nauc_map_at_100_max": 0.194928, + "nauc_map_at_100_std": 0.080479, + "nauc_map_at_100_diff1": 0.525902, + "nauc_map_at_1000_max": 0.194607, + "nauc_map_at_1000_std": 0.079869, + "nauc_map_at_1000_diff1": 0.525687, + "nauc_recall_at_1_max": 0.245524, + "nauc_recall_at_1_std": 0.115891, + "nauc_recall_at_1_diff1": 0.604003, + "nauc_recall_at_3_max": 0.228505, + "nauc_recall_at_3_std": 0.078344, + "nauc_recall_at_3_diff1": 0.564595, + "nauc_recall_at_5_max": 0.045708, + "nauc_recall_at_5_std": 0.004936, + "nauc_recall_at_5_diff1": 0.370659, + "nauc_recall_at_10_max": 0.208043, + "nauc_recall_at_10_std": 0.105402, + "nauc_recall_at_10_diff1": 0.410705, + "nauc_recall_at_20_max": 0.323767, + "nauc_recall_at_20_std": 0.205907, + "nauc_recall_at_20_diff1": 0.447664, + "nauc_recall_at_100_max": 0.679856, + "nauc_recall_at_100_std": 0.843207, + "nauc_recall_at_100_diff1": 0.696413, + "nauc_recall_at_1000_max": 0.842674, + "nauc_recall_at_1000_std": 0.869837, + "nauc_recall_at_1000_diff1": 0.915777, + "nauc_precision_at_1_max": 0.13145, + "nauc_precision_at_1_std": 0.002078, + "nauc_precision_at_1_diff1": 0.545901, + "nauc_precision_at_3_max": 0.032248, + "nauc_precision_at_3_std": -0.069026, + "nauc_precision_at_3_diff1": 0.235192, + "nauc_precision_at_5_max": -0.136555, + "nauc_precision_at_5_std": -0.091741, + "nauc_precision_at_5_diff1": -0.00514, + "nauc_precision_at_10_max": -0.131963, + "nauc_precision_at_10_std": -0.144115, + "nauc_precision_at_10_diff1": -0.055513, + "nauc_precision_at_20_max": -0.1697, + "nauc_precision_at_20_std": -0.1645, + "nauc_precision_at_20_diff1": -0.107052, + "nauc_precision_at_100_max": -0.233892, + "nauc_precision_at_100_std": -0.2575, + "nauc_precision_at_100_diff1": -0.228518, + "nauc_precision_at_1000_max": -0.256225, + "nauc_precision_at_1000_std": -0.299781, + "nauc_precision_at_1000_diff1": -0.270527, + "nauc_cv_recall_at_1_max": 0.13145, + "nauc_cv_recall_at_1_std": 0.002078, + "nauc_cv_recall_at_1_diff1": 0.545901, + "nauc_cv_recall_at_3_max": 0.065719, + "nauc_cv_recall_at_3_std": -0.121293, + "nauc_cv_recall_at_3_diff1": 0.55569, + "nauc_cv_recall_at_5_max": -0.238517, + "nauc_cv_recall_at_5_std": -0.276743, + "nauc_cv_recall_at_5_diff1": 0.393676, + "nauc_cv_recall_at_10_max": -0.169853, + "nauc_cv_recall_at_10_std": -0.30031, + "nauc_cv_recall_at_10_diff1": 0.596348, + "nauc_cv_recall_at_20_max": 0.086058, + "nauc_cv_recall_at_20_std": -0.610674, + "nauc_cv_recall_at_20_diff1": 0.67845, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.13145, + "nauc_mrr_at_1_std": 0.002078, + "nauc_mrr_at_1_diff1": 0.545901, + "nauc_mrr_at_3_max": 0.091233, + "nauc_mrr_at_3_std": -0.054021, + "nauc_mrr_at_3_diff1": 0.539954, + "nauc_mrr_at_5_max": 0.045065, + "nauc_mrr_at_5_std": -0.075921, + "nauc_mrr_at_5_diff1": 0.515762, + "nauc_mrr_at_10_max": 0.055635, + "nauc_mrr_at_10_std": -0.072236, + "nauc_mrr_at_10_diff1": 0.528078, + "nauc_mrr_at_20_max": 0.063681, + "nauc_mrr_at_20_std": -0.068463, + "nauc_mrr_at_20_diff1": 0.526726, + "nauc_mrr_at_100_max": 0.063129, + "nauc_mrr_at_100_std": -0.066319, + "nauc_mrr_at_100_diff1": 0.525689, + "nauc_mrr_at_1000_max": 0.063129, + "nauc_mrr_at_1000_std": -0.066319, + "nauc_mrr_at_1000_diff1": 0.525689, + "main_score": 0.47898, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 541.4709386825562, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..21409c0374 --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.15789, + "ndcg_at_3": 0.25089, + "ndcg_at_5": 0.28123, + "ndcg_at_10": 0.32599, + "ndcg_at_20": 0.37258, + "ndcg_at_100": 0.47197, + "ndcg_at_1000": 0.47879, + "map_at_1": 0.10073, + "map_at_3": 0.18558, + "map_at_5": 0.20837, + "map_at_10": 0.234, + "map_at_20": 0.25529, + "map_at_100": 0.28898, + "map_at_1000": 0.2897, + "recall_at_1": 0.10073, + "recall_at_3": 0.26886, + "recall_at_5": 0.35058, + "recall_at_10": 0.46145, + "recall_at_20": 0.59173, + "recall_at_100": 0.91711, + "recall_at_1000": 0.95614, + "cv_recall_at_1": 0.15789, + "cv_recall_at_3": 0.45614, + "cv_recall_at_5": 0.57895, + "cv_recall_at_10": 0.70175, + "cv_recall_at_20": 0.78947, + "cv_recall_at_100": 0.94737, + "cv_recall_at_1000": 0.96491, + "precision_at_1": 0.15789, + "precision_at_3": 0.17544, + "precision_at_5": 0.14737, + "precision_at_10": 0.10877, + "precision_at_20": 0.08421, + "precision_at_100": 0.03719, + "precision_at_1000": 0.00384, + "mrr_at_1": 0.157895, + "mrr_at_3": 0.28655, + "mrr_at_5": 0.312865, + "mrr_at_10": 0.332185, + "mrr_at_20": 0.338185, + "mrr_at_100": 0.343713, + "mrr_at_1000": 0.34376, + "nauc_ndcg_at_1_max": 0.236088, + "nauc_ndcg_at_1_std": 0.097475, + "nauc_ndcg_at_1_diff1": -0.006335, + "nauc_ndcg_at_3_max": 0.17021, + "nauc_ndcg_at_3_std": 0.083403, + "nauc_ndcg_at_3_diff1": -0.094166, + "nauc_ndcg_at_5_max": 0.166278, + "nauc_ndcg_at_5_std": 0.094613, + "nauc_ndcg_at_5_diff1": -0.112751, + "nauc_ndcg_at_10_max": 0.164662, + "nauc_ndcg_at_10_std": 0.057183, + "nauc_ndcg_at_10_diff1": -0.146936, + "nauc_ndcg_at_20_max": 0.14341, + "nauc_ndcg_at_20_std": 0.076096, + "nauc_ndcg_at_20_diff1": -0.136816, + "nauc_ndcg_at_100_max": 0.003679, + "nauc_ndcg_at_100_std": 0.103083, + "nauc_ndcg_at_100_diff1": -0.238703, + "nauc_ndcg_at_1000_max": 0.012826, + "nauc_ndcg_at_1000_std": 0.086963, + "nauc_ndcg_at_1000_diff1": -0.206116, + "nauc_map_at_1_max": 0.208861, + "nauc_map_at_1_std": 0.07854, + "nauc_map_at_1_diff1": -0.102331, + "nauc_map_at_3_max": 0.212457, + "nauc_map_at_3_std": 0.111023, + "nauc_map_at_3_diff1": -0.109194, + "nauc_map_at_5_max": 0.225115, + "nauc_map_at_5_std": 0.114243, + "nauc_map_at_5_diff1": -0.104274, + "nauc_map_at_10_max": 0.205878, + "nauc_map_at_10_std": 0.081933, + "nauc_map_at_10_diff1": -0.126277, + "nauc_map_at_20_max": 0.179942, + "nauc_map_at_20_std": 0.07478, + "nauc_map_at_20_diff1": -0.113695, + "nauc_map_at_100_max": 0.12506, + "nauc_map_at_100_std": 0.079458, + "nauc_map_at_100_diff1": -0.15126, + "nauc_map_at_1000_max": 0.124744, + "nauc_map_at_1000_std": 0.078353, + "nauc_map_at_1000_diff1": -0.149186, + "nauc_recall_at_1_max": 0.208861, + "nauc_recall_at_1_std": 0.07854, + "nauc_recall_at_1_diff1": -0.102331, + "nauc_recall_at_3_max": 0.184876, + "nauc_recall_at_3_std": 0.081176, + "nauc_recall_at_3_diff1": -0.102022, + "nauc_recall_at_5_max": 0.211657, + "nauc_recall_at_5_std": 0.092539, + "nauc_recall_at_5_diff1": -0.112232, + "nauc_recall_at_10_max": 0.1956, + "nauc_recall_at_10_std": 0.019692, + "nauc_recall_at_10_diff1": -0.187969, + "nauc_recall_at_20_max": 0.183643, + "nauc_recall_at_20_std": 0.092326, + "nauc_recall_at_20_diff1": -0.179466, + "nauc_recall_at_100_max": -0.558993, + "nauc_recall_at_100_std": 0.366868, + "nauc_recall_at_100_diff1": -1.028197, + "nauc_recall_at_1000_max": -0.517674, + "nauc_recall_at_1000_std": 0.160799, + "nauc_recall_at_1000_diff1": -0.741204, + "nauc_precision_at_1_max": 0.236088, + "nauc_precision_at_1_std": 0.097475, + "nauc_precision_at_1_diff1": -0.006335, + "nauc_precision_at_3_max": 0.075364, + "nauc_precision_at_3_std": 0.070935, + "nauc_precision_at_3_diff1": -0.089293, + "nauc_precision_at_5_max": 0.014931, + "nauc_precision_at_5_std": 0.0884, + "nauc_precision_at_5_diff1": -0.114089, + "nauc_precision_at_10_max": -0.157514, + "nauc_precision_at_10_std": -0.059314, + "nauc_precision_at_10_diff1": -0.164706, + "nauc_precision_at_20_max": -0.298278, + "nauc_precision_at_20_std": -0.083686, + "nauc_precision_at_20_diff1": -0.090909, + "nauc_precision_at_100_max": -0.412087, + "nauc_precision_at_100_std": -0.045504, + "nauc_precision_at_100_diff1": -0.222713, + "nauc_precision_at_1000_max": -0.408535, + "nauc_precision_at_1000_std": -0.061471, + "nauc_precision_at_1000_diff1": -0.193165, + "nauc_cv_recall_at_1_max": 0.236088, + "nauc_cv_recall_at_1_std": 0.097475, + "nauc_cv_recall_at_1_diff1": -0.006335, + "nauc_cv_recall_at_3_max": 0.000471, + "nauc_cv_recall_at_3_std": -0.00532, + "nauc_cv_recall_at_3_diff1": -0.095285, + "nauc_cv_recall_at_5_max": -0.058578, + "nauc_cv_recall_at_5_std": 0.022011, + "nauc_cv_recall_at_5_diff1": -0.169085, + "nauc_cv_recall_at_10_max": -0.028813, + "nauc_cv_recall_at_10_std": 0.140536, + "nauc_cv_recall_at_10_diff1": -0.333005, + "nauc_cv_recall_at_20_max": -0.070939, + "nauc_cv_recall_at_20_std": 0.366935, + "nauc_cv_recall_at_20_diff1": -0.590672, + "nauc_cv_recall_at_100_max": -1.1521, + "nauc_cv_recall_at_100_std": 0.346518, + "nauc_cv_recall_at_100_diff1": -1.338374, + "nauc_cv_recall_at_1000_max": -0.864221, + "nauc_cv_recall_at_1000_std": 0.242235, + "nauc_cv_recall_at_1000_diff1": -1.143633, + "nauc_mrr_at_1_max": 0.236088, + "nauc_mrr_at_1_std": 0.097475, + "nauc_mrr_at_1_diff1": -0.006335, + "nauc_mrr_at_3_max": 0.126408, + "nauc_mrr_at_3_std": 0.052328, + "nauc_mrr_at_3_diff1": -0.050981, + "nauc_mrr_at_5_max": 0.115257, + "nauc_mrr_at_5_std": 0.062811, + "nauc_mrr_at_5_diff1": -0.070468, + "nauc_mrr_at_10_max": 0.129339, + "nauc_mrr_at_10_std": 0.083012, + "nauc_mrr_at_10_diff1": -0.090904, + "nauc_mrr_at_20_max": 0.127326, + "nauc_mrr_at_20_std": 0.089324, + "nauc_mrr_at_20_diff1": -0.098097, + "nauc_mrr_at_100_max": 0.124808, + "nauc_mrr_at_100_std": 0.086218, + "nauc_mrr_at_100_diff1": -0.093056, + "nauc_mrr_at_1000_max": 0.125131, + "nauc_mrr_at_1000_std": 0.086136, + "nauc_mrr_at_1000_diff1": -0.092771, + "main_score": 0.28123, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.24561, + "ndcg_at_3": 0.25525, + "ndcg_at_5": 0.2829, + "ndcg_at_10": 0.33844, + "ndcg_at_20": 0.41699, + "ndcg_at_100": 0.49232, + "ndcg_at_1000": 0.49864, + "map_at_1": 0.13728, + "map_at_3": 0.19632, + "map_at_5": 0.22074, + "map_at_10": 0.25096, + "map_at_20": 0.2788, + "map_at_100": 0.31007, + "map_at_1000": 0.3107, + "recall_at_1": 0.13728, + "recall_at_3": 0.25029, + "recall_at_5": 0.32558, + "recall_at_10": 0.46963, + "recall_at_20": 0.71005, + "recall_at_100": 0.93193, + "recall_at_1000": 0.97368, + "cv_recall_at_1": 0.24561, + "cv_recall_at_3": 0.38596, + "cv_recall_at_5": 0.49123, + "cv_recall_at_10": 0.66667, + "cv_recall_at_20": 0.91228, + "cv_recall_at_100": 0.94737, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.24561, + "precision_at_3": 0.15205, + "precision_at_5": 0.13684, + "precision_at_10": 0.10877, + "precision_at_20": 0.09649, + "precision_at_100": 0.03737, + "precision_at_1000": 0.00386, + "mrr_at_1": 0.245614, + "mrr_at_3": 0.307018, + "mrr_at_5": 0.329825, + "mrr_at_10": 0.355723, + "mrr_at_20": 0.372514, + "mrr_at_100": 0.3736, + "mrr_at_1000": 0.373762, + "nauc_ndcg_at_1_max": 0.17543, + "nauc_ndcg_at_1_std": 0.106063, + "nauc_ndcg_at_1_diff1": 0.314142, + "nauc_ndcg_at_3_max": 0.091467, + "nauc_ndcg_at_3_std": -0.005684, + "nauc_ndcg_at_3_diff1": 0.302701, + "nauc_ndcg_at_5_max": 0.14282, + "nauc_ndcg_at_5_std": -0.011351, + "nauc_ndcg_at_5_diff1": 0.313193, + "nauc_ndcg_at_10_max": 0.178967, + "nauc_ndcg_at_10_std": -0.013821, + "nauc_ndcg_at_10_diff1": 0.353415, + "nauc_ndcg_at_20_max": 0.166357, + "nauc_ndcg_at_20_std": -0.030882, + "nauc_ndcg_at_20_diff1": 0.3719, + "nauc_ndcg_at_100_max": 0.083801, + "nauc_ndcg_at_100_std": 0.037827, + "nauc_ndcg_at_100_diff1": 0.277288, + "nauc_ndcg_at_1000_max": 0.077265, + "nauc_ndcg_at_1000_std": 0.033141, + "nauc_ndcg_at_1000_diff1": 0.267868, + "nauc_map_at_1_max": 0.161215, + "nauc_map_at_1_std": 0.009565, + "nauc_map_at_1_diff1": 0.272032, + "nauc_map_at_3_max": 0.106262, + "nauc_map_at_3_std": -0.039439, + "nauc_map_at_3_diff1": 0.26921, + "nauc_map_at_5_max": 0.150367, + "nauc_map_at_5_std": -0.014703, + "nauc_map_at_5_diff1": 0.299513, + "nauc_map_at_10_max": 0.149669, + "nauc_map_at_10_std": -0.026601, + "nauc_map_at_10_diff1": 0.308375, + "nauc_map_at_20_max": 0.141577, + "nauc_map_at_20_std": -0.031851, + "nauc_map_at_20_diff1": 0.316038, + "nauc_map_at_100_max": 0.088624, + "nauc_map_at_100_std": -0.02177, + "nauc_map_at_100_diff1": 0.268579, + "nauc_map_at_1000_max": 0.088047, + "nauc_map_at_1000_std": -0.021966, + "nauc_map_at_1000_diff1": 0.267822, + "nauc_recall_at_1_max": 0.161215, + "nauc_recall_at_1_std": 0.009565, + "nauc_recall_at_1_diff1": 0.272032, + "nauc_recall_at_3_max": 0.059323, + "nauc_recall_at_3_std": -0.131379, + "nauc_recall_at_3_diff1": 0.307917, + "nauc_recall_at_5_max": 0.198369, + "nauc_recall_at_5_std": -0.025405, + "nauc_recall_at_5_diff1": 0.309538, + "nauc_recall_at_10_max": 0.298188, + "nauc_recall_at_10_std": 0.026327, + "nauc_recall_at_10_diff1": 0.407135, + "nauc_recall_at_20_max": 0.323214, + "nauc_recall_at_20_std": -0.11038, + "nauc_recall_at_20_diff1": 0.518881, + "nauc_recall_at_100_max": 0.400478, + "nauc_recall_at_100_std": 0.33104, + "nauc_recall_at_100_diff1": 0.525783, + "nauc_recall_at_1000_max": 0.528073, + "nauc_recall_at_1000_std": 0.524027, + "nauc_recall_at_1000_diff1": 0.280774, + "nauc_precision_at_1_max": 0.17543, + "nauc_precision_at_1_std": 0.106063, + "nauc_precision_at_1_diff1": 0.314142, + "nauc_precision_at_3_max": 0.107126, + "nauc_precision_at_3_std": 0.056146, + "nauc_precision_at_3_diff1": 0.308698, + "nauc_precision_at_5_max": 0.117842, + "nauc_precision_at_5_std": 0.026135, + "nauc_precision_at_5_diff1": 0.275421, + "nauc_precision_at_10_max": -0.020236, + "nauc_precision_at_10_std": -0.049504, + "nauc_precision_at_10_diff1": 0.226006, + "nauc_precision_at_20_max": -0.234922, + "nauc_precision_at_20_std": -0.052045, + "nauc_precision_at_20_diff1": 0.006913, + "nauc_precision_at_100_max": -0.307617, + "nauc_precision_at_100_std": 0.067184, + "nauc_precision_at_100_diff1": -0.24651, + "nauc_precision_at_1000_max": -0.312232, + "nauc_precision_at_1000_std": 0.05895, + "nauc_precision_at_1000_diff1": -0.248609, + "nauc_cv_recall_at_1_max": 0.17543, + "nauc_cv_recall_at_1_std": 0.106063, + "nauc_cv_recall_at_1_diff1": 0.314142, + "nauc_cv_recall_at_3_max": 0.07758, + "nauc_cv_recall_at_3_std": -0.059183, + "nauc_cv_recall_at_3_diff1": 0.418468, + "nauc_cv_recall_at_5_max": 0.132164, + "nauc_cv_recall_at_5_std": 0.004288, + "nauc_cv_recall_at_5_diff1": 0.285515, + "nauc_cv_recall_at_10_max": 0.246, + "nauc_cv_recall_at_10_std": 0.128988, + "nauc_cv_recall_at_10_diff1": 0.546689, + "nauc_cv_recall_at_20_max": 0.141933, + "nauc_cv_recall_at_20_std": 0.184038, + "nauc_cv_recall_at_20_diff1": 0.667307, + "nauc_cv_recall_at_100_max": 0.346518, + "nauc_cv_recall_at_100_std": 0.450994, + "nauc_cv_recall_at_100_diff1": 0.742122, + "nauc_cv_recall_at_1000_max": 0.357853, + "nauc_cv_recall_at_1000_std": 0.868512, + "nauc_cv_recall_at_1000_diff1": 0.357853, + "nauc_mrr_at_1_max": 0.17543, + "nauc_mrr_at_1_std": 0.106063, + "nauc_mrr_at_1_diff1": 0.314142, + "nauc_mrr_at_3_max": 0.12491, + "nauc_mrr_at_3_std": 0.038351, + "nauc_mrr_at_3_diff1": 0.353999, + "nauc_mrr_at_5_max": 0.133629, + "nauc_mrr_at_5_std": 0.051002, + "nauc_mrr_at_5_diff1": 0.32044, + "nauc_mrr_at_10_max": 0.144736, + "nauc_mrr_at_10_std": 0.063555, + "nauc_mrr_at_10_diff1": 0.353407, + "nauc_mrr_at_20_max": 0.140854, + "nauc_mrr_at_20_std": 0.065355, + "nauc_mrr_at_20_diff1": 0.347488, + "nauc_mrr_at_100_max": 0.14193, + "nauc_mrr_at_100_std": 0.066439, + "nauc_mrr_at_100_diff1": 0.346758, + "nauc_mrr_at_1000_max": 0.141906, + "nauc_mrr_at_1000_std": 0.066303, + "nauc_mrr_at_1000_diff1": 0.346424, + "main_score": 0.2829, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.36842, + "ndcg_at_3": 0.45192, + "ndcg_at_5": 0.46115, + "ndcg_at_10": 0.52427, + "ndcg_at_20": 0.55841, + "ndcg_at_100": 0.61484, + "ndcg_at_1000": 0.62099, + "map_at_1": 0.17987, + "map_at_3": 0.32499, + "map_at_5": 0.35353, + "map_at_10": 0.40175, + "map_at_20": 0.42167, + "map_at_100": 0.44626, + "map_at_1000": 0.44693, + "recall_at_1": 0.17987, + "recall_at_3": 0.43864, + "recall_at_5": 0.51627, + "recall_at_10": 0.70008, + "recall_at_20": 0.8005, + "recall_at_100": 0.96418, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.36842, + "cv_recall_at_3": 0.68421, + "cv_recall_at_5": 0.75439, + "cv_recall_at_10": 0.89474, + "cv_recall_at_20": 0.92982, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.36842, + "precision_at_3": 0.31579, + "precision_at_5": 0.23509, + "precision_at_10": 0.17368, + "precision_at_20": 0.11228, + "precision_at_100": 0.03789, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.368421, + "mrr_at_3": 0.51462, + "mrr_at_5": 0.530409, + "mrr_at_10": 0.550313, + "mrr_at_20": 0.553125, + "mrr_at_100": 0.555195, + "mrr_at_1000": 0.555278, + "nauc_ndcg_at_1_max": -0.147531, + "nauc_ndcg_at_1_std": 0.016426, + "nauc_ndcg_at_1_diff1": 0.261603, + "nauc_ndcg_at_3_max": 0.022107, + "nauc_ndcg_at_3_std": 0.053721, + "nauc_ndcg_at_3_diff1": 0.089023, + "nauc_ndcg_at_5_max": 0.069293, + "nauc_ndcg_at_5_std": 0.081217, + "nauc_ndcg_at_5_diff1": 0.046331, + "nauc_ndcg_at_10_max": 0.101499, + "nauc_ndcg_at_10_std": 0.064595, + "nauc_ndcg_at_10_diff1": 0.075736, + "nauc_ndcg_at_20_max": 0.107239, + "nauc_ndcg_at_20_std": 0.04574, + "nauc_ndcg_at_20_diff1": 0.069648, + "nauc_ndcg_at_100_max": -0.011534, + "nauc_ndcg_at_100_std": 0.043423, + "nauc_ndcg_at_100_diff1": 0.033712, + "nauc_ndcg_at_1000_max": -0.015115, + "nauc_ndcg_at_1000_std": 0.040265, + "nauc_ndcg_at_1000_diff1": 0.066368, + "nauc_map_at_1_max": 0.043491, + "nauc_map_at_1_std": 0.056735, + "nauc_map_at_1_diff1": 0.334348, + "nauc_map_at_3_max": 0.097288, + "nauc_map_at_3_std": 0.091838, + "nauc_map_at_3_diff1": 0.14608, + "nauc_map_at_5_max": 0.122467, + "nauc_map_at_5_std": 0.110284, + "nauc_map_at_5_diff1": 0.113426, + "nauc_map_at_10_max": 0.130078, + "nauc_map_at_10_std": 0.107236, + "nauc_map_at_10_diff1": 0.120472, + "nauc_map_at_20_max": 0.121502, + "nauc_map_at_20_std": 0.096232, + "nauc_map_at_20_diff1": 0.109723, + "nauc_map_at_100_max": 0.078096, + "nauc_map_at_100_std": 0.092873, + "nauc_map_at_100_diff1": 0.103371, + "nauc_map_at_1000_max": 0.076536, + "nauc_map_at_1000_std": 0.091451, + "nauc_map_at_1000_diff1": 0.104867, + "nauc_recall_at_1_max": 0.043491, + "nauc_recall_at_1_std": 0.056735, + "nauc_recall_at_1_diff1": 0.334348, + "nauc_recall_at_3_max": 0.169159, + "nauc_recall_at_3_std": 0.085457, + "nauc_recall_at_3_diff1": -0.000881, + "nauc_recall_at_5_max": 0.257041, + "nauc_recall_at_5_std": 0.172647, + "nauc_recall_at_5_diff1": -0.093168, + "nauc_recall_at_10_max": 0.283414, + "nauc_recall_at_10_std": 0.051576, + "nauc_recall_at_10_diff1": -0.041172, + "nauc_recall_at_20_max": 0.404915, + "nauc_recall_at_20_std": -0.016679, + "nauc_recall_at_20_diff1": -0.051775, + "nauc_recall_at_100_max": -0.146387, + "nauc_recall_at_100_std": -0.116344, + "nauc_recall_at_100_diff1": -1.285556, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.147531, + "nauc_precision_at_1_std": 0.016426, + "nauc_precision_at_1_diff1": 0.261603, + "nauc_precision_at_3_max": -0.083749, + "nauc_precision_at_3_std": 0.003727, + "nauc_precision_at_3_diff1": -0.082319, + "nauc_precision_at_5_max": -0.053423, + "nauc_precision_at_5_std": 0.018803, + "nauc_precision_at_5_diff1": -0.169465, + "nauc_precision_at_10_max": -0.102364, + "nauc_precision_at_10_std": -0.046139, + "nauc_precision_at_10_diff1": -0.12248, + "nauc_precision_at_20_max": -0.208674, + "nauc_precision_at_20_std": -0.095476, + "nauc_precision_at_20_diff1": -0.134436, + "nauc_precision_at_100_max": -0.322954, + "nauc_precision_at_100_std": -0.078318, + "nauc_precision_at_100_diff1": -0.076379, + "nauc_precision_at_1000_max": -0.334002, + "nauc_precision_at_1000_std": -0.092343, + "nauc_precision_at_1000_diff1": -0.056384, + "nauc_cv_recall_at_1_max": -0.147531, + "nauc_cv_recall_at_1_std": 0.016426, + "nauc_cv_recall_at_1_diff1": 0.261603, + "nauc_cv_recall_at_3_max": -0.05537, + "nauc_cv_recall_at_3_std": -0.068004, + "nauc_cv_recall_at_3_diff1": -0.095262, + "nauc_cv_recall_at_5_max": 0.048654, + "nauc_cv_recall_at_5_std": 0.157916, + "nauc_cv_recall_at_5_diff1": -0.308187, + "nauc_cv_recall_at_10_max": 0.110283, + "nauc_cv_recall_at_10_std": -0.076409, + "nauc_cv_recall_at_10_diff1": -0.13067, + "nauc_cv_recall_at_20_max": 0.055277, + "nauc_cv_recall_at_20_std": -0.454076, + "nauc_cv_recall_at_20_diff1": 0.13749, + "nauc_cv_recall_at_100_max": -1.169034, + "nauc_cv_recall_at_100_std": -1.169034, + "nauc_cv_recall_at_100_diff1": -1.727857, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.147531, + "nauc_mrr_at_1_std": 0.016426, + "nauc_mrr_at_1_diff1": 0.261603, + "nauc_mrr_at_3_max": -0.11624, + "nauc_mrr_at_3_std": -0.022557, + "nauc_mrr_at_3_diff1": 0.125067, + "nauc_mrr_at_5_max": -0.10161, + "nauc_mrr_at_5_std": 0.014288, + "nauc_mrr_at_5_diff1": 0.097482, + "nauc_mrr_at_10_max": -0.110398, + "nauc_mrr_at_10_std": -0.005118, + "nauc_mrr_at_10_diff1": 0.128078, + "nauc_mrr_at_20_max": -0.113321, + "nauc_mrr_at_20_std": -0.011669, + "nauc_mrr_at_20_diff1": 0.136033, + "nauc_mrr_at_100_max": -0.116965, + "nauc_mrr_at_100_std": -0.00908, + "nauc_mrr_at_100_diff1": 0.13171, + "nauc_mrr_at_1000_max": -0.116659, + "nauc_mrr_at_1000_std": -0.008742, + "nauc_mrr_at_1000_diff1": 0.13225, + "main_score": 0.46115, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.17544, + "ndcg_at_3": 0.17788, + "ndcg_at_5": 0.20436, + "ndcg_at_10": 0.26929, + "ndcg_at_20": 0.31666, + "ndcg_at_100": 0.40765, + "ndcg_at_1000": 0.4252, + "map_at_1": 0.06784, + "map_at_3": 0.11425, + "map_at_5": 0.13326, + "map_at_10": 0.17098, + "map_at_20": 0.1888, + "map_at_100": 0.21769, + "map_at_1000": 0.21978, + "recall_at_1": 0.06784, + "recall_at_3": 0.16213, + "recall_at_5": 0.23889, + "recall_at_10": 0.4056, + "recall_at_20": 0.56548, + "recall_at_100": 0.88223, + "recall_at_1000": 0.97193, + "cv_recall_at_1": 0.17544, + "cv_recall_at_3": 0.35088, + "cv_recall_at_5": 0.45614, + "cv_recall_at_10": 0.64912, + "cv_recall_at_20": 0.77193, + "cv_recall_at_100": 0.94737, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.17544, + "precision_at_3": 0.12865, + "precision_at_5": 0.11579, + "precision_at_10": 0.10877, + "precision_at_20": 0.08158, + "precision_at_100": 0.03421, + "precision_at_1000": 0.00384, + "mrr_at_1": 0.175439, + "mrr_at_3": 0.254386, + "mrr_at_5": 0.276316, + "mrr_at_10": 0.301824, + "mrr_at_20": 0.309461, + "mrr_at_100": 0.315057, + "mrr_at_1000": 0.315263, + "nauc_ndcg_at_1_max": 0.151408, + "nauc_ndcg_at_1_std": 0.182665, + "nauc_ndcg_at_1_diff1": 0.245479, + "nauc_ndcg_at_3_max": 0.090247, + "nauc_ndcg_at_3_std": 0.143169, + "nauc_ndcg_at_3_diff1": 0.160624, + "nauc_ndcg_at_5_max": 0.110215, + "nauc_ndcg_at_5_std": 0.195932, + "nauc_ndcg_at_5_diff1": 0.096174, + "nauc_ndcg_at_10_max": 0.120819, + "nauc_ndcg_at_10_std": 0.172881, + "nauc_ndcg_at_10_diff1": 0.123174, + "nauc_ndcg_at_20_max": 0.100911, + "nauc_ndcg_at_20_std": 0.179797, + "nauc_ndcg_at_20_diff1": 0.114554, + "nauc_ndcg_at_100_max": 0.015961, + "nauc_ndcg_at_100_std": 0.167726, + "nauc_ndcg_at_100_diff1": 0.090973, + "nauc_ndcg_at_1000_max": 0.064507, + "nauc_ndcg_at_1000_std": 0.148064, + "nauc_ndcg_at_1000_diff1": 0.129981, + "nauc_map_at_1_max": 0.249252, + "nauc_map_at_1_std": 0.050098, + "nauc_map_at_1_diff1": 0.016103, + "nauc_map_at_3_max": 0.16378, + "nauc_map_at_3_std": 0.130459, + "nauc_map_at_3_diff1": 0.109886, + "nauc_map_at_5_max": 0.168105, + "nauc_map_at_5_std": 0.163493, + "nauc_map_at_5_diff1": 0.073407, + "nauc_map_at_10_max": 0.165315, + "nauc_map_at_10_std": 0.159788, + "nauc_map_at_10_diff1": 0.097634, + "nauc_map_at_20_max": 0.152618, + "nauc_map_at_20_std": 0.157514, + "nauc_map_at_20_diff1": 0.094339, + "nauc_map_at_100_max": 0.130213, + "nauc_map_at_100_std": 0.162874, + "nauc_map_at_100_diff1": 0.092614, + "nauc_map_at_1000_max": 0.130855, + "nauc_map_at_1000_std": 0.157997, + "nauc_map_at_1000_diff1": 0.092815, + "nauc_recall_at_1_max": 0.249252, + "nauc_recall_at_1_std": 0.050098, + "nauc_recall_at_1_diff1": 0.016103, + "nauc_recall_at_3_max": 0.068987, + "nauc_recall_at_3_std": 0.123515, + "nauc_recall_at_3_diff1": 0.105848, + "nauc_recall_at_5_max": 0.07841, + "nauc_recall_at_5_std": 0.185898, + "nauc_recall_at_5_diff1": 0.008505, + "nauc_recall_at_10_max": 0.154785, + "nauc_recall_at_10_std": 0.206354, + "nauc_recall_at_10_diff1": 0.075997, + "nauc_recall_at_20_max": 0.135776, + "nauc_recall_at_20_std": 0.218815, + "nauc_recall_at_20_diff1": 0.024269, + "nauc_recall_at_100_max": -0.322611, + "nauc_recall_at_100_std": 0.223295, + "nauc_recall_at_100_diff1": -0.217627, + "nauc_recall_at_1000_max": 0.557568, + "nauc_recall_at_1000_std": 0.181202, + "nauc_recall_at_1000_diff1": 0.156001, + "nauc_precision_at_1_max": 0.151408, + "nauc_precision_at_1_std": 0.182665, + "nauc_precision_at_1_diff1": 0.245479, + "nauc_precision_at_3_max": 0.01407, + "nauc_precision_at_3_std": 0.173658, + "nauc_precision_at_3_diff1": 0.241831, + "nauc_precision_at_5_max": 0.068013, + "nauc_precision_at_5_std": 0.251041, + "nauc_precision_at_5_diff1": 0.139382, + "nauc_precision_at_10_max": 0.043924, + "nauc_precision_at_10_std": 0.154548, + "nauc_precision_at_10_diff1": 0.153654, + "nauc_precision_at_20_max": -0.086932, + "nauc_precision_at_20_std": 0.037269, + "nauc_precision_at_20_diff1": 0.135007, + "nauc_precision_at_100_max": -0.121165, + "nauc_precision_at_100_std": -0.054906, + "nauc_precision_at_100_diff1": 0.076706, + "nauc_precision_at_1000_max": -0.0735, + "nauc_precision_at_1000_std": -0.094423, + "nauc_precision_at_1000_diff1": 0.115688, + "nauc_cv_recall_at_1_max": 0.151408, + "nauc_cv_recall_at_1_std": 0.182665, + "nauc_cv_recall_at_1_diff1": 0.245479, + "nauc_cv_recall_at_3_max": -0.127892, + "nauc_cv_recall_at_3_std": 0.089568, + "nauc_cv_recall_at_3_diff1": 0.111245, + "nauc_cv_recall_at_5_max": -0.132474, + "nauc_cv_recall_at_5_std": 0.15251, + "nauc_cv_recall_at_5_diff1": 0.062287, + "nauc_cv_recall_at_10_max": 0.098433, + "nauc_cv_recall_at_10_std": 0.155955, + "nauc_cv_recall_at_10_diff1": 0.23006, + "nauc_cv_recall_at_20_max": 0.133924, + "nauc_cv_recall_at_20_std": 0.305464, + "nauc_cv_recall_at_20_diff1": 0.220189, + "nauc_cv_recall_at_100_max": -0.253654, + "nauc_cv_recall_at_100_std": 0.14524, + "nauc_cv_recall_at_100_diff1": 0.269439, + "nauc_cv_recall_at_1000_max": 0.357853, + "nauc_cv_recall_at_1000_std": 0.126616, + "nauc_cv_recall_at_1000_diff1": 0.126616, + "nauc_mrr_at_1_max": 0.151408, + "nauc_mrr_at_1_std": 0.182665, + "nauc_mrr_at_1_diff1": 0.245479, + "nauc_mrr_at_3_max": 0.008137, + "nauc_mrr_at_3_std": 0.141169, + "nauc_mrr_at_3_diff1": 0.179404, + "nauc_mrr_at_5_max": 0.004833, + "nauc_mrr_at_5_std": 0.154526, + "nauc_mrr_at_5_diff1": 0.171281, + "nauc_mrr_at_10_max": 0.036162, + "nauc_mrr_at_10_std": 0.152943, + "nauc_mrr_at_10_diff1": 0.193286, + "nauc_mrr_at_20_max": 0.037471, + "nauc_mrr_at_20_std": 0.159845, + "nauc_mrr_at_20_diff1": 0.192831, + "nauc_mrr_at_100_max": 0.030518, + "nauc_mrr_at_100_std": 0.157094, + "nauc_mrr_at_100_diff1": 0.189612, + "nauc_mrr_at_1000_max": 0.030938, + "nauc_mrr_at_1000_std": 0.156787, + "nauc_mrr_at_1000_diff1": 0.189411, + "main_score": 0.20436, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 2118.100417137146, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..0092a2ca8e --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.22414, + "ndcg_at_3": 0.22626, + "ndcg_at_5": 0.20743, + "ndcg_at_10": 0.20367, + "ndcg_at_20": 0.22628, + "ndcg_at_100": 0.34541, + "ndcg_at_1000": 0.47161, + "map_at_1": 0.01948, + "map_at_3": 0.04055, + "map_at_5": 0.052, + "map_at_10": 0.07164, + "map_at_20": 0.09832, + "map_at_100": 0.1486, + "map_at_1000": 0.17478, + "recall_at_1": 0.01948, + "recall_at_3": 0.05321, + "recall_at_5": 0.07794, + "recall_at_10": 0.13961, + "recall_at_20": 0.22585, + "recall_at_100": 0.57307, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.22414, + "cv_recall_at_3": 0.43103, + "cv_recall_at_5": 0.53448, + "cv_recall_at_10": 0.62069, + "cv_recall_at_20": 0.7069, + "cv_recall_at_100": 0.91379, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.22414, + "precision_at_3": 0.22414, + "precision_at_5": 0.19655, + "precision_at_10": 0.17414, + "precision_at_20": 0.15345, + "precision_at_100": 0.08207, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.224138, + "mrr_at_3": 0.316092, + "mrr_at_5": 0.341092, + "mrr_at_10": 0.353681, + "mrr_at_20": 0.360103, + "mrr_at_100": 0.364651, + "mrr_at_1000": 0.36532, + "nauc_ndcg_at_1_max": 0.374113, + "nauc_ndcg_at_1_std": 0.283863, + "nauc_ndcg_at_1_diff1": 0.220561, + "nauc_ndcg_at_3_max": 0.381219, + "nauc_ndcg_at_3_std": 0.431786, + "nauc_ndcg_at_3_diff1": 0.024415, + "nauc_ndcg_at_5_max": 0.349058, + "nauc_ndcg_at_5_std": 0.428871, + "nauc_ndcg_at_5_diff1": -0.05818, + "nauc_ndcg_at_10_max": 0.363332, + "nauc_ndcg_at_10_std": 0.420164, + "nauc_ndcg_at_10_diff1": -0.071775, + "nauc_ndcg_at_20_max": 0.422986, + "nauc_ndcg_at_20_std": 0.428037, + "nauc_ndcg_at_20_diff1": -0.027846, + "nauc_ndcg_at_100_max": 0.414936, + "nauc_ndcg_at_100_std": 0.439154, + "nauc_ndcg_at_100_diff1": 0.006569, + "nauc_ndcg_at_1000_max": 0.292796, + "nauc_ndcg_at_1000_std": 0.327898, + "nauc_ndcg_at_1000_diff1": -0.002208, + "nauc_map_at_1_max": 0.496009, + "nauc_map_at_1_std": 0.269008, + "nauc_map_at_1_diff1": 0.405714, + "nauc_map_at_3_max": 0.492623, + "nauc_map_at_3_std": 0.478705, + "nauc_map_at_3_diff1": 0.179132, + "nauc_map_at_5_max": 0.460811, + "nauc_map_at_5_std": 0.504849, + "nauc_map_at_5_diff1": 0.112751, + "nauc_map_at_10_max": 0.484689, + "nauc_map_at_10_std": 0.503823, + "nauc_map_at_10_diff1": 0.07179, + "nauc_map_at_20_max": 0.45594, + "nauc_map_at_20_std": 0.457193, + "nauc_map_at_20_diff1": 0.061902, + "nauc_map_at_100_max": 0.395602, + "nauc_map_at_100_std": 0.452182, + "nauc_map_at_100_diff1": -0.012268, + "nauc_map_at_1000_max": 0.316569, + "nauc_map_at_1000_std": 0.385011, + "nauc_map_at_1000_diff1": -0.031211, + "nauc_recall_at_1_max": 0.496009, + "nauc_recall_at_1_std": 0.269008, + "nauc_recall_at_1_diff1": 0.405714, + "nauc_recall_at_3_max": 0.482781, + "nauc_recall_at_3_std": 0.530619, + "nauc_recall_at_3_diff1": 0.105346, + "nauc_recall_at_5_max": 0.418353, + "nauc_recall_at_5_std": 0.517703, + "nauc_recall_at_5_diff1": -0.000712, + "nauc_recall_at_10_max": 0.485086, + "nauc_recall_at_10_std": 0.445475, + "nauc_recall_at_10_diff1": -0.007481, + "nauc_recall_at_20_max": 0.474644, + "nauc_recall_at_20_std": 0.403833, + "nauc_recall_at_20_diff1": 0.003619, + "nauc_recall_at_100_max": 0.411788, + "nauc_recall_at_100_std": 0.378619, + "nauc_recall_at_100_diff1": 0.034106, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.374113, + "nauc_precision_at_1_std": 0.283863, + "nauc_precision_at_1_diff1": 0.220561, + "nauc_precision_at_3_max": 0.368165, + "nauc_precision_at_3_std": 0.470277, + "nauc_precision_at_3_diff1": -0.032012, + "nauc_precision_at_5_max": 0.300599, + "nauc_precision_at_5_std": 0.442207, + "nauc_precision_at_5_diff1": -0.154018, + "nauc_precision_at_10_max": 0.287627, + "nauc_precision_at_10_std": 0.401126, + "nauc_precision_at_10_diff1": -0.162971, + "nauc_precision_at_20_max": 0.210863, + "nauc_precision_at_20_std": 0.283126, + "nauc_precision_at_20_diff1": -0.143078, + "nauc_precision_at_100_max": -0.026336, + "nauc_precision_at_100_std": 0.090262, + "nauc_precision_at_100_diff1": -0.106647, + "nauc_precision_at_1000_max": -0.245648, + "nauc_precision_at_1000_std": -0.141959, + "nauc_precision_at_1000_diff1": -0.132479, + "nauc_cv_recall_at_1_max": 0.374113, + "nauc_cv_recall_at_1_std": 0.283863, + "nauc_cv_recall_at_1_diff1": 0.220561, + "nauc_cv_recall_at_3_max": 0.453986, + "nauc_cv_recall_at_3_std": 0.461241, + "nauc_cv_recall_at_3_diff1": 0.011901, + "nauc_cv_recall_at_5_max": 0.373563, + "nauc_cv_recall_at_5_std": 0.402422, + "nauc_cv_recall_at_5_diff1": -0.164775, + "nauc_cv_recall_at_10_max": 0.368093, + "nauc_cv_recall_at_10_std": 0.369609, + "nauc_cv_recall_at_10_diff1": -0.094712, + "nauc_cv_recall_at_20_max": 0.221761, + "nauc_cv_recall_at_20_std": 0.191081, + "nauc_cv_recall_at_20_diff1": 0.013025, + "nauc_cv_recall_at_100_max": 0.35593, + "nauc_cv_recall_at_100_std": -0.195414, + "nauc_cv_recall_at_100_diff1": 0.479177, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.374113, + "nauc_mrr_at_1_std": 0.283863, + "nauc_mrr_at_1_diff1": 0.220561, + "nauc_mrr_at_3_max": 0.429901, + "nauc_mrr_at_3_std": 0.387353, + "nauc_mrr_at_3_diff1": 0.119861, + "nauc_mrr_at_5_max": 0.405315, + "nauc_mrr_at_5_std": 0.367591, + "nauc_mrr_at_5_diff1": 0.077735, + "nauc_mrr_at_10_max": 0.406804, + "nauc_mrr_at_10_std": 0.362869, + "nauc_mrr_at_10_diff1": 0.088131, + "nauc_mrr_at_20_max": 0.398372, + "nauc_mrr_at_20_std": 0.352349, + "nauc_mrr_at_20_diff1": 0.09533, + "nauc_mrr_at_100_max": 0.401254, + "nauc_mrr_at_100_std": 0.351739, + "nauc_mrr_at_100_diff1": 0.099089, + "nauc_mrr_at_1000_max": 0.401474, + "nauc_mrr_at_1000_std": 0.353073, + "nauc_mrr_at_1000_diff1": 0.098447, + "main_score": 0.20743, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.22414, + "ndcg_at_3": 0.20882, + "ndcg_at_5": 0.21891, + "ndcg_at_10": 0.2192, + "ndcg_at_20": 0.25838, + "ndcg_at_100": 0.40068, + "ndcg_at_1000": 0.50185, + "map_at_1": 0.02388, + "map_at_3": 0.04951, + "map_at_5": 0.06959, + "map_at_10": 0.09646, + "map_at_20": 0.12512, + "map_at_100": 0.17637, + "map_at_1000": 0.20082, + "recall_at_1": 0.02388, + "recall_at_3": 0.0656, + "recall_at_5": 0.10849, + "recall_at_10": 0.18094, + "recall_at_20": 0.30465, + "recall_at_100": 0.69844, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.22414, + "cv_recall_at_3": 0.44828, + "cv_recall_at_5": 0.58621, + "cv_recall_at_10": 0.7069, + "cv_recall_at_20": 0.7931, + "cv_recall_at_100": 0.94828, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.22414, + "precision_at_3": 0.2069, + "precision_at_5": 0.21724, + "precision_at_10": 0.17931, + "precision_at_20": 0.15603, + "precision_at_100": 0.08638, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.224138, + "mrr_at_3": 0.313218, + "mrr_at_5": 0.345115, + "mrr_at_10": 0.360844, + "mrr_at_20": 0.367291, + "mrr_at_100": 0.371428, + "mrr_at_1000": 0.371814, + "nauc_ndcg_at_1_max": -0.004184, + "nauc_ndcg_at_1_std": -0.164655, + "nauc_ndcg_at_1_diff1": 0.028425, + "nauc_ndcg_at_3_max": 0.080357, + "nauc_ndcg_at_3_std": -0.010528, + "nauc_ndcg_at_3_diff1": -0.157915, + "nauc_ndcg_at_5_max": 0.122122, + "nauc_ndcg_at_5_std": 0.052578, + "nauc_ndcg_at_5_diff1": -0.127838, + "nauc_ndcg_at_10_max": 0.162097, + "nauc_ndcg_at_10_std": 0.104871, + "nauc_ndcg_at_10_diff1": -0.160487, + "nauc_ndcg_at_20_max": 0.278316, + "nauc_ndcg_at_20_std": 0.185434, + "nauc_ndcg_at_20_diff1": -0.146542, + "nauc_ndcg_at_100_max": 0.316214, + "nauc_ndcg_at_100_std": 0.236113, + "nauc_ndcg_at_100_diff1": -0.061281, + "nauc_ndcg_at_1000_max": 0.183521, + "nauc_ndcg_at_1000_std": 0.131913, + "nauc_ndcg_at_1000_diff1": -0.022011, + "nauc_map_at_1_max": 0.023922, + "nauc_map_at_1_std": -0.099324, + "nauc_map_at_1_diff1": -0.003338, + "nauc_map_at_3_max": 0.106877, + "nauc_map_at_3_std": 0.004626, + "nauc_map_at_3_diff1": -0.175413, + "nauc_map_at_5_max": 0.07702, + "nauc_map_at_5_std": -0.020959, + "nauc_map_at_5_diff1": -0.101545, + "nauc_map_at_10_max": 0.159921, + "nauc_map_at_10_std": 0.067565, + "nauc_map_at_10_diff1": -0.149193, + "nauc_map_at_20_max": 0.216543, + "nauc_map_at_20_std": 0.104694, + "nauc_map_at_20_diff1": -0.145622, + "nauc_map_at_100_max": 0.267026, + "nauc_map_at_100_std": 0.176993, + "nauc_map_at_100_diff1": -0.103046, + "nauc_map_at_1000_max": 0.220098, + "nauc_map_at_1000_std": 0.131004, + "nauc_map_at_1000_diff1": -0.07829, + "nauc_recall_at_1_max": 0.023922, + "nauc_recall_at_1_std": -0.099324, + "nauc_recall_at_1_diff1": -0.003338, + "nauc_recall_at_3_max": 0.138594, + "nauc_recall_at_3_std": 0.073026, + "nauc_recall_at_3_diff1": -0.219043, + "nauc_recall_at_5_max": 0.101394, + "nauc_recall_at_5_std": 0.036645, + "nauc_recall_at_5_diff1": -0.088174, + "nauc_recall_at_10_max": 0.199729, + "nauc_recall_at_10_std": 0.132433, + "nauc_recall_at_10_diff1": -0.141739, + "nauc_recall_at_20_max": 0.226967, + "nauc_recall_at_20_std": 0.146526, + "nauc_recall_at_20_diff1": -0.155191, + "nauc_recall_at_100_max": 0.327415, + "nauc_recall_at_100_std": 0.233226, + "nauc_recall_at_100_diff1": -0.063753, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.004184, + "nauc_precision_at_1_std": -0.164655, + "nauc_precision_at_1_diff1": 0.028425, + "nauc_precision_at_3_max": 0.112017, + "nauc_precision_at_3_std": 0.044898, + "nauc_precision_at_3_diff1": -0.244043, + "nauc_precision_at_5_max": 0.173857, + "nauc_precision_at_5_std": 0.139364, + "nauc_precision_at_5_diff1": -0.165854, + "nauc_precision_at_10_max": 0.272408, + "nauc_precision_at_10_std": 0.256062, + "nauc_precision_at_10_diff1": -0.152896, + "nauc_precision_at_20_max": 0.32864, + "nauc_precision_at_20_std": 0.273467, + "nauc_precision_at_20_diff1": -0.089354, + "nauc_precision_at_100_max": 0.106296, + "nauc_precision_at_100_std": 0.109022, + "nauc_precision_at_100_diff1": 0.100056, + "nauc_precision_at_1000_max": -0.138593, + "nauc_precision_at_1000_std": -0.134807, + "nauc_precision_at_1000_diff1": 0.151301, + "nauc_cv_recall_at_1_max": -0.004184, + "nauc_cv_recall_at_1_std": -0.164655, + "nauc_cv_recall_at_1_diff1": 0.028425, + "nauc_cv_recall_at_3_max": 0.172502, + "nauc_cv_recall_at_3_std": 0.153149, + "nauc_cv_recall_at_3_diff1": -0.226535, + "nauc_cv_recall_at_5_max": 0.238005, + "nauc_cv_recall_at_5_std": 0.27172, + "nauc_cv_recall_at_5_diff1": -0.174098, + "nauc_cv_recall_at_10_max": 0.380018, + "nauc_cv_recall_at_10_std": 0.381808, + "nauc_cv_recall_at_10_diff1": -0.087516, + "nauc_cv_recall_at_20_max": 0.39104, + "nauc_cv_recall_at_20_std": 0.444117, + "nauc_cv_recall_at_20_diff1": -0.205331, + "nauc_cv_recall_at_100_max": -0.155051, + "nauc_cv_recall_at_100_std": -0.341612, + "nauc_cv_recall_at_100_diff1": 0.192505, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.004184, + "nauc_mrr_at_1_std": -0.164655, + "nauc_mrr_at_1_diff1": 0.028425, + "nauc_mrr_at_3_max": 0.075006, + "nauc_mrr_at_3_std": -0.016395, + "nauc_mrr_at_3_diff1": -0.061698, + "nauc_mrr_at_5_max": 0.092734, + "nauc_mrr_at_5_std": 0.011888, + "nauc_mrr_at_5_diff1": -0.040861, + "nauc_mrr_at_10_max": 0.111226, + "nauc_mrr_at_10_std": 0.022622, + "nauc_mrr_at_10_diff1": -0.030096, + "nauc_mrr_at_20_max": 0.106367, + "nauc_mrr_at_20_std": 0.018416, + "nauc_mrr_at_20_diff1": -0.034384, + "nauc_mrr_at_100_max": 0.100429, + "nauc_mrr_at_100_std": 0.009273, + "nauc_mrr_at_100_diff1": -0.029427, + "nauc_mrr_at_1000_max": 0.100886, + "nauc_mrr_at_1000_std": 0.009826, + "nauc_mrr_at_1000_diff1": -0.029448, + "main_score": 0.21891, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.55172, + "ndcg_at_3": 0.54325, + "ndcg_at_5": 0.5217, + "ndcg_at_10": 0.52584, + "ndcg_at_20": 0.5496, + "ndcg_at_100": 0.65527, + "ndcg_at_1000": 0.69524, + "map_at_1": 0.05811, + "map_at_3": 0.14922, + "map_at_5": 0.20238, + "map_at_10": 0.275, + "map_at_20": 0.32746, + "map_at_100": 0.41291, + "map_at_1000": 0.43008, + "recall_at_1": 0.05811, + "recall_at_3": 0.18856, + "recall_at_5": 0.2879, + "recall_at_10": 0.4427, + "recall_at_20": 0.58832, + "recall_at_100": 0.89442, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.55172, + "cv_recall_at_3": 0.81034, + "cv_recall_at_5": 0.89655, + "cv_recall_at_10": 0.96552, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.55172, + "precision_at_3": 0.52874, + "precision_at_5": 0.47241, + "precision_at_10": 0.39655, + "precision_at_20": 0.29741, + "precision_at_100": 0.12379, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.551724, + "mrr_at_3": 0.672414, + "mrr_at_5": 0.691379, + "mrr_at_10": 0.702463, + "mrr_at_20": 0.705045, + "mrr_at_100": 0.705045, + "mrr_at_1000": 0.705045, + "nauc_ndcg_at_1_max": -0.289438, + "nauc_ndcg_at_1_std": -0.218144, + "nauc_ndcg_at_1_diff1": 0.140358, + "nauc_ndcg_at_3_max": -0.067005, + "nauc_ndcg_at_3_std": 0.05441, + "nauc_ndcg_at_3_diff1": -0.010138, + "nauc_ndcg_at_5_max": -0.128996, + "nauc_ndcg_at_5_std": -0.044523, + "nauc_ndcg_at_5_diff1": 0.030752, + "nauc_ndcg_at_10_max": -0.179051, + "nauc_ndcg_at_10_std": -0.077349, + "nauc_ndcg_at_10_diff1": 0.037327, + "nauc_ndcg_at_20_max": -0.106924, + "nauc_ndcg_at_20_std": -0.035843, + "nauc_ndcg_at_20_diff1": 0.001125, + "nauc_ndcg_at_100_max": -0.047187, + "nauc_ndcg_at_100_std": 0.116164, + "nauc_ndcg_at_100_diff1": -0.060653, + "nauc_ndcg_at_1000_max": -0.064489, + "nauc_ndcg_at_1000_std": 0.077317, + "nauc_ndcg_at_1000_diff1": -0.022415, + "nauc_map_at_1_max": -0.169807, + "nauc_map_at_1_std": -0.105269, + "nauc_map_at_1_diff1": 0.081972, + "nauc_map_at_3_max": -0.041361, + "nauc_map_at_3_std": 0.002435, + "nauc_map_at_3_diff1": -0.003212, + "nauc_map_at_5_max": -0.009439, + "nauc_map_at_5_std": 0.006109, + "nauc_map_at_5_diff1": 0.039741, + "nauc_map_at_10_max": -0.059297, + "nauc_map_at_10_std": 0.012568, + "nauc_map_at_10_diff1": -0.028718, + "nauc_map_at_20_max": -0.054764, + "nauc_map_at_20_std": 0.028308, + "nauc_map_at_20_diff1": -0.054917, + "nauc_map_at_100_max": -0.063987, + "nauc_map_at_100_std": 0.077295, + "nauc_map_at_100_diff1": -0.081188, + "nauc_map_at_1000_max": -0.072102, + "nauc_map_at_1000_std": 0.057408, + "nauc_map_at_1000_diff1": -0.057805, + "nauc_recall_at_1_max": -0.169807, + "nauc_recall_at_1_std": -0.105269, + "nauc_recall_at_1_diff1": 0.081972, + "nauc_recall_at_3_max": 0.017116, + "nauc_recall_at_3_std": 0.00176, + "nauc_recall_at_3_diff1": -0.014583, + "nauc_recall_at_5_max": -0.053167, + "nauc_recall_at_5_std": -0.099906, + "nauc_recall_at_5_diff1": 0.077042, + "nauc_recall_at_10_max": -0.144177, + "nauc_recall_at_10_std": -0.146937, + "nauc_recall_at_10_diff1": -0.000232, + "nauc_recall_at_20_max": -0.074555, + "nauc_recall_at_20_std": -0.111883, + "nauc_recall_at_20_diff1": 0.044814, + "nauc_recall_at_100_max": 0.072708, + "nauc_recall_at_100_std": 0.178096, + "nauc_recall_at_100_diff1": -0.043012, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.289438, + "nauc_precision_at_1_std": -0.218144, + "nauc_precision_at_1_diff1": 0.140358, + "nauc_precision_at_3_max": 0.002321, + "nauc_precision_at_3_std": 0.151697, + "nauc_precision_at_3_diff1": -0.05603, + "nauc_precision_at_5_max": -0.059261, + "nauc_precision_at_5_std": 0.045878, + "nauc_precision_at_5_diff1": -0.038064, + "nauc_precision_at_10_max": -0.066538, + "nauc_precision_at_10_std": 0.056436, + "nauc_precision_at_10_diff1": -0.060023, + "nauc_precision_at_20_max": -0.04209, + "nauc_precision_at_20_std": 0.040247, + "nauc_precision_at_20_diff1": -0.048776, + "nauc_precision_at_100_max": -0.081268, + "nauc_precision_at_100_std": -0.022902, + "nauc_precision_at_100_diff1": 0.020354, + "nauc_precision_at_1000_max": -0.075792, + "nauc_precision_at_1000_std": -0.061838, + "nauc_precision_at_1000_diff1": 0.079419, + "nauc_cv_recall_at_1_max": -0.289438, + "nauc_cv_recall_at_1_std": -0.218144, + "nauc_cv_recall_at_1_diff1": 0.140358, + "nauc_cv_recall_at_3_max": 0.347749, + "nauc_cv_recall_at_3_std": 0.484301, + "nauc_cv_recall_at_3_diff1": 0.075407, + "nauc_cv_recall_at_5_max": 0.062392, + "nauc_cv_recall_at_5_std": 0.13852, + "nauc_cv_recall_at_5_diff1": -0.078972, + "nauc_cv_recall_at_10_max": -0.373606, + "nauc_cv_recall_at_10_std": -0.574298, + "nauc_cv_recall_at_10_diff1": 0.14724, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.289438, + "nauc_mrr_at_1_std": -0.218144, + "nauc_mrr_at_1_diff1": 0.140358, + "nauc_mrr_at_3_max": -0.101041, + "nauc_mrr_at_3_std": -0.002743, + "nauc_mrr_at_3_diff1": 0.110378, + "nauc_mrr_at_5_max": -0.157518, + "nauc_mrr_at_5_std": -0.067795, + "nauc_mrr_at_5_diff1": 0.09404, + "nauc_mrr_at_10_max": -0.179769, + "nauc_mrr_at_10_std": -0.096336, + "nauc_mrr_at_10_diff1": 0.107631, + "nauc_mrr_at_20_max": -0.178086, + "nauc_mrr_at_20_std": -0.091002, + "nauc_mrr_at_20_diff1": 0.105464, + "nauc_mrr_at_100_max": -0.178086, + "nauc_mrr_at_100_std": -0.091002, + "nauc_mrr_at_100_diff1": 0.105464, + "nauc_mrr_at_1000_max": -0.178086, + "nauc_mrr_at_1000_std": -0.091002, + "nauc_mrr_at_1000_diff1": 0.105464, + "main_score": 0.5217, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.12069, + "ndcg_at_3": 0.1191, + "ndcg_at_5": 0.13598, + "ndcg_at_10": 0.15608, + "ndcg_at_20": 0.18638, + "ndcg_at_100": 0.29049, + "ndcg_at_1000": 0.43918, + "map_at_1": 0.0162, + "map_at_3": 0.03892, + "map_at_5": 0.05484, + "map_at_10": 0.07084, + "map_at_20": 0.08821, + "map_at_100": 0.12351, + "map_at_1000": 0.15144, + "recall_at_1": 0.0162, + "recall_at_3": 0.05746, + "recall_at_5": 0.09207, + "recall_at_10": 0.14033, + "recall_at_20": 0.22868, + "recall_at_100": 0.51234, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.12069, + "cv_recall_at_3": 0.24138, + "cv_recall_at_5": 0.43103, + "cv_recall_at_10": 0.5, + "cv_recall_at_20": 0.7069, + "cv_recall_at_100": 0.86207, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.12069, + "precision_at_3": 0.11494, + "precision_at_5": 0.13448, + "precision_at_10": 0.12759, + "precision_at_20": 0.11034, + "precision_at_100": 0.06517, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.12069, + "mrr_at_3": 0.166667, + "mrr_at_5": 0.20977, + "mrr_at_10": 0.219178, + "mrr_at_20": 0.233584, + "mrr_at_100": 0.236566, + "mrr_at_1000": 0.23743, + "nauc_ndcg_at_1_max": 0.305034, + "nauc_ndcg_at_1_std": 0.277205, + "nauc_ndcg_at_1_diff1": 0.406184, + "nauc_ndcg_at_3_max": 0.262351, + "nauc_ndcg_at_3_std": 0.234158, + "nauc_ndcg_at_3_diff1": 0.427673, + "nauc_ndcg_at_5_max": 0.225834, + "nauc_ndcg_at_5_std": 0.213536, + "nauc_ndcg_at_5_diff1": 0.339903, + "nauc_ndcg_at_10_max": 0.20876, + "nauc_ndcg_at_10_std": 0.22053, + "nauc_ndcg_at_10_diff1": 0.282209, + "nauc_ndcg_at_20_max": 0.222509, + "nauc_ndcg_at_20_std": 0.245968, + "nauc_ndcg_at_20_diff1": 0.271367, + "nauc_ndcg_at_100_max": 0.260911, + "nauc_ndcg_at_100_std": 0.364595, + "nauc_ndcg_at_100_diff1": 0.202568, + "nauc_ndcg_at_1000_max": 0.193914, + "nauc_ndcg_at_1000_std": 0.223251, + "nauc_ndcg_at_1000_diff1": 0.329213, + "nauc_map_at_1_max": 0.220247, + "nauc_map_at_1_std": 0.285059, + "nauc_map_at_1_diff1": 0.324472, + "nauc_map_at_3_max": 0.087339, + "nauc_map_at_3_std": 0.153532, + "nauc_map_at_3_diff1": 0.196188, + "nauc_map_at_5_max": 0.097604, + "nauc_map_at_5_std": 0.166088, + "nauc_map_at_5_diff1": 0.22827, + "nauc_map_at_10_max": 0.108636, + "nauc_map_at_10_std": 0.176488, + "nauc_map_at_10_diff1": 0.234598, + "nauc_map_at_20_max": 0.117714, + "nauc_map_at_20_std": 0.177098, + "nauc_map_at_20_diff1": 0.252189, + "nauc_map_at_100_max": 0.174657, + "nauc_map_at_100_std": 0.2586, + "nauc_map_at_100_diff1": 0.23368, + "nauc_map_at_1000_max": 0.171998, + "nauc_map_at_1000_std": 0.223119, + "nauc_map_at_1000_diff1": 0.291315, + "nauc_recall_at_1_max": 0.220247, + "nauc_recall_at_1_std": 0.285059, + "nauc_recall_at_1_diff1": 0.324472, + "nauc_recall_at_3_max": -0.012746, + "nauc_recall_at_3_std": 0.01881, + "nauc_recall_at_3_diff1": 0.061331, + "nauc_recall_at_5_max": 0.052761, + "nauc_recall_at_5_std": 0.098572, + "nauc_recall_at_5_diff1": 0.082057, + "nauc_recall_at_10_max": 0.065282, + "nauc_recall_at_10_std": 0.117244, + "nauc_recall_at_10_diff1": 0.089338, + "nauc_recall_at_20_max": 0.164924, + "nauc_recall_at_20_std": 0.215856, + "nauc_recall_at_20_diff1": 0.055645, + "nauc_recall_at_100_max": 0.284742, + "nauc_recall_at_100_std": 0.431384, + "nauc_recall_at_100_diff1": 0.061462, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.305034, + "nauc_precision_at_1_std": 0.277205, + "nauc_precision_at_1_diff1": 0.406184, + "nauc_precision_at_3_max": 0.277833, + "nauc_precision_at_3_std": 0.257743, + "nauc_precision_at_3_diff1": 0.47783, + "nauc_precision_at_5_max": 0.255351, + "nauc_precision_at_5_std": 0.254956, + "nauc_precision_at_5_diff1": 0.353406, + "nauc_precision_at_10_max": 0.249364, + "nauc_precision_at_10_std": 0.251872, + "nauc_precision_at_10_diff1": 0.28102, + "nauc_precision_at_20_max": 0.248292, + "nauc_precision_at_20_std": 0.236292, + "nauc_precision_at_20_diff1": 0.344348, + "nauc_precision_at_100_max": 0.208421, + "nauc_precision_at_100_std": 0.224883, + "nauc_precision_at_100_diff1": 0.273833, + "nauc_precision_at_1000_max": 0.039504, + "nauc_precision_at_1000_std": -0.056462, + "nauc_precision_at_1000_diff1": 0.312188, + "nauc_cv_recall_at_1_max": 0.305034, + "nauc_cv_recall_at_1_std": 0.277205, + "nauc_cv_recall_at_1_diff1": 0.406184, + "nauc_cv_recall_at_3_max": 0.223091, + "nauc_cv_recall_at_3_std": 0.20238, + "nauc_cv_recall_at_3_diff1": 0.436988, + "nauc_cv_recall_at_5_max": 0.291603, + "nauc_cv_recall_at_5_std": 0.320592, + "nauc_cv_recall_at_5_diff1": 0.193135, + "nauc_cv_recall_at_10_max": 0.26416, + "nauc_cv_recall_at_10_std": 0.301165, + "nauc_cv_recall_at_10_diff1": 0.125872, + "nauc_cv_recall_at_20_max": 0.407074, + "nauc_cv_recall_at_20_std": 0.466177, + "nauc_cv_recall_at_20_diff1": 0.167172, + "nauc_cv_recall_at_100_max": 0.243071, + "nauc_cv_recall_at_100_std": 0.281843, + "nauc_cv_recall_at_100_diff1": 0.693317, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.305034, + "nauc_mrr_at_1_std": 0.277205, + "nauc_mrr_at_1_diff1": 0.406184, + "nauc_mrr_at_3_max": 0.259127, + "nauc_mrr_at_3_std": 0.231619, + "nauc_mrr_at_3_diff1": 0.412614, + "nauc_mrr_at_5_max": 0.271561, + "nauc_mrr_at_5_std": 0.263468, + "nauc_mrr_at_5_diff1": 0.337274, + "nauc_mrr_at_10_max": 0.270876, + "nauc_mrr_at_10_std": 0.264376, + "nauc_mrr_at_10_diff1": 0.326489, + "nauc_mrr_at_20_max": 0.277236, + "nauc_mrr_at_20_std": 0.272121, + "nauc_mrr_at_20_diff1": 0.336904, + "nauc_mrr_at_100_max": 0.274448, + "nauc_mrr_at_100_std": 0.268327, + "nauc_mrr_at_100_diff1": 0.339255, + "nauc_mrr_at_1000_max": 0.274267, + "nauc_mrr_at_1000_std": 0.268023, + "nauc_mrr_at_1000_diff1": 0.337848, + "main_score": 0.13598, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 553.2975287437439, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..208d0ffd7b --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", + "task_name": "VidoreArxivQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.664, + "ndcg_at_3": 0.71728, + "ndcg_at_5": 0.72864, + "ndcg_at_10": 0.74933, + "ndcg_at_20": 0.76007, + "ndcg_at_100": 0.7756, + "ndcg_at_1000": 0.77875, + "map_at_1": 0.664, + "map_at_3": 0.70333, + "map_at_5": 0.70953, + "map_at_10": 0.71807, + "map_at_20": 0.72108, + "map_at_100": 0.72321, + "map_at_1000": 0.72334, + "recall_at_1": 0.664, + "recall_at_3": 0.758, + "recall_at_5": 0.786, + "recall_at_10": 0.85, + "recall_at_20": 0.892, + "recall_at_100": 0.976, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.664, + "cv_recall_at_3": 0.758, + "cv_recall_at_5": 0.786, + "cv_recall_at_10": 0.85, + "cv_recall_at_20": 0.892, + "cv_recall_at_100": 0.976, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.664, + "precision_at_3": 0.25267, + "precision_at_5": 0.1572, + "precision_at_10": 0.085, + "precision_at_20": 0.0446, + "precision_at_100": 0.00976, + "precision_at_1000": 0.001, + "mrr_at_1": 0.664, + "mrr_at_3": 0.703333, + "mrr_at_5": 0.709533, + "mrr_at_10": 0.718066, + "mrr_at_20": 0.721081, + "mrr_at_100": 0.723213, + "mrr_at_1000": 0.723343, + "nauc_ndcg_at_1_max": 0.611811, + "nauc_ndcg_at_1_std": 0.004607, + "nauc_ndcg_at_1_diff1": 0.914314, + "nauc_ndcg_at_3_max": 0.636503, + "nauc_ndcg_at_3_std": 0.034246, + "nauc_ndcg_at_3_diff1": 0.889811, + "nauc_ndcg_at_5_max": 0.640787, + "nauc_ndcg_at_5_std": 0.038142, + "nauc_ndcg_at_5_diff1": 0.881488, + "nauc_ndcg_at_10_max": 0.655205, + "nauc_ndcg_at_10_std": 0.057542, + "nauc_ndcg_at_10_diff1": 0.878909, + "nauc_ndcg_at_20_max": 0.650285, + "nauc_ndcg_at_20_std": 0.047323, + "nauc_ndcg_at_20_diff1": 0.877326, + "nauc_ndcg_at_100_max": 0.645084, + "nauc_ndcg_at_100_std": 0.04755, + "nauc_ndcg_at_100_diff1": 0.883361, + "nauc_ndcg_at_1000_max": 0.64177, + "nauc_ndcg_at_1000_std": 0.041445, + "nauc_ndcg_at_1000_diff1": 0.884931, + "nauc_map_at_1_max": 0.611811, + "nauc_map_at_1_std": 0.004607, + "nauc_map_at_1_diff1": 0.914314, + "nauc_map_at_3_max": 0.629806, + "nauc_map_at_3_std": 0.026865, + "nauc_map_at_3_diff1": 0.895277, + "nauc_map_at_5_max": 0.631702, + "nauc_map_at_5_std": 0.028434, + "nauc_map_at_5_diff1": 0.891045, + "nauc_map_at_10_max": 0.636751, + "nauc_map_at_10_std": 0.035402, + "nauc_map_at_10_diff1": 0.890027, + "nauc_map_at_20_max": 0.635589, + "nauc_map_at_20_std": 0.032916, + "nauc_map_at_20_diff1": 0.889782, + "nauc_map_at_100_max": 0.634975, + "nauc_map_at_100_std": 0.033029, + "nauc_map_at_100_diff1": 0.890366, + "nauc_map_at_1000_max": 0.634868, + "nauc_map_at_1000_std": 0.032872, + "nauc_map_at_1000_diff1": 0.890412, + "nauc_recall_at_1_max": 0.611811, + "nauc_recall_at_1_std": 0.004607, + "nauc_recall_at_1_diff1": 0.914314, + "nauc_recall_at_3_max": 0.659143, + "nauc_recall_at_3_std": 0.05889, + "nauc_recall_at_3_diff1": 0.871892, + "nauc_recall_at_5_max": 0.675177, + "nauc_recall_at_5_std": 0.074577, + "nauc_recall_at_5_diff1": 0.845571, + "nauc_recall_at_10_max": 0.756668, + "nauc_recall_at_10_std": 0.179935, + "nauc_recall_at_10_diff1": 0.822909, + "nauc_recall_at_20_max": 0.749441, + "nauc_recall_at_20_std": 0.139129, + "nauc_recall_at_20_diff1": 0.791223, + "nauc_recall_at_100_max": 0.849829, + "nauc_recall_at_100_std": 0.442227, + "nauc_recall_at_100_diff1": 0.784664, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.611811, + "nauc_precision_at_1_std": 0.004607, + "nauc_precision_at_1_diff1": 0.914314, + "nauc_precision_at_3_max": 0.659143, + "nauc_precision_at_3_std": 0.05889, + "nauc_precision_at_3_diff1": 0.871892, + "nauc_precision_at_5_max": 0.675177, + "nauc_precision_at_5_std": 0.074577, + "nauc_precision_at_5_diff1": 0.845571, + "nauc_precision_at_10_max": 0.756668, + "nauc_precision_at_10_std": 0.179935, + "nauc_precision_at_10_diff1": 0.822909, + "nauc_precision_at_20_max": 0.749441, + "nauc_precision_at_20_std": 0.139129, + "nauc_precision_at_20_diff1": 0.791223, + "nauc_precision_at_100_max": 0.849829, + "nauc_precision_at_100_std": 0.442227, + "nauc_precision_at_100_diff1": 0.784664, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_cv_recall_at_1_max": 0.611811, + "nauc_cv_recall_at_1_std": 0.004607, + "nauc_cv_recall_at_1_diff1": 0.914314, + "nauc_cv_recall_at_3_max": 0.659143, + "nauc_cv_recall_at_3_std": 0.05889, + "nauc_cv_recall_at_3_diff1": 0.871892, + "nauc_cv_recall_at_5_max": 0.675177, + "nauc_cv_recall_at_5_std": 0.074577, + "nauc_cv_recall_at_5_diff1": 0.845571, + "nauc_cv_recall_at_10_max": 0.756668, + "nauc_cv_recall_at_10_std": 0.179935, + "nauc_cv_recall_at_10_diff1": 0.822909, + "nauc_cv_recall_at_20_max": 0.749441, + "nauc_cv_recall_at_20_std": 0.139129, + "nauc_cv_recall_at_20_diff1": 0.791223, + "nauc_cv_recall_at_100_max": 0.849829, + "nauc_cv_recall_at_100_std": 0.442227, + "nauc_cv_recall_at_100_diff1": 0.784664, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.611811, + "nauc_mrr_at_1_std": 0.004607, + "nauc_mrr_at_1_diff1": 0.914314, + "nauc_mrr_at_3_max": 0.629806, + "nauc_mrr_at_3_std": 0.026865, + "nauc_mrr_at_3_diff1": 0.895277, + "nauc_mrr_at_5_max": 0.631702, + "nauc_mrr_at_5_std": 0.028434, + "nauc_mrr_at_5_diff1": 0.891045, + "nauc_mrr_at_10_max": 0.636751, + "nauc_mrr_at_10_std": 0.035402, + "nauc_mrr_at_10_diff1": 0.890027, + "nauc_mrr_at_20_max": 0.635589, + "nauc_mrr_at_20_std": 0.032916, + "nauc_mrr_at_20_diff1": 0.889782, + "nauc_mrr_at_100_max": 0.634975, + "nauc_mrr_at_100_std": 0.033029, + "nauc_mrr_at_100_diff1": 0.890366, + "nauc_mrr_at_1000_max": 0.634868, + "nauc_mrr_at_1000_std": 0.032872, + "nauc_mrr_at_1000_diff1": 0.890412, + "main_score": 0.72864, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 143.66198325157166, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..bf1e0b5b1f --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", + "task_name": "VidoreDocVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.4745, + "ndcg_at_3": 0.53941, + "ndcg_at_5": 0.57063, + "ndcg_at_10": 0.58752, + "ndcg_at_20": 0.60664, + "ndcg_at_100": 0.62571, + "ndcg_at_1000": 0.64161, + "map_at_1": 0.47339, + "map_at_3": 0.52214, + "map_at_5": 0.53999, + "map_at_10": 0.54698, + "map_at_20": 0.55261, + "map_at_100": 0.55527, + "map_at_1000": 0.55596, + "recall_at_1": 0.47339, + "recall_at_3": 0.58581, + "recall_at_5": 0.6612, + "recall_at_10": 0.71279, + "recall_at_20": 0.784, + "recall_at_100": 0.88383, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.4745, + "cv_recall_at_3": 0.59202, + "cv_recall_at_5": 0.66519, + "cv_recall_at_10": 0.71619, + "cv_recall_at_20": 0.78936, + "cv_recall_at_100": 0.89357, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.4745, + "precision_at_3": 0.19734, + "precision_at_5": 0.13392, + "precision_at_10": 0.07295, + "precision_at_20": 0.04091, + "precision_at_100": 0.00938, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.474501, + "mrr_at_3": 0.525129, + "mrr_at_5": 0.541981, + "mrr_at_10": 0.548542, + "mrr_at_20": 0.55376, + "mrr_at_100": 0.556379, + "mrr_at_1000": 0.556872, + "nauc_ndcg_at_1_max": 0.248287, + "nauc_ndcg_at_1_std": 0.354907, + "nauc_ndcg_at_1_diff1": 0.879106, + "nauc_ndcg_at_3_max": 0.224594, + "nauc_ndcg_at_3_std": 0.362387, + "nauc_ndcg_at_3_diff1": 0.841466, + "nauc_ndcg_at_5_max": 0.210751, + "nauc_ndcg_at_5_std": 0.402634, + "nauc_ndcg_at_5_diff1": 0.83609, + "nauc_ndcg_at_10_max": 0.190577, + "nauc_ndcg_at_10_std": 0.406, + "nauc_ndcg_at_10_diff1": 0.828997, + "nauc_ndcg_at_20_max": 0.171539, + "nauc_ndcg_at_20_std": 0.408228, + "nauc_ndcg_at_20_diff1": 0.831251, + "nauc_ndcg_at_100_max": 0.181168, + "nauc_ndcg_at_100_std": 0.422588, + "nauc_ndcg_at_100_diff1": 0.83432, + "nauc_ndcg_at_1000_max": 0.192686, + "nauc_ndcg_at_1000_std": 0.398287, + "nauc_ndcg_at_1000_diff1": 0.837132, + "nauc_map_at_1_max": 0.247653, + "nauc_map_at_1_std": 0.353, + "nauc_map_at_1_diff1": 0.881032, + "nauc_map_at_3_max": 0.231224, + "nauc_map_at_3_std": 0.360996, + "nauc_map_at_3_diff1": 0.85215, + "nauc_map_at_5_max": 0.223934, + "nauc_map_at_5_std": 0.382468, + "nauc_map_at_5_diff1": 0.848707, + "nauc_map_at_10_max": 0.215996, + "nauc_map_at_10_std": 0.38308, + "nauc_map_at_10_diff1": 0.845876, + "nauc_map_at_20_max": 0.211004, + "nauc_map_at_20_std": 0.383066, + "nauc_map_at_20_diff1": 0.846567, + "nauc_map_at_100_max": 0.211961, + "nauc_map_at_100_std": 0.385639, + "nauc_map_at_100_diff1": 0.84698, + "nauc_map_at_1000_max": 0.212325, + "nauc_map_at_1000_std": 0.384641, + "nauc_map_at_1000_diff1": 0.847003, + "nauc_recall_at_1_max": 0.247653, + "nauc_recall_at_1_std": 0.353, + "nauc_recall_at_1_diff1": 0.881032, + "nauc_recall_at_3_max": 0.209318, + "nauc_recall_at_3_std": 0.366623, + "nauc_recall_at_3_diff1": 0.812581, + "nauc_recall_at_5_max": 0.166752, + "nauc_recall_at_5_std": 0.477906, + "nauc_recall_at_5_diff1": 0.794718, + "nauc_recall_at_10_max": 0.086652, + "nauc_recall_at_10_std": 0.503355, + "nauc_recall_at_10_diff1": 0.763621, + "nauc_recall_at_20_max": -0.040558, + "nauc_recall_at_20_std": 0.544083, + "nauc_recall_at_20_diff1": 0.763645, + "nauc_recall_at_100_max": -0.041577, + "nauc_recall_at_100_std": 0.814584, + "nauc_recall_at_100_diff1": 0.760481, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.248287, + "nauc_precision_at_1_std": 0.354907, + "nauc_precision_at_1_diff1": 0.879106, + "nauc_precision_at_3_max": 0.199257, + "nauc_precision_at_3_std": 0.364778, + "nauc_precision_at_3_diff1": 0.806688, + "nauc_precision_at_5_max": 0.148615, + "nauc_precision_at_5_std": 0.466637, + "nauc_precision_at_5_diff1": 0.760914, + "nauc_precision_at_10_max": 0.043977, + "nauc_precision_at_10_std": 0.438896, + "nauc_precision_at_10_diff1": 0.660425, + "nauc_precision_at_20_max": -0.125145, + "nauc_precision_at_20_std": 0.400356, + "nauc_precision_at_20_diff1": 0.531093, + "nauc_precision_at_100_max": -0.208873, + "nauc_precision_at_100_std": 0.358136, + "nauc_precision_at_100_diff1": 0.257515, + "nauc_precision_at_1000_max": -0.309706, + "nauc_precision_at_1000_std": -0.428274, + "nauc_precision_at_1000_diff1": -0.479815, + "nauc_cv_recall_at_1_max": 0.248287, + "nauc_cv_recall_at_1_std": 0.354907, + "nauc_cv_recall_at_1_diff1": 0.879106, + "nauc_cv_recall_at_3_max": 0.199257, + "nauc_cv_recall_at_3_std": 0.364778, + "nauc_cv_recall_at_3_diff1": 0.806688, + "nauc_cv_recall_at_5_max": 0.157132, + "nauc_cv_recall_at_5_std": 0.473735, + "nauc_cv_recall_at_5_diff1": 0.793576, + "nauc_cv_recall_at_10_max": 0.077427, + "nauc_cv_recall_at_10_std": 0.500672, + "nauc_cv_recall_at_10_diff1": 0.763704, + "nauc_cv_recall_at_20_max": -0.060612, + "nauc_cv_recall_at_20_std": 0.536132, + "nauc_cv_recall_at_20_diff1": 0.754676, + "nauc_cv_recall_at_100_max": -0.075757, + "nauc_cv_recall_at_100_std": 0.791491, + "nauc_cv_recall_at_100_diff1": 0.735497, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.248287, + "nauc_mrr_at_1_std": 0.354907, + "nauc_mrr_at_1_diff1": 0.879106, + "nauc_mrr_at_3_max": 0.228156, + "nauc_mrr_at_3_std": 0.36181, + "nauc_mrr_at_3_diff1": 0.848598, + "nauc_mrr_at_5_max": 0.221681, + "nauc_mrr_at_5_std": 0.38219, + "nauc_mrr_at_5_diff1": 0.846978, + "nauc_mrr_at_10_max": 0.214537, + "nauc_mrr_at_10_std": 0.383485, + "nauc_mrr_at_10_diff1": 0.844464, + "nauc_mrr_at_20_max": 0.210427, + "nauc_mrr_at_20_std": 0.383401, + "nauc_mrr_at_20_diff1": 0.844932, + "nauc_mrr_at_100_max": 0.211589, + "nauc_mrr_at_100_std": 0.385709, + "nauc_mrr_at_100_diff1": 0.845387, + "nauc_mrr_at_1000_max": 0.2121, + "nauc_mrr_at_1000_std": 0.385114, + "nauc_mrr_at_1000_diff1": 0.845558, + "main_score": 0.57063, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 173.05761194229126, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..5f0ec2a9da --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", + "task_name": "VidoreInfoVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.78947, + "ndcg_at_3": 0.83866, + "ndcg_at_5": 0.84667, + "ndcg_at_10": 0.85515, + "ndcg_at_20": 0.86452, + "ndcg_at_100": 0.86879, + "ndcg_at_1000": 0.8721, + "map_at_1": 0.78846, + "map_at_3": 0.82692, + "map_at_5": 0.83117, + "map_at_10": 0.83466, + "map_at_20": 0.83729, + "map_at_100": 0.83799, + "map_at_1000": 0.83814, + "recall_at_1": 0.78846, + "recall_at_3": 0.87247, + "recall_at_5": 0.89271, + "recall_at_10": 0.91903, + "recall_at_20": 0.95486, + "recall_at_100": 0.97591, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.78947, + "cv_recall_at_3": 0.87247, + "cv_recall_at_5": 0.89271, + "cv_recall_at_10": 0.91903, + "cv_recall_at_20": 0.95749, + "cv_recall_at_100": 0.97773, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.78947, + "precision_at_3": 0.2915, + "precision_at_5": 0.17895, + "precision_at_10": 0.09211, + "precision_at_20": 0.04798, + "precision_at_100": 0.00984, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.789474, + "mrr_at_3": 0.826923, + "mrr_at_5": 0.831174, + "mrr_at_10": 0.834656, + "mrr_at_20": 0.837459, + "mrr_at_100": 0.838118, + "mrr_at_1000": 0.838252, + "nauc_ndcg_at_1_max": 0.536193, + "nauc_ndcg_at_1_std": 0.045634, + "nauc_ndcg_at_1_diff1": 0.889737, + "nauc_ndcg_at_3_max": 0.551942, + "nauc_ndcg_at_3_std": 0.034953, + "nauc_ndcg_at_3_diff1": 0.868162, + "nauc_ndcg_at_5_max": 0.558609, + "nauc_ndcg_at_5_std": 0.053876, + "nauc_ndcg_at_5_diff1": 0.862961, + "nauc_ndcg_at_10_max": 0.548847, + "nauc_ndcg_at_10_std": 0.04019, + "nauc_ndcg_at_10_diff1": 0.864481, + "nauc_ndcg_at_20_max": 0.564886, + "nauc_ndcg_at_20_std": 0.07312, + "nauc_ndcg_at_20_diff1": 0.869353, + "nauc_ndcg_at_100_max": 0.562027, + "nauc_ndcg_at_100_std": 0.071897, + "nauc_ndcg_at_100_diff1": 0.871188, + "nauc_ndcg_at_1000_max": 0.554703, + "nauc_ndcg_at_1000_std": 0.053632, + "nauc_ndcg_at_1000_diff1": 0.870971, + "nauc_map_at_1_max": 0.537059, + "nauc_map_at_1_std": 0.039498, + "nauc_map_at_1_diff1": 0.891589, + "nauc_map_at_3_max": 0.548735, + "nauc_map_at_3_std": 0.036621, + "nauc_map_at_3_diff1": 0.873529, + "nauc_map_at_5_max": 0.551553, + "nauc_map_at_5_std": 0.045412, + "nauc_map_at_5_diff1": 0.871019, + "nauc_map_at_10_max": 0.547252, + "nauc_map_at_10_std": 0.038925, + "nauc_map_at_10_diff1": 0.871721, + "nauc_map_at_20_max": 0.55141, + "nauc_map_at_20_std": 0.04752, + "nauc_map_at_20_diff1": 0.873034, + "nauc_map_at_100_max": 0.550928, + "nauc_map_at_100_std": 0.046819, + "nauc_map_at_100_diff1": 0.873297, + "nauc_map_at_1000_max": 0.550621, + "nauc_map_at_1000_std": 0.046094, + "nauc_map_at_1000_diff1": 0.873297, + "nauc_recall_at_1_max": 0.537059, + "nauc_recall_at_1_std": 0.039498, + "nauc_recall_at_1_diff1": 0.891589, + "nauc_recall_at_3_max": 0.563379, + "nauc_recall_at_3_std": 0.029393, + "nauc_recall_at_3_diff1": 0.848275, + "nauc_recall_at_5_max": 0.590557, + "nauc_recall_at_5_std": 0.096899, + "nauc_recall_at_5_diff1": 0.827599, + "nauc_recall_at_10_max": 0.55522, + "nauc_recall_at_10_std": 0.051765, + "nauc_recall_at_10_diff1": 0.824814, + "nauc_recall_at_20_max": 0.734799, + "nauc_recall_at_20_std": 0.41441, + "nauc_recall_at_20_diff1": 0.847973, + "nauc_recall_at_100_max": 0.820362, + "nauc_recall_at_100_std": 0.730764, + "nauc_recall_at_100_diff1": 0.879207, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.536193, + "nauc_precision_at_1_std": 0.045634, + "nauc_precision_at_1_diff1": 0.889737, + "nauc_precision_at_3_max": 0.538955, + "nauc_precision_at_3_std": 0.046867, + "nauc_precision_at_3_diff1": 0.808871, + "nauc_precision_at_5_max": 0.561344, + "nauc_precision_at_5_std": 0.115004, + "nauc_precision_at_5_diff1": 0.782543, + "nauc_precision_at_10_max": 0.51875, + "nauc_precision_at_10_std": 0.077166, + "nauc_precision_at_10_diff1": 0.766405, + "nauc_precision_at_20_max": 0.641807, + "nauc_precision_at_20_std": 0.419045, + "nauc_precision_at_20_diff1": 0.733177, + "nauc_precision_at_100_max": 0.392665, + "nauc_precision_at_100_std": 0.455018, + "nauc_precision_at_100_diff1": 0.408348, + "nauc_precision_at_1000_max": -0.518427, + "nauc_precision_at_1000_std": -0.284086, + "nauc_precision_at_1000_diff1": -0.56103, + "nauc_cv_recall_at_1_max": 0.536193, + "nauc_cv_recall_at_1_std": 0.045634, + "nauc_cv_recall_at_1_diff1": 0.889737, + "nauc_cv_recall_at_3_max": 0.563379, + "nauc_cv_recall_at_3_std": 0.029393, + "nauc_cv_recall_at_3_diff1": 0.848275, + "nauc_cv_recall_at_5_max": 0.590557, + "nauc_cv_recall_at_5_std": 0.096899, + "nauc_cv_recall_at_5_diff1": 0.827599, + "nauc_cv_recall_at_10_max": 0.55522, + "nauc_cv_recall_at_10_std": 0.051765, + "nauc_cv_recall_at_10_diff1": 0.824814, + "nauc_cv_recall_at_20_max": 0.721491, + "nauc_cv_recall_at_20_std": 0.399048, + "nauc_cv_recall_at_20_diff1": 0.841672, + "nauc_cv_recall_at_100_max": 0.8116, + "nauc_cv_recall_at_100_std": 0.748615, + "nauc_cv_recall_at_100_diff1": 0.87526, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.536193, + "nauc_mrr_at_1_std": 0.045634, + "nauc_mrr_at_1_diff1": 0.889737, + "nauc_mrr_at_3_max": 0.548735, + "nauc_mrr_at_3_std": 0.036621, + "nauc_mrr_at_3_diff1": 0.873529, + "nauc_mrr_at_5_max": 0.551553, + "nauc_mrr_at_5_std": 0.045412, + "nauc_mrr_at_5_diff1": 0.871019, + "nauc_mrr_at_10_max": 0.547252, + "nauc_mrr_at_10_std": 0.038925, + "nauc_mrr_at_10_diff1": 0.871721, + "nauc_mrr_at_20_max": 0.550981, + "nauc_mrr_at_20_std": 0.046859, + "nauc_mrr_at_20_diff1": 0.872951, + "nauc_mrr_at_100_max": 0.550601, + "nauc_mrr_at_100_std": 0.046373, + "nauc_mrr_at_100_diff1": 0.873243, + "nauc_mrr_at_1000_max": 0.550346, + "nauc_mrr_at_1000_std": 0.045729, + "nauc_mrr_at_1000_diff1": 0.873253, + "main_score": 0.84667, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 127.96320414543152, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..f3060ef734 --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "84a382e05c4473fed9cff2bbae95fe2379416117", + "task_name": "VidoreShiftProjectRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.42, + "ndcg_at_3": 0.58595, + "ndcg_at_5": 0.61865, + "ndcg_at_10": 0.65502, + "ndcg_at_20": 0.66265, + "ndcg_at_100": 0.67222, + "ndcg_at_1000": 0.67516, + "map_at_1": 0.42, + "map_at_3": 0.54333, + "map_at_5": 0.56133, + "map_at_10": 0.57683, + "map_at_20": 0.57895, + "map_at_100": 0.58036, + "map_at_1000": 0.58054, + "recall_at_1": 0.42, + "recall_at_3": 0.71, + "recall_at_5": 0.79, + "recall_at_10": 0.9, + "recall_at_20": 0.93, + "recall_at_100": 0.98, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.42, + "cv_recall_at_3": 0.71, + "cv_recall_at_5": 0.79, + "cv_recall_at_10": 0.9, + "cv_recall_at_20": 0.93, + "cv_recall_at_100": 0.98, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.42, + "precision_at_3": 0.23667, + "precision_at_5": 0.158, + "precision_at_10": 0.09, + "precision_at_20": 0.0465, + "precision_at_100": 0.0098, + "precision_at_1000": 0.001, + "mrr_at_1": 0.42, + "mrr_at_3": 0.543333, + "mrr_at_5": 0.561333, + "mrr_at_10": 0.576825, + "mrr_at_20": 0.578949, + "mrr_at_100": 0.580362, + "mrr_at_1000": 0.580542, + "nauc_ndcg_at_1_max": 0.111426, + "nauc_ndcg_at_1_std": -0.051002, + "nauc_ndcg_at_1_diff1": 0.611323, + "nauc_ndcg_at_3_max": 0.0761, + "nauc_ndcg_at_3_std": -0.098556, + "nauc_ndcg_at_3_diff1": 0.568481, + "nauc_ndcg_at_5_max": 0.075435, + "nauc_ndcg_at_5_std": -0.106925, + "nauc_ndcg_at_5_diff1": 0.543971, + "nauc_ndcg_at_10_max": 0.101624, + "nauc_ndcg_at_10_std": -0.095496, + "nauc_ndcg_at_10_diff1": 0.553051, + "nauc_ndcg_at_20_max": 0.087854, + "nauc_ndcg_at_20_std": -0.093918, + "nauc_ndcg_at_20_diff1": 0.549252, + "nauc_ndcg_at_100_max": 0.087665, + "nauc_ndcg_at_100_std": -0.089865, + "nauc_ndcg_at_100_diff1": 0.569238, + "nauc_ndcg_at_1000_max": 0.091757, + "nauc_ndcg_at_1000_std": -0.088079, + "nauc_ndcg_at_1000_diff1": 0.567431, + "nauc_map_at_1_max": 0.111426, + "nauc_map_at_1_std": -0.051002, + "nauc_map_at_1_diff1": 0.611323, + "nauc_map_at_3_max": 0.087955, + "nauc_map_at_3_std": -0.085339, + "nauc_map_at_3_diff1": 0.581156, + "nauc_map_at_5_max": 0.087463, + "nauc_map_at_5_std": -0.089841, + "nauc_map_at_5_diff1": 0.569985, + "nauc_map_at_10_max": 0.098223, + "nauc_map_at_10_std": -0.083043, + "nauc_map_at_10_diff1": 0.574368, + "nauc_map_at_20_max": 0.095202, + "nauc_map_at_20_std": -0.082342, + "nauc_map_at_20_diff1": 0.573947, + "nauc_map_at_100_max": 0.09476, + "nauc_map_at_100_std": -0.082567, + "nauc_map_at_100_diff1": 0.575922, + "nauc_map_at_1000_max": 0.094979, + "nauc_map_at_1000_std": -0.08246, + "nauc_map_at_1000_diff1": 0.575847, + "nauc_recall_at_1_max": 0.111426, + "nauc_recall_at_1_std": -0.051002, + "nauc_recall_at_1_diff1": 0.611323, + "nauc_recall_at_3_max": 0.031818, + "nauc_recall_at_3_std": -0.146767, + "nauc_recall_at_3_diff1": 0.521858, + "nauc_recall_at_5_max": 0.020631, + "nauc_recall_at_5_std": -0.184697, + "nauc_recall_at_5_diff1": 0.418592, + "nauc_recall_at_10_max": 0.15098, + "nauc_recall_at_10_std": -0.203221, + "nauc_recall_at_10_diff1": 0.370075, + "nauc_recall_at_20_max": -0.017807, + "nauc_recall_at_20_std": -0.236495, + "nauc_recall_at_20_diff1": 0.231493, + "nauc_recall_at_100_max": -0.220355, + "nauc_recall_at_100_std": -0.220355, + "nauc_recall_at_100_diff1": 0.711718, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.111426, + "nauc_precision_at_1_std": -0.051002, + "nauc_precision_at_1_diff1": 0.611323, + "nauc_precision_at_3_max": 0.031818, + "nauc_precision_at_3_std": -0.146767, + "nauc_precision_at_3_diff1": 0.521858, + "nauc_precision_at_5_max": 0.020631, + "nauc_precision_at_5_std": -0.184697, + "nauc_precision_at_5_diff1": 0.418592, + "nauc_precision_at_10_max": 0.15098, + "nauc_precision_at_10_std": -0.203221, + "nauc_precision_at_10_diff1": 0.370075, + "nauc_precision_at_20_max": -0.017807, + "nauc_precision_at_20_std": -0.236495, + "nauc_precision_at_20_diff1": 0.231493, + "nauc_precision_at_100_max": -0.220355, + "nauc_precision_at_100_std": -0.220355, + "nauc_precision_at_100_diff1": 0.711718, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.111426, + "nauc_cv_recall_at_1_std": -0.051002, + "nauc_cv_recall_at_1_diff1": 0.611323, + "nauc_cv_recall_at_3_max": 0.031818, + "nauc_cv_recall_at_3_std": -0.146767, + "nauc_cv_recall_at_3_diff1": 0.521858, + "nauc_cv_recall_at_5_max": 0.020631, + "nauc_cv_recall_at_5_std": -0.184697, + "nauc_cv_recall_at_5_diff1": 0.418592, + "nauc_cv_recall_at_10_max": 0.15098, + "nauc_cv_recall_at_10_std": -0.203221, + "nauc_cv_recall_at_10_diff1": 0.370075, + "nauc_cv_recall_at_20_max": -0.017807, + "nauc_cv_recall_at_20_std": -0.236495, + "nauc_cv_recall_at_20_diff1": 0.231493, + "nauc_cv_recall_at_100_max": -0.220355, + "nauc_cv_recall_at_100_std": -0.220355, + "nauc_cv_recall_at_100_diff1": 0.711718, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.111426, + "nauc_mrr_at_1_std": -0.051002, + "nauc_mrr_at_1_diff1": 0.611323, + "nauc_mrr_at_3_max": 0.087955, + "nauc_mrr_at_3_std": -0.085339, + "nauc_mrr_at_3_diff1": 0.581156, + "nauc_mrr_at_5_max": 0.087463, + "nauc_mrr_at_5_std": -0.089841, + "nauc_mrr_at_5_diff1": 0.569985, + "nauc_mrr_at_10_max": 0.098223, + "nauc_mrr_at_10_std": -0.083043, + "nauc_mrr_at_10_diff1": 0.574368, + "nauc_mrr_at_20_max": 0.095202, + "nauc_mrr_at_20_std": -0.082342, + "nauc_mrr_at_20_diff1": 0.573947, + "nauc_mrr_at_100_max": 0.09476, + "nauc_mrr_at_100_std": -0.082567, + "nauc_mrr_at_100_diff1": 0.575922, + "nauc_mrr_at_1000_max": 0.094979, + "nauc_mrr_at_1000_std": -0.08246, + "nauc_mrr_at_1000_diff1": 0.575847, + "main_score": 0.61865, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 291.00110912323, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..7705627a38 --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", + "task_name": "VidoreSyntheticDocQAAIRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.93, + "ndcg_at_3": 0.96024, + "ndcg_at_5": 0.96454, + "ndcg_at_10": 0.96454, + "ndcg_at_20": 0.96454, + "ndcg_at_100": 0.96608, + "ndcg_at_1000": 0.96608, + "map_at_1": 0.93, + "map_at_3": 0.95333, + "map_at_5": 0.95583, + "map_at_10": 0.95583, + "map_at_20": 0.95583, + "map_at_100": 0.95595, + "map_at_1000": 0.95595, + "recall_at_1": 0.93, + "recall_at_3": 0.98, + "recall_at_5": 0.99, + "recall_at_10": 0.99, + "recall_at_20": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.93, + "cv_recall_at_3": 0.98, + "cv_recall_at_5": 0.99, + "cv_recall_at_10": 0.99, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.93, + "precision_at_3": 0.32667, + "precision_at_5": 0.198, + "precision_at_10": 0.099, + "precision_at_20": 0.0495, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.93, + "mrr_at_3": 0.953333, + "mrr_at_5": 0.955833, + "mrr_at_10": 0.955833, + "mrr_at_20": 0.955833, + "mrr_at_100": 0.955946, + "mrr_at_1000": 0.955946, + "nauc_ndcg_at_1_max": 0.688209, + "nauc_ndcg_at_1_std": 0.189276, + "nauc_ndcg_at_1_diff1": 0.917634, + "nauc_ndcg_at_3_max": 0.746791, + "nauc_ndcg_at_3_std": 0.295491, + "nauc_ndcg_at_3_diff1": 0.854999, + "nauc_ndcg_at_5_max": 0.731913, + "nauc_ndcg_at_5_std": 0.209916, + "nauc_ndcg_at_5_diff1": 0.891542, + "nauc_ndcg_at_10_max": 0.731913, + "nauc_ndcg_at_10_std": 0.209916, + "nauc_ndcg_at_10_diff1": 0.891542, + "nauc_ndcg_at_20_max": 0.731913, + "nauc_ndcg_at_20_std": 0.209916, + "nauc_ndcg_at_20_diff1": 0.891542, + "nauc_ndcg_at_100_max": 0.725674, + "nauc_ndcg_at_100_std": 0.213874, + "nauc_ndcg_at_100_diff1": 0.892553, + "nauc_ndcg_at_1000_max": 0.725674, + "nauc_ndcg_at_1000_std": 0.213874, + "nauc_ndcg_at_1000_diff1": 0.892553, + "nauc_map_at_1_max": 0.688209, + "nauc_map_at_1_std": 0.189276, + "nauc_map_at_1_diff1": 0.917634, + "nauc_map_at_3_max": 0.728225, + "nauc_map_at_3_std": 0.256136, + "nauc_map_at_3_diff1": 0.876451, + "nauc_map_at_5_max": 0.72024, + "nauc_map_at_5_std": 0.21403, + "nauc_map_at_5_diff1": 0.894694, + "nauc_map_at_10_max": 0.72024, + "nauc_map_at_10_std": 0.21403, + "nauc_map_at_10_diff1": 0.894694, + "nauc_map_at_20_max": 0.72024, + "nauc_map_at_20_std": 0.21403, + "nauc_map_at_20_diff1": 0.894694, + "nauc_map_at_100_max": 0.71986, + "nauc_map_at_100_std": 0.214263, + "nauc_map_at_100_diff1": 0.894759, + "nauc_map_at_1000_max": 0.71986, + "nauc_map_at_1000_std": 0.214263, + "nauc_map_at_1000_diff1": 0.894759, + "nauc_recall_at_1_max": 0.688209, + "nauc_recall_at_1_std": 0.189276, + "nauc_recall_at_1_diff1": 0.917634, + "nauc_recall_at_3_max": 0.869281, + "nauc_recall_at_3_std": 0.561391, + "nauc_recall_at_3_diff1": 0.711718, + "nauc_recall_at_5_max": 0.869281, + "nauc_recall_at_5_std": 0.122782, + "nauc_recall_at_5_diff1": 0.869281, + "nauc_recall_at_10_max": 0.869281, + "nauc_recall_at_10_std": 0.122782, + "nauc_recall_at_10_diff1": 0.869281, + "nauc_recall_at_20_max": 0.869281, + "nauc_recall_at_20_std": 0.122782, + "nauc_recall_at_20_diff1": 0.869281, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.688209, + "nauc_precision_at_1_std": 0.189276, + "nauc_precision_at_1_diff1": 0.917634, + "nauc_precision_at_3_max": 0.869281, + "nauc_precision_at_3_std": 0.561391, + "nauc_precision_at_3_diff1": 0.711718, + "nauc_precision_at_5_max": 0.869281, + "nauc_precision_at_5_std": 0.122782, + "nauc_precision_at_5_diff1": 0.869281, + "nauc_precision_at_10_max": 0.869281, + "nauc_precision_at_10_std": 0.122782, + "nauc_precision_at_10_diff1": 0.869281, + "nauc_precision_at_20_max": 0.869281, + "nauc_precision_at_20_std": 0.122782, + "nauc_precision_at_20_diff1": 0.869281, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.688209, + "nauc_cv_recall_at_1_std": 0.189276, + "nauc_cv_recall_at_1_diff1": 0.917634, + "nauc_cv_recall_at_3_max": 0.869281, + "nauc_cv_recall_at_3_std": 0.561391, + "nauc_cv_recall_at_3_diff1": 0.711718, + "nauc_cv_recall_at_5_max": 0.869281, + "nauc_cv_recall_at_5_std": 0.122782, + "nauc_cv_recall_at_5_diff1": 0.869281, + "nauc_cv_recall_at_10_max": 0.869281, + "nauc_cv_recall_at_10_std": 0.122782, + "nauc_cv_recall_at_10_diff1": 0.869281, + "nauc_cv_recall_at_20_max": 0.869281, + "nauc_cv_recall_at_20_std": 0.122782, + "nauc_cv_recall_at_20_diff1": 0.869281, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.688209, + "nauc_mrr_at_1_std": 0.189276, + "nauc_mrr_at_1_diff1": 0.917634, + "nauc_mrr_at_3_max": 0.728225, + "nauc_mrr_at_3_std": 0.256136, + "nauc_mrr_at_3_diff1": 0.876451, + "nauc_mrr_at_5_max": 0.72024, + "nauc_mrr_at_5_std": 0.21403, + "nauc_mrr_at_5_diff1": 0.894694, + "nauc_mrr_at_10_max": 0.72024, + "nauc_mrr_at_10_std": 0.21403, + "nauc_mrr_at_10_diff1": 0.894694, + "nauc_mrr_at_20_max": 0.72024, + "nauc_mrr_at_20_std": 0.21403, + "nauc_mrr_at_20_diff1": 0.894694, + "nauc_mrr_at_100_max": 0.71986, + "nauc_mrr_at_100_std": 0.214263, + "nauc_mrr_at_100_diff1": 0.894759, + "nauc_mrr_at_1000_max": 0.71986, + "nauc_mrr_at_1000_std": 0.214263, + "nauc_mrr_at_1000_diff1": 0.894759, + "main_score": 0.96454, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 334.5020217895508, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..7188c328d8 --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", + "task_name": "VidoreSyntheticDocQAEnergyRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.9, + "ndcg_at_3": 0.92893, + "ndcg_at_5": 0.93323, + "ndcg_at_10": 0.93613, + "ndcg_at_20": 0.94118, + "ndcg_at_100": 0.94322, + "ndcg_at_1000": 0.94322, + "map_at_1": 0.9, + "map_at_3": 0.92167, + "map_at_5": 0.92417, + "map_at_10": 0.92517, + "map_at_20": 0.92655, + "map_at_100": 0.92689, + "map_at_1000": 0.92689, + "recall_at_1": 0.9, + "recall_at_3": 0.95, + "recall_at_5": 0.96, + "recall_at_10": 0.97, + "recall_at_20": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.9, + "cv_recall_at_3": 0.95, + "cv_recall_at_5": 0.96, + "cv_recall_at_10": 0.97, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.9, + "precision_at_3": 0.31667, + "precision_at_5": 0.192, + "precision_at_10": 0.097, + "precision_at_20": 0.0495, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.9, + "mrr_at_3": 0.921667, + "mrr_at_5": 0.924167, + "mrr_at_10": 0.925167, + "mrr_at_20": 0.926548, + "mrr_at_100": 0.926892, + "mrr_at_1000": 0.926892, + "nauc_ndcg_at_1_max": 0.544258, + "nauc_ndcg_at_1_std": -0.514332, + "nauc_ndcg_at_1_diff1": 0.886788, + "nauc_ndcg_at_3_max": 0.619191, + "nauc_ndcg_at_3_std": -0.379375, + "nauc_ndcg_at_3_diff1": 0.891616, + "nauc_ndcg_at_5_max": 0.603058, + "nauc_ndcg_at_5_std": -0.459921, + "nauc_ndcg_at_5_diff1": 0.884625, + "nauc_ndcg_at_10_max": 0.614145, + "nauc_ndcg_at_10_std": -0.401992, + "nauc_ndcg_at_10_diff1": 0.879403, + "nauc_ndcg_at_20_max": 0.586641, + "nauc_ndcg_at_20_std": -0.435606, + "nauc_ndcg_at_20_diff1": 0.881118, + "nauc_ndcg_at_100_max": 0.587807, + "nauc_ndcg_at_100_std": -0.445092, + "nauc_ndcg_at_100_diff1": 0.881542, + "nauc_ndcg_at_1000_max": 0.587807, + "nauc_ndcg_at_1000_std": -0.445092, + "nauc_ndcg_at_1000_diff1": 0.881542, + "nauc_map_at_1_max": 0.544258, + "nauc_map_at_1_std": -0.514332, + "nauc_map_at_1_diff1": 0.886788, + "nauc_map_at_3_max": 0.594831, + "nauc_map_at_3_std": -0.417297, + "nauc_map_at_3_diff1": 0.886267, + "nauc_map_at_5_max": 0.585783, + "nauc_map_at_5_std": -0.459712, + "nauc_map_at_5_diff1": 0.882517, + "nauc_map_at_10_max": 0.588826, + "nauc_map_at_10_std": -0.442604, + "nauc_map_at_10_diff1": 0.880947, + "nauc_map_at_20_max": 0.582367, + "nauc_map_at_20_std": -0.450475, + "nauc_map_at_20_diff1": 0.88141, + "nauc_map_at_100_max": 0.5825, + "nauc_map_at_100_std": -0.451792, + "nauc_map_at_100_diff1": 0.881467, + "nauc_map_at_1000_max": 0.5825, + "nauc_map_at_1000_std": -0.451792, + "nauc_map_at_1000_diff1": 0.881467, + "nauc_recall_at_1_max": 0.544258, + "nauc_recall_at_1_std": -0.514332, + "nauc_recall_at_1_diff1": 0.886788, + "nauc_recall_at_3_max": 0.730159, + "nauc_recall_at_3_std": -0.210271, + "nauc_recall_at_3_diff1": 0.918301, + "nauc_recall_at_5_max": 0.695378, + "nauc_recall_at_5_std": -0.480159, + "nauc_recall_at_5_diff1": 0.897876, + "nauc_recall_at_10_max": 0.807812, + "nauc_recall_at_10_std": -0.060224, + "nauc_recall_at_10_diff1": 0.863834, + "nauc_recall_at_20_max": 0.554155, + "nauc_recall_at_20_std": -0.171335, + "nauc_recall_at_20_diff1": 0.869281, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.544258, + "nauc_precision_at_1_std": -0.514332, + "nauc_precision_at_1_diff1": 0.886788, + "nauc_precision_at_3_max": 0.730159, + "nauc_precision_at_3_std": -0.210271, + "nauc_precision_at_3_diff1": 0.918301, + "nauc_precision_at_5_max": 0.695378, + "nauc_precision_at_5_std": -0.480159, + "nauc_precision_at_5_diff1": 0.897876, + "nauc_precision_at_10_max": 0.807812, + "nauc_precision_at_10_std": -0.060224, + "nauc_precision_at_10_diff1": 0.863834, + "nauc_precision_at_20_max": 0.554155, + "nauc_precision_at_20_std": -0.171335, + "nauc_precision_at_20_diff1": 0.869281, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.544258, + "nauc_cv_recall_at_1_std": -0.514332, + "nauc_cv_recall_at_1_diff1": 0.886788, + "nauc_cv_recall_at_3_max": 0.730159, + "nauc_cv_recall_at_3_std": -0.210271, + "nauc_cv_recall_at_3_diff1": 0.918301, + "nauc_cv_recall_at_5_max": 0.695378, + "nauc_cv_recall_at_5_std": -0.480159, + "nauc_cv_recall_at_5_diff1": 0.897876, + "nauc_cv_recall_at_10_max": 0.807812, + "nauc_cv_recall_at_10_std": -0.060224, + "nauc_cv_recall_at_10_diff1": 0.863834, + "nauc_cv_recall_at_20_max": 0.554155, + "nauc_cv_recall_at_20_std": -0.171335, + "nauc_cv_recall_at_20_diff1": 0.869281, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.544258, + "nauc_mrr_at_1_std": -0.514332, + "nauc_mrr_at_1_diff1": 0.886788, + "nauc_mrr_at_3_max": 0.594831, + "nauc_mrr_at_3_std": -0.417297, + "nauc_mrr_at_3_diff1": 0.886267, + "nauc_mrr_at_5_max": 0.585783, + "nauc_mrr_at_5_std": -0.459712, + "nauc_mrr_at_5_diff1": 0.882517, + "nauc_mrr_at_10_max": 0.588826, + "nauc_mrr_at_10_std": -0.442604, + "nauc_mrr_at_10_diff1": 0.880947, + "nauc_mrr_at_20_max": 0.582367, + "nauc_mrr_at_20_std": -0.450475, + "nauc_mrr_at_20_diff1": 0.88141, + "nauc_mrr_at_100_max": 0.5825, + "nauc_mrr_at_100_std": -0.451792, + "nauc_mrr_at_100_diff1": 0.881467, + "nauc_mrr_at_1000_max": 0.5825, + "nauc_mrr_at_1000_std": -0.451792, + "nauc_mrr_at_1000_diff1": 0.881467, + "main_score": 0.93323, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 335.38410663604736, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..0e8c0e2f31 --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b4909afa930f81282fd20601e860668073ad02aa", + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.86, + "ndcg_at_3": 0.91917, + "ndcg_at_5": 0.92778, + "ndcg_at_10": 0.9345, + "ndcg_at_20": 0.9345, + "ndcg_at_100": 0.9345, + "ndcg_at_1000": 0.9345, + "map_at_1": 0.86, + "map_at_3": 0.905, + "map_at_5": 0.91, + "map_at_10": 0.91292, + "map_at_20": 0.91292, + "map_at_100": 0.91292, + "map_at_1000": 0.91292, + "recall_at_1": 0.86, + "recall_at_3": 0.96, + "recall_at_5": 0.98, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.86, + "cv_recall_at_3": 0.96, + "cv_recall_at_5": 0.98, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.86, + "precision_at_3": 0.32, + "precision_at_5": 0.196, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.86, + "mrr_at_3": 0.905, + "mrr_at_5": 0.91, + "mrr_at_10": 0.912917, + "mrr_at_20": 0.912917, + "mrr_at_100": 0.912917, + "mrr_at_1000": 0.912917, + "nauc_ndcg_at_1_max": 0.483269, + "nauc_ndcg_at_1_std": -0.051476, + "nauc_ndcg_at_1_diff1": 0.867154, + "nauc_ndcg_at_3_max": 0.588709, + "nauc_ndcg_at_3_std": 0.116537, + "nauc_ndcg_at_3_diff1": 0.794446, + "nauc_ndcg_at_5_max": 0.57764, + "nauc_ndcg_at_5_std": 0.101355, + "nauc_ndcg_at_5_diff1": 0.816597, + "nauc_ndcg_at_10_max": 0.540341, + "nauc_ndcg_at_10_std": 0.028536, + "nauc_ndcg_at_10_diff1": 0.835984, + "nauc_ndcg_at_20_max": 0.540341, + "nauc_ndcg_at_20_std": 0.028536, + "nauc_ndcg_at_20_diff1": 0.835984, + "nauc_ndcg_at_100_max": 0.540341, + "nauc_ndcg_at_100_std": 0.028536, + "nauc_ndcg_at_100_diff1": 0.835984, + "nauc_ndcg_at_1000_max": 0.540341, + "nauc_ndcg_at_1000_std": 0.028536, + "nauc_ndcg_at_1000_diff1": 0.835984, + "nauc_map_at_1_max": 0.483269, + "nauc_map_at_1_std": -0.051476, + "nauc_map_at_1_diff1": 0.867154, + "nauc_map_at_3_max": 0.555435, + "nauc_map_at_3_std": 0.057658, + "nauc_map_at_3_diff1": 0.819765, + "nauc_map_at_5_max": 0.548363, + "nauc_map_at_5_std": 0.047201, + "nauc_map_at_5_diff1": 0.831557, + "nauc_map_at_10_max": 0.535347, + "nauc_map_at_10_std": 0.020899, + "nauc_map_at_10_diff1": 0.838648, + "nauc_map_at_20_max": 0.535347, + "nauc_map_at_20_std": 0.020899, + "nauc_map_at_20_diff1": 0.838648, + "nauc_map_at_100_max": 0.535347, + "nauc_map_at_100_std": 0.020899, + "nauc_map_at_100_diff1": 0.838648, + "nauc_map_at_1000_max": 0.535347, + "nauc_map_at_1000_std": 0.020899, + "nauc_map_at_1000_diff1": 0.838648, + "nauc_recall_at_1_max": 0.483269, + "nauc_recall_at_1_std": -0.051476, + "nauc_recall_at_1_diff1": 0.867154, + "nauc_recall_at_3_max": 0.806839, + "nauc_recall_at_3_std": 0.508754, + "nauc_recall_at_3_diff1": 0.625934, + "nauc_recall_at_5_max": 0.934641, + "nauc_recall_at_5_std": 0.777077, + "nauc_recall_at_5_diff1": 0.638189, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.483269, + "nauc_precision_at_1_std": -0.051476, + "nauc_precision_at_1_diff1": 0.867154, + "nauc_precision_at_3_max": 0.806839, + "nauc_precision_at_3_std": 0.508754, + "nauc_precision_at_3_diff1": 0.625934, + "nauc_precision_at_5_max": 0.934641, + "nauc_precision_at_5_std": 0.777077, + "nauc_precision_at_5_diff1": 0.638189, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.483269, + "nauc_cv_recall_at_1_std": -0.051476, + "nauc_cv_recall_at_1_diff1": 0.867154, + "nauc_cv_recall_at_3_max": 0.806839, + "nauc_cv_recall_at_3_std": 0.508754, + "nauc_cv_recall_at_3_diff1": 0.625934, + "nauc_cv_recall_at_5_max": 0.934641, + "nauc_cv_recall_at_5_std": 0.777077, + "nauc_cv_recall_at_5_diff1": 0.638189, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.483269, + "nauc_mrr_at_1_std": -0.051476, + "nauc_mrr_at_1_diff1": 0.867154, + "nauc_mrr_at_3_max": 0.555435, + "nauc_mrr_at_3_std": 0.057658, + "nauc_mrr_at_3_diff1": 0.819765, + "nauc_mrr_at_5_max": 0.548363, + "nauc_mrr_at_5_std": 0.047201, + "nauc_mrr_at_5_diff1": 0.831557, + "nauc_mrr_at_10_max": 0.535347, + "nauc_mrr_at_10_std": 0.020899, + "nauc_mrr_at_10_diff1": 0.838648, + "nauc_mrr_at_20_max": 0.535347, + "nauc_mrr_at_20_std": 0.020899, + "nauc_mrr_at_20_diff1": 0.838648, + "nauc_mrr_at_100_max": 0.535347, + "nauc_mrr_at_100_std": 0.020899, + "nauc_mrr_at_100_diff1": 0.838648, + "nauc_mrr_at_1000_max": 0.535347, + "nauc_mrr_at_1000_std": 0.020899, + "nauc_mrr_at_1000_diff1": 0.838648, + "main_score": 0.92778, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 335.1275272369385, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..2a96e83218 --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.91, + "ndcg_at_3": 0.96155, + "ndcg_at_5": 0.96155, + "ndcg_at_10": 0.96155, + "ndcg_at_20": 0.96155, + "ndcg_at_100": 0.96155, + "ndcg_at_1000": 0.96155, + "map_at_1": 0.91, + "map_at_3": 0.94833, + "map_at_5": 0.94833, + "map_at_10": 0.94833, + "map_at_20": 0.94833, + "map_at_100": 0.94833, + "map_at_1000": 0.94833, + "recall_at_1": 0.91, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.91, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.91, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.91, + "mrr_at_3": 0.948333, + "mrr_at_5": 0.948333, + "mrr_at_10": 0.948333, + "mrr_at_20": 0.948333, + "mrr_at_100": 0.948333, + "mrr_at_1000": 0.948333, + "nauc_ndcg_at_1_max": 0.685133, + "nauc_ndcg_at_1_std": 0.005291, + "nauc_ndcg_at_1_diff1": 0.927378, + "nauc_ndcg_at_3_max": 0.688198, + "nauc_ndcg_at_3_std": -0.040796, + "nauc_ndcg_at_3_diff1": 0.923917, + "nauc_ndcg_at_5_max": 0.688198, + "nauc_ndcg_at_5_std": -0.040796, + "nauc_ndcg_at_5_diff1": 0.923917, + "nauc_ndcg_at_10_max": 0.688198, + "nauc_ndcg_at_10_std": -0.040796, + "nauc_ndcg_at_10_diff1": 0.923917, + "nauc_ndcg_at_20_max": 0.688198, + "nauc_ndcg_at_20_std": -0.040796, + "nauc_ndcg_at_20_diff1": 0.923917, + "nauc_ndcg_at_100_max": 0.688198, + "nauc_ndcg_at_100_std": -0.040796, + "nauc_ndcg_at_100_diff1": 0.923917, + "nauc_ndcg_at_1000_max": 0.688198, + "nauc_ndcg_at_1000_std": -0.040796, + "nauc_ndcg_at_1000_diff1": 0.923917, + "nauc_map_at_1_max": 0.685133, + "nauc_map_at_1_std": 0.005291, + "nauc_map_at_1_diff1": 0.927378, + "nauc_map_at_3_max": 0.688037, + "nauc_map_at_3_std": -0.038372, + "nauc_map_at_3_diff1": 0.924099, + "nauc_map_at_5_max": 0.688037, + "nauc_map_at_5_std": -0.038372, + "nauc_map_at_5_diff1": 0.924099, + "nauc_map_at_10_max": 0.688037, + "nauc_map_at_10_std": -0.038372, + "nauc_map_at_10_diff1": 0.924099, + "nauc_map_at_20_max": 0.688037, + "nauc_map_at_20_std": -0.038372, + "nauc_map_at_20_diff1": 0.924099, + "nauc_map_at_100_max": 0.688037, + "nauc_map_at_100_std": -0.038372, + "nauc_map_at_100_diff1": 0.924099, + "nauc_map_at_1000_max": 0.688037, + "nauc_map_at_1000_std": -0.038372, + "nauc_map_at_1000_diff1": 0.924099, + "nauc_recall_at_1_max": 0.685133, + "nauc_recall_at_1_std": 0.005291, + "nauc_recall_at_1_diff1": 0.927378, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.685133, + "nauc_precision_at_1_std": 0.005291, + "nauc_precision_at_1_diff1": 0.927378, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.685133, + "nauc_cv_recall_at_1_std": 0.005291, + "nauc_cv_recall_at_1_diff1": 0.927378, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.685133, + "nauc_mrr_at_1_std": 0.005291, + "nauc_mrr_at_1_diff1": 0.927378, + "nauc_mrr_at_3_max": 0.688037, + "nauc_mrr_at_3_std": -0.038372, + "nauc_mrr_at_3_diff1": 0.924099, + "nauc_mrr_at_5_max": 0.688037, + "nauc_mrr_at_5_std": -0.038372, + "nauc_mrr_at_5_diff1": 0.924099, + "nauc_mrr_at_10_max": 0.688037, + "nauc_mrr_at_10_std": -0.038372, + "nauc_mrr_at_10_diff1": 0.924099, + "nauc_mrr_at_20_max": 0.688037, + "nauc_mrr_at_20_std": -0.038372, + "nauc_mrr_at_20_diff1": 0.924099, + "nauc_mrr_at_100_max": 0.688037, + "nauc_mrr_at_100_std": -0.038372, + "nauc_mrr_at_100_diff1": 0.924099, + "nauc_mrr_at_1000_max": 0.688037, + "nauc_mrr_at_1000_std": -0.038372, + "nauc_mrr_at_1000_diff1": 0.924099, + "main_score": 0.96155, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 320.76485085487366, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..3a4ede1175 --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", + "task_name": "VidoreTabfquadRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.55357, + "ndcg_at_3": 0.62411, + "ndcg_at_5": 0.65208, + "ndcg_at_10": 0.66514, + "ndcg_at_20": 0.68222, + "ndcg_at_100": 0.70918, + "ndcg_at_1000": 0.70918, + "map_at_1": 0.55357, + "map_at_3": 0.60655, + "map_at_5": 0.62208, + "map_at_10": 0.62702, + "map_at_20": 0.63167, + "map_at_100": 0.63525, + "map_at_1000": 0.63525, + "recall_at_1": 0.55357, + "recall_at_3": 0.675, + "recall_at_5": 0.74286, + "recall_at_10": 0.78571, + "recall_at_20": 0.85357, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.55357, + "cv_recall_at_3": 0.675, + "cv_recall_at_5": 0.74286, + "cv_recall_at_10": 0.78571, + "cv_recall_at_20": 0.85357, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.55357, + "precision_at_3": 0.225, + "precision_at_5": 0.14857, + "precision_at_10": 0.07857, + "precision_at_20": 0.04268, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.553571, + "mrr_at_3": 0.606548, + "mrr_at_5": 0.622083, + "mrr_at_10": 0.627018, + "mrr_at_20": 0.631669, + "mrr_at_100": 0.635248, + "mrr_at_1000": 0.635248, + "nauc_ndcg_at_1_max": 0.169618, + "nauc_ndcg_at_1_std": -0.006691, + "nauc_ndcg_at_1_diff1": 0.722159, + "nauc_ndcg_at_3_max": 0.194177, + "nauc_ndcg_at_3_std": 0.037506, + "nauc_ndcg_at_3_diff1": 0.67735, + "nauc_ndcg_at_5_max": 0.17545, + "nauc_ndcg_at_5_std": 0.02779, + "nauc_ndcg_at_5_diff1": 0.663072, + "nauc_ndcg_at_10_max": 0.163984, + "nauc_ndcg_at_10_std": 0.026986, + "nauc_ndcg_at_10_diff1": 0.651379, + "nauc_ndcg_at_20_max": 0.151583, + "nauc_ndcg_at_20_std": 0.013562, + "nauc_ndcg_at_20_diff1": 0.656537, + "nauc_ndcg_at_100_max": 0.168273, + "nauc_ndcg_at_100_std": 0.018422, + "nauc_ndcg_at_100_diff1": 0.671545, + "nauc_ndcg_at_1000_max": 0.168273, + "nauc_ndcg_at_1000_std": 0.018422, + "nauc_ndcg_at_1000_diff1": 0.671545, + "nauc_map_at_1_max": 0.169618, + "nauc_map_at_1_std": -0.006691, + "nauc_map_at_1_diff1": 0.722159, + "nauc_map_at_3_max": 0.188512, + "nauc_map_at_3_std": 0.028831, + "nauc_map_at_3_diff1": 0.688968, + "nauc_map_at_5_max": 0.178469, + "nauc_map_at_5_std": 0.023263, + "nauc_map_at_5_diff1": 0.681994, + "nauc_map_at_10_max": 0.174947, + "nauc_map_at_10_std": 0.023129, + "nauc_map_at_10_diff1": 0.678522, + "nauc_map_at_20_max": 0.171435, + "nauc_map_at_20_std": 0.019334, + "nauc_map_at_20_diff1": 0.679969, + "nauc_map_at_100_max": 0.173266, + "nauc_map_at_100_std": 0.019344, + "nauc_map_at_100_diff1": 0.68141, + "nauc_map_at_1000_max": 0.173266, + "nauc_map_at_1000_std": 0.019344, + "nauc_map_at_1000_diff1": 0.68141, + "nauc_recall_at_1_max": 0.169618, + "nauc_recall_at_1_std": -0.006691, + "nauc_recall_at_1_diff1": 0.722159, + "nauc_recall_at_3_max": 0.212182, + "nauc_recall_at_3_std": 0.06428, + "nauc_recall_at_3_diff1": 0.639752, + "nauc_recall_at_5_max": 0.160283, + "nauc_recall_at_5_std": 0.04164, + "nauc_recall_at_5_diff1": 0.589373, + "nauc_recall_at_10_max": 0.102832, + "nauc_recall_at_10_std": 0.038487, + "nauc_recall_at_10_diff1": 0.52235, + "nauc_recall_at_20_max": -0.001485, + "nauc_recall_at_20_std": -0.046221, + "nauc_recall_at_20_diff1": 0.509681, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.169618, + "nauc_precision_at_1_std": -0.006691, + "nauc_precision_at_1_diff1": 0.722159, + "nauc_precision_at_3_max": 0.212182, + "nauc_precision_at_3_std": 0.06428, + "nauc_precision_at_3_diff1": 0.639752, + "nauc_precision_at_5_max": 0.160283, + "nauc_precision_at_5_std": 0.04164, + "nauc_precision_at_5_diff1": 0.589373, + "nauc_precision_at_10_max": 0.102832, + "nauc_precision_at_10_std": 0.038487, + "nauc_precision_at_10_diff1": 0.52235, + "nauc_precision_at_20_max": -0.001485, + "nauc_precision_at_20_std": -0.046221, + "nauc_precision_at_20_diff1": 0.509681, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.169618, + "nauc_cv_recall_at_1_std": -0.006691, + "nauc_cv_recall_at_1_diff1": 0.722159, + "nauc_cv_recall_at_3_max": 0.212182, + "nauc_cv_recall_at_3_std": 0.06428, + "nauc_cv_recall_at_3_diff1": 0.639752, + "nauc_cv_recall_at_5_max": 0.160283, + "nauc_cv_recall_at_5_std": 0.04164, + "nauc_cv_recall_at_5_diff1": 0.589373, + "nauc_cv_recall_at_10_max": 0.102832, + "nauc_cv_recall_at_10_std": 0.038487, + "nauc_cv_recall_at_10_diff1": 0.52235, + "nauc_cv_recall_at_20_max": -0.001485, + "nauc_cv_recall_at_20_std": -0.046221, + "nauc_cv_recall_at_20_diff1": 0.509681, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.169618, + "nauc_mrr_at_1_std": -0.006691, + "nauc_mrr_at_1_diff1": 0.722159, + "nauc_mrr_at_3_max": 0.188512, + "nauc_mrr_at_3_std": 0.028831, + "nauc_mrr_at_3_diff1": 0.688968, + "nauc_mrr_at_5_max": 0.178469, + "nauc_mrr_at_5_std": 0.023263, + "nauc_mrr_at_5_diff1": 0.681994, + "nauc_mrr_at_10_max": 0.174947, + "nauc_mrr_at_10_std": 0.023129, + "nauc_mrr_at_10_diff1": 0.678522, + "nauc_mrr_at_20_max": 0.171435, + "nauc_mrr_at_20_std": 0.019334, + "nauc_mrr_at_20_diff1": 0.679969, + "nauc_mrr_at_100_max": 0.173266, + "nauc_mrr_at_100_std": 0.019344, + "nauc_mrr_at_100_diff1": 0.68141, + "nauc_mrr_at_1000_max": 0.173266, + "nauc_mrr_at_1000_std": 0.019344, + "nauc_mrr_at_1000_diff1": 0.68141, + "main_score": 0.65208, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 17.597821474075317, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..aff2f340cd --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", + "task_name": "VidoreTatdqaRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.64702, + "ndcg_at_3": 0.73624, + "ndcg_at_5": 0.76363, + "ndcg_at_10": 0.78415, + "ndcg_at_20": 0.79184, + "ndcg_at_100": 0.79787, + "ndcg_at_1000": 0.79947, + "map_at_1": 0.64581, + "map_at_3": 0.71441, + "map_at_5": 0.72979, + "map_at_10": 0.73854, + "map_at_20": 0.74066, + "map_at_100": 0.7415, + "map_at_1000": 0.74159, + "recall_at_1": 0.64581, + "recall_at_3": 0.7986, + "recall_at_5": 0.86428, + "recall_at_10": 0.92655, + "recall_at_20": 0.95693, + "recall_at_100": 0.989, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.64702, + "cv_recall_at_3": 0.79951, + "cv_recall_at_5": 0.86513, + "cv_recall_at_10": 0.9271, + "cv_recall_at_20": 0.95747, + "cv_recall_at_100": 0.98967, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.64702, + "precision_at_3": 0.26711, + "precision_at_5": 0.17363, + "precision_at_10": 0.09307, + "precision_at_20": 0.04806, + "precision_at_100": 0.00996, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.647023, + "mrr_at_3": 0.715067, + "mrr_at_5": 0.730346, + "mrr_at_10": 0.739007, + "mrr_at_20": 0.741123, + "mrr_at_100": 0.741967, + "mrr_at_1000": 0.742042, + "nauc_ndcg_at_1_max": 0.21364, + "nauc_ndcg_at_1_std": -0.260832, + "nauc_ndcg_at_1_diff1": 0.772919, + "nauc_ndcg_at_3_max": 0.216383, + "nauc_ndcg_at_3_std": -0.263178, + "nauc_ndcg_at_3_diff1": 0.739007, + "nauc_ndcg_at_5_max": 0.229064, + "nauc_ndcg_at_5_std": -0.247688, + "nauc_ndcg_at_5_diff1": 0.734617, + "nauc_ndcg_at_10_max": 0.233123, + "nauc_ndcg_at_10_std": -0.232368, + "nauc_ndcg_at_10_diff1": 0.734627, + "nauc_ndcg_at_20_max": 0.230332, + "nauc_ndcg_at_20_std": -0.225301, + "nauc_ndcg_at_20_diff1": 0.735396, + "nauc_ndcg_at_100_max": 0.230483, + "nauc_ndcg_at_100_std": -0.231544, + "nauc_ndcg_at_100_diff1": 0.741774, + "nauc_ndcg_at_1000_max": 0.225804, + "nauc_ndcg_at_1000_std": -0.241169, + "nauc_ndcg_at_1000_diff1": 0.742216, + "nauc_map_at_1_max": 0.21083, + "nauc_map_at_1_std": -0.262051, + "nauc_map_at_1_diff1": 0.773419, + "nauc_map_at_3_max": 0.214079, + "nauc_map_at_3_std": -0.26411, + "nauc_map_at_3_diff1": 0.748399, + "nauc_map_at_5_max": 0.220304, + "nauc_map_at_5_std": -0.256308, + "nauc_map_at_5_diff1": 0.746701, + "nauc_map_at_10_max": 0.221919, + "nauc_map_at_10_std": -0.250447, + "nauc_map_at_10_diff1": 0.7468, + "nauc_map_at_20_max": 0.221282, + "nauc_map_at_20_std": -0.248892, + "nauc_map_at_20_diff1": 0.746948, + "nauc_map_at_100_max": 0.221343, + "nauc_map_at_100_std": -0.249463, + "nauc_map_at_100_diff1": 0.747615, + "nauc_map_at_1000_max": 0.221121, + "nauc_map_at_1000_std": -0.24992, + "nauc_map_at_1000_diff1": 0.747626, + "nauc_recall_at_1_max": 0.21083, + "nauc_recall_at_1_std": -0.262051, + "nauc_recall_at_1_diff1": 0.773419, + "nauc_recall_at_3_max": 0.219715, + "nauc_recall_at_3_std": -0.264408, + "nauc_recall_at_3_diff1": 0.704704, + "nauc_recall_at_5_max": 0.272151, + "nauc_recall_at_5_std": -0.202578, + "nauc_recall_at_5_diff1": 0.673609, + "nauc_recall_at_10_max": 0.330599, + "nauc_recall_at_10_std": -0.060932, + "nauc_recall_at_10_diff1": 0.630817, + "nauc_recall_at_20_max": 0.349691, + "nauc_recall_at_20_std": 0.170518, + "nauc_recall_at_20_diff1": 0.575683, + "nauc_recall_at_100_max": 0.726978, + "nauc_recall_at_100_std": 0.793566, + "nauc_recall_at_100_diff1": 0.682123, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.21364, + "nauc_precision_at_1_std": -0.260832, + "nauc_precision_at_1_diff1": 0.772919, + "nauc_precision_at_3_max": 0.215106, + "nauc_precision_at_3_std": -0.260414, + "nauc_precision_at_3_diff1": 0.685683, + "nauc_precision_at_5_max": 0.259253, + "nauc_precision_at_5_std": -0.193954, + "nauc_precision_at_5_diff1": 0.629494, + "nauc_precision_at_10_max": 0.306628, + "nauc_precision_at_10_std": -0.046666, + "nauc_precision_at_10_diff1": 0.545698, + "nauc_precision_at_20_max": 0.308556, + "nauc_precision_at_20_std": 0.16622, + "nauc_precision_at_20_diff1": 0.444187, + "nauc_precision_at_100_max": 0.261231, + "nauc_precision_at_100_std": 0.332767, + "nauc_precision_at_100_diff1": 0.088921, + "nauc_precision_at_1000_max": -0.340818, + "nauc_precision_at_1000_std": -0.29837, + "nauc_precision_at_1000_diff1": -0.513805, + "nauc_cv_recall_at_1_max": 0.21364, + "nauc_cv_recall_at_1_std": -0.260832, + "nauc_cv_recall_at_1_diff1": 0.772919, + "nauc_cv_recall_at_3_max": 0.225229, + "nauc_cv_recall_at_3_std": -0.258783, + "nauc_cv_recall_at_3_diff1": 0.703489, + "nauc_cv_recall_at_5_max": 0.273552, + "nauc_cv_recall_at_5_std": -0.20387, + "nauc_cv_recall_at_5_diff1": 0.671944, + "nauc_cv_recall_at_10_max": 0.325578, + "nauc_cv_recall_at_10_std": -0.068889, + "nauc_cv_recall_at_10_diff1": 0.628048, + "nauc_cv_recall_at_20_max": 0.341329, + "nauc_cv_recall_at_20_std": 0.159853, + "nauc_cv_recall_at_20_diff1": 0.570228, + "nauc_cv_recall_at_100_max": 0.709311, + "nauc_cv_recall_at_100_std": 0.780208, + "nauc_cv_recall_at_100_diff1": 0.661554, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.21364, + "nauc_mrr_at_1_std": -0.260832, + "nauc_mrr_at_1_diff1": 0.772919, + "nauc_mrr_at_3_max": 0.218667, + "nauc_mrr_at_3_std": -0.259931, + "nauc_mrr_at_3_diff1": 0.747773, + "nauc_mrr_at_5_max": 0.223914, + "nauc_mrr_at_5_std": -0.253642, + "nauc_mrr_at_5_diff1": 0.746233, + "nauc_mrr_at_10_max": 0.224943, + "nauc_mrr_at_10_std": -0.248311, + "nauc_mrr_at_10_diff1": 0.746402, + "nauc_mrr_at_20_max": 0.224335, + "nauc_mrr_at_20_std": -0.246734, + "nauc_mrr_at_20_diff1": 0.746547, + "nauc_mrr_at_100_max": 0.224394, + "nauc_mrr_at_100_std": -0.247318, + "nauc_mrr_at_100_diff1": 0.747208, + "nauc_mrr_at_1000_max": 0.224244, + "nauc_mrr_at_1000_std": -0.247661, + "nauc_mrr_at_1000_diff1": 0.747243, + "main_score": 0.76363, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 101.59077453613281, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json new file mode 100644 index 0000000000..e9261691fe --- /dev/null +++ b/results/vidore__colSmol-256M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json @@ -0,0 +1 @@ +{"name": "vidore/colSmol-256M", "revision": "530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f", "release_date": "2025-01-22", "languages": ["eng-Latn"], "n_parameters": 256000000, "memory_usage_mb": 800.0, "max_tokens": 8192.0, "embed_dim": 128, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/illuin-tech/colpali", "public_training_data": "https://huggingface.co/datasets/vidore/colpali_train_set", "framework": ["ColPali"], "reference": "https://huggingface.co/vidore/colSmol-256M", "similarity_fn_name": "max_sim", "use_instructions": true, "training_datasets": {"DocVQA": ["train"], "InfoVQA": ["train"], "TATDQA": ["train"], "arXivQA": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "ColSmolWrapper"} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..636e4e184b --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.43125, + "ndcg_at_3": 0.42407, + "ndcg_at_5": 0.45107, + "ndcg_at_10": 0.48172, + "ndcg_at_20": 0.51957, + "ndcg_at_100": 0.56948, + "ndcg_at_1000": 0.59789, + "map_at_1": 0.27356, + "map_at_3": 0.34799, + "map_at_5": 0.37505, + "map_at_10": 0.3983, + "map_at_20": 0.41569, + "map_at_100": 0.42841, + "map_at_1000": 0.43089, + "recall_at_1": 0.27356, + "recall_at_3": 0.4033, + "recall_at_5": 0.48903, + "recall_at_10": 0.57132, + "recall_at_20": 0.68109, + "recall_at_100": 0.8568, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.43125, + "cv_recall_at_3": 0.61875, + "cv_recall_at_5": 0.71875, + "cv_recall_at_10": 0.76875, + "cv_recall_at_20": 0.85625, + "cv_recall_at_100": 0.95, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.43125, + "precision_at_3": 0.24167, + "precision_at_5": 0.18875, + "precision_at_10": 0.125, + "precision_at_20": 0.08188, + "precision_at_100": 0.02413, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.43125, + "mrr_at_3": 0.514583, + "mrr_at_5": 0.536458, + "mrr_at_10": 0.543251, + "mrr_at_20": 0.549771, + "mrr_at_100": 0.552833, + "mrr_at_1000": 0.553042, + "nauc_ndcg_at_1_max": 0.189977, + "nauc_ndcg_at_1_std": -0.105918, + "nauc_ndcg_at_1_diff1": 0.507559, + "nauc_ndcg_at_3_max": 0.231673, + "nauc_ndcg_at_3_std": -0.000578, + "nauc_ndcg_at_3_diff1": 0.487291, + "nauc_ndcg_at_5_max": 0.269636, + "nauc_ndcg_at_5_std": 0.034255, + "nauc_ndcg_at_5_diff1": 0.51545, + "nauc_ndcg_at_10_max": 0.254756, + "nauc_ndcg_at_10_std": 0.040128, + "nauc_ndcg_at_10_diff1": 0.494146, + "nauc_ndcg_at_20_max": 0.267001, + "nauc_ndcg_at_20_std": 0.033192, + "nauc_ndcg_at_20_diff1": 0.50129, + "nauc_ndcg_at_100_max": 0.255287, + "nauc_ndcg_at_100_std": 0.033227, + "nauc_ndcg_at_100_diff1": 0.495504, + "nauc_ndcg_at_1000_max": 0.234269, + "nauc_ndcg_at_1000_std": 0.002367, + "nauc_ndcg_at_1000_diff1": 0.483463, + "nauc_map_at_1_max": 0.240645, + "nauc_map_at_1_std": 0.008756, + "nauc_map_at_1_diff1": 0.529453, + "nauc_map_at_3_max": 0.244358, + "nauc_map_at_3_std": 0.01523, + "nauc_map_at_3_diff1": 0.50197, + "nauc_map_at_5_max": 0.260469, + "nauc_map_at_5_std": 0.020905, + "nauc_map_at_5_diff1": 0.518104, + "nauc_map_at_10_max": 0.242697, + "nauc_map_at_10_std": 0.016461, + "nauc_map_at_10_diff1": 0.499988, + "nauc_map_at_20_max": 0.244788, + "nauc_map_at_20_std": 0.011587, + "nauc_map_at_20_diff1": 0.504209, + "nauc_map_at_100_max": 0.242111, + "nauc_map_at_100_std": 0.012232, + "nauc_map_at_100_diff1": 0.505223, + "nauc_map_at_1000_max": 0.240501, + "nauc_map_at_1000_std": 0.009913, + "nauc_map_at_1000_diff1": 0.504324, + "nauc_recall_at_1_max": 0.240645, + "nauc_recall_at_1_std": 0.008756, + "nauc_recall_at_1_diff1": 0.529453, + "nauc_recall_at_3_max": 0.26002, + "nauc_recall_at_3_std": 0.075703, + "nauc_recall_at_3_diff1": 0.463381, + "nauc_recall_at_5_max": 0.311617, + "nauc_recall_at_5_std": 0.115748, + "nauc_recall_at_5_diff1": 0.492397, + "nauc_recall_at_10_max": 0.280685, + "nauc_recall_at_10_std": 0.141863, + "nauc_recall_at_10_diff1": 0.428683, + "nauc_recall_at_20_max": 0.334588, + "nauc_recall_at_20_std": 0.142963, + "nauc_recall_at_20_diff1": 0.435019, + "nauc_recall_at_100_max": 0.335951, + "nauc_recall_at_100_std": 0.255412, + "nauc_recall_at_100_diff1": 0.381867, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.189977, + "nauc_precision_at_1_std": -0.105918, + "nauc_precision_at_1_diff1": 0.507559, + "nauc_precision_at_3_max": 0.139929, + "nauc_precision_at_3_std": -0.061615, + "nauc_precision_at_3_diff1": 0.313825, + "nauc_precision_at_5_max": 0.107924, + "nauc_precision_at_5_std": -0.052637, + "nauc_precision_at_5_diff1": 0.256728, + "nauc_precision_at_10_max": -0.039857, + "nauc_precision_at_10_std": -0.075141, + "nauc_precision_at_10_diff1": 0.065544, + "nauc_precision_at_20_max": -0.064007, + "nauc_precision_at_20_std": -0.086783, + "nauc_precision_at_20_diff1": 0.023407, + "nauc_precision_at_100_max": -0.131464, + "nauc_precision_at_100_std": -0.078541, + "nauc_precision_at_100_diff1": -0.12149, + "nauc_precision_at_1000_max": -0.236934, + "nauc_precision_at_1000_std": -0.172656, + "nauc_precision_at_1000_diff1": -0.273263, + "nauc_cv_recall_at_1_max": 0.189977, + "nauc_cv_recall_at_1_std": -0.105918, + "nauc_cv_recall_at_1_diff1": 0.507559, + "nauc_cv_recall_at_3_max": 0.269562, + "nauc_cv_recall_at_3_std": 0.051579, + "nauc_cv_recall_at_3_diff1": 0.478469, + "nauc_cv_recall_at_5_max": 0.284265, + "nauc_cv_recall_at_5_std": 0.102691, + "nauc_cv_recall_at_5_diff1": 0.498735, + "nauc_cv_recall_at_10_max": 0.386398, + "nauc_cv_recall_at_10_std": 0.19985, + "nauc_cv_recall_at_10_diff1": 0.465578, + "nauc_cv_recall_at_20_max": 0.506469, + "nauc_cv_recall_at_20_std": 0.216146, + "nauc_cv_recall_at_20_diff1": 0.508265, + "nauc_cv_recall_at_100_max": 0.567694, + "nauc_cv_recall_at_100_std": 0.393266, + "nauc_cv_recall_at_100_diff1": 0.139648, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.189977, + "nauc_mrr_at_1_std": -0.105918, + "nauc_mrr_at_1_diff1": 0.507559, + "nauc_mrr_at_3_max": 0.221919, + "nauc_mrr_at_3_std": -0.039934, + "nauc_mrr_at_3_diff1": 0.480114, + "nauc_mrr_at_5_max": 0.221141, + "nauc_mrr_at_5_std": -0.035142, + "nauc_mrr_at_5_diff1": 0.481645, + "nauc_mrr_at_10_max": 0.230688, + "nauc_mrr_at_10_std": -0.026683, + "nauc_mrr_at_10_diff1": 0.479026, + "nauc_mrr_at_20_max": 0.233049, + "nauc_mrr_at_20_std": -0.029266, + "nauc_mrr_at_20_diff1": 0.480191, + "nauc_mrr_at_100_max": 0.231053, + "nauc_mrr_at_100_std": -0.030419, + "nauc_mrr_at_100_diff1": 0.478234, + "nauc_mrr_at_1000_max": 0.230717, + "nauc_mrr_at_1000_std": -0.03075, + "nauc_mrr_at_1000_diff1": 0.478345, + "main_score": 0.45107, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.3625, + "ndcg_at_3": 0.38879, + "ndcg_at_5": 0.41631, + "ndcg_at_10": 0.45674, + "ndcg_at_20": 0.48627, + "ndcg_at_100": 0.53061, + "ndcg_at_1000": 0.56642, + "map_at_1": 0.23964, + "map_at_3": 0.31359, + "map_at_5": 0.34217, + "map_at_10": 0.36944, + "map_at_20": 0.38136, + "map_at_100": 0.39233, + "map_at_1000": 0.39533, + "recall_at_1": 0.23964, + "recall_at_3": 0.38204, + "recall_at_5": 0.46464, + "recall_at_10": 0.57016, + "recall_at_20": 0.66298, + "recall_at_100": 0.81922, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.3625, + "cv_recall_at_3": 0.6, + "cv_recall_at_5": 0.6625, + "cv_recall_at_10": 0.7625, + "cv_recall_at_20": 0.83125, + "cv_recall_at_100": 0.9375, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.3625, + "precision_at_3": 0.23125, + "precision_at_5": 0.18, + "precision_at_10": 0.12187, + "precision_at_20": 0.07531, + "precision_at_100": 0.02225, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.3625, + "mrr_at_3": 0.465625, + "mrr_at_5": 0.479687, + "mrr_at_10": 0.494256, + "mrr_at_20": 0.499335, + "mrr_at_100": 0.502429, + "mrr_at_1000": 0.502718, + "nauc_ndcg_at_1_max": 0.265976, + "nauc_ndcg_at_1_std": 0.097923, + "nauc_ndcg_at_1_diff1": 0.413602, + "nauc_ndcg_at_3_max": 0.213391, + "nauc_ndcg_at_3_std": 0.067645, + "nauc_ndcg_at_3_diff1": 0.363852, + "nauc_ndcg_at_5_max": 0.222446, + "nauc_ndcg_at_5_std": 0.074225, + "nauc_ndcg_at_5_diff1": 0.36797, + "nauc_ndcg_at_10_max": 0.237576, + "nauc_ndcg_at_10_std": 0.09071, + "nauc_ndcg_at_10_diff1": 0.333976, + "nauc_ndcg_at_20_max": 0.233856, + "nauc_ndcg_at_20_std": 0.095778, + "nauc_ndcg_at_20_diff1": 0.319055, + "nauc_ndcg_at_100_max": 0.235047, + "nauc_ndcg_at_100_std": 0.099202, + "nauc_ndcg_at_100_diff1": 0.328263, + "nauc_ndcg_at_1000_max": 0.221946, + "nauc_ndcg_at_1000_std": 0.080959, + "nauc_ndcg_at_1000_diff1": 0.333705, + "nauc_map_at_1_max": 0.273074, + "nauc_map_at_1_std": 0.118605, + "nauc_map_at_1_diff1": 0.457081, + "nauc_map_at_3_max": 0.248, + "nauc_map_at_3_std": 0.090281, + "nauc_map_at_3_diff1": 0.383039, + "nauc_map_at_5_max": 0.245219, + "nauc_map_at_5_std": 0.085963, + "nauc_map_at_5_diff1": 0.381541, + "nauc_map_at_10_max": 0.240375, + "nauc_map_at_10_std": 0.083293, + "nauc_map_at_10_diff1": 0.354371, + "nauc_map_at_20_max": 0.238454, + "nauc_map_at_20_std": 0.083643, + "nauc_map_at_20_diff1": 0.349721, + "nauc_map_at_100_max": 0.236248, + "nauc_map_at_100_std": 0.082524, + "nauc_map_at_100_diff1": 0.354045, + "nauc_map_at_1000_max": 0.234816, + "nauc_map_at_1000_std": 0.081526, + "nauc_map_at_1000_diff1": 0.354769, + "nauc_recall_at_1_max": 0.273074, + "nauc_recall_at_1_std": 0.118605, + "nauc_recall_at_1_diff1": 0.457081, + "nauc_recall_at_3_max": 0.204268, + "nauc_recall_at_3_std": 0.082551, + "nauc_recall_at_3_diff1": 0.348682, + "nauc_recall_at_5_max": 0.194138, + "nauc_recall_at_5_std": 0.07553, + "nauc_recall_at_5_diff1": 0.319767, + "nauc_recall_at_10_max": 0.209267, + "nauc_recall_at_10_std": 0.107767, + "nauc_recall_at_10_diff1": 0.239348, + "nauc_recall_at_20_max": 0.203921, + "nauc_recall_at_20_std": 0.135145, + "nauc_recall_at_20_diff1": 0.178366, + "nauc_recall_at_100_max": 0.226174, + "nauc_recall_at_100_std": 0.189488, + "nauc_recall_at_100_diff1": 0.154765, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.265976, + "nauc_precision_at_1_std": 0.097923, + "nauc_precision_at_1_diff1": 0.413602, + "nauc_precision_at_3_max": 0.09921, + "nauc_precision_at_3_std": -0.018062, + "nauc_precision_at_3_diff1": 0.133563, + "nauc_precision_at_5_max": 0.030328, + "nauc_precision_at_5_std": -0.041488, + "nauc_precision_at_5_diff1": 0.081425, + "nauc_precision_at_10_max": -0.015409, + "nauc_precision_at_10_std": -0.054602, + "nauc_precision_at_10_diff1": -0.029046, + "nauc_precision_at_20_max": -0.073696, + "nauc_precision_at_20_std": -0.063021, + "nauc_precision_at_20_diff1": -0.054164, + "nauc_precision_at_100_max": -0.11446, + "nauc_precision_at_100_std": -0.053879, + "nauc_precision_at_100_diff1": -0.082888, + "nauc_precision_at_1000_max": -0.213614, + "nauc_precision_at_1000_std": -0.156362, + "nauc_precision_at_1000_diff1": -0.174862, + "nauc_cv_recall_at_1_max": 0.265976, + "nauc_cv_recall_at_1_std": 0.097923, + "nauc_cv_recall_at_1_diff1": 0.413602, + "nauc_cv_recall_at_3_max": 0.151185, + "nauc_cv_recall_at_3_std": 0.026779, + "nauc_cv_recall_at_3_diff1": 0.308567, + "nauc_cv_recall_at_5_max": 0.136182, + "nauc_cv_recall_at_5_std": 0.036805, + "nauc_cv_recall_at_5_diff1": 0.239717, + "nauc_cv_recall_at_10_max": 0.26135, + "nauc_cv_recall_at_10_std": 0.183814, + "nauc_cv_recall_at_10_diff1": 0.30183, + "nauc_cv_recall_at_20_max": 0.256182, + "nauc_cv_recall_at_20_std": 0.234595, + "nauc_cv_recall_at_20_diff1": 0.224952, + "nauc_cv_recall_at_100_max": 0.262045, + "nauc_cv_recall_at_100_std": 0.291317, + "nauc_cv_recall_at_100_diff1": 0.062232, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.265976, + "nauc_mrr_at_1_std": 0.097923, + "nauc_mrr_at_1_diff1": 0.413602, + "nauc_mrr_at_3_max": 0.221899, + "nauc_mrr_at_3_std": 0.070113, + "nauc_mrr_at_3_diff1": 0.358566, + "nauc_mrr_at_5_max": 0.221547, + "nauc_mrr_at_5_std": 0.074409, + "nauc_mrr_at_5_diff1": 0.34686, + "nauc_mrr_at_10_max": 0.239326, + "nauc_mrr_at_10_std": 0.092081, + "nauc_mrr_at_10_diff1": 0.356204, + "nauc_mrr_at_20_max": 0.23837, + "nauc_mrr_at_20_std": 0.092788, + "nauc_mrr_at_20_diff1": 0.353424, + "nauc_mrr_at_100_max": 0.240161, + "nauc_mrr_at_100_std": 0.093064, + "nauc_mrr_at_100_diff1": 0.354529, + "nauc_mrr_at_1000_max": 0.239991, + "nauc_mrr_at_1000_std": 0.092863, + "nauc_mrr_at_1000_diff1": 0.354561, + "main_score": 0.41631, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.48125, + "ndcg_at_3": 0.51436, + "ndcg_at_5": 0.54361, + "ndcg_at_10": 0.56866, + "ndcg_at_20": 0.59565, + "ndcg_at_100": 0.63965, + "ndcg_at_1000": 0.66057, + "map_at_1": 0.30037, + "map_at_3": 0.41256, + "map_at_5": 0.44815, + "map_at_10": 0.47271, + "map_at_20": 0.48611, + "map_at_100": 0.49922, + "map_at_1000": 0.50124, + "recall_at_1": 0.30037, + "recall_at_3": 0.50001, + "recall_at_5": 0.58715, + "recall_at_10": 0.67326, + "recall_at_20": 0.74612, + "recall_at_100": 0.8976, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.48125, + "cv_recall_at_3": 0.74375, + "cv_recall_at_5": 0.8375, + "cv_recall_at_10": 0.8875, + "cv_recall_at_20": 0.93125, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.48125, + "precision_at_3": 0.31458, + "precision_at_5": 0.24625, + "precision_at_10": 0.15125, + "precision_at_20": 0.09344, + "precision_at_100": 0.02612, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.48125, + "mrr_at_3": 0.596875, + "mrr_at_5": 0.619375, + "mrr_at_10": 0.626349, + "mrr_at_20": 0.629167, + "mrr_at_100": 0.630377, + "mrr_at_1000": 0.630439, + "nauc_ndcg_at_1_max": 0.249195, + "nauc_ndcg_at_1_std": -0.038405, + "nauc_ndcg_at_1_diff1": 0.422342, + "nauc_ndcg_at_3_max": 0.256388, + "nauc_ndcg_at_3_std": -0.070418, + "nauc_ndcg_at_3_diff1": 0.354556, + "nauc_ndcg_at_5_max": 0.256311, + "nauc_ndcg_at_5_std": -0.083349, + "nauc_ndcg_at_5_diff1": 0.365457, + "nauc_ndcg_at_10_max": 0.259308, + "nauc_ndcg_at_10_std": -0.058573, + "nauc_ndcg_at_10_diff1": 0.360081, + "nauc_ndcg_at_20_max": 0.25905, + "nauc_ndcg_at_20_std": -0.053358, + "nauc_ndcg_at_20_diff1": 0.344859, + "nauc_ndcg_at_100_max": 0.250667, + "nauc_ndcg_at_100_std": -0.049076, + "nauc_ndcg_at_100_diff1": 0.35479, + "nauc_ndcg_at_1000_max": 0.244618, + "nauc_ndcg_at_1000_std": -0.042288, + "nauc_ndcg_at_1000_diff1": 0.347924, + "nauc_map_at_1_max": 0.267458, + "nauc_map_at_1_std": -0.027375, + "nauc_map_at_1_diff1": 0.495127, + "nauc_map_at_3_max": 0.251785, + "nauc_map_at_3_std": -0.073183, + "nauc_map_at_3_diff1": 0.415073, + "nauc_map_at_5_max": 0.253158, + "nauc_map_at_5_std": -0.08516, + "nauc_map_at_5_diff1": 0.403497, + "nauc_map_at_10_max": 0.255022, + "nauc_map_at_10_std": -0.075445, + "nauc_map_at_10_diff1": 0.389553, + "nauc_map_at_20_max": 0.25651, + "nauc_map_at_20_std": -0.073269, + "nauc_map_at_20_diff1": 0.38399, + "nauc_map_at_100_max": 0.257752, + "nauc_map_at_100_std": -0.069785, + "nauc_map_at_100_diff1": 0.382438, + "nauc_map_at_1000_max": 0.257453, + "nauc_map_at_1000_std": -0.068932, + "nauc_map_at_1000_diff1": 0.381958, + "nauc_recall_at_1_max": 0.267458, + "nauc_recall_at_1_std": -0.027375, + "nauc_recall_at_1_diff1": 0.495127, + "nauc_recall_at_3_max": 0.229121, + "nauc_recall_at_3_std": -0.06798, + "nauc_recall_at_3_diff1": 0.317871, + "nauc_recall_at_5_max": 0.208816, + "nauc_recall_at_5_std": -0.107908, + "nauc_recall_at_5_diff1": 0.297928, + "nauc_recall_at_10_max": 0.232562, + "nauc_recall_at_10_std": -0.031183, + "nauc_recall_at_10_diff1": 0.226356, + "nauc_recall_at_20_max": 0.23404, + "nauc_recall_at_20_std": -0.014768, + "nauc_recall_at_20_diff1": 0.175342, + "nauc_recall_at_100_max": 0.158037, + "nauc_recall_at_100_std": -0.088969, + "nauc_recall_at_100_diff1": 0.289666, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.249195, + "nauc_precision_at_1_std": -0.038405, + "nauc_precision_at_1_diff1": 0.422342, + "nauc_precision_at_3_max": 0.116865, + "nauc_precision_at_3_std": -0.05316, + "nauc_precision_at_3_diff1": 0.033224, + "nauc_precision_at_5_max": 0.069041, + "nauc_precision_at_5_std": -0.049347, + "nauc_precision_at_5_diff1": -0.098051, + "nauc_precision_at_10_max": 0.028655, + "nauc_precision_at_10_std": 0.000373, + "nauc_precision_at_10_diff1": -0.169935, + "nauc_precision_at_20_max": -0.007887, + "nauc_precision_at_20_std": 0.054979, + "nauc_precision_at_20_diff1": -0.200328, + "nauc_precision_at_100_max": -0.07224, + "nauc_precision_at_100_std": 0.111312, + "nauc_precision_at_100_diff1": -0.205019, + "nauc_precision_at_1000_max": -0.140352, + "nauc_precision_at_1000_std": 0.107918, + "nauc_precision_at_1000_diff1": -0.238999, + "nauc_cv_recall_at_1_max": 0.249195, + "nauc_cv_recall_at_1_std": -0.038405, + "nauc_cv_recall_at_1_diff1": 0.422342, + "nauc_cv_recall_at_3_max": 0.275533, + "nauc_cv_recall_at_3_std": -0.018606, + "nauc_cv_recall_at_3_diff1": 0.278957, + "nauc_cv_recall_at_5_max": 0.144332, + "nauc_cv_recall_at_5_std": -0.151214, + "nauc_cv_recall_at_5_diff1": 0.231755, + "nauc_cv_recall_at_10_max": 0.206285, + "nauc_cv_recall_at_10_std": 0.054611, + "nauc_cv_recall_at_10_diff1": 0.041346, + "nauc_cv_recall_at_20_max": 0.158773, + "nauc_cv_recall_at_20_std": 0.255793, + "nauc_cv_recall_at_20_diff1": -0.172566, + "nauc_cv_recall_at_100_max": -0.726891, + "nauc_cv_recall_at_100_std": -0.400093, + "nauc_cv_recall_at_100_diff1": 0.04155, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.249195, + "nauc_mrr_at_1_std": -0.038405, + "nauc_mrr_at_1_diff1": 0.422342, + "nauc_mrr_at_3_max": 0.248191, + "nauc_mrr_at_3_std": -0.043249, + "nauc_mrr_at_3_diff1": 0.374598, + "nauc_mrr_at_5_max": 0.229538, + "nauc_mrr_at_5_std": -0.061024, + "nauc_mrr_at_5_diff1": 0.374057, + "nauc_mrr_at_10_max": 0.235429, + "nauc_mrr_at_10_std": -0.044834, + "nauc_mrr_at_10_diff1": 0.36662, + "nauc_mrr_at_20_max": 0.235664, + "nauc_mrr_at_20_std": -0.042051, + "nauc_mrr_at_20_diff1": 0.366318, + "nauc_mrr_at_100_max": 0.235718, + "nauc_mrr_at_100_std": -0.043683, + "nauc_mrr_at_100_diff1": 0.368024, + "nauc_mrr_at_1000_max": 0.235879, + "nauc_mrr_at_1000_std": -0.043546, + "nauc_mrr_at_1000_diff1": 0.36806, + "main_score": 0.54361, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.25, + "ndcg_at_3": 0.27204, + "ndcg_at_5": 0.28902, + "ndcg_at_10": 0.3365, + "ndcg_at_20": 0.35622, + "ndcg_at_100": 0.41693, + "ndcg_at_1000": 0.46483, + "map_at_1": 0.15787, + "map_at_3": 0.21608, + "map_at_5": 0.23389, + "map_at_10": 0.25954, + "map_at_20": 0.26783, + "map_at_100": 0.28093, + "map_at_1000": 0.28431, + "recall_at_1": 0.15787, + "recall_at_3": 0.27078, + "recall_at_5": 0.32677, + "recall_at_10": 0.4582, + "recall_at_20": 0.50941, + "recall_at_100": 0.73498, + "recall_at_1000": 0.99875, + "cv_recall_at_1": 0.25, + "cv_recall_at_3": 0.43125, + "cv_recall_at_5": 0.48125, + "cv_recall_at_10": 0.63125, + "cv_recall_at_20": 0.68125, + "cv_recall_at_100": 0.88125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.25, + "precision_at_3": 0.16458, + "precision_at_5": 0.1225, + "precision_at_10": 0.09375, + "precision_at_20": 0.05937, + "precision_at_100": 0.02038, + "precision_at_1000": 0.00321, + "mrr_at_1": 0.25, + "mrr_at_3": 0.328125, + "mrr_at_5": 0.340313, + "mrr_at_10": 0.35963, + "mrr_at_20": 0.363081, + "mrr_at_100": 0.368614, + "mrr_at_1000": 0.369145, + "nauc_ndcg_at_1_max": 0.245329, + "nauc_ndcg_at_1_std": 0.124301, + "nauc_ndcg_at_1_diff1": 0.590097, + "nauc_ndcg_at_3_max": 0.299954, + "nauc_ndcg_at_3_std": 0.170265, + "nauc_ndcg_at_3_diff1": 0.482631, + "nauc_ndcg_at_5_max": 0.316506, + "nauc_ndcg_at_5_std": 0.184984, + "nauc_ndcg_at_5_diff1": 0.452494, + "nauc_ndcg_at_10_max": 0.330334, + "nauc_ndcg_at_10_std": 0.209232, + "nauc_ndcg_at_10_diff1": 0.436525, + "nauc_ndcg_at_20_max": 0.327655, + "nauc_ndcg_at_20_std": 0.214318, + "nauc_ndcg_at_20_diff1": 0.427158, + "nauc_ndcg_at_100_max": 0.311161, + "nauc_ndcg_at_100_std": 0.207691, + "nauc_ndcg_at_100_diff1": 0.463152, + "nauc_ndcg_at_1000_max": 0.303505, + "nauc_ndcg_at_1000_std": 0.19531, + "nauc_ndcg_at_1000_diff1": 0.486814, + "nauc_map_at_1_max": 0.300317, + "nauc_map_at_1_std": 0.103933, + "nauc_map_at_1_diff1": 0.679943, + "nauc_map_at_3_max": 0.304975, + "nauc_map_at_3_std": 0.134118, + "nauc_map_at_3_diff1": 0.511335, + "nauc_map_at_5_max": 0.307778, + "nauc_map_at_5_std": 0.147384, + "nauc_map_at_5_diff1": 0.491804, + "nauc_map_at_10_max": 0.312044, + "nauc_map_at_10_std": 0.160317, + "nauc_map_at_10_diff1": 0.48308, + "nauc_map_at_20_max": 0.305719, + "nauc_map_at_20_std": 0.159947, + "nauc_map_at_20_diff1": 0.475956, + "nauc_map_at_100_max": 0.302531, + "nauc_map_at_100_std": 0.16024, + "nauc_map_at_100_diff1": 0.482567, + "nauc_map_at_1000_max": 0.301506, + "nauc_map_at_1000_std": 0.159555, + "nauc_map_at_1000_diff1": 0.485534, + "nauc_recall_at_1_max": 0.300317, + "nauc_recall_at_1_std": 0.103933, + "nauc_recall_at_1_diff1": 0.679943, + "nauc_recall_at_3_max": 0.31819, + "nauc_recall_at_3_std": 0.17947, + "nauc_recall_at_3_diff1": 0.37566, + "nauc_recall_at_5_max": 0.328185, + "nauc_recall_at_5_std": 0.204677, + "nauc_recall_at_5_diff1": 0.298305, + "nauc_recall_at_10_max": 0.325178, + "nauc_recall_at_10_std": 0.237305, + "nauc_recall_at_10_diff1": 0.229595, + "nauc_recall_at_20_max": 0.332772, + "nauc_recall_at_20_std": 0.253605, + "nauc_recall_at_20_diff1": 0.210043, + "nauc_recall_at_100_max": 0.278109, + "nauc_recall_at_100_std": 0.229791, + "nauc_recall_at_100_diff1": 0.293403, + "nauc_recall_at_1000_max": 0.554155, + "nauc_recall_at_1000_std": 0.358077, + "nauc_recall_at_1000_diff1": 1.0, + "nauc_precision_at_1_max": 0.245329, + "nauc_precision_at_1_std": 0.124301, + "nauc_precision_at_1_diff1": 0.590097, + "nauc_precision_at_3_max": 0.24705, + "nauc_precision_at_3_std": 0.185281, + "nauc_precision_at_3_diff1": 0.338448, + "nauc_precision_at_5_max": 0.213367, + "nauc_precision_at_5_std": 0.192798, + "nauc_precision_at_5_diff1": 0.27306, + "nauc_precision_at_10_max": 0.158823, + "nauc_precision_at_10_std": 0.20616, + "nauc_precision_at_10_diff1": 0.182578, + "nauc_precision_at_20_max": 0.082091, + "nauc_precision_at_20_std": 0.176745, + "nauc_precision_at_20_diff1": 0.097397, + "nauc_precision_at_100_max": -0.044267, + "nauc_precision_at_100_std": 0.09641, + "nauc_precision_at_100_diff1": 0.108544, + "nauc_precision_at_1000_max": -0.164133, + "nauc_precision_at_1000_std": -0.065241, + "nauc_precision_at_1000_diff1": 0.030839, + "nauc_cv_recall_at_1_max": 0.245329, + "nauc_cv_recall_at_1_std": 0.124301, + "nauc_cv_recall_at_1_diff1": 0.590097, + "nauc_cv_recall_at_3_max": 0.349044, + "nauc_cv_recall_at_3_std": 0.283355, + "nauc_cv_recall_at_3_diff1": 0.453579, + "nauc_cv_recall_at_5_max": 0.39676, + "nauc_cv_recall_at_5_std": 0.308383, + "nauc_cv_recall_at_5_diff1": 0.398481, + "nauc_cv_recall_at_10_max": 0.416715, + "nauc_cv_recall_at_10_std": 0.403178, + "nauc_cv_recall_at_10_diff1": 0.340542, + "nauc_cv_recall_at_20_max": 0.458913, + "nauc_cv_recall_at_20_std": 0.444667, + "nauc_cv_recall_at_20_diff1": 0.360133, + "nauc_cv_recall_at_100_max": 0.366652, + "nauc_cv_recall_at_100_std": 0.372773, + "nauc_cv_recall_at_100_diff1": 0.333935, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.245329, + "nauc_mrr_at_1_std": 0.124301, + "nauc_mrr_at_1_diff1": 0.590097, + "nauc_mrr_at_3_max": 0.292454, + "nauc_mrr_at_3_std": 0.200921, + "nauc_mrr_at_3_diff1": 0.515734, + "nauc_mrr_at_5_max": 0.30416, + "nauc_mrr_at_5_std": 0.207204, + "nauc_mrr_at_5_diff1": 0.502308, + "nauc_mrr_at_10_max": 0.303775, + "nauc_mrr_at_10_std": 0.21393, + "nauc_mrr_at_10_diff1": 0.498485, + "nauc_mrr_at_20_max": 0.305146, + "nauc_mrr_at_20_std": 0.214696, + "nauc_mrr_at_20_diff1": 0.500836, + "nauc_mrr_at_100_max": 0.301665, + "nauc_mrr_at_100_std": 0.210662, + "nauc_mrr_at_100_diff1": 0.501947, + "nauc_mrr_at_1000_max": 0.301344, + "nauc_mrr_at_1000_std": 0.210098, + "nauc_mrr_at_1000_diff1": 0.502214, + "main_score": 0.28902, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 1123.2319576740265, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..f56743bd42 --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.42949, + "ndcg_at_3": 0.52209, + "ndcg_at_5": 0.52727, + "ndcg_at_10": 0.59063, + "ndcg_at_20": 0.62104, + "ndcg_at_100": 0.65055, + "ndcg_at_1000": 0.65762, + "map_at_1": 0.28558, + "map_at_3": 0.44049, + "map_at_5": 0.46432, + "map_at_10": 0.5045, + "map_at_20": 0.51637, + "map_at_100": 0.52447, + "map_at_1000": 0.52529, + "recall_at_1": 0.28558, + "recall_at_3": 0.55417, + "recall_at_5": 0.58718, + "recall_at_10": 0.76364, + "recall_at_20": 0.85948, + "recall_at_100": 0.96941, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.44231, + "cv_recall_at_3": 0.71154, + "cv_recall_at_5": 0.71154, + "cv_recall_at_10": 0.84615, + "cv_recall_at_20": 0.92308, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.44231, + "precision_at_3": 0.32051, + "precision_at_5": 0.22692, + "precision_at_10": 0.15, + "precision_at_20": 0.0875, + "precision_at_100": 0.02231, + "precision_at_1000": 0.00246, + "mrr_at_1": 0.442308, + "mrr_at_3": 0.567308, + "mrr_at_5": 0.567308, + "mrr_at_10": 0.586958, + "mrr_at_20": 0.59239, + "mrr_at_100": 0.594742, + "mrr_at_1000": 0.594742, + "nauc_ndcg_at_1_max": 0.112291, + "nauc_ndcg_at_1_std": 0.191724, + "nauc_ndcg_at_1_diff1": 0.251871, + "nauc_ndcg_at_3_max": 0.239055, + "nauc_ndcg_at_3_std": 0.247307, + "nauc_ndcg_at_3_diff1": 0.252434, + "nauc_ndcg_at_5_max": 0.222158, + "nauc_ndcg_at_5_std": 0.234168, + "nauc_ndcg_at_5_diff1": 0.240474, + "nauc_ndcg_at_10_max": 0.145684, + "nauc_ndcg_at_10_std": 0.166776, + "nauc_ndcg_at_10_diff1": 0.175705, + "nauc_ndcg_at_20_max": 0.141847, + "nauc_ndcg_at_20_std": 0.151449, + "nauc_ndcg_at_20_diff1": 0.211327, + "nauc_ndcg_at_100_max": 0.143513, + "nauc_ndcg_at_100_std": 0.174175, + "nauc_ndcg_at_100_diff1": 0.223204, + "nauc_ndcg_at_1000_max": 0.145447, + "nauc_ndcg_at_1000_std": 0.170515, + "nauc_ndcg_at_1000_diff1": 0.214673, + "nauc_map_at_1_max": 0.259827, + "nauc_map_at_1_std": 0.253746, + "nauc_map_at_1_diff1": 0.371509, + "nauc_map_at_3_max": 0.240423, + "nauc_map_at_3_std": 0.222171, + "nauc_map_at_3_diff1": 0.287563, + "nauc_map_at_5_max": 0.244661, + "nauc_map_at_5_std": 0.235875, + "nauc_map_at_5_diff1": 0.271656, + "nauc_map_at_10_max": 0.202353, + "nauc_map_at_10_std": 0.207818, + "nauc_map_at_10_diff1": 0.228854, + "nauc_map_at_20_max": 0.195157, + "nauc_map_at_20_std": 0.201139, + "nauc_map_at_20_diff1": 0.240766, + "nauc_map_at_100_max": 0.192712, + "nauc_map_at_100_std": 0.204475, + "nauc_map_at_100_diff1": 0.238466, + "nauc_map_at_1000_max": 0.194208, + "nauc_map_at_1000_std": 0.205127, + "nauc_map_at_1000_diff1": 0.237516, + "nauc_recall_at_1_max": 0.259827, + "nauc_recall_at_1_std": 0.253746, + "nauc_recall_at_1_diff1": 0.371509, + "nauc_recall_at_3_max": 0.277131, + "nauc_recall_at_3_std": 0.215856, + "nauc_recall_at_3_diff1": 0.252289, + "nauc_recall_at_5_max": 0.275682, + "nauc_recall_at_5_std": 0.225214, + "nauc_recall_at_5_diff1": 0.228642, + "nauc_recall_at_10_max": -0.0014, + "nauc_recall_at_10_std": 0.01237, + "nauc_recall_at_10_diff1": -0.002478, + "nauc_recall_at_20_max": 0.019575, + "nauc_recall_at_20_std": -0.07948, + "nauc_recall_at_20_diff1": 0.075214, + "nauc_recall_at_100_max": 0.100621, + "nauc_recall_at_100_std": 0.287753, + "nauc_recall_at_100_diff1": 0.419798, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.092931, + "nauc_precision_at_1_std": 0.179777, + "nauc_precision_at_1_diff1": 0.27274, + "nauc_precision_at_3_max": 0.008391, + "nauc_precision_at_3_std": 0.114287, + "nauc_precision_at_3_diff1": -0.036405, + "nauc_precision_at_5_max": -0.022606, + "nauc_precision_at_5_std": 0.095073, + "nauc_precision_at_5_diff1": -0.094588, + "nauc_precision_at_10_max": -0.135944, + "nauc_precision_at_10_std": -0.007078, + "nauc_precision_at_10_diff1": -0.251289, + "nauc_precision_at_20_max": -0.216056, + "nauc_precision_at_20_std": -0.088915, + "nauc_precision_at_20_diff1": -0.20253, + "nauc_precision_at_100_max": -0.25475, + "nauc_precision_at_100_std": -0.119878, + "nauc_precision_at_100_diff1": -0.246016, + "nauc_precision_at_1000_max": -0.220455, + "nauc_precision_at_1000_std": -0.134025, + "nauc_precision_at_1000_diff1": -0.255261, + "nauc_cv_recall_at_1_max": 0.092931, + "nauc_cv_recall_at_1_std": 0.179777, + "nauc_cv_recall_at_1_diff1": 0.27274, + "nauc_cv_recall_at_3_max": 0.216308, + "nauc_cv_recall_at_3_std": 0.23353, + "nauc_cv_recall_at_3_diff1": 0.270691, + "nauc_cv_recall_at_5_max": 0.216308, + "nauc_cv_recall_at_5_std": 0.23353, + "nauc_cv_recall_at_5_diff1": 0.270691, + "nauc_cv_recall_at_10_max": 0.063235, + "nauc_cv_recall_at_10_std": 0.006969, + "nauc_cv_recall_at_10_diff1": 0.009054, + "nauc_cv_recall_at_20_max": 0.250602, + "nauc_cv_recall_at_20_std": -0.124236, + "nauc_cv_recall_at_20_diff1": -0.013778, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.092931, + "nauc_mrr_at_1_std": 0.179777, + "nauc_mrr_at_1_diff1": 0.27274, + "nauc_mrr_at_3_max": 0.132802, + "nauc_mrr_at_3_std": 0.181419, + "nauc_mrr_at_3_diff1": 0.280699, + "nauc_mrr_at_5_max": 0.132802, + "nauc_mrr_at_5_std": 0.181419, + "nauc_mrr_at_5_diff1": 0.280699, + "nauc_mrr_at_10_max": 0.118679, + "nauc_mrr_at_10_std": 0.162886, + "nauc_mrr_at_10_diff1": 0.260521, + "nauc_mrr_at_20_max": 0.12117, + "nauc_mrr_at_20_std": 0.161691, + "nauc_mrr_at_20_diff1": 0.263749, + "nauc_mrr_at_100_max": 0.119816, + "nauc_mrr_at_100_std": 0.163992, + "nauc_mrr_at_100_diff1": 0.26511, + "nauc_mrr_at_1000_max": 0.119816, + "nauc_mrr_at_1000_std": 0.163992, + "nauc_mrr_at_1000_diff1": 0.26511, + "main_score": 0.52727, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 535.5030071735382, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..a29a817b2f --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.33333, + "ndcg_at_3": 0.37996, + "ndcg_at_5": 0.39835, + "ndcg_at_10": 0.46452, + "ndcg_at_20": 0.52943, + "ndcg_at_100": 0.58305, + "ndcg_at_1000": 0.58915, + "map_at_1": 0.17822, + "map_at_3": 0.28097, + "map_at_5": 0.30664, + "map_at_10": 0.34839, + "map_at_20": 0.38186, + "map_at_100": 0.40894, + "map_at_1000": 0.40932, + "recall_at_1": 0.17822, + "recall_at_3": 0.36076, + "recall_at_5": 0.43317, + "recall_at_10": 0.60343, + "recall_at_20": 0.79975, + "recall_at_100": 0.95541, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.33333, + "cv_recall_at_3": 0.59649, + "cv_recall_at_5": 0.66667, + "cv_recall_at_10": 0.85965, + "cv_recall_at_20": 0.92982, + "cv_recall_at_100": 0.96491, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.33333, + "precision_at_3": 0.25731, + "precision_at_5": 0.19649, + "precision_at_10": 0.15965, + "precision_at_20": 0.12018, + "precision_at_100": 0.03825, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.333333, + "mrr_at_3": 0.453216, + "mrr_at_5": 0.47076, + "mrr_at_10": 0.496199, + "mrr_at_20": 0.501522, + "mrr_at_100": 0.502769, + "mrr_at_1000": 0.502916, + "nauc_ndcg_at_1_max": -0.063725, + "nauc_ndcg_at_1_std": -0.01695, + "nauc_ndcg_at_1_diff1": 0.082232, + "nauc_ndcg_at_3_max": 0.075277, + "nauc_ndcg_at_3_std": 0.079168, + "nauc_ndcg_at_3_diff1": -0.020036, + "nauc_ndcg_at_5_max": 0.062667, + "nauc_ndcg_at_5_std": 0.00041, + "nauc_ndcg_at_5_diff1": -0.066201, + "nauc_ndcg_at_10_max": 0.045095, + "nauc_ndcg_at_10_std": -0.015981, + "nauc_ndcg_at_10_diff1": 0.004134, + "nauc_ndcg_at_20_max": 0.016567, + "nauc_ndcg_at_20_std": 0.001338, + "nauc_ndcg_at_20_diff1": -0.052952, + "nauc_ndcg_at_100_max": -0.071222, + "nauc_ndcg_at_100_std": 0.018057, + "nauc_ndcg_at_100_diff1": -0.102458, + "nauc_ndcg_at_1000_max": -0.073406, + "nauc_ndcg_at_1000_std": -0.003, + "nauc_ndcg_at_1000_diff1": -0.079493, + "nauc_map_at_1_max": 0.049967, + "nauc_map_at_1_std": -0.078227, + "nauc_map_at_1_diff1": 0.137626, + "nauc_map_at_3_max": 0.117767, + "nauc_map_at_3_std": 0.009497, + "nauc_map_at_3_diff1": 0.050122, + "nauc_map_at_5_max": 0.114466, + "nauc_map_at_5_std": -0.012005, + "nauc_map_at_5_diff1": 0.014316, + "nauc_map_at_10_max": 0.082317, + "nauc_map_at_10_std": -0.022096, + "nauc_map_at_10_diff1": 0.038853, + "nauc_map_at_20_max": 0.05652, + "nauc_map_at_20_std": -0.026803, + "nauc_map_at_20_diff1": 0.012172, + "nauc_map_at_100_max": 0.014935, + "nauc_map_at_100_std": -0.014457, + "nauc_map_at_100_diff1": -0.019369, + "nauc_map_at_1000_max": 0.014272, + "nauc_map_at_1000_std": -0.015608, + "nauc_map_at_1000_diff1": -0.018386, + "nauc_recall_at_1_max": 0.049967, + "nauc_recall_at_1_std": -0.078227, + "nauc_recall_at_1_diff1": 0.137626, + "nauc_recall_at_3_max": 0.191708, + "nauc_recall_at_3_std": 0.07735, + "nauc_recall_at_3_diff1": -0.026037, + "nauc_recall_at_5_max": 0.175867, + "nauc_recall_at_5_std": 0.00548, + "nauc_recall_at_5_diff1": -0.115848, + "nauc_recall_at_10_max": 0.200565, + "nauc_recall_at_10_std": 0.0573, + "nauc_recall_at_10_diff1": 0.073166, + "nauc_recall_at_20_max": 0.245961, + "nauc_recall_at_20_std": 0.201803, + "nauc_recall_at_20_diff1": -0.168798, + "nauc_recall_at_100_max": -0.097743, + "nauc_recall_at_100_std": 0.764587, + "nauc_recall_at_100_diff1": -0.854104, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.063725, + "nauc_precision_at_1_std": -0.01695, + "nauc_precision_at_1_diff1": 0.082232, + "nauc_precision_at_3_max": -0.041618, + "nauc_precision_at_3_std": 0.088478, + "nauc_precision_at_3_diff1": -0.117826, + "nauc_precision_at_5_max": -0.1124, + "nauc_precision_at_5_std": -0.02714, + "nauc_precision_at_5_diff1": -0.233452, + "nauc_precision_at_10_max": -0.285615, + "nauc_precision_at_10_std": -0.083963, + "nauc_precision_at_10_diff1": -0.176993, + "nauc_precision_at_20_max": -0.363717, + "nauc_precision_at_20_std": -0.008865, + "nauc_precision_at_20_diff1": -0.264108, + "nauc_precision_at_100_max": -0.326021, + "nauc_precision_at_100_std": 0.092001, + "nauc_precision_at_100_diff1": -0.263207, + "nauc_precision_at_1000_max": -0.332815, + "nauc_precision_at_1000_std": 0.077564, + "nauc_precision_at_1000_diff1": -0.252661, + "nauc_cv_recall_at_1_max": -0.063725, + "nauc_cv_recall_at_1_std": -0.01695, + "nauc_cv_recall_at_1_diff1": 0.082232, + "nauc_cv_recall_at_3_max": -0.098934, + "nauc_cv_recall_at_3_std": 0.006272, + "nauc_cv_recall_at_3_diff1": -0.11246, + "nauc_cv_recall_at_5_max": -0.124791, + "nauc_cv_recall_at_5_std": -0.043107, + "nauc_cv_recall_at_5_diff1": -0.253422, + "nauc_cv_recall_at_10_max": -0.403156, + "nauc_cv_recall_at_10_std": -0.204006, + "nauc_cv_recall_at_10_diff1": -0.143086, + "nauc_cv_recall_at_20_max": -0.231477, + "nauc_cv_recall_at_20_std": 0.5518, + "nauc_cv_recall_at_20_diff1": -0.742811, + "nauc_cv_recall_at_100_max": -0.362176, + "nauc_cv_recall_at_100_std": 0.861366, + "nauc_cv_recall_at_100_diff1": -0.800621, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.063725, + "nauc_mrr_at_1_std": -0.01695, + "nauc_mrr_at_1_diff1": 0.082232, + "nauc_mrr_at_3_max": -0.0594, + "nauc_mrr_at_3_std": 0.014589, + "nauc_mrr_at_3_diff1": -0.005388, + "nauc_mrr_at_5_max": -0.063699, + "nauc_mrr_at_5_std": 0.004379, + "nauc_mrr_at_5_diff1": -0.032053, + "nauc_mrr_at_10_max": -0.076315, + "nauc_mrr_at_10_std": -0.002101, + "nauc_mrr_at_10_diff1": -0.008116, + "nauc_mrr_at_20_max": -0.065601, + "nauc_mrr_at_20_std": 0.01429, + "nauc_mrr_at_20_diff1": -0.015393, + "nauc_mrr_at_100_max": -0.065781, + "nauc_mrr_at_100_std": 0.013433, + "nauc_mrr_at_100_diff1": -0.013375, + "nauc_mrr_at_1000_max": -0.065691, + "nauc_mrr_at_1000_std": 0.012984, + "nauc_mrr_at_1000_diff1": -0.01274, + "main_score": 0.39835, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.35088, + "ndcg_at_3": 0.36478, + "ndcg_at_5": 0.40913, + "ndcg_at_10": 0.4729, + "ndcg_at_20": 0.54157, + "ndcg_at_100": 0.58599, + "ndcg_at_1000": 0.58907, + "map_at_1": 0.15556, + "map_at_3": 0.26161, + "map_at_5": 0.30309, + "map_at_10": 0.34392, + "map_at_20": 0.38256, + "map_at_100": 0.40736, + "map_at_1000": 0.40743, + "recall_at_1": 0.15556, + "recall_at_3": 0.34478, + "recall_at_5": 0.46734, + "recall_at_10": 0.64002, + "recall_at_20": 0.84344, + "recall_at_100": 0.97368, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.35088, + "cv_recall_at_3": 0.5614, + "cv_recall_at_5": 0.70175, + "cv_recall_at_10": 0.87719, + "cv_recall_at_20": 0.96491, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.35088, + "precision_at_3": 0.25146, + "precision_at_5": 0.22105, + "precision_at_10": 0.17193, + "precision_at_20": 0.1307, + "precision_at_100": 0.0386, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.350877, + "mrr_at_3": 0.44152, + "mrr_at_5": 0.473977, + "mrr_at_10": 0.498308, + "mrr_at_20": 0.504176, + "mrr_at_100": 0.504939, + "mrr_at_1000": 0.504959, + "nauc_ndcg_at_1_max": -0.010077, + "nauc_ndcg_at_1_std": 0.055629, + "nauc_ndcg_at_1_diff1": 0.308658, + "nauc_ndcg_at_3_max": 0.083503, + "nauc_ndcg_at_3_std": 0.076438, + "nauc_ndcg_at_3_diff1": 0.251225, + "nauc_ndcg_at_5_max": 0.075306, + "nauc_ndcg_at_5_std": 0.069929, + "nauc_ndcg_at_5_diff1": 0.264661, + "nauc_ndcg_at_10_max": 0.085552, + "nauc_ndcg_at_10_std": 0.05012, + "nauc_ndcg_at_10_diff1": 0.317298, + "nauc_ndcg_at_20_max": 0.099715, + "nauc_ndcg_at_20_std": 0.083135, + "nauc_ndcg_at_20_diff1": 0.289867, + "nauc_ndcg_at_100_max": 0.050473, + "nauc_ndcg_at_100_std": 0.142555, + "nauc_ndcg_at_100_diff1": 0.217466, + "nauc_ndcg_at_1000_max": 0.03966, + "nauc_ndcg_at_1000_std": 0.132619, + "nauc_ndcg_at_1000_diff1": 0.229246, + "nauc_map_at_1_max": -0.004034, + "nauc_map_at_1_std": -0.022015, + "nauc_map_at_1_diff1": 0.152345, + "nauc_map_at_3_max": 0.109359, + "nauc_map_at_3_std": 0.029871, + "nauc_map_at_3_diff1": 0.196613, + "nauc_map_at_5_max": 0.117408, + "nauc_map_at_5_std": 0.026465, + "nauc_map_at_5_diff1": 0.251434, + "nauc_map_at_10_max": 0.104737, + "nauc_map_at_10_std": 0.036786, + "nauc_map_at_10_diff1": 0.281497, + "nauc_map_at_20_max": 0.088963, + "nauc_map_at_20_std": 0.04783, + "nauc_map_at_20_diff1": 0.268212, + "nauc_map_at_100_max": 0.060889, + "nauc_map_at_100_std": 0.088162, + "nauc_map_at_100_diff1": 0.238391, + "nauc_map_at_1000_max": 0.060671, + "nauc_map_at_1000_std": 0.087992, + "nauc_map_at_1000_diff1": 0.23871, + "nauc_recall_at_1_max": -0.004034, + "nauc_recall_at_1_std": -0.022015, + "nauc_recall_at_1_diff1": 0.152345, + "nauc_recall_at_3_max": 0.128928, + "nauc_recall_at_3_std": 0.010039, + "nauc_recall_at_3_diff1": 0.174393, + "nauc_recall_at_5_max": 0.141008, + "nauc_recall_at_5_std": -0.009451, + "nauc_recall_at_5_diff1": 0.229098, + "nauc_recall_at_10_max": 0.161908, + "nauc_recall_at_10_std": -0.099971, + "nauc_recall_at_10_diff1": 0.338674, + "nauc_recall_at_20_max": 0.394066, + "nauc_recall_at_20_std": 0.032128, + "nauc_recall_at_20_diff1": 0.324594, + "nauc_recall_at_100_max": 0.771325, + "nauc_recall_at_100_std": 0.851695, + "nauc_recall_at_100_diff1": -0.499641, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.010077, + "nauc_precision_at_1_std": 0.055629, + "nauc_precision_at_1_diff1": 0.308658, + "nauc_precision_at_3_max": 0.109914, + "nauc_precision_at_3_std": 0.112265, + "nauc_precision_at_3_diff1": 0.324048, + "nauc_precision_at_5_max": 0.022625, + "nauc_precision_at_5_std": 0.164499, + "nauc_precision_at_5_diff1": 0.290828, + "nauc_precision_at_10_max": -0.083058, + "nauc_precision_at_10_std": 0.217122, + "nauc_precision_at_10_diff1": 0.171436, + "nauc_precision_at_20_max": -0.137204, + "nauc_precision_at_20_std": 0.237473, + "nauc_precision_at_20_diff1": -0.012792, + "nauc_precision_at_100_max": -0.146616, + "nauc_precision_at_100_std": 0.296003, + "nauc_precision_at_100_diff1": -0.134264, + "nauc_precision_at_1000_max": -0.155358, + "nauc_precision_at_1000_std": 0.290743, + "nauc_precision_at_1000_diff1": -0.128858, + "nauc_cv_recall_at_1_max": -0.010077, + "nauc_cv_recall_at_1_std": 0.055629, + "nauc_cv_recall_at_1_diff1": 0.308658, + "nauc_cv_recall_at_3_max": 0.03137, + "nauc_cv_recall_at_3_std": 0.070507, + "nauc_cv_recall_at_3_diff1": 0.346301, + "nauc_cv_recall_at_5_max": 0.07705, + "nauc_cv_recall_at_5_std": 0.272061, + "nauc_cv_recall_at_5_diff1": 0.26043, + "nauc_cv_recall_at_10_max": 0.014382, + "nauc_cv_recall_at_10_std": 0.05475, + "nauc_cv_recall_at_10_diff1": 0.196298, + "nauc_cv_recall_at_20_max": 0.722732, + "nauc_cv_recall_at_20_std": 0.934256, + "nauc_cv_recall_at_20_diff1": 0.278894, + "nauc_cv_recall_at_100_max": 0.722732, + "nauc_cv_recall_at_100_std": 1.0, + "nauc_cv_recall_at_100_diff1": -0.164944, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.010077, + "nauc_mrr_at_1_std": 0.055629, + "nauc_mrr_at_1_diff1": 0.308658, + "nauc_mrr_at_3_max": 0.01963, + "nauc_mrr_at_3_std": 0.070244, + "nauc_mrr_at_3_diff1": 0.322357, + "nauc_mrr_at_5_max": 0.030013, + "nauc_mrr_at_5_std": 0.105918, + "nauc_mrr_at_5_diff1": 0.30691, + "nauc_mrr_at_10_max": 0.021817, + "nauc_mrr_at_10_std": 0.083865, + "nauc_mrr_at_10_diff1": 0.305421, + "nauc_mrr_at_20_max": 0.02821, + "nauc_mrr_at_20_std": 0.091901, + "nauc_mrr_at_20_diff1": 0.307423, + "nauc_mrr_at_100_max": 0.02646, + "nauc_mrr_at_100_std": 0.089944, + "nauc_mrr_at_100_diff1": 0.306377, + "nauc_mrr_at_1000_max": 0.026412, + "nauc_mrr_at_1000_std": 0.089882, + "nauc_mrr_at_1000_diff1": 0.306409, + "main_score": 0.40913, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.52632, + "ndcg_at_3": 0.47736, + "ndcg_at_5": 0.51366, + "ndcg_at_10": 0.58317, + "ndcg_at_20": 0.62079, + "ndcg_at_100": 0.66233, + "ndcg_at_1000": 0.66866, + "map_at_1": 0.25589, + "map_at_3": 0.35167, + "map_at_5": 0.40232, + "map_at_10": 0.45466, + "map_at_20": 0.4808, + "map_at_100": 0.50208, + "map_at_1000": 0.50306, + "recall_at_1": 0.25589, + "recall_at_3": 0.41003, + "recall_at_5": 0.52494, + "recall_at_10": 0.74524, + "recall_at_20": 0.85384, + "recall_at_100": 0.96564, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.52632, + "cv_recall_at_3": 0.7193, + "cv_recall_at_5": 0.78947, + "cv_recall_at_10": 0.91228, + "cv_recall_at_20": 0.94737, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.52632, + "precision_at_3": 0.32164, + "precision_at_5": 0.26667, + "precision_at_10": 0.19123, + "precision_at_20": 0.12544, + "precision_at_100": 0.03754, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.526316, + "mrr_at_3": 0.602339, + "mrr_at_5": 0.618129, + "mrr_at_10": 0.635304, + "mrr_at_20": 0.63757, + "mrr_at_100": 0.63924, + "mrr_at_1000": 0.63931, + "nauc_ndcg_at_1_max": -0.092409, + "nauc_ndcg_at_1_std": 0.019272, + "nauc_ndcg_at_1_diff1": 0.258157, + "nauc_ndcg_at_3_max": 0.037795, + "nauc_ndcg_at_3_std": 0.048561, + "nauc_ndcg_at_3_diff1": 0.116329, + "nauc_ndcg_at_5_max": 0.03285, + "nauc_ndcg_at_5_std": 0.020011, + "nauc_ndcg_at_5_diff1": 0.097859, + "nauc_ndcg_at_10_max": -0.015726, + "nauc_ndcg_at_10_std": -0.022416, + "nauc_ndcg_at_10_diff1": 0.093922, + "nauc_ndcg_at_20_max": 0.010416, + "nauc_ndcg_at_20_std": -0.031875, + "nauc_ndcg_at_20_diff1": 0.085385, + "nauc_ndcg_at_100_max": -0.058737, + "nauc_ndcg_at_100_std": -0.017136, + "nauc_ndcg_at_100_diff1": 0.078955, + "nauc_ndcg_at_1000_max": -0.064775, + "nauc_ndcg_at_1000_std": -0.027182, + "nauc_ndcg_at_1000_diff1": 0.110225, + "nauc_map_at_1_max": 0.105774, + "nauc_map_at_1_std": 0.091641, + "nauc_map_at_1_diff1": 0.183475, + "nauc_map_at_3_max": 0.099683, + "nauc_map_at_3_std": 0.058162, + "nauc_map_at_3_diff1": 0.098584, + "nauc_map_at_5_max": 0.108768, + "nauc_map_at_5_std": 0.062821, + "nauc_map_at_5_diff1": 0.124895, + "nauc_map_at_10_max": 0.063363, + "nauc_map_at_10_std": 0.026954, + "nauc_map_at_10_diff1": 0.123992, + "nauc_map_at_20_max": 0.055983, + "nauc_map_at_20_std": 0.015417, + "nauc_map_at_20_diff1": 0.112931, + "nauc_map_at_100_max": 0.030548, + "nauc_map_at_100_std": 0.0271, + "nauc_map_at_100_diff1": 0.107274, + "nauc_map_at_1000_max": 0.027791, + "nauc_map_at_1000_std": 0.024517, + "nauc_map_at_1000_diff1": 0.110086, + "nauc_recall_at_1_max": 0.105774, + "nauc_recall_at_1_std": 0.091641, + "nauc_recall_at_1_diff1": 0.183475, + "nauc_recall_at_3_max": 0.128432, + "nauc_recall_at_3_std": 0.017578, + "nauc_recall_at_3_diff1": -0.001445, + "nauc_recall_at_5_max": 0.178619, + "nauc_recall_at_5_std": 0.043412, + "nauc_recall_at_5_diff1": -0.021752, + "nauc_recall_at_10_max": -0.017105, + "nauc_recall_at_10_std": -0.132196, + "nauc_recall_at_10_diff1": -0.122593, + "nauc_recall_at_20_max": 0.211934, + "nauc_recall_at_20_std": -0.09312, + "nauc_recall_at_20_diff1": -0.229612, + "nauc_recall_at_100_max": -0.195169, + "nauc_recall_at_100_std": 0.076123, + "nauc_recall_at_100_diff1": -1.081456, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.092409, + "nauc_precision_at_1_std": 0.019272, + "nauc_precision_at_1_diff1": 0.258157, + "nauc_precision_at_3_max": -0.065699, + "nauc_precision_at_3_std": 0.01703, + "nauc_precision_at_3_diff1": 0.032384, + "nauc_precision_at_5_max": -0.138032, + "nauc_precision_at_5_std": -0.056468, + "nauc_precision_at_5_diff1": 0.073076, + "nauc_precision_at_10_max": -0.297123, + "nauc_precision_at_10_std": -0.153919, + "nauc_precision_at_10_diff1": -0.009571, + "nauc_precision_at_20_max": -0.273595, + "nauc_precision_at_20_std": -0.138613, + "nauc_precision_at_20_diff1": -0.078002, + "nauc_precision_at_100_max": -0.274164, + "nauc_precision_at_100_std": -0.009483, + "nauc_precision_at_100_diff1": -0.072023, + "nauc_precision_at_1000_max": -0.290149, + "nauc_precision_at_1000_std": -0.034785, + "nauc_precision_at_1000_diff1": -0.042626, + "nauc_cv_recall_at_1_max": -0.092409, + "nauc_cv_recall_at_1_std": 0.019272, + "nauc_cv_recall_at_1_diff1": 0.258157, + "nauc_cv_recall_at_3_max": -0.175625, + "nauc_cv_recall_at_3_std": -0.16949, + "nauc_cv_recall_at_3_diff1": -0.000479, + "nauc_cv_recall_at_5_max": -0.173165, + "nauc_cv_recall_at_5_std": -0.183149, + "nauc_cv_recall_at_5_diff1": -0.084204, + "nauc_cv_recall_at_10_max": -0.692124, + "nauc_cv_recall_at_10_std": -0.577701, + "nauc_cv_recall_at_10_diff1": -0.316856, + "nauc_cv_recall_at_20_max": -0.391119, + "nauc_cv_recall_at_20_std": -0.429088, + "nauc_cv_recall_at_20_diff1": -0.286643, + "nauc_cv_recall_at_100_max": -1.169034, + "nauc_cv_recall_at_100_std": -0.559408, + "nauc_cv_recall_at_100_diff1": -1.169034, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.092409, + "nauc_mrr_at_1_std": 0.019272, + "nauc_mrr_at_1_diff1": 0.258157, + "nauc_mrr_at_3_max": -0.122634, + "nauc_mrr_at_3_std": -0.052042, + "nauc_mrr_at_3_diff1": 0.185794, + "nauc_mrr_at_5_max": -0.115347, + "nauc_mrr_at_5_std": -0.046026, + "nauc_mrr_at_5_diff1": 0.178589, + "nauc_mrr_at_10_max": -0.130396, + "nauc_mrr_at_10_std": -0.05591, + "nauc_mrr_at_10_diff1": 0.187944, + "nauc_mrr_at_20_max": -0.122243, + "nauc_mrr_at_20_std": -0.05005, + "nauc_mrr_at_20_diff1": 0.192402, + "nauc_mrr_at_100_max": -0.122974, + "nauc_mrr_at_100_std": -0.048141, + "nauc_mrr_at_100_diff1": 0.192632, + "nauc_mrr_at_1000_max": -0.122711, + "nauc_mrr_at_1000_std": -0.048013, + "nauc_mrr_at_1000_diff1": 0.192975, + "main_score": 0.51366, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.15789, + "ndcg_at_3": 0.27467, + "ndcg_at_5": 0.30818, + "ndcg_at_10": 0.37458, + "ndcg_at_20": 0.43482, + "ndcg_at_100": 0.50971, + "ndcg_at_1000": 0.51124, + "map_at_1": 0.08845, + "map_at_3": 0.19183, + "map_at_5": 0.21494, + "map_at_10": 0.25988, + "map_at_20": 0.29082, + "map_at_100": 0.32013, + "map_at_1000": 0.32059, + "recall_at_1": 0.08845, + "recall_at_3": 0.28718, + "recall_at_5": 0.36802, + "recall_at_10": 0.56222, + "recall_at_20": 0.73663, + "recall_at_100": 0.98684, + "recall_at_1000": 0.99123, + "cv_recall_at_1": 0.15789, + "cv_recall_at_3": 0.4386, + "cv_recall_at_5": 0.63158, + "cv_recall_at_10": 0.75439, + "cv_recall_at_20": 0.89474, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.15789, + "precision_at_3": 0.20468, + "precision_at_5": 0.17895, + "precision_at_10": 0.14386, + "precision_at_20": 0.11316, + "precision_at_100": 0.03807, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.157895, + "mrr_at_3": 0.289474, + "mrr_at_5": 0.334211, + "mrr_at_10": 0.34938, + "mrr_at_20": 0.359619, + "mrr_at_100": 0.363383, + "mrr_at_1000": 0.363383, + "nauc_ndcg_at_1_max": 0.180172, + "nauc_ndcg_at_1_std": 0.023992, + "nauc_ndcg_at_1_diff1": -0.092128, + "nauc_ndcg_at_3_max": 0.226214, + "nauc_ndcg_at_3_std": 0.066745, + "nauc_ndcg_at_3_diff1": 0.025945, + "nauc_ndcg_at_5_max": 0.162827, + "nauc_ndcg_at_5_std": 0.069859, + "nauc_ndcg_at_5_diff1": 0.040129, + "nauc_ndcg_at_10_max": 0.127131, + "nauc_ndcg_at_10_std": 0.020301, + "nauc_ndcg_at_10_diff1": 0.022157, + "nauc_ndcg_at_20_max": 0.134318, + "nauc_ndcg_at_20_std": 0.056236, + "nauc_ndcg_at_20_diff1": -0.018305, + "nauc_ndcg_at_100_max": 0.127811, + "nauc_ndcg_at_100_std": 0.097525, + "nauc_ndcg_at_100_diff1": -0.048292, + "nauc_ndcg_at_1000_max": 0.120026, + "nauc_ndcg_at_1000_std": 0.089853, + "nauc_ndcg_at_1000_diff1": -0.05291, + "nauc_map_at_1_max": 0.242805, + "nauc_map_at_1_std": 0.073259, + "nauc_map_at_1_diff1": 0.083819, + "nauc_map_at_3_max": 0.198954, + "nauc_map_at_3_std": 0.024923, + "nauc_map_at_3_diff1": 0.063557, + "nauc_map_at_5_max": 0.190244, + "nauc_map_at_5_std": 0.025255, + "nauc_map_at_5_diff1": 0.056316, + "nauc_map_at_10_max": 0.187075, + "nauc_map_at_10_std": 0.032028, + "nauc_map_at_10_diff1": 0.042362, + "nauc_map_at_20_max": 0.191051, + "nauc_map_at_20_std": 0.056491, + "nauc_map_at_20_diff1": 0.030398, + "nauc_map_at_100_max": 0.18519, + "nauc_map_at_100_std": 0.102398, + "nauc_map_at_100_diff1": -0.004711, + "nauc_map_at_1000_max": 0.183287, + "nauc_map_at_1000_std": 0.100483, + "nauc_map_at_1000_diff1": -0.005902, + "nauc_recall_at_1_max": 0.242805, + "nauc_recall_at_1_std": 0.073259, + "nauc_recall_at_1_diff1": 0.083819, + "nauc_recall_at_3_max": 0.181555, + "nauc_recall_at_3_std": 0.049729, + "nauc_recall_at_3_diff1": 0.149976, + "nauc_recall_at_5_max": 0.090943, + "nauc_recall_at_5_std": 0.003663, + "nauc_recall_at_5_diff1": 0.136693, + "nauc_recall_at_10_max": 0.061214, + "nauc_recall_at_10_std": -0.068706, + "nauc_recall_at_10_diff1": 0.063132, + "nauc_recall_at_20_max": 0.102432, + "nauc_recall_at_20_std": 0.05133, + "nauc_recall_at_20_diff1": -0.066724, + "nauc_recall_at_100_max": 0.978085, + "nauc_recall_at_100_std": 0.978085, + "nauc_recall_at_100_diff1": 0.694791, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 0.722732, + "nauc_precision_at_1_max": 0.180172, + "nauc_precision_at_1_std": 0.023992, + "nauc_precision_at_1_diff1": -0.092128, + "nauc_precision_at_3_max": 0.205545, + "nauc_precision_at_3_std": 0.085078, + "nauc_precision_at_3_diff1": -0.064913, + "nauc_precision_at_5_max": 0.115897, + "nauc_precision_at_5_std": 0.149039, + "nauc_precision_at_5_diff1": -0.096989, + "nauc_precision_at_10_max": 0.043649, + "nauc_precision_at_10_std": 0.154008, + "nauc_precision_at_10_diff1": -0.119829, + "nauc_precision_at_20_max": -0.018898, + "nauc_precision_at_20_std": 0.193456, + "nauc_precision_at_20_diff1": -0.202106, + "nauc_precision_at_100_max": -0.097008, + "nauc_precision_at_100_std": 0.222407, + "nauc_precision_at_100_diff1": -0.280493, + "nauc_precision_at_1000_max": -0.112594, + "nauc_precision_at_1000_std": 0.197097, + "nauc_precision_at_1000_diff1": -0.283397, + "nauc_cv_recall_at_1_max": 0.180172, + "nauc_cv_recall_at_1_std": 0.023992, + "nauc_cv_recall_at_1_diff1": -0.092128, + "nauc_cv_recall_at_3_max": 0.172178, + "nauc_cv_recall_at_3_std": 0.075636, + "nauc_cv_recall_at_3_diff1": 0.077779, + "nauc_cv_recall_at_5_max": -0.101773, + "nauc_cv_recall_at_5_std": 0.050692, + "nauc_cv_recall_at_5_diff1": 0.063357, + "nauc_cv_recall_at_10_max": -0.118169, + "nauc_cv_recall_at_10_std": -0.039215, + "nauc_cv_recall_at_10_diff1": 0.010475, + "nauc_cv_recall_at_20_max": -0.251807, + "nauc_cv_recall_at_20_std": -0.024552, + "nauc_cv_recall_at_20_diff1": -0.486863, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.180172, + "nauc_mrr_at_1_std": 0.023992, + "nauc_mrr_at_1_diff1": -0.092128, + "nauc_mrr_at_3_max": 0.158394, + "nauc_mrr_at_3_std": 0.033572, + "nauc_mrr_at_3_diff1": -0.018191, + "nauc_mrr_at_5_max": 0.094242, + "nauc_mrr_at_5_std": 0.030277, + "nauc_mrr_at_5_diff1": -0.025628, + "nauc_mrr_at_10_max": 0.099534, + "nauc_mrr_at_10_std": 0.023288, + "nauc_mrr_at_10_diff1": -0.030121, + "nauc_mrr_at_20_max": 0.102527, + "nauc_mrr_at_20_std": 0.024659, + "nauc_mrr_at_20_diff1": -0.045833, + "nauc_mrr_at_100_max": 0.109351, + "nauc_mrr_at_100_std": 0.02656, + "nauc_mrr_at_100_diff1": -0.042251, + "nauc_mrr_at_1000_max": 0.109351, + "nauc_mrr_at_1000_std": 0.02656, + "nauc_mrr_at_1000_diff1": -0.042251, + "main_score": 0.30818, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 2090.009434223175, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..60c4606fdc --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.39655, + "ndcg_at_3": 0.39086, + "ndcg_at_5": 0.3663, + "ndcg_at_10": 0.35816, + "ndcg_at_20": 0.40554, + "ndcg_at_100": 0.52492, + "ndcg_at_1000": 0.59018, + "map_at_1": 0.03223, + "map_at_3": 0.08982, + "map_at_5": 0.11547, + "map_at_10": 0.15452, + "map_at_20": 0.20434, + "map_at_100": 0.2763, + "map_at_1000": 0.29787, + "recall_at_1": 0.03223, + "recall_at_3": 0.12472, + "recall_at_5": 0.17175, + "recall_at_10": 0.27161, + "recall_at_20": 0.45808, + "recall_at_100": 0.81325, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.39655, + "cv_recall_at_3": 0.67241, + "cv_recall_at_5": 0.77586, + "cv_recall_at_10": 0.86207, + "cv_recall_at_20": 0.93103, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.39655, + "precision_at_3": 0.37356, + "precision_at_5": 0.33448, + "precision_at_10": 0.28448, + "precision_at_20": 0.24138, + "precision_at_100": 0.11121, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.396552, + "mrr_at_3": 0.517241, + "mrr_at_5": 0.539655, + "mrr_at_10": 0.549398, + "mrr_at_20": 0.554071, + "mrr_at_100": 0.555689, + "mrr_at_1000": 0.555689, + "nauc_ndcg_at_1_max": -0.165525, + "nauc_ndcg_at_1_std": 0.027054, + "nauc_ndcg_at_1_diff1": -0.148031, + "nauc_ndcg_at_3_max": -0.060826, + "nauc_ndcg_at_3_std": 0.071611, + "nauc_ndcg_at_3_diff1": -0.185057, + "nauc_ndcg_at_5_max": -0.075842, + "nauc_ndcg_at_5_std": 0.056838, + "nauc_ndcg_at_5_diff1": -0.160277, + "nauc_ndcg_at_10_max": -0.063011, + "nauc_ndcg_at_10_std": 0.061782, + "nauc_ndcg_at_10_diff1": -0.190992, + "nauc_ndcg_at_20_max": 0.027071, + "nauc_ndcg_at_20_std": 0.110724, + "nauc_ndcg_at_20_diff1": -0.141972, + "nauc_ndcg_at_100_max": 0.114095, + "nauc_ndcg_at_100_std": 0.195439, + "nauc_ndcg_at_100_diff1": -0.184123, + "nauc_ndcg_at_1000_max": 0.007542, + "nauc_ndcg_at_1000_std": 0.109816, + "nauc_ndcg_at_1000_diff1": -0.215038, + "nauc_map_at_1_max": -0.093682, + "nauc_map_at_1_std": 0.045479, + "nauc_map_at_1_diff1": -0.033537, + "nauc_map_at_3_max": -0.049018, + "nauc_map_at_3_std": 0.062872, + "nauc_map_at_3_diff1": 0.019123, + "nauc_map_at_5_max": -0.037736, + "nauc_map_at_5_std": 0.066935, + "nauc_map_at_5_diff1": 0.050831, + "nauc_map_at_10_max": -0.005613, + "nauc_map_at_10_std": 0.095798, + "nauc_map_at_10_diff1": 0.018163, + "nauc_map_at_20_max": 0.03961, + "nauc_map_at_20_std": 0.125525, + "nauc_map_at_20_diff1": 0.00066, + "nauc_map_at_100_max": 0.027998, + "nauc_map_at_100_std": 0.117801, + "nauc_map_at_100_diff1": -0.14123, + "nauc_map_at_1000_max": -0.019325, + "nauc_map_at_1000_std": 0.070015, + "nauc_map_at_1000_diff1": -0.176547, + "nauc_recall_at_1_max": -0.093682, + "nauc_recall_at_1_std": 0.045479, + "nauc_recall_at_1_diff1": -0.033537, + "nauc_recall_at_3_max": 0.009173, + "nauc_recall_at_3_std": 0.110279, + "nauc_recall_at_3_diff1": 0.052578, + "nauc_recall_at_5_max": 0.015316, + "nauc_recall_at_5_std": 0.095666, + "nauc_recall_at_5_diff1": 0.109234, + "nauc_recall_at_10_max": 0.037285, + "nauc_recall_at_10_std": 0.108673, + "nauc_recall_at_10_diff1": 0.024551, + "nauc_recall_at_20_max": 0.135271, + "nauc_recall_at_20_std": 0.145249, + "nauc_recall_at_20_diff1": 0.096886, + "nauc_recall_at_100_max": 0.311144, + "nauc_recall_at_100_std": 0.260301, + "nauc_recall_at_100_diff1": -0.006666, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.165525, + "nauc_precision_at_1_std": 0.027054, + "nauc_precision_at_1_diff1": -0.148031, + "nauc_precision_at_3_max": -0.01393, + "nauc_precision_at_3_std": 0.095969, + "nauc_precision_at_3_diff1": -0.223839, + "nauc_precision_at_5_max": -0.036747, + "nauc_precision_at_5_std": 0.062307, + "nauc_precision_at_5_diff1": -0.177568, + "nauc_precision_at_10_max": -0.013819, + "nauc_precision_at_10_std": 0.086149, + "nauc_precision_at_10_diff1": -0.246528, + "nauc_precision_at_20_max": -0.013627, + "nauc_precision_at_20_std": 0.044393, + "nauc_precision_at_20_diff1": -0.270937, + "nauc_precision_at_100_max": -0.150148, + "nauc_precision_at_100_std": -0.121739, + "nauc_precision_at_100_diff1": -0.345136, + "nauc_precision_at_1000_max": -0.2308, + "nauc_precision_at_1000_std": -0.200874, + "nauc_precision_at_1000_diff1": -0.33126, + "nauc_cv_recall_at_1_max": -0.165525, + "nauc_cv_recall_at_1_std": 0.027054, + "nauc_cv_recall_at_1_diff1": -0.148031, + "nauc_cv_recall_at_3_max": 0.168656, + "nauc_cv_recall_at_3_std": 0.310053, + "nauc_cv_recall_at_3_diff1": -0.226482, + "nauc_cv_recall_at_5_max": 0.314601, + "nauc_cv_recall_at_5_std": 0.394024, + "nauc_cv_recall_at_5_diff1": -0.181211, + "nauc_cv_recall_at_10_max": 0.154036, + "nauc_cv_recall_at_10_std": 0.223794, + "nauc_cv_recall_at_10_diff1": -0.195117, + "nauc_cv_recall_at_20_max": 0.562469, + "nauc_cv_recall_at_20_std": 0.51373, + "nauc_cv_recall_at_20_diff1": -0.357922, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.165525, + "nauc_mrr_at_1_std": 0.027054, + "nauc_mrr_at_1_diff1": -0.148031, + "nauc_mrr_at_3_max": -0.025129, + "nauc_mrr_at_3_std": 0.133633, + "nauc_mrr_at_3_diff1": -0.154591, + "nauc_mrr_at_5_max": -0.016384, + "nauc_mrr_at_5_std": 0.135004, + "nauc_mrr_at_5_diff1": -0.1423, + "nauc_mrr_at_10_max": -0.03307, + "nauc_mrr_at_10_std": 0.119036, + "nauc_mrr_at_10_diff1": -0.142397, + "nauc_mrr_at_20_max": -0.027936, + "nauc_mrr_at_20_std": 0.122139, + "nauc_mrr_at_20_diff1": -0.145592, + "nauc_mrr_at_100_max": -0.029759, + "nauc_mrr_at_100_std": 0.121306, + "nauc_mrr_at_100_diff1": -0.146641, + "nauc_mrr_at_1000_max": -0.029759, + "nauc_mrr_at_1000_std": 0.121306, + "nauc_mrr_at_1000_diff1": -0.146641, + "main_score": 0.3663, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.32759, + "ndcg_at_3": 0.34149, + "ndcg_at_5": 0.33293, + "ndcg_at_10": 0.34472, + "ndcg_at_20": 0.382, + "ndcg_at_100": 0.51583, + "ndcg_at_1000": 0.58631, + "map_at_1": 0.04958, + "map_at_3": 0.09333, + "map_at_5": 0.1177, + "map_at_10": 0.15848, + "map_at_20": 0.19559, + "map_at_100": 0.27041, + "map_at_1000": 0.29253, + "recall_at_1": 0.04958, + "recall_at_3": 0.12954, + "recall_at_5": 0.17068, + "recall_at_10": 0.29329, + "recall_at_20": 0.4313, + "recall_at_100": 0.80251, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.32759, + "cv_recall_at_3": 0.67241, + "cv_recall_at_5": 0.7069, + "cv_recall_at_10": 0.87931, + "cv_recall_at_20": 0.91379, + "cv_recall_at_100": 0.98276, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.32759, + "precision_at_3": 0.32759, + "precision_at_5": 0.3069, + "precision_at_10": 0.26897, + "precision_at_20": 0.2181, + "precision_at_100": 0.10517, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.327586, + "mrr_at_3": 0.474138, + "mrr_at_5": 0.481897, + "mrr_at_10": 0.50481, + "mrr_at_20": 0.507191, + "mrr_at_100": 0.509566, + "mrr_at_1000": 0.509708, + "nauc_ndcg_at_1_max": 0.270928, + "nauc_ndcg_at_1_std": 0.198479, + "nauc_ndcg_at_1_diff1": -0.149039, + "nauc_ndcg_at_3_max": 0.157418, + "nauc_ndcg_at_3_std": 0.128521, + "nauc_ndcg_at_3_diff1": -0.193814, + "nauc_ndcg_at_5_max": 0.174792, + "nauc_ndcg_at_5_std": 0.140359, + "nauc_ndcg_at_5_diff1": -0.147713, + "nauc_ndcg_at_10_max": 0.1159, + "nauc_ndcg_at_10_std": 0.033577, + "nauc_ndcg_at_10_diff1": -0.111715, + "nauc_ndcg_at_20_max": 0.191215, + "nauc_ndcg_at_20_std": 0.110134, + "nauc_ndcg_at_20_diff1": -0.115386, + "nauc_ndcg_at_100_max": 0.328049, + "nauc_ndcg_at_100_std": 0.310915, + "nauc_ndcg_at_100_diff1": -0.214328, + "nauc_ndcg_at_1000_max": 0.244018, + "nauc_ndcg_at_1000_std": 0.158387, + "nauc_ndcg_at_1000_diff1": -0.078292, + "nauc_map_at_1_max": 0.249981, + "nauc_map_at_1_std": 0.195261, + "nauc_map_at_1_diff1": -0.103843, + "nauc_map_at_3_max": 0.184395, + "nauc_map_at_3_std": 0.141626, + "nauc_map_at_3_diff1": -0.129368, + "nauc_map_at_5_max": 0.212756, + "nauc_map_at_5_std": 0.169446, + "nauc_map_at_5_diff1": -0.118045, + "nauc_map_at_10_max": 0.194121, + "nauc_map_at_10_std": 0.101253, + "nauc_map_at_10_diff1": -0.156305, + "nauc_map_at_20_max": 0.211802, + "nauc_map_at_20_std": 0.125147, + "nauc_map_at_20_diff1": -0.211894, + "nauc_map_at_100_max": 0.250801, + "nauc_map_at_100_std": 0.171423, + "nauc_map_at_100_diff1": -0.208027, + "nauc_map_at_1000_max": 0.215105, + "nauc_map_at_1000_std": 0.114551, + "nauc_map_at_1000_diff1": -0.125267, + "nauc_recall_at_1_max": 0.249981, + "nauc_recall_at_1_std": 0.195261, + "nauc_recall_at_1_diff1": -0.103843, + "nauc_recall_at_3_max": 0.095095, + "nauc_recall_at_3_std": 0.119545, + "nauc_recall_at_3_diff1": -0.038633, + "nauc_recall_at_5_max": 0.157798, + "nauc_recall_at_5_std": 0.17166, + "nauc_recall_at_5_diff1": -0.019923, + "nauc_recall_at_10_max": -0.017644, + "nauc_recall_at_10_std": -0.062036, + "nauc_recall_at_10_diff1": 0.019386, + "nauc_recall_at_20_max": 0.019628, + "nauc_recall_at_20_std": 0.002902, + "nauc_recall_at_20_diff1": -0.080416, + "nauc_recall_at_100_max": 0.294317, + "nauc_recall_at_100_std": 0.458447, + "nauc_recall_at_100_diff1": -0.261077, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.270928, + "nauc_precision_at_1_std": 0.198479, + "nauc_precision_at_1_diff1": -0.149039, + "nauc_precision_at_3_max": 0.104789, + "nauc_precision_at_3_std": 0.089712, + "nauc_precision_at_3_diff1": -0.211858, + "nauc_precision_at_5_max": 0.148456, + "nauc_precision_at_5_std": 0.109003, + "nauc_precision_at_5_diff1": -0.126109, + "nauc_precision_at_10_max": 0.097476, + "nauc_precision_at_10_std": 0.002499, + "nauc_precision_at_10_diff1": -0.137064, + "nauc_precision_at_20_max": 0.089225, + "nauc_precision_at_20_std": 0.021286, + "nauc_precision_at_20_diff1": -0.086474, + "nauc_precision_at_100_max": 0.004955, + "nauc_precision_at_100_std": -0.054442, + "nauc_precision_at_100_diff1": 0.185236, + "nauc_precision_at_1000_max": -0.099323, + "nauc_precision_at_1000_std": -0.19564, + "nauc_precision_at_1000_diff1": 0.314803, + "nauc_cv_recall_at_1_max": 0.270928, + "nauc_cv_recall_at_1_std": 0.198479, + "nauc_cv_recall_at_1_diff1": -0.149039, + "nauc_cv_recall_at_3_max": 0.42719, + "nauc_cv_recall_at_3_std": 0.532387, + "nauc_cv_recall_at_3_diff1": -0.158074, + "nauc_cv_recall_at_5_max": 0.489552, + "nauc_cv_recall_at_5_std": 0.499339, + "nauc_cv_recall_at_5_diff1": 0.02278, + "nauc_cv_recall_at_10_max": 0.166176, + "nauc_cv_recall_at_10_std": 0.166779, + "nauc_cv_recall_at_10_diff1": 0.173012, + "nauc_cv_recall_at_20_max": 0.131722, + "nauc_cv_recall_at_20_std": 0.152894, + "nauc_cv_recall_at_20_diff1": 0.178781, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 1.0, + "nauc_cv_recall_at_100_diff1": -1.142926, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.270928, + "nauc_mrr_at_1_std": 0.198479, + "nauc_mrr_at_1_diff1": -0.149039, + "nauc_mrr_at_3_max": 0.307339, + "nauc_mrr_at_3_std": 0.307644, + "nauc_mrr_at_3_diff1": -0.163084, + "nauc_mrr_at_5_max": 0.317874, + "nauc_mrr_at_5_std": 0.299173, + "nauc_mrr_at_5_diff1": -0.129892, + "nauc_mrr_at_10_max": 0.290963, + "nauc_mrr_at_10_std": 0.270294, + "nauc_mrr_at_10_diff1": -0.134215, + "nauc_mrr_at_20_max": 0.291167, + "nauc_mrr_at_20_std": 0.270682, + "nauc_mrr_at_20_diff1": -0.136576, + "nauc_mrr_at_100_max": 0.294941, + "nauc_mrr_at_100_std": 0.274203, + "nauc_mrr_at_100_diff1": -0.142074, + "nauc_mrr_at_1000_max": 0.294603, + "nauc_mrr_at_1000_std": 0.273856, + "nauc_mrr_at_1000_diff1": -0.141595, + "main_score": 0.33293, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.58621, + "ndcg_at_3": 0.5396, + "ndcg_at_5": 0.52277, + "ndcg_at_10": 0.5268, + "ndcg_at_20": 0.56589, + "ndcg_at_100": 0.66741, + "ndcg_at_1000": 0.70488, + "map_at_1": 0.08279, + "map_at_3": 0.15934, + "map_at_5": 0.21082, + "map_at_10": 0.28582, + "map_at_20": 0.34698, + "map_at_100": 0.42931, + "map_at_1000": 0.44636, + "recall_at_1": 0.08279, + "recall_at_3": 0.19262, + "recall_at_5": 0.28174, + "recall_at_10": 0.43802, + "recall_at_20": 0.60131, + "recall_at_100": 0.90259, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.58621, + "cv_recall_at_3": 0.7931, + "cv_recall_at_5": 0.89655, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.58621, + "precision_at_3": 0.50575, + "precision_at_5": 0.46207, + "precision_at_10": 0.38793, + "precision_at_20": 0.30776, + "precision_at_100": 0.12483, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.586207, + "mrr_at_3": 0.681034, + "mrr_at_5": 0.70431, + "mrr_at_10": 0.712521, + "mrr_at_20": 0.715969, + "mrr_at_100": 0.715969, + "mrr_at_1000": 0.715969, + "nauc_ndcg_at_1_max": -0.418368, + "nauc_ndcg_at_1_std": -0.313566, + "nauc_ndcg_at_1_diff1": 0.133035, + "nauc_ndcg_at_3_max": -0.328667, + "nauc_ndcg_at_3_std": -0.263635, + "nauc_ndcg_at_3_diff1": -0.055735, + "nauc_ndcg_at_5_max": -0.251502, + "nauc_ndcg_at_5_std": -0.202687, + "nauc_ndcg_at_5_diff1": -0.0183, + "nauc_ndcg_at_10_max": -0.24943, + "nauc_ndcg_at_10_std": -0.237595, + "nauc_ndcg_at_10_diff1": -0.049073, + "nauc_ndcg_at_20_max": -0.173145, + "nauc_ndcg_at_20_std": -0.172462, + "nauc_ndcg_at_20_diff1": -0.083702, + "nauc_ndcg_at_100_max": -0.170149, + "nauc_ndcg_at_100_std": -0.096871, + "nauc_ndcg_at_100_diff1": -0.214855, + "nauc_ndcg_at_1000_max": -0.18345, + "nauc_ndcg_at_1000_std": -0.127083, + "nauc_ndcg_at_1000_diff1": -0.15749, + "nauc_map_at_1_max": -0.266384, + "nauc_map_at_1_std": -0.217922, + "nauc_map_at_1_diff1": -0.052473, + "nauc_map_at_3_max": -0.295967, + "nauc_map_at_3_std": -0.250767, + "nauc_map_at_3_diff1": -0.112414, + "nauc_map_at_5_max": -0.256465, + "nauc_map_at_5_std": -0.191604, + "nauc_map_at_5_diff1": -0.11411, + "nauc_map_at_10_max": -0.199903, + "nauc_map_at_10_std": -0.138739, + "nauc_map_at_10_diff1": -0.162374, + "nauc_map_at_20_max": -0.153281, + "nauc_map_at_20_std": -0.099971, + "nauc_map_at_20_diff1": -0.204666, + "nauc_map_at_100_max": -0.162646, + "nauc_map_at_100_std": -0.113248, + "nauc_map_at_100_diff1": -0.215115, + "nauc_map_at_1000_max": -0.167343, + "nauc_map_at_1000_std": -0.133211, + "nauc_map_at_1000_diff1": -0.174838, + "nauc_recall_at_1_max": -0.266384, + "nauc_recall_at_1_std": -0.217922, + "nauc_recall_at_1_diff1": -0.052473, + "nauc_recall_at_3_max": -0.287614, + "nauc_recall_at_3_std": -0.25401, + "nauc_recall_at_3_diff1": -0.136901, + "nauc_recall_at_5_max": -0.205904, + "nauc_recall_at_5_std": -0.173228, + "nauc_recall_at_5_diff1": -0.03404, + "nauc_recall_at_10_max": -0.194911, + "nauc_recall_at_10_std": -0.170706, + "nauc_recall_at_10_diff1": -0.145872, + "nauc_recall_at_20_max": -0.040781, + "nauc_recall_at_20_std": -0.065903, + "nauc_recall_at_20_diff1": -0.138332, + "nauc_recall_at_100_max": 0.062665, + "nauc_recall_at_100_std": 0.104145, + "nauc_recall_at_100_diff1": -0.22222, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.418368, + "nauc_precision_at_1_std": -0.313566, + "nauc_precision_at_1_diff1": 0.133035, + "nauc_precision_at_3_max": -0.24354, + "nauc_precision_at_3_std": -0.186849, + "nauc_precision_at_3_diff1": -0.071058, + "nauc_precision_at_5_max": -0.069911, + "nauc_precision_at_5_std": -0.03257, + "nauc_precision_at_5_diff1": -0.017258, + "nauc_precision_at_10_max": -0.014819, + "nauc_precision_at_10_std": -0.022014, + "nauc_precision_at_10_diff1": -0.007162, + "nauc_precision_at_20_max": 0.068417, + "nauc_precision_at_20_std": 0.029629, + "nauc_precision_at_20_diff1": -0.038623, + "nauc_precision_at_100_max": -0.054757, + "nauc_precision_at_100_std": -0.106864, + "nauc_precision_at_100_diff1": 0.068355, + "nauc_precision_at_1000_max": -0.055472, + "nauc_precision_at_1000_std": -0.128965, + "nauc_precision_at_1000_diff1": 0.140612, + "nauc_cv_recall_at_1_max": -0.418368, + "nauc_cv_recall_at_1_std": -0.313566, + "nauc_cv_recall_at_1_diff1": 0.133035, + "nauc_cv_recall_at_3_max": -0.422953, + "nauc_cv_recall_at_3_std": -0.265028, + "nauc_cv_recall_at_3_diff1": -0.164577, + "nauc_cv_recall_at_5_max": 0.04345, + "nauc_cv_recall_at_5_std": 0.050279, + "nauc_cv_recall_at_5_diff1": 0.10543, + "nauc_cv_recall_at_10_max": -0.577279, + "nauc_cv_recall_at_10_std": -0.508471, + "nauc_cv_recall_at_10_diff1": -0.409579, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.418368, + "nauc_mrr_at_1_std": -0.313566, + "nauc_mrr_at_1_diff1": 0.133035, + "nauc_mrr_at_3_max": -0.416764, + "nauc_mrr_at_3_std": -0.289983, + "nauc_mrr_at_3_diff1": 0.017064, + "nauc_mrr_at_5_max": -0.372125, + "nauc_mrr_at_5_std": -0.265023, + "nauc_mrr_at_5_diff1": 0.05693, + "nauc_mrr_at_10_max": -0.409448, + "nauc_mrr_at_10_std": -0.297075, + "nauc_mrr_at_10_diff1": 0.037586, + "nauc_mrr_at_20_max": -0.405519, + "nauc_mrr_at_20_std": -0.291993, + "nauc_mrr_at_20_diff1": 0.041695, + "nauc_mrr_at_100_max": -0.405519, + "nauc_mrr_at_100_std": -0.291993, + "nauc_mrr_at_100_diff1": 0.041695, + "nauc_mrr_at_1000_max": -0.405519, + "nauc_mrr_at_1000_std": -0.291993, + "nauc_mrr_at_1000_diff1": 0.041695, + "main_score": 0.52277, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.22414, + "ndcg_at_3": 0.2535, + "ndcg_at_5": 0.25105, + "ndcg_at_10": 0.25593, + "ndcg_at_20": 0.28542, + "ndcg_at_100": 0.42413, + "ndcg_at_1000": 0.52256, + "map_at_1": 0.02211, + "map_at_3": 0.07045, + "map_at_5": 0.09193, + "map_at_10": 0.12283, + "map_at_20": 0.14392, + "map_at_100": 0.19816, + "map_at_1000": 0.22444, + "recall_at_1": 0.02211, + "recall_at_3": 0.10722, + "recall_at_5": 0.15576, + "recall_at_10": 0.23542, + "recall_at_20": 0.33228, + "recall_at_100": 0.71496, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.22414, + "cv_recall_at_3": 0.53448, + "cv_recall_at_5": 0.63793, + "cv_recall_at_10": 0.72414, + "cv_recall_at_20": 0.81034, + "cv_recall_at_100": 0.98276, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.22414, + "precision_at_3": 0.25287, + "precision_at_5": 0.23103, + "precision_at_10": 0.18793, + "precision_at_20": 0.15, + "precision_at_100": 0.08948, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.224138, + "mrr_at_3": 0.359195, + "mrr_at_5": 0.383333, + "mrr_at_10": 0.396743, + "mrr_at_20": 0.402282, + "mrr_at_100": 0.407529, + "mrr_at_1000": 0.40765, + "nauc_ndcg_at_1_max": 0.174564, + "nauc_ndcg_at_1_std": 0.243337, + "nauc_ndcg_at_1_diff1": 0.290228, + "nauc_ndcg_at_3_max": 0.359243, + "nauc_ndcg_at_3_std": 0.348692, + "nauc_ndcg_at_3_diff1": 0.338115, + "nauc_ndcg_at_5_max": 0.338769, + "nauc_ndcg_at_5_std": 0.295553, + "nauc_ndcg_at_5_diff1": 0.305799, + "nauc_ndcg_at_10_max": 0.29944, + "nauc_ndcg_at_10_std": 0.228547, + "nauc_ndcg_at_10_diff1": 0.362154, + "nauc_ndcg_at_20_max": 0.359576, + "nauc_ndcg_at_20_std": 0.277564, + "nauc_ndcg_at_20_diff1": 0.361522, + "nauc_ndcg_at_100_max": 0.40197, + "nauc_ndcg_at_100_std": 0.362522, + "nauc_ndcg_at_100_diff1": 0.407742, + "nauc_ndcg_at_1000_max": 0.325935, + "nauc_ndcg_at_1000_std": 0.292519, + "nauc_ndcg_at_1000_diff1": 0.305113, + "nauc_map_at_1_max": 0.127883, + "nauc_map_at_1_std": 0.220933, + "nauc_map_at_1_diff1": 0.442526, + "nauc_map_at_3_max": 0.234258, + "nauc_map_at_3_std": 0.225165, + "nauc_map_at_3_diff1": 0.399697, + "nauc_map_at_5_max": 0.213827, + "nauc_map_at_5_std": 0.191815, + "nauc_map_at_5_diff1": 0.389737, + "nauc_map_at_10_max": 0.226075, + "nauc_map_at_10_std": 0.162384, + "nauc_map_at_10_diff1": 0.42866, + "nauc_map_at_20_max": 0.260105, + "nauc_map_at_20_std": 0.198397, + "nauc_map_at_20_diff1": 0.409319, + "nauc_map_at_100_max": 0.311242, + "nauc_map_at_100_std": 0.253076, + "nauc_map_at_100_diff1": 0.402785, + "nauc_map_at_1000_max": 0.300661, + "nauc_map_at_1000_std": 0.22238, + "nauc_map_at_1000_diff1": 0.362287, + "nauc_recall_at_1_max": 0.127883, + "nauc_recall_at_1_std": 0.220933, + "nauc_recall_at_1_diff1": 0.442526, + "nauc_recall_at_3_max": 0.289727, + "nauc_recall_at_3_std": 0.223364, + "nauc_recall_at_3_diff1": 0.310097, + "nauc_recall_at_5_max": 0.184567, + "nauc_recall_at_5_std": 0.11288, + "nauc_recall_at_5_diff1": 0.244606, + "nauc_recall_at_10_max": 0.154117, + "nauc_recall_at_10_std": 0.052104, + "nauc_recall_at_10_diff1": 0.347132, + "nauc_recall_at_20_max": 0.254583, + "nauc_recall_at_20_std": 0.154333, + "nauc_recall_at_20_diff1": 0.266667, + "nauc_recall_at_100_max": 0.373385, + "nauc_recall_at_100_std": 0.303238, + "nauc_recall_at_100_diff1": 0.368188, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.174564, + "nauc_precision_at_1_std": 0.243337, + "nauc_precision_at_1_diff1": 0.290228, + "nauc_precision_at_3_max": 0.437141, + "nauc_precision_at_3_std": 0.390312, + "nauc_precision_at_3_diff1": 0.37839, + "nauc_precision_at_5_max": 0.394153, + "nauc_precision_at_5_std": 0.312563, + "nauc_precision_at_5_diff1": 0.334412, + "nauc_precision_at_10_max": 0.303292, + "nauc_precision_at_10_std": 0.22795, + "nauc_precision_at_10_diff1": 0.3063, + "nauc_precision_at_20_max": 0.335279, + "nauc_precision_at_20_std": 0.271456, + "nauc_precision_at_20_diff1": 0.156543, + "nauc_precision_at_100_max": 0.166959, + "nauc_precision_at_100_std": 0.104281, + "nauc_precision_at_100_diff1": -0.012945, + "nauc_precision_at_1000_max": 0.039334, + "nauc_precision_at_1000_std": -0.058102, + "nauc_precision_at_1000_diff1": -0.127429, + "nauc_cv_recall_at_1_max": 0.174564, + "nauc_cv_recall_at_1_std": 0.243337, + "nauc_cv_recall_at_1_diff1": 0.290228, + "nauc_cv_recall_at_3_max": 0.595396, + "nauc_cv_recall_at_3_std": 0.532907, + "nauc_cv_recall_at_3_diff1": 0.283135, + "nauc_cv_recall_at_5_max": 0.608705, + "nauc_cv_recall_at_5_std": 0.478027, + "nauc_cv_recall_at_5_diff1": 0.359928, + "nauc_cv_recall_at_10_max": 0.575592, + "nauc_cv_recall_at_10_std": 0.480566, + "nauc_cv_recall_at_10_diff1": 0.582327, + "nauc_cv_recall_at_20_max": 0.664733, + "nauc_cv_recall_at_20_std": 0.551341, + "nauc_cv_recall_at_20_diff1": 0.459392, + "nauc_cv_recall_at_100_max": 0.868778, + "nauc_cv_recall_at_100_std": 0.868778, + "nauc_cv_recall_at_100_diff1": 1.0, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.174564, + "nauc_mrr_at_1_std": 0.243337, + "nauc_mrr_at_1_diff1": 0.290228, + "nauc_mrr_at_3_max": 0.357028, + "nauc_mrr_at_3_std": 0.37904, + "nauc_mrr_at_3_diff1": 0.264958, + "nauc_mrr_at_5_max": 0.352716, + "nauc_mrr_at_5_std": 0.364461, + "nauc_mrr_at_5_diff1": 0.283263, + "nauc_mrr_at_10_max": 0.34278, + "nauc_mrr_at_10_std": 0.363105, + "nauc_mrr_at_10_diff1": 0.309283, + "nauc_mrr_at_20_max": 0.342859, + "nauc_mrr_at_20_std": 0.364188, + "nauc_mrr_at_20_diff1": 0.300673, + "nauc_mrr_at_100_max": 0.336979, + "nauc_mrr_at_100_std": 0.360139, + "nauc_mrr_at_100_diff1": 0.299881, + "nauc_mrr_at_1000_max": 0.336763, + "nauc_mrr_at_1000_std": 0.359932, + "nauc_mrr_at_1000_diff1": 0.299596, + "main_score": 0.25105, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 555.9574897289276, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..31bce050be --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", + "task_name": "VidoreArxivQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.684, + "ndcg_at_3": 0.73802, + "ndcg_at_5": 0.7511, + "ndcg_at_10": 0.76657, + "ndcg_at_20": 0.77597, + "ndcg_at_100": 0.79244, + "ndcg_at_1000": 0.79467, + "map_at_1": 0.684, + "map_at_3": 0.72367, + "map_at_5": 0.73087, + "map_at_10": 0.73722, + "map_at_20": 0.73995, + "map_at_100": 0.74228, + "map_at_1000": 0.7424, + "recall_at_1": 0.684, + "recall_at_3": 0.78, + "recall_at_5": 0.812, + "recall_at_10": 0.86, + "recall_at_20": 0.896, + "recall_at_100": 0.984, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.684, + "cv_recall_at_3": 0.78, + "cv_recall_at_5": 0.812, + "cv_recall_at_10": 0.86, + "cv_recall_at_20": 0.896, + "cv_recall_at_100": 0.984, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.684, + "precision_at_3": 0.26, + "precision_at_5": 0.1624, + "precision_at_10": 0.086, + "precision_at_20": 0.0448, + "precision_at_100": 0.00984, + "precision_at_1000": 0.001, + "mrr_at_1": 0.684, + "mrr_at_3": 0.723667, + "mrr_at_5": 0.730867, + "mrr_at_10": 0.737221, + "mrr_at_20": 0.739954, + "mrr_at_100": 0.742282, + "mrr_at_1000": 0.742398, + "nauc_ndcg_at_1_max": 0.536329, + "nauc_ndcg_at_1_std": -0.101123, + "nauc_ndcg_at_1_diff1": 0.906367, + "nauc_ndcg_at_3_max": 0.571858, + "nauc_ndcg_at_3_std": -0.081504, + "nauc_ndcg_at_3_diff1": 0.891387, + "nauc_ndcg_at_5_max": 0.576703, + "nauc_ndcg_at_5_std": -0.06827, + "nauc_ndcg_at_5_diff1": 0.888419, + "nauc_ndcg_at_10_max": 0.587327, + "nauc_ndcg_at_10_std": -0.055473, + "nauc_ndcg_at_10_diff1": 0.887749, + "nauc_ndcg_at_20_max": 0.58456, + "nauc_ndcg_at_20_std": -0.05867, + "nauc_ndcg_at_20_diff1": 0.883995, + "nauc_ndcg_at_100_max": 0.578346, + "nauc_ndcg_at_100_std": -0.068653, + "nauc_ndcg_at_100_diff1": 0.889375, + "nauc_ndcg_at_1000_max": 0.574502, + "nauc_ndcg_at_1000_std": -0.070231, + "nauc_ndcg_at_1000_diff1": 0.890246, + "nauc_map_at_1_max": 0.536329, + "nauc_map_at_1_std": -0.101123, + "nauc_map_at_1_diff1": 0.906367, + "nauc_map_at_3_max": 0.563661, + "nauc_map_at_3_std": -0.086199, + "nauc_map_at_3_diff1": 0.895072, + "nauc_map_at_5_max": 0.566057, + "nauc_map_at_5_std": -0.079086, + "nauc_map_at_5_diff1": 0.893511, + "nauc_map_at_10_max": 0.569614, + "nauc_map_at_10_std": -0.074522, + "nauc_map_at_10_diff1": 0.893368, + "nauc_map_at_20_max": 0.568587, + "nauc_map_at_20_std": -0.075639, + "nauc_map_at_20_diff1": 0.892497, + "nauc_map_at_100_max": 0.56772, + "nauc_map_at_100_std": -0.07709, + "nauc_map_at_100_diff1": 0.89318, + "nauc_map_at_1000_max": 0.567565, + "nauc_map_at_1000_std": -0.077086, + "nauc_map_at_1000_diff1": 0.893221, + "nauc_recall_at_1_max": 0.536329, + "nauc_recall_at_1_std": -0.101123, + "nauc_recall_at_1_diff1": 0.906367, + "nauc_recall_at_3_max": 0.599471, + "nauc_recall_at_3_std": -0.065591, + "nauc_recall_at_3_diff1": 0.878843, + "nauc_recall_at_5_max": 0.617686, + "nauc_recall_at_5_std": -0.025049, + "nauc_recall_at_5_diff1": 0.868644, + "nauc_recall_at_10_max": 0.681987, + "nauc_recall_at_10_std": 0.048351, + "nauc_recall_at_10_diff1": 0.859429, + "nauc_recall_at_20_max": 0.692251, + "nauc_recall_at_20_std": 0.057794, + "nauc_recall_at_20_diff1": 0.822782, + "nauc_recall_at_100_max": 0.89525, + "nauc_recall_at_100_std": 0.101832, + "nauc_recall_at_100_diff1": 0.821137, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.536329, + "nauc_precision_at_1_std": -0.101123, + "nauc_precision_at_1_diff1": 0.906367, + "nauc_precision_at_3_max": 0.599471, + "nauc_precision_at_3_std": -0.065591, + "nauc_precision_at_3_diff1": 0.878843, + "nauc_precision_at_5_max": 0.617686, + "nauc_precision_at_5_std": -0.025049, + "nauc_precision_at_5_diff1": 0.868644, + "nauc_precision_at_10_max": 0.681987, + "nauc_precision_at_10_std": 0.048351, + "nauc_precision_at_10_diff1": 0.859429, + "nauc_precision_at_20_max": 0.692251, + "nauc_precision_at_20_std": 0.057794, + "nauc_precision_at_20_diff1": 0.822782, + "nauc_precision_at_100_max": 0.89525, + "nauc_precision_at_100_std": 0.101832, + "nauc_precision_at_100_diff1": 0.821137, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_cv_recall_at_1_max": 0.536329, + "nauc_cv_recall_at_1_std": -0.101123, + "nauc_cv_recall_at_1_diff1": 0.906367, + "nauc_cv_recall_at_3_max": 0.599471, + "nauc_cv_recall_at_3_std": -0.065591, + "nauc_cv_recall_at_3_diff1": 0.878843, + "nauc_cv_recall_at_5_max": 0.617686, + "nauc_cv_recall_at_5_std": -0.025049, + "nauc_cv_recall_at_5_diff1": 0.868644, + "nauc_cv_recall_at_10_max": 0.681987, + "nauc_cv_recall_at_10_std": 0.048351, + "nauc_cv_recall_at_10_diff1": 0.859429, + "nauc_cv_recall_at_20_max": 0.692251, + "nauc_cv_recall_at_20_std": 0.057794, + "nauc_cv_recall_at_20_diff1": 0.822782, + "nauc_cv_recall_at_100_max": 0.89525, + "nauc_cv_recall_at_100_std": 0.101832, + "nauc_cv_recall_at_100_diff1": 0.821137, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.536329, + "nauc_mrr_at_1_std": -0.101123, + "nauc_mrr_at_1_diff1": 0.906367, + "nauc_mrr_at_3_max": 0.563661, + "nauc_mrr_at_3_std": -0.086199, + "nauc_mrr_at_3_diff1": 0.895072, + "nauc_mrr_at_5_max": 0.566057, + "nauc_mrr_at_5_std": -0.079086, + "nauc_mrr_at_5_diff1": 0.893511, + "nauc_mrr_at_10_max": 0.569614, + "nauc_mrr_at_10_std": -0.074522, + "nauc_mrr_at_10_diff1": 0.893368, + "nauc_mrr_at_20_max": 0.568587, + "nauc_mrr_at_20_std": -0.075639, + "nauc_mrr_at_20_diff1": 0.892497, + "nauc_mrr_at_100_max": 0.56772, + "nauc_mrr_at_100_std": -0.07709, + "nauc_mrr_at_100_diff1": 0.89318, + "nauc_mrr_at_1000_max": 0.567565, + "nauc_mrr_at_1000_std": -0.077086, + "nauc_mrr_at_1000_diff1": 0.893221, + "main_score": 0.7511, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 153.20932531356812, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..67023dc908 --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", + "task_name": "VidoreDocVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.51441, + "ndcg_at_3": 0.56034, + "ndcg_at_5": 0.58542, + "ndcg_at_10": 0.60963, + "ndcg_at_20": 0.62642, + "ndcg_at_100": 0.64637, + "ndcg_at_1000": 0.66086, + "map_at_1": 0.51183, + "map_at_3": 0.54755, + "map_at_5": 0.56192, + "map_at_10": 0.57219, + "map_at_20": 0.57732, + "map_at_100": 0.57998, + "map_at_1000": 0.58059, + "recall_at_1": 0.51183, + "recall_at_3": 0.59239, + "recall_at_5": 0.65092, + "recall_at_10": 0.72417, + "recall_at_20": 0.78695, + "recall_at_100": 0.89381, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.51441, + "cv_recall_at_3": 0.60089, + "cv_recall_at_5": 0.66075, + "cv_recall_at_10": 0.73392, + "cv_recall_at_20": 0.79601, + "cv_recall_at_100": 0.90687, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.51441, + "precision_at_3": 0.2003, + "precision_at_5": 0.13348, + "precision_at_10": 0.07472, + "precision_at_20": 0.04113, + "precision_at_100": 0.00953, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.514412, + "mrr_at_3": 0.552476, + "mrr_at_5": 0.566445, + "mrr_at_10": 0.576278, + "mrr_at_20": 0.580811, + "mrr_at_100": 0.583318, + "mrr_at_1000": 0.583739, + "nauc_ndcg_at_1_max": 0.277802, + "nauc_ndcg_at_1_std": 0.319777, + "nauc_ndcg_at_1_diff1": 0.833206, + "nauc_ndcg_at_3_max": 0.275393, + "nauc_ndcg_at_3_std": 0.338402, + "nauc_ndcg_at_3_diff1": 0.802123, + "nauc_ndcg_at_5_max": 0.250229, + "nauc_ndcg_at_5_std": 0.336473, + "nauc_ndcg_at_5_diff1": 0.795878, + "nauc_ndcg_at_10_max": 0.2227, + "nauc_ndcg_at_10_std": 0.349691, + "nauc_ndcg_at_10_diff1": 0.793449, + "nauc_ndcg_at_20_max": 0.205874, + "nauc_ndcg_at_20_std": 0.367397, + "nauc_ndcg_at_20_diff1": 0.792376, + "nauc_ndcg_at_100_max": 0.209368, + "nauc_ndcg_at_100_std": 0.378549, + "nauc_ndcg_at_100_diff1": 0.794356, + "nauc_ndcg_at_1000_max": 0.225161, + "nauc_ndcg_at_1000_std": 0.353525, + "nauc_ndcg_at_1000_diff1": 0.797008, + "nauc_map_at_1_max": 0.281921, + "nauc_map_at_1_std": 0.317044, + "nauc_map_at_1_diff1": 0.836374, + "nauc_map_at_3_max": 0.27865, + "nauc_map_at_3_std": 0.329771, + "nauc_map_at_3_diff1": 0.812978, + "nauc_map_at_5_max": 0.26503, + "nauc_map_at_5_std": 0.328466, + "nauc_map_at_5_diff1": 0.809302, + "nauc_map_at_10_max": 0.253894, + "nauc_map_at_10_std": 0.333092, + "nauc_map_at_10_diff1": 0.808356, + "nauc_map_at_20_max": 0.249007, + "nauc_map_at_20_std": 0.337667, + "nauc_map_at_20_diff1": 0.807451, + "nauc_map_at_100_max": 0.249556, + "nauc_map_at_100_std": 0.339716, + "nauc_map_at_100_diff1": 0.807594, + "nauc_map_at_1000_max": 0.249858, + "nauc_map_at_1000_std": 0.338706, + "nauc_map_at_1000_diff1": 0.807632, + "nauc_recall_at_1_max": 0.281921, + "nauc_recall_at_1_std": 0.317044, + "nauc_recall_at_1_diff1": 0.836374, + "nauc_recall_at_3_max": 0.273626, + "nauc_recall_at_3_std": 0.356711, + "nauc_recall_at_3_diff1": 0.776762, + "nauc_recall_at_5_max": 0.208392, + "nauc_recall_at_5_std": 0.35452, + "nauc_recall_at_5_diff1": 0.758177, + "nauc_recall_at_10_max": 0.098267, + "nauc_recall_at_10_std": 0.411865, + "nauc_recall_at_10_diff1": 0.739994, + "nauc_recall_at_20_max": -0.016904, + "nauc_recall_at_20_std": 0.527684, + "nauc_recall_at_20_diff1": 0.730329, + "nauc_recall_at_100_max": -0.140439, + "nauc_recall_at_100_std": 0.809657, + "nauc_recall_at_100_diff1": 0.714447, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.277802, + "nauc_precision_at_1_std": 0.319777, + "nauc_precision_at_1_diff1": 0.833206, + "nauc_precision_at_3_max": 0.259367, + "nauc_precision_at_3_std": 0.374135, + "nauc_precision_at_3_diff1": 0.762167, + "nauc_precision_at_5_max": 0.165886, + "nauc_precision_at_5_std": 0.346616, + "nauc_precision_at_5_diff1": 0.694382, + "nauc_precision_at_10_max": 0.030862, + "nauc_precision_at_10_std": 0.374188, + "nauc_precision_at_10_diff1": 0.620671, + "nauc_precision_at_20_max": -0.118728, + "nauc_precision_at_20_std": 0.40867, + "nauc_precision_at_20_diff1": 0.481079, + "nauc_precision_at_100_max": -0.318878, + "nauc_precision_at_100_std": 0.334615, + "nauc_precision_at_100_diff1": 0.134254, + "nauc_precision_at_1000_max": -0.380987, + "nauc_precision_at_1000_std": -0.380671, + "nauc_precision_at_1000_diff1": -0.507936, + "nauc_cv_recall_at_1_max": 0.277802, + "nauc_cv_recall_at_1_std": 0.319777, + "nauc_cv_recall_at_1_diff1": 0.833206, + "nauc_cv_recall_at_3_max": 0.259367, + "nauc_cv_recall_at_3_std": 0.374135, + "nauc_cv_recall_at_3_diff1": 0.762167, + "nauc_cv_recall_at_5_max": 0.186985, + "nauc_cv_recall_at_5_std": 0.371996, + "nauc_cv_recall_at_5_diff1": 0.743255, + "nauc_cv_recall_at_10_max": 0.075459, + "nauc_cv_recall_at_10_std": 0.429309, + "nauc_cv_recall_at_10_diff1": 0.727273, + "nauc_cv_recall_at_20_max": -0.04504, + "nauc_cv_recall_at_20_std": 0.526659, + "nauc_cv_recall_at_20_diff1": 0.718466, + "nauc_cv_recall_at_100_max": -0.251152, + "nauc_cv_recall_at_100_std": 0.808979, + "nauc_cv_recall_at_100_diff1": 0.696669, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.277802, + "nauc_mrr_at_1_std": 0.319777, + "nauc_mrr_at_1_diff1": 0.833206, + "nauc_mrr_at_3_max": 0.270805, + "nauc_mrr_at_3_std": 0.336867, + "nauc_mrr_at_3_diff1": 0.805695, + "nauc_mrr_at_5_max": 0.257005, + "nauc_mrr_at_5_std": 0.33568, + "nauc_mrr_at_5_diff1": 0.802896, + "nauc_mrr_at_10_max": 0.247002, + "nauc_mrr_at_10_std": 0.339934, + "nauc_mrr_at_10_diff1": 0.802522, + "nauc_mrr_at_20_max": 0.243256, + "nauc_mrr_at_20_std": 0.343521, + "nauc_mrr_at_20_diff1": 0.802768, + "nauc_mrr_at_100_max": 0.243987, + "nauc_mrr_at_100_std": 0.345213, + "nauc_mrr_at_100_diff1": 0.803195, + "nauc_mrr_at_1000_max": 0.244585, + "nauc_mrr_at_1000_std": 0.344607, + "nauc_mrr_at_1000_diff1": 0.80335, + "main_score": 0.58542, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 199.35491967201233, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..a288b92afd --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", + "task_name": "VidoreInfoVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.82389, + "ndcg_at_3": 0.86364, + "ndcg_at_5": 0.87419, + "ndcg_at_10": 0.88124, + "ndcg_at_20": 0.88613, + "ndcg_at_100": 0.89013, + "ndcg_at_1000": 0.89289, + "map_at_1": 0.82389, + "map_at_3": 0.85374, + "map_at_5": 0.85982, + "map_at_10": 0.86264, + "map_at_20": 0.86403, + "map_at_100": 0.86452, + "map_at_1000": 0.86464, + "recall_at_1": 0.82389, + "recall_at_3": 0.8917, + "recall_at_5": 0.917, + "recall_at_10": 0.93927, + "recall_at_20": 0.95789, + "recall_at_100": 0.97955, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.82389, + "cv_recall_at_3": 0.89271, + "cv_recall_at_5": 0.917, + "cv_recall_at_10": 0.93927, + "cv_recall_at_20": 0.95951, + "cv_recall_at_100": 0.98178, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.82389, + "precision_at_3": 0.29757, + "precision_at_5": 0.18381, + "precision_at_10": 0.09413, + "precision_at_20": 0.04808, + "precision_at_100": 0.00986, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.823887, + "mrr_at_3": 0.854251, + "mrr_at_5": 0.859818, + "mrr_at_10": 0.86264, + "mrr_at_20": 0.864109, + "mrr_at_100": 0.864607, + "mrr_at_1000": 0.864697, + "nauc_ndcg_at_1_max": 0.505735, + "nauc_ndcg_at_1_std": -0.152264, + "nauc_ndcg_at_1_diff1": 0.898703, + "nauc_ndcg_at_3_max": 0.514164, + "nauc_ndcg_at_3_std": -0.184167, + "nauc_ndcg_at_3_diff1": 0.882428, + "nauc_ndcg_at_5_max": 0.520644, + "nauc_ndcg_at_5_std": -0.171507, + "nauc_ndcg_at_5_diff1": 0.879305, + "nauc_ndcg_at_10_max": 0.552922, + "nauc_ndcg_at_10_std": -0.109676, + "nauc_ndcg_at_10_diff1": 0.882008, + "nauc_ndcg_at_20_max": 0.543338, + "nauc_ndcg_at_20_std": -0.124161, + "nauc_ndcg_at_20_diff1": 0.881452, + "nauc_ndcg_at_100_max": 0.53524, + "nauc_ndcg_at_100_std": -0.126639, + "nauc_ndcg_at_100_diff1": 0.882093, + "nauc_ndcg_at_1000_max": 0.525579, + "nauc_ndcg_at_1000_std": -0.146195, + "nauc_ndcg_at_1000_diff1": 0.883989, + "nauc_map_at_1_max": 0.505735, + "nauc_map_at_1_std": -0.152264, + "nauc_map_at_1_diff1": 0.898703, + "nauc_map_at_3_max": 0.506948, + "nauc_map_at_3_std": -0.181754, + "nauc_map_at_3_diff1": 0.887508, + "nauc_map_at_5_max": 0.509635, + "nauc_map_at_5_std": -0.173927, + "nauc_map_at_5_diff1": 0.885772, + "nauc_map_at_10_max": 0.519935, + "nauc_map_at_10_std": -0.15305, + "nauc_map_at_10_diff1": 0.88673, + "nauc_map_at_20_max": 0.517352, + "nauc_map_at_20_std": -0.15686, + "nauc_map_at_20_diff1": 0.886512, + "nauc_map_at_100_max": 0.516349, + "nauc_map_at_100_std": -0.157717, + "nauc_map_at_100_diff1": 0.886615, + "nauc_map_at_1000_max": 0.515986, + "nauc_map_at_1000_std": -0.158398, + "nauc_map_at_1000_diff1": 0.886674, + "nauc_recall_at_1_max": 0.505735, + "nauc_recall_at_1_std": -0.152264, + "nauc_recall_at_1_diff1": 0.898703, + "nauc_recall_at_3_max": 0.543493, + "nauc_recall_at_3_std": -0.195265, + "nauc_recall_at_3_diff1": 0.864338, + "nauc_recall_at_5_max": 0.577384, + "nauc_recall_at_5_std": -0.155757, + "nauc_recall_at_5_diff1": 0.848948, + "nauc_recall_at_10_max": 0.798631, + "nauc_recall_at_10_std": 0.225448, + "nauc_recall_at_10_diff1": 0.855403, + "nauc_recall_at_20_max": 0.813195, + "nauc_recall_at_20_std": 0.233763, + "nauc_recall_at_20_diff1": 0.839659, + "nauc_recall_at_100_max": 0.886814, + "nauc_recall_at_100_std": 0.604347, + "nauc_recall_at_100_diff1": 0.809982, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.505735, + "nauc_precision_at_1_std": -0.152264, + "nauc_precision_at_1_diff1": 0.898703, + "nauc_precision_at_3_max": 0.544584, + "nauc_precision_at_3_std": -0.185919, + "nauc_precision_at_3_diff1": 0.861969, + "nauc_precision_at_5_max": 0.545457, + "nauc_precision_at_5_std": -0.122485, + "nauc_precision_at_5_diff1": 0.790922, + "nauc_precision_at_10_max": 0.743487, + "nauc_precision_at_10_std": 0.249311, + "nauc_precision_at_10_diff1": 0.776914, + "nauc_precision_at_20_max": 0.724649, + "nauc_precision_at_20_std": 0.244547, + "nauc_precision_at_20_diff1": 0.730655, + "nauc_precision_at_100_max": 0.525669, + "nauc_precision_at_100_std": 0.476397, + "nauc_precision_at_100_diff1": 0.447695, + "nauc_precision_at_1000_max": -0.49966, + "nauc_precision_at_1000_std": -0.206095, + "nauc_precision_at_1000_diff1": -0.45484, + "nauc_cv_recall_at_1_max": 0.505735, + "nauc_cv_recall_at_1_std": -0.152264, + "nauc_cv_recall_at_1_diff1": 0.898703, + "nauc_cv_recall_at_3_max": 0.544584, + "nauc_cv_recall_at_3_std": -0.185919, + "nauc_cv_recall_at_3_diff1": 0.861969, + "nauc_cv_recall_at_5_max": 0.577384, + "nauc_cv_recall_at_5_std": -0.155757, + "nauc_cv_recall_at_5_diff1": 0.848948, + "nauc_cv_recall_at_10_max": 0.798631, + "nauc_cv_recall_at_10_std": 0.225448, + "nauc_cv_recall_at_10_diff1": 0.855403, + "nauc_cv_recall_at_20_max": 0.805722, + "nauc_cv_recall_at_20_std": 0.208337, + "nauc_cv_recall_at_20_diff1": 0.844355, + "nauc_cv_recall_at_100_max": 0.880236, + "nauc_cv_recall_at_100_std": 0.629744, + "nauc_cv_recall_at_100_diff1": 0.812528, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.505735, + "nauc_mrr_at_1_std": -0.152264, + "nauc_mrr_at_1_diff1": 0.898703, + "nauc_mrr_at_3_max": 0.507248, + "nauc_mrr_at_3_std": -0.178157, + "nauc_mrr_at_3_diff1": 0.886681, + "nauc_mrr_at_5_max": 0.509635, + "nauc_mrr_at_5_std": -0.173927, + "nauc_mrr_at_5_diff1": 0.885772, + "nauc_mrr_at_10_max": 0.519935, + "nauc_mrr_at_10_std": -0.15305, + "nauc_mrr_at_10_diff1": 0.88673, + "nauc_mrr_at_20_max": 0.517051, + "nauc_mrr_at_20_std": -0.157498, + "nauc_mrr_at_20_diff1": 0.886614, + "nauc_mrr_at_100_max": 0.516067, + "nauc_mrr_at_100_std": -0.158134, + "nauc_mrr_at_100_diff1": 0.886686, + "nauc_mrr_at_1000_max": 0.515798, + "nauc_mrr_at_1000_std": -0.15866, + "nauc_mrr_at_1000_diff1": 0.88672, + "main_score": 0.87419, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 149.08781051635742, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..9a517d9cd2 --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "84a382e05c4473fed9cff2bbae95fe2379416117", + "task_name": "VidoreShiftProjectRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.48, + "ndcg_at_3": 0.60964, + "ndcg_at_5": 0.66344, + "ndcg_at_10": 0.69674, + "ndcg_at_20": 0.70687, + "ndcg_at_100": 0.7108, + "ndcg_at_1000": 0.71224, + "map_at_1": 0.48, + "map_at_3": 0.57833, + "map_at_5": 0.60833, + "map_at_10": 0.62262, + "map_at_20": 0.62539, + "map_at_100": 0.62601, + "map_at_1000": 0.62609, + "recall_at_1": 0.48, + "recall_at_3": 0.7, + "recall_at_5": 0.83, + "recall_at_10": 0.93, + "recall_at_20": 0.97, + "recall_at_100": 0.99, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.48, + "cv_recall_at_3": 0.7, + "cv_recall_at_5": 0.83, + "cv_recall_at_10": 0.93, + "cv_recall_at_20": 0.97, + "cv_recall_at_100": 0.99, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.48, + "precision_at_3": 0.23333, + "precision_at_5": 0.166, + "precision_at_10": 0.093, + "precision_at_20": 0.0485, + "precision_at_100": 0.0099, + "precision_at_1000": 0.001, + "mrr_at_1": 0.48, + "mrr_at_3": 0.578333, + "mrr_at_5": 0.608333, + "mrr_at_10": 0.622619, + "mrr_at_20": 0.625394, + "mrr_at_100": 0.626012, + "mrr_at_1000": 0.626094, + "nauc_ndcg_at_1_max": -0.04209, + "nauc_ndcg_at_1_std": -0.232494, + "nauc_ndcg_at_1_diff1": 0.708982, + "nauc_ndcg_at_3_max": 0.017954, + "nauc_ndcg_at_3_std": -0.269007, + "nauc_ndcg_at_3_diff1": 0.588364, + "nauc_ndcg_at_5_max": 0.036526, + "nauc_ndcg_at_5_std": -0.190808, + "nauc_ndcg_at_5_diff1": 0.607584, + "nauc_ndcg_at_10_max": -0.031259, + "nauc_ndcg_at_10_std": -0.23241, + "nauc_ndcg_at_10_diff1": 0.638735, + "nauc_ndcg_at_20_max": -0.028668, + "nauc_ndcg_at_20_std": -0.248248, + "nauc_ndcg_at_20_diff1": 0.646858, + "nauc_ndcg_at_100_max": -0.018116, + "nauc_ndcg_at_100_std": -0.239387, + "nauc_ndcg_at_100_diff1": 0.644809, + "nauc_ndcg_at_1000_max": -0.017076, + "nauc_ndcg_at_1000_std": -0.241845, + "nauc_ndcg_at_1000_diff1": 0.642398, + "nauc_map_at_1_max": -0.04209, + "nauc_map_at_1_std": -0.232494, + "nauc_map_at_1_diff1": 0.708982, + "nauc_map_at_3_max": 0.0007, + "nauc_map_at_3_std": -0.263607, + "nauc_map_at_3_diff1": 0.621582, + "nauc_map_at_5_max": 0.008457, + "nauc_map_at_5_std": -0.224621, + "nauc_map_at_5_diff1": 0.632865, + "nauc_map_at_10_max": -0.017006, + "nauc_map_at_10_std": -0.239862, + "nauc_map_at_10_diff1": 0.645005, + "nauc_map_at_20_max": -0.016336, + "nauc_map_at_20_std": -0.243565, + "nauc_map_at_20_diff1": 0.646698, + "nauc_map_at_100_max": -0.015514, + "nauc_map_at_100_std": -0.242912, + "nauc_map_at_100_diff1": 0.646553, + "nauc_map_at_1000_max": -0.015467, + "nauc_map_at_1000_std": -0.243022, + "nauc_map_at_1000_diff1": 0.646446, + "nauc_recall_at_1_max": -0.04209, + "nauc_recall_at_1_std": -0.232494, + "nauc_recall_at_1_diff1": 0.708982, + "nauc_recall_at_3_max": 0.078436, + "nauc_recall_at_3_std": -0.285534, + "nauc_recall_at_3_diff1": 0.4726, + "nauc_recall_at_5_max": 0.188396, + "nauc_recall_at_5_std": 0.009839, + "nauc_recall_at_5_diff1": 0.48276, + "nauc_recall_at_10_max": -0.272909, + "nauc_recall_at_10_std": -0.15713, + "nauc_recall_at_10_diff1": 0.625917, + "nauc_recall_at_20_max": -0.51774, + "nauc_recall_at_20_std": -0.498133, + "nauc_recall_at_20_diff1": 0.851385, + "nauc_recall_at_100_max": -0.171335, + "nauc_recall_at_100_std": 0.122782, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.04209, + "nauc_precision_at_1_std": -0.232494, + "nauc_precision_at_1_diff1": 0.708982, + "nauc_precision_at_3_max": 0.078436, + "nauc_precision_at_3_std": -0.285534, + "nauc_precision_at_3_diff1": 0.4726, + "nauc_precision_at_5_max": 0.188396, + "nauc_precision_at_5_std": 0.009839, + "nauc_precision_at_5_diff1": 0.48276, + "nauc_precision_at_10_max": -0.272909, + "nauc_precision_at_10_std": -0.15713, + "nauc_precision_at_10_diff1": 0.625917, + "nauc_precision_at_20_max": -0.51774, + "nauc_precision_at_20_std": -0.498133, + "nauc_precision_at_20_diff1": 0.851385, + "nauc_precision_at_100_max": -0.171335, + "nauc_precision_at_100_std": 0.122782, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": -0.04209, + "nauc_cv_recall_at_1_std": -0.232494, + "nauc_cv_recall_at_1_diff1": 0.708982, + "nauc_cv_recall_at_3_max": 0.078436, + "nauc_cv_recall_at_3_std": -0.285534, + "nauc_cv_recall_at_3_diff1": 0.4726, + "nauc_cv_recall_at_5_max": 0.188396, + "nauc_cv_recall_at_5_std": 0.009839, + "nauc_cv_recall_at_5_diff1": 0.48276, + "nauc_cv_recall_at_10_max": -0.272909, + "nauc_cv_recall_at_10_std": -0.15713, + "nauc_cv_recall_at_10_diff1": 0.625917, + "nauc_cv_recall_at_20_max": -0.51774, + "nauc_cv_recall_at_20_std": -0.498133, + "nauc_cv_recall_at_20_diff1": 0.851385, + "nauc_cv_recall_at_100_max": -0.171335, + "nauc_cv_recall_at_100_std": 0.122782, + "nauc_cv_recall_at_100_diff1": 1.0, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.04209, + "nauc_mrr_at_1_std": -0.232494, + "nauc_mrr_at_1_diff1": 0.708982, + "nauc_mrr_at_3_max": 0.0007, + "nauc_mrr_at_3_std": -0.263607, + "nauc_mrr_at_3_diff1": 0.621582, + "nauc_mrr_at_5_max": 0.008457, + "nauc_mrr_at_5_std": -0.224621, + "nauc_mrr_at_5_diff1": 0.632865, + "nauc_mrr_at_10_max": -0.017006, + "nauc_mrr_at_10_std": -0.239862, + "nauc_mrr_at_10_diff1": 0.645005, + "nauc_mrr_at_20_max": -0.016336, + "nauc_mrr_at_20_std": -0.243565, + "nauc_mrr_at_20_diff1": 0.646698, + "nauc_mrr_at_100_max": -0.015514, + "nauc_mrr_at_100_std": -0.242912, + "nauc_mrr_at_100_diff1": 0.646553, + "nauc_mrr_at_1000_max": -0.015467, + "nauc_mrr_at_1000_std": -0.243022, + "nauc_mrr_at_1000_diff1": 0.646446, + "main_score": 0.66344, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 332.99056482315063, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..1d7e120677 --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", + "task_name": "VidoreSyntheticDocQAAIRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.96, + "ndcg_at_3": 0.98393, + "ndcg_at_5": 0.98393, + "ndcg_at_10": 0.98393, + "ndcg_at_20": 0.98393, + "ndcg_at_100": 0.98393, + "ndcg_at_1000": 0.98393, + "map_at_1": 0.96, + "map_at_3": 0.97833, + "map_at_5": 0.97833, + "map_at_10": 0.97833, + "map_at_20": 0.97833, + "map_at_100": 0.97833, + "map_at_1000": 0.97833, + "recall_at_1": 0.96, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.96, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.96, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.96, + "mrr_at_3": 0.978333, + "mrr_at_5": 0.978333, + "mrr_at_10": 0.978333, + "mrr_at_20": 0.978333, + "mrr_at_100": 0.978333, + "mrr_at_1000": 0.978333, + "nauc_ndcg_at_1_max": 0.715336, + "nauc_ndcg_at_1_std": 0.320378, + "nauc_ndcg_at_1_diff1": 0.934641, + "nauc_ndcg_at_3_max": 0.727877, + "nauc_ndcg_at_3_std": 0.304281, + "nauc_ndcg_at_3_diff1": 0.939965, + "nauc_ndcg_at_5_max": 0.727877, + "nauc_ndcg_at_5_std": 0.304281, + "nauc_ndcg_at_5_diff1": 0.939965, + "nauc_ndcg_at_10_max": 0.727877, + "nauc_ndcg_at_10_std": 0.304281, + "nauc_ndcg_at_10_diff1": 0.939965, + "nauc_ndcg_at_20_max": 0.727877, + "nauc_ndcg_at_20_std": 0.304281, + "nauc_ndcg_at_20_diff1": 0.939965, + "nauc_ndcg_at_100_max": 0.727877, + "nauc_ndcg_at_100_std": 0.304281, + "nauc_ndcg_at_100_diff1": 0.939965, + "nauc_ndcg_at_1000_max": 0.727877, + "nauc_ndcg_at_1000_std": 0.304281, + "nauc_ndcg_at_1000_diff1": 0.939965, + "nauc_map_at_1_max": 0.715336, + "nauc_map_at_1_std": 0.320378, + "nauc_map_at_1_diff1": 0.934641, + "nauc_map_at_3_max": 0.727178, + "nauc_map_at_3_std": 0.305178, + "nauc_map_at_3_diff1": 0.939668, + "nauc_map_at_5_max": 0.727178, + "nauc_map_at_5_std": 0.305178, + "nauc_map_at_5_diff1": 0.939668, + "nauc_map_at_10_max": 0.727178, + "nauc_map_at_10_std": 0.305178, + "nauc_map_at_10_diff1": 0.939668, + "nauc_map_at_20_max": 0.727178, + "nauc_map_at_20_std": 0.305178, + "nauc_map_at_20_diff1": 0.939668, + "nauc_map_at_100_max": 0.727178, + "nauc_map_at_100_std": 0.305178, + "nauc_map_at_100_diff1": 0.939668, + "nauc_map_at_1000_max": 0.727178, + "nauc_map_at_1000_std": 0.305178, + "nauc_map_at_1000_diff1": 0.939668, + "nauc_recall_at_1_max": 0.715336, + "nauc_recall_at_1_std": 0.320378, + "nauc_recall_at_1_diff1": 0.934641, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.715336, + "nauc_precision_at_1_std": 0.320378, + "nauc_precision_at_1_diff1": 0.934641, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.715336, + "nauc_cv_recall_at_1_std": 0.320378, + "nauc_cv_recall_at_1_diff1": 0.934641, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.715336, + "nauc_mrr_at_1_std": 0.320378, + "nauc_mrr_at_1_diff1": 0.934641, + "nauc_mrr_at_3_max": 0.727178, + "nauc_mrr_at_3_std": 0.305178, + "nauc_mrr_at_3_diff1": 0.939668, + "nauc_mrr_at_5_max": 0.727178, + "nauc_mrr_at_5_std": 0.305178, + "nauc_mrr_at_5_diff1": 0.939668, + "nauc_mrr_at_10_max": 0.727178, + "nauc_mrr_at_10_std": 0.305178, + "nauc_mrr_at_10_diff1": 0.939668, + "nauc_mrr_at_20_max": 0.727178, + "nauc_mrr_at_20_std": 0.305178, + "nauc_mrr_at_20_diff1": 0.939668, + "nauc_mrr_at_100_max": 0.727178, + "nauc_mrr_at_100_std": 0.305178, + "nauc_mrr_at_100_diff1": 0.939668, + "nauc_mrr_at_1000_max": 0.727178, + "nauc_mrr_at_1000_std": 0.305178, + "nauc_mrr_at_1000_diff1": 0.939668, + "main_score": 0.98393, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 361.18219089508057, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..5066d74b8b --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", + "task_name": "VidoreSyntheticDocQAEnergyRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.9, + "ndcg_at_3": 0.94155, + "ndcg_at_5": 0.94585, + "ndcg_at_10": 0.94585, + "ndcg_at_20": 0.94585, + "ndcg_at_100": 0.94949, + "ndcg_at_1000": 0.94949, + "map_at_1": 0.9, + "map_at_3": 0.93167, + "map_at_5": 0.93417, + "map_at_10": 0.93417, + "map_at_20": 0.93417, + "map_at_100": 0.93466, + "map_at_1000": 0.93466, + "recall_at_1": 0.9, + "recall_at_3": 0.97, + "recall_at_5": 0.98, + "recall_at_10": 0.98, + "recall_at_20": 0.98, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.9, + "cv_recall_at_3": 0.97, + "cv_recall_at_5": 0.98, + "cv_recall_at_10": 0.98, + "cv_recall_at_20": 0.98, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.9, + "precision_at_3": 0.32333, + "precision_at_5": 0.196, + "precision_at_10": 0.098, + "precision_at_20": 0.049, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.9, + "mrr_at_3": 0.931667, + "mrr_at_5": 0.934167, + "mrr_at_10": 0.934167, + "mrr_at_20": 0.934167, + "mrr_at_100": 0.934662, + "mrr_at_1000": 0.934662, + "nauc_ndcg_at_1_max": 0.270915, + "nauc_ndcg_at_1_std": -0.381839, + "nauc_ndcg_at_1_diff1": 0.960784, + "nauc_ndcg_at_3_max": 0.41038, + "nauc_ndcg_at_3_std": -0.220183, + "nauc_ndcg_at_3_diff1": 0.96113, + "nauc_ndcg_at_5_max": 0.37388, + "nauc_ndcg_at_5_std": -0.306837, + "nauc_ndcg_at_5_diff1": 0.958038, + "nauc_ndcg_at_10_max": 0.37388, + "nauc_ndcg_at_10_std": -0.306837, + "nauc_ndcg_at_10_diff1": 0.958038, + "nauc_ndcg_at_20_max": 0.37388, + "nauc_ndcg_at_20_std": -0.306837, + "nauc_ndcg_at_20_diff1": 0.958038, + "nauc_ndcg_at_100_max": 0.355489, + "nauc_ndcg_at_100_std": -0.302964, + "nauc_ndcg_at_100_diff1": 0.958992, + "nauc_ndcg_at_1000_max": 0.355489, + "nauc_ndcg_at_1000_std": -0.302964, + "nauc_ndcg_at_1000_diff1": 0.958992, + "nauc_map_at_1_max": 0.270915, + "nauc_map_at_1_std": -0.381839, + "nauc_map_at_1_diff1": 0.960784, + "nauc_map_at_3_max": 0.360024, + "nauc_map_at_3_std": -0.276138, + "nauc_map_at_3_diff1": 0.961741, + "nauc_map_at_5_max": 0.340685, + "nauc_map_at_5_std": -0.319635, + "nauc_map_at_5_diff1": 0.960288, + "nauc_map_at_10_max": 0.340685, + "nauc_map_at_10_std": -0.319635, + "nauc_map_at_10_diff1": 0.960288, + "nauc_map_at_20_max": 0.340685, + "nauc_map_at_20_std": -0.319635, + "nauc_map_at_20_diff1": 0.960288, + "nauc_map_at_100_max": 0.339461, + "nauc_map_at_100_std": -0.316509, + "nauc_map_at_100_diff1": 0.960209, + "nauc_map_at_1000_max": 0.339461, + "nauc_map_at_1000_std": -0.316509, + "nauc_map_at_1000_diff1": 0.960209, + "nauc_recall_at_1_max": 0.270915, + "nauc_recall_at_1_std": -0.381839, + "nauc_recall_at_1_diff1": 0.960784, + "nauc_recall_at_3_max": 0.742453, + "nauc_recall_at_3_std": 0.146592, + "nauc_recall_at_3_diff1": 0.956427, + "nauc_recall_at_5_max": 0.679038, + "nauc_recall_at_5_std": -0.214753, + "nauc_recall_at_5_diff1": 0.934641, + "nauc_recall_at_10_max": 0.679038, + "nauc_recall_at_10_std": -0.214753, + "nauc_recall_at_10_diff1": 0.934641, + "nauc_recall_at_20_max": 0.679038, + "nauc_recall_at_20_std": -0.214753, + "nauc_recall_at_20_diff1": 0.934641, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.270915, + "nauc_precision_at_1_std": -0.381839, + "nauc_precision_at_1_diff1": 0.960784, + "nauc_precision_at_3_max": 0.742453, + "nauc_precision_at_3_std": 0.146592, + "nauc_precision_at_3_diff1": 0.956427, + "nauc_precision_at_5_max": 0.679038, + "nauc_precision_at_5_std": -0.214753, + "nauc_precision_at_5_diff1": 0.934641, + "nauc_precision_at_10_max": 0.679038, + "nauc_precision_at_10_std": -0.214753, + "nauc_precision_at_10_diff1": 0.934641, + "nauc_precision_at_20_max": 0.679038, + "nauc_precision_at_20_std": -0.214753, + "nauc_precision_at_20_diff1": 0.934641, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.270915, + "nauc_cv_recall_at_1_std": -0.381839, + "nauc_cv_recall_at_1_diff1": 0.960784, + "nauc_cv_recall_at_3_max": 0.742453, + "nauc_cv_recall_at_3_std": 0.146592, + "nauc_cv_recall_at_3_diff1": 0.956427, + "nauc_cv_recall_at_5_max": 0.679038, + "nauc_cv_recall_at_5_std": -0.214753, + "nauc_cv_recall_at_5_diff1": 0.934641, + "nauc_cv_recall_at_10_max": 0.679038, + "nauc_cv_recall_at_10_std": -0.214753, + "nauc_cv_recall_at_10_diff1": 0.934641, + "nauc_cv_recall_at_20_max": 0.679038, + "nauc_cv_recall_at_20_std": -0.214753, + "nauc_cv_recall_at_20_diff1": 0.934641, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.270915, + "nauc_mrr_at_1_std": -0.381839, + "nauc_mrr_at_1_diff1": 0.960784, + "nauc_mrr_at_3_max": 0.360024, + "nauc_mrr_at_3_std": -0.276138, + "nauc_mrr_at_3_diff1": 0.961741, + "nauc_mrr_at_5_max": 0.340685, + "nauc_mrr_at_5_std": -0.319635, + "nauc_mrr_at_5_diff1": 0.960288, + "nauc_mrr_at_10_max": 0.340685, + "nauc_mrr_at_10_std": -0.319635, + "nauc_mrr_at_10_diff1": 0.960288, + "nauc_mrr_at_20_max": 0.340685, + "nauc_mrr_at_20_std": -0.319635, + "nauc_mrr_at_20_diff1": 0.960288, + "nauc_mrr_at_100_max": 0.339461, + "nauc_mrr_at_100_std": -0.316509, + "nauc_mrr_at_100_diff1": 0.960209, + "nauc_mrr_at_1000_max": 0.339461, + "nauc_mrr_at_1000_std": -0.316509, + "nauc_mrr_at_1000_diff1": 0.960209, + "main_score": 0.94585, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 357.2190451622009, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..02b4719b1d --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b4909afa930f81282fd20601e860668073ad02aa", + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.88, + "ndcg_at_3": 0.93547, + "ndcg_at_5": 0.94796, + "ndcg_at_10": 0.94796, + "ndcg_at_20": 0.94796, + "ndcg_at_100": 0.94796, + "ndcg_at_1000": 0.94796, + "map_at_1": 0.88, + "map_at_3": 0.92333, + "map_at_5": 0.93033, + "map_at_10": 0.93033, + "map_at_20": 0.93033, + "map_at_100": 0.93033, + "map_at_1000": 0.93033, + "recall_at_1": 0.88, + "recall_at_3": 0.97, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.88, + "cv_recall_at_3": 0.97, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.88, + "precision_at_3": 0.32333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.88, + "mrr_at_3": 0.923333, + "mrr_at_5": 0.930333, + "mrr_at_10": 0.930333, + "mrr_at_20": 0.930333, + "mrr_at_100": 0.930333, + "mrr_at_1000": 0.930333, + "nauc_ndcg_at_1_max": 0.511534, + "nauc_ndcg_at_1_std": 0.205178, + "nauc_ndcg_at_1_diff1": 0.789015, + "nauc_ndcg_at_3_max": 0.607132, + "nauc_ndcg_at_3_std": 0.209058, + "nauc_ndcg_at_3_diff1": 0.778639, + "nauc_ndcg_at_5_max": 0.559741, + "nauc_ndcg_at_5_std": 0.222767, + "nauc_ndcg_at_5_diff1": 0.783175, + "nauc_ndcg_at_10_max": 0.559741, + "nauc_ndcg_at_10_std": 0.222767, + "nauc_ndcg_at_10_diff1": 0.783175, + "nauc_ndcg_at_20_max": 0.559741, + "nauc_ndcg_at_20_std": 0.222767, + "nauc_ndcg_at_20_diff1": 0.783175, + "nauc_ndcg_at_100_max": 0.559741, + "nauc_ndcg_at_100_std": 0.222767, + "nauc_ndcg_at_100_diff1": 0.783175, + "nauc_ndcg_at_1000_max": 0.559741, + "nauc_ndcg_at_1000_std": 0.222767, + "nauc_ndcg_at_1000_diff1": 0.783175, + "nauc_map_at_1_max": 0.511534, + "nauc_map_at_1_std": 0.205178, + "nauc_map_at_1_diff1": 0.789015, + "nauc_map_at_3_max": 0.578419, + "nauc_map_at_3_std": 0.213315, + "nauc_map_at_3_diff1": 0.781522, + "nauc_map_at_5_max": 0.556334, + "nauc_map_at_5_std": 0.221541, + "nauc_map_at_5_diff1": 0.783582, + "nauc_map_at_10_max": 0.556334, + "nauc_map_at_10_std": 0.221541, + "nauc_map_at_10_diff1": 0.783582, + "nauc_map_at_20_max": 0.556334, + "nauc_map_at_20_std": 0.221541, + "nauc_map_at_20_diff1": 0.783582, + "nauc_map_at_100_max": 0.556334, + "nauc_map_at_100_std": 0.221541, + "nauc_map_at_100_diff1": 0.783582, + "nauc_map_at_1000_max": 0.556334, + "nauc_map_at_1000_std": 0.221541, + "nauc_map_at_1000_diff1": 0.783582, + "nauc_recall_at_1_max": 0.511534, + "nauc_recall_at_1_std": 0.205178, + "nauc_recall_at_1_diff1": 0.789015, + "nauc_recall_at_3_max": 0.807812, + "nauc_recall_at_3_std": 0.172269, + "nauc_recall_at_3_diff1": 0.758792, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.511534, + "nauc_precision_at_1_std": 0.205178, + "nauc_precision_at_1_diff1": 0.789015, + "nauc_precision_at_3_max": 0.807812, + "nauc_precision_at_3_std": 0.172269, + "nauc_precision_at_3_diff1": 0.758792, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.511534, + "nauc_cv_recall_at_1_std": 0.205178, + "nauc_cv_recall_at_1_diff1": 0.789015, + "nauc_cv_recall_at_3_max": 0.807812, + "nauc_cv_recall_at_3_std": 0.172269, + "nauc_cv_recall_at_3_diff1": 0.758792, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.511534, + "nauc_mrr_at_1_std": 0.205178, + "nauc_mrr_at_1_diff1": 0.789015, + "nauc_mrr_at_3_max": 0.578419, + "nauc_mrr_at_3_std": 0.213315, + "nauc_mrr_at_3_diff1": 0.781522, + "nauc_mrr_at_5_max": 0.556334, + "nauc_mrr_at_5_std": 0.221541, + "nauc_mrr_at_5_diff1": 0.783582, + "nauc_mrr_at_10_max": 0.556334, + "nauc_mrr_at_10_std": 0.221541, + "nauc_mrr_at_10_diff1": 0.783582, + "nauc_mrr_at_20_max": 0.556334, + "nauc_mrr_at_20_std": 0.221541, + "nauc_mrr_at_20_diff1": 0.783582, + "nauc_mrr_at_100_max": 0.556334, + "nauc_mrr_at_100_std": 0.221541, + "nauc_mrr_at_100_diff1": 0.783582, + "nauc_mrr_at_1000_max": 0.556334, + "nauc_mrr_at_1000_std": 0.221541, + "nauc_mrr_at_1000_diff1": 0.783582, + "main_score": 0.94796, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 381.84849095344543, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..09a7bfc670 --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.96, + "ndcg_at_3": 0.98131, + "ndcg_at_5": 0.98131, + "ndcg_at_10": 0.98131, + "ndcg_at_20": 0.98131, + "ndcg_at_100": 0.98131, + "ndcg_at_1000": 0.98131, + "map_at_1": 0.96, + "map_at_3": 0.975, + "map_at_5": 0.975, + "map_at_10": 0.975, + "map_at_20": 0.975, + "map_at_100": 0.975, + "map_at_1000": 0.975, + "recall_at_1": 0.96, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.96, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.96, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.96, + "mrr_at_3": 0.975, + "mrr_at_5": 0.975, + "mrr_at_10": 0.975, + "mrr_at_20": 0.975, + "mrr_at_100": 0.975, + "mrr_at_1000": 0.975, + "nauc_ndcg_at_1_max": 0.508754, + "nauc_ndcg_at_1_std": -0.161531, + "nauc_ndcg_at_1_diff1": 0.897876, + "nauc_ndcg_at_3_max": 0.519309, + "nauc_ndcg_at_3_std": -0.160845, + "nauc_ndcg_at_3_diff1": 0.890722, + "nauc_ndcg_at_5_max": 0.519309, + "nauc_ndcg_at_5_std": -0.160845, + "nauc_ndcg_at_5_diff1": 0.890722, + "nauc_ndcg_at_10_max": 0.519309, + "nauc_ndcg_at_10_std": -0.160845, + "nauc_ndcg_at_10_diff1": 0.890722, + "nauc_ndcg_at_20_max": 0.519309, + "nauc_ndcg_at_20_std": -0.160845, + "nauc_ndcg_at_20_diff1": 0.890722, + "nauc_ndcg_at_100_max": 0.519309, + "nauc_ndcg_at_100_std": -0.160845, + "nauc_ndcg_at_100_diff1": 0.890722, + "nauc_ndcg_at_1000_max": 0.519309, + "nauc_ndcg_at_1000_std": -0.160845, + "nauc_ndcg_at_1000_diff1": 0.890722, + "nauc_map_at_1_max": 0.508754, + "nauc_map_at_1_std": -0.161531, + "nauc_map_at_1_diff1": 0.897876, + "nauc_map_at_3_max": 0.518799, + "nauc_map_at_3_std": -0.160878, + "nauc_map_at_3_diff1": 0.891068, + "nauc_map_at_5_max": 0.518799, + "nauc_map_at_5_std": -0.160878, + "nauc_map_at_5_diff1": 0.891068, + "nauc_map_at_10_max": 0.518799, + "nauc_map_at_10_std": -0.160878, + "nauc_map_at_10_diff1": 0.891068, + "nauc_map_at_20_max": 0.518799, + "nauc_map_at_20_std": -0.160878, + "nauc_map_at_20_diff1": 0.891068, + "nauc_map_at_100_max": 0.518799, + "nauc_map_at_100_std": -0.160878, + "nauc_map_at_100_diff1": 0.891068, + "nauc_map_at_1000_max": 0.518799, + "nauc_map_at_1000_std": -0.160878, + "nauc_map_at_1000_diff1": 0.891068, + "nauc_recall_at_1_max": 0.508754, + "nauc_recall_at_1_std": -0.161531, + "nauc_recall_at_1_diff1": 0.897876, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.508754, + "nauc_precision_at_1_std": -0.161531, + "nauc_precision_at_1_diff1": 0.897876, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.508754, + "nauc_cv_recall_at_1_std": -0.161531, + "nauc_cv_recall_at_1_diff1": 0.897876, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.508754, + "nauc_mrr_at_1_std": -0.161531, + "nauc_mrr_at_1_diff1": 0.897876, + "nauc_mrr_at_3_max": 0.518799, + "nauc_mrr_at_3_std": -0.160878, + "nauc_mrr_at_3_diff1": 0.891068, + "nauc_mrr_at_5_max": 0.518799, + "nauc_mrr_at_5_std": -0.160878, + "nauc_mrr_at_5_diff1": 0.891068, + "nauc_mrr_at_10_max": 0.518799, + "nauc_mrr_at_10_std": -0.160878, + "nauc_mrr_at_10_diff1": 0.891068, + "nauc_mrr_at_20_max": 0.518799, + "nauc_mrr_at_20_std": -0.160878, + "nauc_mrr_at_20_diff1": 0.891068, + "nauc_mrr_at_100_max": 0.518799, + "nauc_mrr_at_100_std": -0.160878, + "nauc_mrr_at_100_diff1": 0.891068, + "nauc_mrr_at_1000_max": 0.518799, + "nauc_mrr_at_1000_std": -0.160878, + "nauc_mrr_at_1000_diff1": 0.891068, + "main_score": 0.98131, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 359.49987745285034, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..9b2b27c835 --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", + "task_name": "VidoreTabfquadRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.69286, + "ndcg_at_3": 0.74141, + "ndcg_at_5": 0.7534, + "ndcg_at_10": 0.76901, + "ndcg_at_20": 0.78093, + "ndcg_at_100": 0.80024, + "ndcg_at_1000": 0.80024, + "map_at_1": 0.69286, + "map_at_3": 0.72976, + "map_at_5": 0.73655, + "map_at_10": 0.74266, + "map_at_20": 0.74604, + "map_at_100": 0.74894, + "map_at_1000": 0.74894, + "recall_at_1": 0.69286, + "recall_at_3": 0.775, + "recall_at_5": 0.80357, + "recall_at_10": 0.85357, + "recall_at_20": 0.9, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.69286, + "cv_recall_at_3": 0.775, + "cv_recall_at_5": 0.80357, + "cv_recall_at_10": 0.85357, + "cv_recall_at_20": 0.9, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.69286, + "precision_at_3": 0.25833, + "precision_at_5": 0.16071, + "precision_at_10": 0.08536, + "precision_at_20": 0.045, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.692857, + "mrr_at_3": 0.729762, + "mrr_at_5": 0.736548, + "mrr_at_10": 0.742663, + "mrr_at_20": 0.746038, + "mrr_at_100": 0.748945, + "mrr_at_1000": 0.748945, + "nauc_ndcg_at_1_max": 0.321225, + "nauc_ndcg_at_1_std": 0.097933, + "nauc_ndcg_at_1_diff1": 0.795573, + "nauc_ndcg_at_3_max": 0.323562, + "nauc_ndcg_at_3_std": 0.107604, + "nauc_ndcg_at_3_diff1": 0.746401, + "nauc_ndcg_at_5_max": 0.314364, + "nauc_ndcg_at_5_std": 0.097493, + "nauc_ndcg_at_5_diff1": 0.733722, + "nauc_ndcg_at_10_max": 0.309424, + "nauc_ndcg_at_10_std": 0.088003, + "nauc_ndcg_at_10_diff1": 0.735895, + "nauc_ndcg_at_20_max": 0.298385, + "nauc_ndcg_at_20_std": 0.07164, + "nauc_ndcg_at_20_diff1": 0.732738, + "nauc_ndcg_at_100_max": 0.314817, + "nauc_ndcg_at_100_std": 0.092753, + "nauc_ndcg_at_100_diff1": 0.748264, + "nauc_ndcg_at_1000_max": 0.314817, + "nauc_ndcg_at_1000_std": 0.092753, + "nauc_ndcg_at_1000_diff1": 0.748264, + "nauc_map_at_1_max": 0.321225, + "nauc_map_at_1_std": 0.097933, + "nauc_map_at_1_diff1": 0.795573, + "nauc_map_at_3_max": 0.325516, + "nauc_map_at_3_std": 0.106819, + "nauc_map_at_3_diff1": 0.75932, + "nauc_map_at_5_max": 0.320766, + "nauc_map_at_5_std": 0.101832, + "nauc_map_at_5_diff1": 0.752831, + "nauc_map_at_10_max": 0.318497, + "nauc_map_at_10_std": 0.097649, + "nauc_map_at_10_diff1": 0.754011, + "nauc_map_at_20_max": 0.315571, + "nauc_map_at_20_std": 0.09338, + "nauc_map_at_20_diff1": 0.753463, + "nauc_map_at_100_max": 0.318308, + "nauc_map_at_100_std": 0.096767, + "nauc_map_at_100_diff1": 0.75565, + "nauc_map_at_1000_max": 0.318308, + "nauc_map_at_1000_std": 0.096767, + "nauc_map_at_1000_diff1": 0.75565, + "nauc_recall_at_1_max": 0.321225, + "nauc_recall_at_1_std": 0.097933, + "nauc_recall_at_1_diff1": 0.795573, + "nauc_recall_at_3_max": 0.31581, + "nauc_recall_at_3_std": 0.109361, + "nauc_recall_at_3_diff1": 0.703402, + "nauc_recall_at_5_max": 0.287955, + "nauc_recall_at_5_std": 0.078376, + "nauc_recall_at_5_diff1": 0.662141, + "nauc_recall_at_10_max": 0.262396, + "nauc_recall_at_10_std": 0.035661, + "nauc_recall_at_10_diff1": 0.650567, + "nauc_recall_at_20_max": 0.164699, + "nauc_recall_at_20_std": -0.104358, + "nauc_recall_at_20_diff1": 0.59197, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.321225, + "nauc_precision_at_1_std": 0.097933, + "nauc_precision_at_1_diff1": 0.795573, + "nauc_precision_at_3_max": 0.31581, + "nauc_precision_at_3_std": 0.109361, + "nauc_precision_at_3_diff1": 0.703402, + "nauc_precision_at_5_max": 0.287955, + "nauc_precision_at_5_std": 0.078376, + "nauc_precision_at_5_diff1": 0.662141, + "nauc_precision_at_10_max": 0.262396, + "nauc_precision_at_10_std": 0.035661, + "nauc_precision_at_10_diff1": 0.650567, + "nauc_precision_at_20_max": 0.164699, + "nauc_precision_at_20_std": -0.104358, + "nauc_precision_at_20_diff1": 0.59197, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.321225, + "nauc_cv_recall_at_1_std": 0.097933, + "nauc_cv_recall_at_1_diff1": 0.795573, + "nauc_cv_recall_at_3_max": 0.31581, + "nauc_cv_recall_at_3_std": 0.109361, + "nauc_cv_recall_at_3_diff1": 0.703402, + "nauc_cv_recall_at_5_max": 0.287955, + "nauc_cv_recall_at_5_std": 0.078376, + "nauc_cv_recall_at_5_diff1": 0.662141, + "nauc_cv_recall_at_10_max": 0.262396, + "nauc_cv_recall_at_10_std": 0.035661, + "nauc_cv_recall_at_10_diff1": 0.650567, + "nauc_cv_recall_at_20_max": 0.164699, + "nauc_cv_recall_at_20_std": -0.104358, + "nauc_cv_recall_at_20_diff1": 0.59197, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.321225, + "nauc_mrr_at_1_std": 0.097933, + "nauc_mrr_at_1_diff1": 0.795573, + "nauc_mrr_at_3_max": 0.325516, + "nauc_mrr_at_3_std": 0.106819, + "nauc_mrr_at_3_diff1": 0.75932, + "nauc_mrr_at_5_max": 0.320766, + "nauc_mrr_at_5_std": 0.101832, + "nauc_mrr_at_5_diff1": 0.752831, + "nauc_mrr_at_10_max": 0.318497, + "nauc_mrr_at_10_std": 0.097649, + "nauc_mrr_at_10_diff1": 0.754011, + "nauc_mrr_at_20_max": 0.315571, + "nauc_mrr_at_20_std": 0.09338, + "nauc_mrr_at_20_diff1": 0.753463, + "nauc_mrr_at_100_max": 0.318308, + "nauc_mrr_at_100_std": 0.096767, + "nauc_mrr_at_100_diff1": 0.75565, + "nauc_mrr_at_1000_max": 0.318308, + "nauc_mrr_at_1000_std": 0.096767, + "nauc_mrr_at_1000_diff1": 0.75565, + "main_score": 0.7534, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 19.42788553237915, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..82be289fed --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", + "task_name": "VidoreTatdqaRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.66039, + "ndcg_at_3": 0.74882, + "ndcg_at_5": 0.77546, + "ndcg_at_10": 0.79412, + "ndcg_at_20": 0.80051, + "ndcg_at_100": 0.80595, + "ndcg_at_1000": 0.80781, + "map_at_1": 0.65948, + "map_at_3": 0.72676, + "map_at_5": 0.74177, + "map_at_10": 0.74958, + "map_at_20": 0.75136, + "map_at_100": 0.75213, + "map_at_1000": 0.75224, + "recall_at_1": 0.65948, + "recall_at_3": 0.81227, + "recall_at_5": 0.87606, + "recall_at_10": 0.93323, + "recall_at_20": 0.9582, + "recall_at_100": 0.98718, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.66039, + "cv_recall_at_3": 0.81288, + "cv_recall_at_5": 0.87667, + "cv_recall_at_10": 0.93378, + "cv_recall_at_20": 0.95869, + "cv_recall_at_100": 0.98785, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.66039, + "precision_at_3": 0.27136, + "precision_at_5": 0.17582, + "precision_at_10": 0.09374, + "precision_at_20": 0.04815, + "precision_at_100": 0.00993, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.660389, + "mrr_at_3": 0.727217, + "mrr_at_5": 0.742102, + "mrr_at_10": 0.749811, + "mrr_at_20": 0.751589, + "mrr_at_100": 0.752362, + "mrr_at_1000": 0.752445, + "nauc_ndcg_at_1_max": 0.144304, + "nauc_ndcg_at_1_std": -0.305119, + "nauc_ndcg_at_1_diff1": 0.798943, + "nauc_ndcg_at_3_max": 0.15797, + "nauc_ndcg_at_3_std": -0.315925, + "nauc_ndcg_at_3_diff1": 0.761715, + "nauc_ndcg_at_5_max": 0.175843, + "nauc_ndcg_at_5_std": -0.2942, + "nauc_ndcg_at_5_diff1": 0.756229, + "nauc_ndcg_at_10_max": 0.178488, + "nauc_ndcg_at_10_std": -0.258939, + "nauc_ndcg_at_10_diff1": 0.75444, + "nauc_ndcg_at_20_max": 0.176982, + "nauc_ndcg_at_20_std": -0.257364, + "nauc_ndcg_at_20_diff1": 0.759061, + "nauc_ndcg_at_100_max": 0.168948, + "nauc_ndcg_at_100_std": -0.274904, + "nauc_ndcg_at_100_diff1": 0.764387, + "nauc_ndcg_at_1000_max": 0.165688, + "nauc_ndcg_at_1000_std": -0.283304, + "nauc_ndcg_at_1000_diff1": 0.766024, + "nauc_map_at_1_max": 0.14602, + "nauc_map_at_1_std": -0.302564, + "nauc_map_at_1_diff1": 0.799108, + "nauc_map_at_3_max": 0.153764, + "nauc_map_at_3_std": -0.314572, + "nauc_map_at_3_diff1": 0.772901, + "nauc_map_at_5_max": 0.162087, + "nauc_map_at_5_std": -0.304077, + "nauc_map_at_5_diff1": 0.770573, + "nauc_map_at_10_max": 0.162528, + "nauc_map_at_10_std": -0.292048, + "nauc_map_at_10_diff1": 0.770418, + "nauc_map_at_20_max": 0.161877, + "nauc_map_at_20_std": -0.291987, + "nauc_map_at_20_diff1": 0.771576, + "nauc_map_at_100_max": 0.160836, + "nauc_map_at_100_std": -0.293882, + "nauc_map_at_100_diff1": 0.77214, + "nauc_map_at_1000_max": 0.160672, + "nauc_map_at_1000_std": -0.294261, + "nauc_map_at_1000_diff1": 0.772204, + "nauc_recall_at_1_max": 0.14602, + "nauc_recall_at_1_std": -0.302564, + "nauc_recall_at_1_diff1": 0.799108, + "nauc_recall_at_3_max": 0.173493, + "nauc_recall_at_3_std": -0.320388, + "nauc_recall_at_3_diff1": 0.71837, + "nauc_recall_at_5_max": 0.252675, + "nauc_recall_at_5_std": -0.236139, + "nauc_recall_at_5_diff1": 0.678724, + "nauc_recall_at_10_max": 0.339806, + "nauc_recall_at_10_std": 0.105236, + "nauc_recall_at_10_diff1": 0.602447, + "nauc_recall_at_20_max": 0.420215, + "nauc_recall_at_20_std": 0.351498, + "nauc_recall_at_20_diff1": 0.588151, + "nauc_recall_at_100_max": 0.439557, + "nauc_recall_at_100_std": 0.463483, + "nauc_recall_at_100_diff1": 0.604439, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.144304, + "nauc_precision_at_1_std": -0.305119, + "nauc_precision_at_1_diff1": 0.798943, + "nauc_precision_at_3_max": 0.169898, + "nauc_precision_at_3_std": -0.321714, + "nauc_precision_at_3_diff1": 0.708409, + "nauc_precision_at_5_max": 0.244753, + "nauc_precision_at_5_std": -0.224105, + "nauc_precision_at_5_diff1": 0.644408, + "nauc_precision_at_10_max": 0.309387, + "nauc_precision_at_10_std": 0.107568, + "nauc_precision_at_10_diff1": 0.518976, + "nauc_precision_at_20_max": 0.344473, + "nauc_precision_at_20_std": 0.3064, + "nauc_precision_at_20_diff1": 0.440787, + "nauc_precision_at_100_max": 0.159357, + "nauc_precision_at_100_std": 0.220511, + "nauc_precision_at_100_diff1": 0.151279, + "nauc_precision_at_1000_max": -0.354982, + "nauc_precision_at_1000_std": -0.298309, + "nauc_precision_at_1000_diff1": -0.480202, + "nauc_cv_recall_at_1_max": 0.144304, + "nauc_cv_recall_at_1_std": -0.305119, + "nauc_cv_recall_at_1_diff1": 0.798943, + "nauc_cv_recall_at_3_max": 0.175771, + "nauc_cv_recall_at_3_std": -0.319572, + "nauc_cv_recall_at_3_diff1": 0.718587, + "nauc_cv_recall_at_5_max": 0.256775, + "nauc_cv_recall_at_5_std": -0.231358, + "nauc_cv_recall_at_5_diff1": 0.67957, + "nauc_cv_recall_at_10_max": 0.334355, + "nauc_cv_recall_at_10_std": 0.097848, + "nauc_cv_recall_at_10_diff1": 0.599165, + "nauc_cv_recall_at_20_max": 0.413394, + "nauc_cv_recall_at_20_std": 0.343869, + "nauc_cv_recall_at_20_diff1": 0.583305, + "nauc_cv_recall_at_100_max": 0.408733, + "nauc_cv_recall_at_100_std": 0.433975, + "nauc_cv_recall_at_100_diff1": 0.585952, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.144304, + "nauc_mrr_at_1_std": -0.305119, + "nauc_mrr_at_1_diff1": 0.798943, + "nauc_mrr_at_3_max": 0.15414, + "nauc_mrr_at_3_std": -0.314369, + "nauc_mrr_at_3_diff1": 0.772469, + "nauc_mrr_at_5_max": 0.162521, + "nauc_mrr_at_5_std": -0.303648, + "nauc_mrr_at_5_diff1": 0.770553, + "nauc_mrr_at_10_max": 0.162344, + "nauc_mrr_at_10_std": -0.292567, + "nauc_mrr_at_10_diff1": 0.770198, + "nauc_mrr_at_20_max": 0.161717, + "nauc_mrr_at_20_std": -0.29247, + "nauc_mrr_at_20_diff1": 0.771363, + "nauc_mrr_at_100_max": 0.16067, + "nauc_mrr_at_100_std": -0.294377, + "nauc_mrr_at_100_diff1": 0.77193, + "nauc_mrr_at_1000_max": 0.160581, + "nauc_mrr_at_1000_std": -0.29464, + "nauc_mrr_at_1000_diff1": 0.772012, + "main_score": 0.77546, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 116.9430501461029, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json new file mode 100644 index 0000000000..ffbc9ab7d1 --- /dev/null +++ b/results/vidore__colSmol-500M/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json @@ -0,0 +1 @@ +{"name": "vidore/colSmol-500M", "revision": "530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f", "release_date": "2025-01-22", "languages": ["eng-Latn"], "n_parameters": 500000000, "memory_usage_mb": 1200.0, "max_tokens": 8192.0, "embed_dim": 128, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/illuin-tech/colpali", "public_training_data": "https://huggingface.co/datasets/vidore/colpali_train_set", "framework": ["ColPali"], "reference": "https://huggingface.co/vidore/colSmol-500M", "similarity_fn_name": "max_sim", "use_instructions": true, "training_datasets": {"DocVQA": ["train"], "InfoVQA": ["train"], "TATDQA": ["train"], "arXivQA": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "ColSmolWrapper"} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/Vidore2BioMedicalLecturesRetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..a6350ee3a6 --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.45, + "ndcg_at_3": 0.45963, + "ndcg_at_5": 0.4949, + "ndcg_at_10": 0.52721, + "ndcg_at_20": 0.56025, + "ndcg_at_100": 0.60196, + "ndcg_at_1000": 0.62491, + "map_at_1": 0.26724, + "map_at_3": 0.36191, + "map_at_5": 0.40159, + "map_at_10": 0.43073, + "map_at_20": 0.44619, + "map_at_100": 0.45789, + "map_at_1000": 0.46, + "recall_at_1": 0.26724, + "recall_at_3": 0.44094, + "recall_at_5": 0.54726, + "recall_at_10": 0.64376, + "recall_at_20": 0.74245, + "recall_at_100": 0.88632, + "recall_at_1000": 0.99978, + "cv_recall_at_1": 0.45, + "cv_recall_at_3": 0.6625, + "cv_recall_at_5": 0.76875, + "cv_recall_at_10": 0.84375, + "cv_recall_at_20": 0.925, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.45, + "precision_at_3": 0.28542, + "precision_at_5": 0.23, + "precision_at_10": 0.15, + "precision_at_20": 0.09312, + "precision_at_100": 0.02544, + "precision_at_1000": 0.00321, + "mrr_at_1": 0.45, + "mrr_at_3": 0.541667, + "mrr_at_5": 0.565729, + "mrr_at_10": 0.575685, + "mrr_at_20": 0.581001, + "mrr_at_100": 0.582675, + "mrr_at_1000": 0.582721, + "nauc_ndcg_at_1_max": 0.323037, + "nauc_ndcg_at_1_std": 0.114991, + "nauc_ndcg_at_1_diff1": 0.442881, + "nauc_ndcg_at_3_max": 0.389906, + "nauc_ndcg_at_3_std": 0.128216, + "nauc_ndcg_at_3_diff1": 0.312721, + "nauc_ndcg_at_5_max": 0.405545, + "nauc_ndcg_at_5_std": 0.153079, + "nauc_ndcg_at_5_diff1": 0.300555, + "nauc_ndcg_at_10_max": 0.381297, + "nauc_ndcg_at_10_std": 0.145669, + "nauc_ndcg_at_10_diff1": 0.287571, + "nauc_ndcg_at_20_max": 0.382448, + "nauc_ndcg_at_20_std": 0.142253, + "nauc_ndcg_at_20_diff1": 0.254235, + "nauc_ndcg_at_100_max": 0.36308, + "nauc_ndcg_at_100_std": 0.122989, + "nauc_ndcg_at_100_diff1": 0.267692, + "nauc_ndcg_at_1000_max": 0.348583, + "nauc_ndcg_at_1000_std": 0.119794, + "nauc_ndcg_at_1000_diff1": 0.285546, + "nauc_map_at_1_max": 0.341284, + "nauc_map_at_1_std": 0.229387, + "nauc_map_at_1_diff1": 0.395913, + "nauc_map_at_3_max": 0.395477, + "nauc_map_at_3_std": 0.179349, + "nauc_map_at_3_diff1": 0.332683, + "nauc_map_at_5_max": 0.394901, + "nauc_map_at_5_std": 0.169187, + "nauc_map_at_5_diff1": 0.328107, + "nauc_map_at_10_max": 0.371605, + "nauc_map_at_10_std": 0.150784, + "nauc_map_at_10_diff1": 0.309171, + "nauc_map_at_20_max": 0.370742, + "nauc_map_at_20_std": 0.149009, + "nauc_map_at_20_diff1": 0.293812, + "nauc_map_at_100_max": 0.365259, + "nauc_map_at_100_std": 0.144357, + "nauc_map_at_100_diff1": 0.295809, + "nauc_map_at_1000_max": 0.363806, + "nauc_map_at_1000_std": 0.143351, + "nauc_map_at_1000_diff1": 0.296537, + "nauc_recall_at_1_max": 0.341284, + "nauc_recall_at_1_std": 0.229387, + "nauc_recall_at_1_diff1": 0.395913, + "nauc_recall_at_3_max": 0.410367, + "nauc_recall_at_3_std": 0.163028, + "nauc_recall_at_3_diff1": 0.250888, + "nauc_recall_at_5_max": 0.400473, + "nauc_recall_at_5_std": 0.150508, + "nauc_recall_at_5_diff1": 0.207255, + "nauc_recall_at_10_max": 0.369474, + "nauc_recall_at_10_std": 0.151609, + "nauc_recall_at_10_diff1": 0.141339, + "nauc_recall_at_20_max": 0.366998, + "nauc_recall_at_20_std": 0.131321, + "nauc_recall_at_20_diff1": -0.027105, + "nauc_recall_at_100_max": 0.328807, + "nauc_recall_at_100_std": 0.024641, + "nauc_recall_at_100_diff1": -0.090562, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.869281, + "nauc_recall_at_1000_diff1": 0.358077, + "nauc_precision_at_1_max": 0.323037, + "nauc_precision_at_1_std": 0.114991, + "nauc_precision_at_1_diff1": 0.442881, + "nauc_precision_at_3_max": 0.256129, + "nauc_precision_at_3_std": -0.043382, + "nauc_precision_at_3_diff1": 0.119799, + "nauc_precision_at_5_max": 0.146338, + "nauc_precision_at_5_std": -0.072498, + "nauc_precision_at_5_diff1": -0.001215, + "nauc_precision_at_10_max": -0.025978, + "nauc_precision_at_10_std": -0.131804, + "nauc_precision_at_10_diff1": -0.091907, + "nauc_precision_at_20_max": -0.080345, + "nauc_precision_at_20_std": -0.15218, + "nauc_precision_at_20_diff1": -0.176321, + "nauc_precision_at_100_max": -0.197364, + "nauc_precision_at_100_std": -0.193598, + "nauc_precision_at_100_diff1": -0.188067, + "nauc_precision_at_1000_max": -0.290745, + "nauc_precision_at_1000_std": -0.215541, + "nauc_precision_at_1000_diff1": -0.189551, + "nauc_cv_recall_at_1_max": 0.323037, + "nauc_cv_recall_at_1_std": 0.114991, + "nauc_cv_recall_at_1_diff1": 0.442881, + "nauc_cv_recall_at_3_max": 0.439138, + "nauc_cv_recall_at_3_std": 0.102618, + "nauc_cv_recall_at_3_diff1": 0.333028, + "nauc_cv_recall_at_5_max": 0.49218, + "nauc_cv_recall_at_5_std": 0.087076, + "nauc_cv_recall_at_5_diff1": 0.214362, + "nauc_cv_recall_at_10_max": 0.490515, + "nauc_cv_recall_at_10_std": 0.155144, + "nauc_cv_recall_at_10_diff1": 0.07864, + "nauc_cv_recall_at_20_max": 0.514939, + "nauc_cv_recall_at_20_std": 0.106598, + "nauc_cv_recall_at_20_diff1": -0.310924, + "nauc_cv_recall_at_100_max": 0.190165, + "nauc_cv_recall_at_100_std": -0.478058, + "nauc_cv_recall_at_100_diff1": -0.82493, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.323037, + "nauc_mrr_at_1_std": 0.114991, + "nauc_mrr_at_1_diff1": 0.442881, + "nauc_mrr_at_3_max": 0.370803, + "nauc_mrr_at_3_std": 0.11447, + "nauc_mrr_at_3_diff1": 0.399297, + "nauc_mrr_at_5_max": 0.377188, + "nauc_mrr_at_5_std": 0.112451, + "nauc_mrr_at_5_diff1": 0.387636, + "nauc_mrr_at_10_max": 0.374815, + "nauc_mrr_at_10_std": 0.120653, + "nauc_mrr_at_10_diff1": 0.382154, + "nauc_mrr_at_20_max": 0.374027, + "nauc_mrr_at_20_std": 0.119995, + "nauc_mrr_at_20_diff1": 0.380631, + "nauc_mrr_at_100_max": 0.372583, + "nauc_mrr_at_100_std": 0.119042, + "nauc_mrr_at_100_diff1": 0.383767, + "nauc_mrr_at_1000_max": 0.372628, + "nauc_mrr_at_1000_std": 0.119163, + "nauc_mrr_at_1000_diff1": 0.38395, + "main_score": 0.4949, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.4625, + "ndcg_at_3": 0.48174, + "ndcg_at_5": 0.50104, + "ndcg_at_10": 0.53148, + "ndcg_at_20": 0.56281, + "ndcg_at_100": 0.61019, + "ndcg_at_1000": 0.63242, + "map_at_1": 0.2738, + "map_at_3": 0.38128, + "map_at_5": 0.41138, + "map_at_10": 0.44002, + "map_at_20": 0.45462, + "map_at_100": 0.46795, + "map_at_1000": 0.47007, + "recall_at_1": 0.2738, + "recall_at_3": 0.45777, + "recall_at_5": 0.53844, + "recall_at_10": 0.63368, + "recall_at_20": 0.73017, + "recall_at_100": 0.89337, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.4625, + "cv_recall_at_3": 0.675, + "cv_recall_at_5": 0.7625, + "cv_recall_at_10": 0.825, + "cv_recall_at_20": 0.89375, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.4625, + "precision_at_3": 0.30208, + "precision_at_5": 0.22875, + "precision_at_10": 0.14688, + "precision_at_20": 0.09, + "precision_at_100": 0.02556, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.4625, + "mrr_at_3": 0.555208, + "mrr_at_5": 0.575521, + "mrr_at_10": 0.584033, + "mrr_at_20": 0.588886, + "mrr_at_100": 0.591503, + "mrr_at_1000": 0.591613, + "nauc_ndcg_at_1_max": 0.302241, + "nauc_ndcg_at_1_std": 0.057785, + "nauc_ndcg_at_1_diff1": 0.465222, + "nauc_ndcg_at_3_max": 0.325206, + "nauc_ndcg_at_3_std": 0.087965, + "nauc_ndcg_at_3_diff1": 0.360743, + "nauc_ndcg_at_5_max": 0.362622, + "nauc_ndcg_at_5_std": 0.122277, + "nauc_ndcg_at_5_diff1": 0.322754, + "nauc_ndcg_at_10_max": 0.362541, + "nauc_ndcg_at_10_std": 0.097293, + "nauc_ndcg_at_10_diff1": 0.335557, + "nauc_ndcg_at_20_max": 0.352526, + "nauc_ndcg_at_20_std": 0.092059, + "nauc_ndcg_at_20_diff1": 0.316024, + "nauc_ndcg_at_100_max": 0.335759, + "nauc_ndcg_at_100_std": 0.089969, + "nauc_ndcg_at_100_diff1": 0.326865, + "nauc_ndcg_at_1000_max": 0.318192, + "nauc_ndcg_at_1000_std": 0.073614, + "nauc_ndcg_at_1000_diff1": 0.344972, + "nauc_map_at_1_max": 0.359171, + "nauc_map_at_1_std": 0.14196, + "nauc_map_at_1_diff1": 0.435605, + "nauc_map_at_3_max": 0.370141, + "nauc_map_at_3_std": 0.132383, + "nauc_map_at_3_diff1": 0.387566, + "nauc_map_at_5_max": 0.375211, + "nauc_map_at_5_std": 0.131044, + "nauc_map_at_5_diff1": 0.358531, + "nauc_map_at_10_max": 0.360863, + "nauc_map_at_10_std": 0.104123, + "nauc_map_at_10_diff1": 0.354018, + "nauc_map_at_20_max": 0.353617, + "nauc_map_at_20_std": 0.101716, + "nauc_map_at_20_diff1": 0.347366, + "nauc_map_at_100_max": 0.349161, + "nauc_map_at_100_std": 0.099038, + "nauc_map_at_100_diff1": 0.348949, + "nauc_map_at_1000_max": 0.347175, + "nauc_map_at_1000_std": 0.097574, + "nauc_map_at_1000_diff1": 0.349591, + "nauc_recall_at_1_max": 0.359171, + "nauc_recall_at_1_std": 0.14196, + "nauc_recall_at_1_diff1": 0.435605, + "nauc_recall_at_3_max": 0.365964, + "nauc_recall_at_3_std": 0.143916, + "nauc_recall_at_3_diff1": 0.310879, + "nauc_recall_at_5_max": 0.393141, + "nauc_recall_at_5_std": 0.162111, + "nauc_recall_at_5_diff1": 0.209735, + "nauc_recall_at_10_max": 0.38377, + "nauc_recall_at_10_std": 0.114858, + "nauc_recall_at_10_diff1": 0.193388, + "nauc_recall_at_20_max": 0.342147, + "nauc_recall_at_20_std": 0.075034, + "nauc_recall_at_20_diff1": 0.079886, + "nauc_recall_at_100_max": 0.313719, + "nauc_recall_at_100_std": 0.15158, + "nauc_recall_at_100_diff1": -0.05695, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.302241, + "nauc_precision_at_1_std": 0.057785, + "nauc_precision_at_1_diff1": 0.465222, + "nauc_precision_at_3_max": 0.120524, + "nauc_precision_at_3_std": -0.027804, + "nauc_precision_at_3_diff1": 0.148728, + "nauc_precision_at_5_max": 0.062786, + "nauc_precision_at_5_std": -0.030679, + "nauc_precision_at_5_diff1": -0.01407, + "nauc_precision_at_10_max": -0.051847, + "nauc_precision_at_10_std": -0.125226, + "nauc_precision_at_10_diff1": -0.058311, + "nauc_precision_at_20_max": -0.122314, + "nauc_precision_at_20_std": -0.140444, + "nauc_precision_at_20_diff1": -0.107146, + "nauc_precision_at_100_max": -0.248426, + "nauc_precision_at_100_std": -0.177465, + "nauc_precision_at_100_diff1": -0.135649, + "nauc_precision_at_1000_max": -0.332366, + "nauc_precision_at_1000_std": -0.240564, + "nauc_precision_at_1000_diff1": -0.146724, + "nauc_cv_recall_at_1_max": 0.302241, + "nauc_cv_recall_at_1_std": 0.057785, + "nauc_cv_recall_at_1_diff1": 0.465222, + "nauc_cv_recall_at_3_max": 0.345988, + "nauc_cv_recall_at_3_std": 0.071117, + "nauc_cv_recall_at_3_diff1": 0.365974, + "nauc_cv_recall_at_5_max": 0.390914, + "nauc_cv_recall_at_5_std": 0.151106, + "nauc_cv_recall_at_5_diff1": 0.210605, + "nauc_cv_recall_at_10_max": 0.48541, + "nauc_cv_recall_at_10_std": 0.20095, + "nauc_cv_recall_at_10_diff1": 0.203158, + "nauc_cv_recall_at_20_max": 0.55618, + "nauc_cv_recall_at_20_std": 0.189711, + "nauc_cv_recall_at_20_diff1": 0.040737, + "nauc_cv_recall_at_100_max": 0.301587, + "nauc_cv_recall_at_100_std": 0.157446, + "nauc_cv_recall_at_100_diff1": -0.256769, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.302241, + "nauc_mrr_at_1_std": 0.057785, + "nauc_mrr_at_1_diff1": 0.465222, + "nauc_mrr_at_3_max": 0.312595, + "nauc_mrr_at_3_std": 0.065369, + "nauc_mrr_at_3_diff1": 0.428501, + "nauc_mrr_at_5_max": 0.316016, + "nauc_mrr_at_5_std": 0.076352, + "nauc_mrr_at_5_diff1": 0.405946, + "nauc_mrr_at_10_max": 0.320389, + "nauc_mrr_at_10_std": 0.077878, + "nauc_mrr_at_10_diff1": 0.41082, + "nauc_mrr_at_20_max": 0.320055, + "nauc_mrr_at_20_std": 0.076138, + "nauc_mrr_at_20_diff1": 0.4109, + "nauc_mrr_at_100_max": 0.317593, + "nauc_mrr_at_100_std": 0.075666, + "nauc_mrr_at_100_diff1": 0.414034, + "nauc_mrr_at_1000_max": 0.317518, + "nauc_mrr_at_1000_std": 0.075594, + "nauc_mrr_at_1000_diff1": 0.41418, + "main_score": 0.50104, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.48125, + "ndcg_at_3": 0.54944, + "ndcg_at_5": 0.5631, + "ndcg_at_10": 0.5993, + "ndcg_at_20": 0.62358, + "ndcg_at_100": 0.66024, + "ndcg_at_1000": 0.68125, + "map_at_1": 0.29224, + "map_at_3": 0.43651, + "map_at_5": 0.46816, + "map_at_10": 0.50079, + "map_at_20": 0.51463, + "map_at_100": 0.52704, + "map_at_1000": 0.5291, + "recall_at_1": 0.29224, + "recall_at_3": 0.53606, + "recall_at_5": 0.60294, + "recall_at_10": 0.71294, + "recall_at_20": 0.78186, + "recall_at_100": 0.89701, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.48125, + "cv_recall_at_3": 0.78125, + "cv_recall_at_5": 0.8375, + "cv_recall_at_10": 0.91875, + "cv_recall_at_20": 0.94375, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.48125, + "precision_at_3": 0.35208, + "precision_at_5": 0.26125, + "precision_at_10": 0.16813, + "precision_at_20": 0.09938, + "precision_at_100": 0.02644, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.48125, + "mrr_at_3": 0.619792, + "mrr_at_5": 0.633229, + "mrr_at_10": 0.644387, + "mrr_at_20": 0.646284, + "mrr_at_100": 0.647116, + "mrr_at_1000": 0.647263, + "nauc_ndcg_at_1_max": 0.242411, + "nauc_ndcg_at_1_std": 0.041566, + "nauc_ndcg_at_1_diff1": 0.496686, + "nauc_ndcg_at_3_max": 0.27023, + "nauc_ndcg_at_3_std": 0.055435, + "nauc_ndcg_at_3_diff1": 0.366902, + "nauc_ndcg_at_5_max": 0.270364, + "nauc_ndcg_at_5_std": 0.06134, + "nauc_ndcg_at_5_diff1": 0.377327, + "nauc_ndcg_at_10_max": 0.248082, + "nauc_ndcg_at_10_std": 0.033009, + "nauc_ndcg_at_10_diff1": 0.375708, + "nauc_ndcg_at_20_max": 0.245267, + "nauc_ndcg_at_20_std": 0.0375, + "nauc_ndcg_at_20_diff1": 0.354333, + "nauc_ndcg_at_100_max": 0.220233, + "nauc_ndcg_at_100_std": 0.021132, + "nauc_ndcg_at_100_diff1": 0.360625, + "nauc_ndcg_at_1000_max": 0.197819, + "nauc_ndcg_at_1000_std": -0.009223, + "nauc_ndcg_at_1000_diff1": 0.365733, + "nauc_map_at_1_max": 0.256308, + "nauc_map_at_1_std": 0.078927, + "nauc_map_at_1_diff1": 0.457767, + "nauc_map_at_3_max": 0.292686, + "nauc_map_at_3_std": 0.086152, + "nauc_map_at_3_diff1": 0.404572, + "nauc_map_at_5_max": 0.274206, + "nauc_map_at_5_std": 0.065225, + "nauc_map_at_5_diff1": 0.400349, + "nauc_map_at_10_max": 0.256723, + "nauc_map_at_10_std": 0.049269, + "nauc_map_at_10_diff1": 0.3884, + "nauc_map_at_20_max": 0.251147, + "nauc_map_at_20_std": 0.045181, + "nauc_map_at_20_diff1": 0.379046, + "nauc_map_at_100_max": 0.240287, + "nauc_map_at_100_std": 0.034966, + "nauc_map_at_100_diff1": 0.379513, + "nauc_map_at_1000_max": 0.237943, + "nauc_map_at_1000_std": 0.032433, + "nauc_map_at_1000_diff1": 0.379207, + "nauc_recall_at_1_max": 0.256308, + "nauc_recall_at_1_std": 0.078927, + "nauc_recall_at_1_diff1": 0.457767, + "nauc_recall_at_3_max": 0.337651, + "nauc_recall_at_3_std": 0.135944, + "nauc_recall_at_3_diff1": 0.297812, + "nauc_recall_at_5_max": 0.30303, + "nauc_recall_at_5_std": 0.123134, + "nauc_recall_at_5_diff1": 0.28666, + "nauc_recall_at_10_max": 0.238735, + "nauc_recall_at_10_std": 0.06792, + "nauc_recall_at_10_diff1": 0.227767, + "nauc_recall_at_20_max": 0.252613, + "nauc_recall_at_20_std": 0.095625, + "nauc_recall_at_20_diff1": 0.142413, + "nauc_recall_at_100_max": 0.275919, + "nauc_recall_at_100_std": 0.212529, + "nauc_recall_at_100_diff1": 0.141458, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.242411, + "nauc_precision_at_1_std": 0.041566, + "nauc_precision_at_1_diff1": 0.496686, + "nauc_precision_at_3_max": 0.039717, + "nauc_precision_at_3_std": -0.072375, + "nauc_precision_at_3_diff1": 0.093154, + "nauc_precision_at_5_max": -0.065636, + "nauc_precision_at_5_std": -0.135186, + "nauc_precision_at_5_diff1": 0.013927, + "nauc_precision_at_10_max": -0.160175, + "nauc_precision_at_10_std": -0.183117, + "nauc_precision_at_10_diff1": -0.091766, + "nauc_precision_at_20_max": -0.194638, + "nauc_precision_at_20_std": -0.180217, + "nauc_precision_at_20_diff1": -0.155455, + "nauc_precision_at_100_max": -0.288694, + "nauc_precision_at_100_std": -0.229862, + "nauc_precision_at_100_diff1": -0.200462, + "nauc_precision_at_1000_max": -0.358676, + "nauc_precision_at_1000_std": -0.299442, + "nauc_precision_at_1000_diff1": -0.257015, + "nauc_cv_recall_at_1_max": 0.242411, + "nauc_cv_recall_at_1_std": 0.041566, + "nauc_cv_recall_at_1_diff1": 0.496686, + "nauc_cv_recall_at_3_max": 0.329015, + "nauc_cv_recall_at_3_std": 0.049298, + "nauc_cv_recall_at_3_diff1": 0.385722, + "nauc_cv_recall_at_5_max": 0.238146, + "nauc_cv_recall_at_5_std": -0.023236, + "nauc_cv_recall_at_5_diff1": 0.365273, + "nauc_cv_recall_at_10_max": 0.054299, + "nauc_cv_recall_at_10_std": -0.210479, + "nauc_cv_recall_at_10_diff1": 0.171335, + "nauc_cv_recall_at_20_max": 0.107221, + "nauc_cv_recall_at_20_std": -0.166874, + "nauc_cv_recall_at_20_diff1": 0.057164, + "nauc_cv_recall_at_100_max": -0.047269, + "nauc_cv_recall_at_100_std": -0.291083, + "nauc_cv_recall_at_100_diff1": -0.3331, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.242411, + "nauc_mrr_at_1_std": 0.041566, + "nauc_mrr_at_1_diff1": 0.496686, + "nauc_mrr_at_3_max": 0.274783, + "nauc_mrr_at_3_std": 0.050893, + "nauc_mrr_at_3_diff1": 0.452927, + "nauc_mrr_at_5_max": 0.261284, + "nauc_mrr_at_5_std": 0.042414, + "nauc_mrr_at_5_diff1": 0.453528, + "nauc_mrr_at_10_max": 0.254588, + "nauc_mrr_at_10_std": 0.036564, + "nauc_mrr_at_10_diff1": 0.449237, + "nauc_mrr_at_20_max": 0.256251, + "nauc_mrr_at_20_std": 0.038483, + "nauc_mrr_at_20_diff1": 0.448843, + "nauc_mrr_at_100_max": 0.256594, + "nauc_mrr_at_100_std": 0.039031, + "nauc_mrr_at_100_diff1": 0.449141, + "nauc_mrr_at_1000_max": 0.25664, + "nauc_mrr_at_1000_std": 0.039169, + "nauc_mrr_at_1000_diff1": 0.449561, + "main_score": 0.5631, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.45, + "ndcg_at_3": 0.46414, + "ndcg_at_5": 0.48644, + "ndcg_at_10": 0.52012, + "ndcg_at_20": 0.54933, + "ndcg_at_100": 0.5987, + "ndcg_at_1000": 0.62068, + "map_at_1": 0.26293, + "map_at_3": 0.36751, + "map_at_5": 0.40222, + "map_at_10": 0.42805, + "map_at_20": 0.44284, + "map_at_100": 0.45546, + "map_at_1000": 0.4576, + "recall_at_1": 0.26293, + "recall_at_3": 0.44173, + "recall_at_5": 0.52491, + "recall_at_10": 0.63159, + "recall_at_20": 0.71019, + "recall_at_100": 0.8929, + "recall_at_1000": 0.99935, + "cv_recall_at_1": 0.45, + "cv_recall_at_3": 0.6625, + "cv_recall_at_5": 0.73125, + "cv_recall_at_10": 0.84375, + "cv_recall_at_20": 0.8875, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.45, + "precision_at_3": 0.2875, + "precision_at_5": 0.22, + "precision_at_10": 0.1425, + "precision_at_20": 0.08875, + "precision_at_100": 0.02575, + "precision_at_1000": 0.0032, + "mrr_at_1": 0.45, + "mrr_at_3": 0.547917, + "mrr_at_5": 0.563229, + "mrr_at_10": 0.577495, + "mrr_at_20": 0.58068, + "mrr_at_100": 0.582643, + "mrr_at_1000": 0.582702, + "nauc_ndcg_at_1_max": 0.214758, + "nauc_ndcg_at_1_std": 0.004485, + "nauc_ndcg_at_1_diff1": 0.312537, + "nauc_ndcg_at_3_max": 0.311903, + "nauc_ndcg_at_3_std": 0.102675, + "nauc_ndcg_at_3_diff1": 0.271746, + "nauc_ndcg_at_5_max": 0.322179, + "nauc_ndcg_at_5_std": 0.115555, + "nauc_ndcg_at_5_diff1": 0.266529, + "nauc_ndcg_at_10_max": 0.289252, + "nauc_ndcg_at_10_std": 0.06547, + "nauc_ndcg_at_10_diff1": 0.265471, + "nauc_ndcg_at_20_max": 0.293106, + "nauc_ndcg_at_20_std": 0.052217, + "nauc_ndcg_at_20_diff1": 0.262388, + "nauc_ndcg_at_100_max": 0.276874, + "nauc_ndcg_at_100_std": 0.048401, + "nauc_ndcg_at_100_diff1": 0.273323, + "nauc_ndcg_at_1000_max": 0.263956, + "nauc_ndcg_at_1000_std": 0.038382, + "nauc_ndcg_at_1000_diff1": 0.262047, + "nauc_map_at_1_max": 0.295745, + "nauc_map_at_1_std": 0.088994, + "nauc_map_at_1_diff1": 0.40886, + "nauc_map_at_3_max": 0.3372, + "nauc_map_at_3_std": 0.133085, + "nauc_map_at_3_diff1": 0.331151, + "nauc_map_at_5_max": 0.325906, + "nauc_map_at_5_std": 0.11187, + "nauc_map_at_5_diff1": 0.308411, + "nauc_map_at_10_max": 0.296989, + "nauc_map_at_10_std": 0.075903, + "nauc_map_at_10_diff1": 0.288016, + "nauc_map_at_20_max": 0.296749, + "nauc_map_at_20_std": 0.067391, + "nauc_map_at_20_diff1": 0.285015, + "nauc_map_at_100_max": 0.292058, + "nauc_map_at_100_std": 0.064122, + "nauc_map_at_100_diff1": 0.283277, + "nauc_map_at_1000_max": 0.290365, + "nauc_map_at_1000_std": 0.062733, + "nauc_map_at_1000_diff1": 0.282125, + "nauc_recall_at_1_max": 0.295745, + "nauc_recall_at_1_std": 0.088994, + "nauc_recall_at_1_diff1": 0.40886, + "nauc_recall_at_3_max": 0.362656, + "nauc_recall_at_3_std": 0.20435, + "nauc_recall_at_3_diff1": 0.274911, + "nauc_recall_at_5_max": 0.339798, + "nauc_recall_at_5_std": 0.18187, + "nauc_recall_at_5_diff1": 0.229967, + "nauc_recall_at_10_max": 0.245766, + "nauc_recall_at_10_std": 0.054385, + "nauc_recall_at_10_diff1": 0.165875, + "nauc_recall_at_20_max": 0.266094, + "nauc_recall_at_20_std": 0.013506, + "nauc_recall_at_20_diff1": 0.160873, + "nauc_recall_at_100_max": 0.215967, + "nauc_recall_at_100_std": 0.04041, + "nauc_recall_at_100_diff1": 0.259748, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.869281, + "nauc_recall_at_1000_diff1": 0.722222, + "nauc_precision_at_1_max": 0.214758, + "nauc_precision_at_1_std": 0.004485, + "nauc_precision_at_1_diff1": 0.312537, + "nauc_precision_at_3_max": 0.156667, + "nauc_precision_at_3_std": 0.009171, + "nauc_precision_at_3_diff1": 0.037438, + "nauc_precision_at_5_max": 0.068941, + "nauc_precision_at_5_std": -0.046582, + "nauc_precision_at_5_diff1": -0.071025, + "nauc_precision_at_10_max": -0.092166, + "nauc_precision_at_10_std": -0.194563, + "nauc_precision_at_10_diff1": -0.139923, + "nauc_precision_at_20_max": -0.121223, + "nauc_precision_at_20_std": -0.202924, + "nauc_precision_at_20_diff1": -0.181633, + "nauc_precision_at_100_max": -0.233611, + "nauc_precision_at_100_std": -0.199712, + "nauc_precision_at_100_diff1": -0.19627, + "nauc_precision_at_1000_max": -0.29113, + "nauc_precision_at_1000_std": -0.223171, + "nauc_precision_at_1000_diff1": -0.236296, + "nauc_cv_recall_at_1_max": 0.214758, + "nauc_cv_recall_at_1_std": 0.004485, + "nauc_cv_recall_at_1_diff1": 0.312537, + "nauc_cv_recall_at_3_max": 0.3587, + "nauc_cv_recall_at_3_std": 0.196116, + "nauc_cv_recall_at_3_diff1": 0.288843, + "nauc_cv_recall_at_5_max": 0.335504, + "nauc_cv_recall_at_5_std": 0.169547, + "nauc_cv_recall_at_5_diff1": 0.295618, + "nauc_cv_recall_at_10_max": 0.241642, + "nauc_cv_recall_at_10_std": -0.017264, + "nauc_cv_recall_at_10_diff1": 0.259416, + "nauc_cv_recall_at_20_max": 0.319323, + "nauc_cv_recall_at_20_std": -0.001132, + "nauc_cv_recall_at_20_diff1": 0.089983, + "nauc_cv_recall_at_100_max": -0.219888, + "nauc_cv_recall_at_100_std": -0.452381, + "nauc_cv_recall_at_100_diff1": 0.068783, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.214758, + "nauc_mrr_at_1_std": 0.004485, + "nauc_mrr_at_1_diff1": 0.312537, + "nauc_mrr_at_3_max": 0.276985, + "nauc_mrr_at_3_std": 0.08486, + "nauc_mrr_at_3_diff1": 0.295363, + "nauc_mrr_at_5_max": 0.271068, + "nauc_mrr_at_5_std": 0.075597, + "nauc_mrr_at_5_diff1": 0.295829, + "nauc_mrr_at_10_max": 0.265558, + "nauc_mrr_at_10_std": 0.062343, + "nauc_mrr_at_10_diff1": 0.293377, + "nauc_mrr_at_20_max": 0.2693, + "nauc_mrr_at_20_std": 0.065199, + "nauc_mrr_at_20_diff1": 0.288995, + "nauc_mrr_at_100_max": 0.268238, + "nauc_mrr_at_100_std": 0.064243, + "nauc_mrr_at_100_diff1": 0.289512, + "nauc_mrr_at_1000_max": 0.268308, + "nauc_mrr_at_1000_std": 0.064322, + "nauc_mrr_at_1000_diff1": 0.289546, + "main_score": 0.48644, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 157.30244159698486, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/Vidore2ESGReportsHLRetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..6c6e1308f0 --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.52564, + "ndcg_at_3": 0.5324, + "ndcg_at_5": 0.56687, + "ndcg_at_10": 0.60425, + "ndcg_at_20": 0.62471, + "ndcg_at_100": 0.67082, + "ndcg_at_1000": 0.67707, + "map_at_1": 0.34502, + "map_at_3": 0.44316, + "map_at_5": 0.49159, + "map_at_10": 0.51946, + "map_at_20": 0.53046, + "map_at_100": 0.54184, + "map_at_1000": 0.54247, + "recall_at_1": 0.34502, + "recall_at_3": 0.52373, + "recall_at_5": 0.61718, + "recall_at_10": 0.71847, + "recall_at_20": 0.77969, + "recall_at_100": 0.96864, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.53846, + "cv_recall_at_3": 0.75, + "cv_recall_at_5": 0.76923, + "cv_recall_at_10": 0.84615, + "cv_recall_at_20": 0.86538, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.53846, + "precision_at_3": 0.28846, + "precision_at_5": 0.23462, + "precision_at_10": 0.14808, + "precision_at_20": 0.08462, + "precision_at_100": 0.02269, + "precision_at_1000": 0.00246, + "mrr_at_1": 0.538462, + "mrr_at_3": 0.637821, + "mrr_at_5": 0.642628, + "mrr_at_10": 0.653922, + "mrr_at_20": 0.655402, + "mrr_at_100": 0.659347, + "mrr_at_1000": 0.659347, + "nauc_ndcg_at_1_max": 0.08259, + "nauc_ndcg_at_1_std": 0.097936, + "nauc_ndcg_at_1_diff1": 0.485394, + "nauc_ndcg_at_3_max": 0.288381, + "nauc_ndcg_at_3_std": 0.232301, + "nauc_ndcg_at_3_diff1": 0.547798, + "nauc_ndcg_at_5_max": 0.311954, + "nauc_ndcg_at_5_std": 0.262433, + "nauc_ndcg_at_5_diff1": 0.552132, + "nauc_ndcg_at_10_max": 0.340172, + "nauc_ndcg_at_10_std": 0.232176, + "nauc_ndcg_at_10_diff1": 0.524005, + "nauc_ndcg_at_20_max": 0.319755, + "nauc_ndcg_at_20_std": 0.227139, + "nauc_ndcg_at_20_diff1": 0.514302, + "nauc_ndcg_at_100_max": 0.278042, + "nauc_ndcg_at_100_std": 0.220181, + "nauc_ndcg_at_100_diff1": 0.504022, + "nauc_ndcg_at_1000_max": 0.266918, + "nauc_ndcg_at_1000_std": 0.203655, + "nauc_ndcg_at_1000_diff1": 0.505964, + "nauc_map_at_1_max": 0.135923, + "nauc_map_at_1_std": 0.035513, + "nauc_map_at_1_diff1": 0.653538, + "nauc_map_at_3_max": 0.258017, + "nauc_map_at_3_std": 0.14155, + "nauc_map_at_3_diff1": 0.623569, + "nauc_map_at_5_max": 0.301397, + "nauc_map_at_5_std": 0.21372, + "nauc_map_at_5_diff1": 0.599795, + "nauc_map_at_10_max": 0.309121, + "nauc_map_at_10_std": 0.207612, + "nauc_map_at_10_diff1": 0.568438, + "nauc_map_at_20_max": 0.300642, + "nauc_map_at_20_std": 0.213391, + "nauc_map_at_20_diff1": 0.562242, + "nauc_map_at_100_max": 0.297032, + "nauc_map_at_100_std": 0.213619, + "nauc_map_at_100_diff1": 0.559903, + "nauc_map_at_1000_max": 0.29665, + "nauc_map_at_1000_std": 0.212521, + "nauc_map_at_1000_diff1": 0.559767, + "nauc_recall_at_1_max": 0.135923, + "nauc_recall_at_1_std": 0.035513, + "nauc_recall_at_1_diff1": 0.653538, + "nauc_recall_at_3_max": 0.349626, + "nauc_recall_at_3_std": 0.216839, + "nauc_recall_at_3_diff1": 0.552114, + "nauc_recall_at_5_max": 0.377211, + "nauc_recall_at_5_std": 0.308808, + "nauc_recall_at_5_diff1": 0.545139, + "nauc_recall_at_10_max": 0.494671, + "nauc_recall_at_10_std": 0.318656, + "nauc_recall_at_10_diff1": 0.436421, + "nauc_recall_at_20_max": 0.461711, + "nauc_recall_at_20_std": 0.330252, + "nauc_recall_at_20_diff1": 0.422217, + "nauc_recall_at_100_max": 0.456182, + "nauc_recall_at_100_std": 0.711833, + "nauc_recall_at_100_diff1": 0.101294, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.06264, + "nauc_precision_at_1_std": 0.14351, + "nauc_precision_at_1_diff1": 0.498431, + "nauc_precision_at_3_max": 0.352364, + "nauc_precision_at_3_std": 0.377038, + "nauc_precision_at_3_diff1": 0.125682, + "nauc_precision_at_5_max": 0.264769, + "nauc_precision_at_5_std": 0.368529, + "nauc_precision_at_5_diff1": -0.065881, + "nauc_precision_at_10_max": 0.176603, + "nauc_precision_at_10_std": 0.24333, + "nauc_precision_at_10_diff1": -0.232704, + "nauc_precision_at_20_max": 0.077689, + "nauc_precision_at_20_std": 0.244869, + "nauc_precision_at_20_diff1": -0.285425, + "nauc_precision_at_100_max": -0.088105, + "nauc_precision_at_100_std": 0.160633, + "nauc_precision_at_100_diff1": -0.418042, + "nauc_precision_at_1000_max": -0.127343, + "nauc_precision_at_1000_std": 0.064432, + "nauc_precision_at_1000_diff1": -0.400857, + "nauc_cv_recall_at_1_max": 0.06264, + "nauc_cv_recall_at_1_std": 0.14351, + "nauc_cv_recall_at_1_diff1": 0.498431, + "nauc_cv_recall_at_3_max": 0.461616, + "nauc_cv_recall_at_3_std": 0.375519, + "nauc_cv_recall_at_3_diff1": 0.453978, + "nauc_cv_recall_at_5_max": 0.412017, + "nauc_cv_recall_at_5_std": 0.367651, + "nauc_cv_recall_at_5_diff1": 0.535554, + "nauc_cv_recall_at_10_max": 0.516518, + "nauc_cv_recall_at_10_std": 0.369979, + "nauc_cv_recall_at_10_diff1": 0.435939, + "nauc_cv_recall_at_20_max": 0.500945, + "nauc_cv_recall_at_20_std": 0.335493, + "nauc_cv_recall_at_20_diff1": 0.439247, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.06264, + "nauc_mrr_at_1_std": 0.14351, + "nauc_mrr_at_1_diff1": 0.498431, + "nauc_mrr_at_3_max": 0.18406, + "nauc_mrr_at_3_std": 0.239281, + "nauc_mrr_at_3_diff1": 0.47506, + "nauc_mrr_at_5_max": 0.171439, + "nauc_mrr_at_5_std": 0.236135, + "nauc_mrr_at_5_diff1": 0.490415, + "nauc_mrr_at_10_max": 0.164886, + "nauc_mrr_at_10_std": 0.226887, + "nauc_mrr_at_10_diff1": 0.480491, + "nauc_mrr_at_20_max": 0.162638, + "nauc_mrr_at_20_std": 0.224934, + "nauc_mrr_at_20_diff1": 0.480825, + "nauc_mrr_at_100_max": 0.158289, + "nauc_mrr_at_100_std": 0.222427, + "nauc_mrr_at_100_diff1": 0.482187, + "nauc_mrr_at_1000_max": 0.158289, + "nauc_mrr_at_1000_std": 0.222427, + "nauc_mrr_at_1000_diff1": 0.482187, + "main_score": 0.56687, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 151.87894368171692, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/Vidore2ESGReportsRetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..a6ca767b14 --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.42105, + "ndcg_at_3": 0.44143, + "ndcg_at_5": 0.46718, + "ndcg_at_10": 0.50583, + "ndcg_at_20": 0.54571, + "ndcg_at_100": 0.61277, + "ndcg_at_1000": 0.62012, + "map_at_1": 0.20881, + "map_at_3": 0.32159, + "map_at_5": 0.35764, + "map_at_10": 0.39295, + "map_at_20": 0.41714, + "map_at_100": 0.44418, + "map_at_1000": 0.44518, + "recall_at_1": 0.20881, + "recall_at_3": 0.39603, + "recall_at_5": 0.49125, + "recall_at_10": 0.6142, + "recall_at_20": 0.73398, + "recall_at_100": 0.95209, + "recall_at_1000": 0.99561, + "cv_recall_at_1": 0.42105, + "cv_recall_at_3": 0.64912, + "cv_recall_at_5": 0.75439, + "cv_recall_at_10": 0.82456, + "cv_recall_at_20": 0.87719, + "cv_recall_at_100": 0.96491, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.42105, + "precision_at_3": 0.30409, + "precision_at_5": 0.2386, + "precision_at_10": 0.16491, + "precision_at_20": 0.11053, + "precision_at_100": 0.03702, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.421053, + "mrr_at_3": 0.523392, + "mrr_at_5": 0.54883, + "mrr_at_10": 0.55809, + "mrr_at_20": 0.561572, + "mrr_at_100": 0.564706, + "mrr_at_1000": 0.564892, + "nauc_ndcg_at_1_max": 0.066724, + "nauc_ndcg_at_1_std": 0.208172, + "nauc_ndcg_at_1_diff1": 0.105542, + "nauc_ndcg_at_3_max": 0.114351, + "nauc_ndcg_at_3_std": 0.159323, + "nauc_ndcg_at_3_diff1": 0.080309, + "nauc_ndcg_at_5_max": 0.126896, + "nauc_ndcg_at_5_std": 0.137582, + "nauc_ndcg_at_5_diff1": 0.090487, + "nauc_ndcg_at_10_max": 0.102567, + "nauc_ndcg_at_10_std": 0.156126, + "nauc_ndcg_at_10_diff1": 0.0355, + "nauc_ndcg_at_20_max": 0.125024, + "nauc_ndcg_at_20_std": 0.208228, + "nauc_ndcg_at_20_diff1": 0.011044, + "nauc_ndcg_at_100_max": 0.018266, + "nauc_ndcg_at_100_std": 0.20017, + "nauc_ndcg_at_100_diff1": 0.037923, + "nauc_ndcg_at_1000_max": 0.019458, + "nauc_ndcg_at_1000_std": 0.178651, + "nauc_ndcg_at_1000_diff1": 0.069083, + "nauc_map_at_1_max": 0.134312, + "nauc_map_at_1_std": 0.180554, + "nauc_map_at_1_diff1": 0.15377, + "nauc_map_at_3_max": 0.20029, + "nauc_map_at_3_std": 0.20224, + "nauc_map_at_3_diff1": 0.089904, + "nauc_map_at_5_max": 0.225296, + "nauc_map_at_5_std": 0.19695, + "nauc_map_at_5_diff1": 0.101187, + "nauc_map_at_10_max": 0.175064, + "nauc_map_at_10_std": 0.183056, + "nauc_map_at_10_diff1": 0.055731, + "nauc_map_at_20_max": 0.172629, + "nauc_map_at_20_std": 0.20041, + "nauc_map_at_20_diff1": 0.04731, + "nauc_map_at_100_max": 0.124537, + "nauc_map_at_100_std": 0.190971, + "nauc_map_at_100_diff1": 0.08613, + "nauc_map_at_1000_max": 0.121982, + "nauc_map_at_1000_std": 0.188985, + "nauc_map_at_1000_diff1": 0.085604, + "nauc_recall_at_1_max": 0.134312, + "nauc_recall_at_1_std": 0.180554, + "nauc_recall_at_1_diff1": 0.15377, + "nauc_recall_at_3_max": 0.236997, + "nauc_recall_at_3_std": 0.158897, + "nauc_recall_at_3_diff1": 0.076136, + "nauc_recall_at_5_max": 0.265837, + "nauc_recall_at_5_std": 0.085296, + "nauc_recall_at_5_diff1": 0.081526, + "nauc_recall_at_10_max": 0.192447, + "nauc_recall_at_10_std": 0.138243, + "nauc_recall_at_10_diff1": -0.094887, + "nauc_recall_at_20_max": 0.262369, + "nauc_recall_at_20_std": 0.293255, + "nauc_recall_at_20_diff1": -0.181855, + "nauc_recall_at_100_max": -0.286538, + "nauc_recall_at_100_std": 0.826256, + "nauc_recall_at_100_diff1": -1.180689, + "nauc_recall_at_1000_max": 0.722732, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 0.126616, + "nauc_precision_at_1_max": 0.066724, + "nauc_precision_at_1_std": 0.208172, + "nauc_precision_at_1_diff1": 0.105542, + "nauc_precision_at_3_max": -0.012248, + "nauc_precision_at_3_std": 0.073419, + "nauc_precision_at_3_diff1": 0.011187, + "nauc_precision_at_5_max": -0.037871, + "nauc_precision_at_5_std": 0.037974, + "nauc_precision_at_5_diff1": -0.003262, + "nauc_precision_at_10_max": -0.262888, + "nauc_precision_at_10_std": -0.091664, + "nauc_precision_at_10_diff1": -0.052666, + "nauc_precision_at_20_max": -0.239352, + "nauc_precision_at_20_std": -0.012135, + "nauc_precision_at_20_diff1": 0.009892, + "nauc_precision_at_100_max": -0.402911, + "nauc_precision_at_100_std": -0.110698, + "nauc_precision_at_100_diff1": 0.1919, + "nauc_precision_at_1000_max": -0.410845, + "nauc_precision_at_1000_std": -0.135194, + "nauc_precision_at_1000_diff1": 0.174555, + "nauc_cv_recall_at_1_max": 0.066724, + "nauc_cv_recall_at_1_std": 0.208172, + "nauc_cv_recall_at_1_diff1": 0.105542, + "nauc_cv_recall_at_3_max": -0.014448, + "nauc_cv_recall_at_3_std": 0.119459, + "nauc_cv_recall_at_3_diff1": 0.230495, + "nauc_cv_recall_at_5_max": -0.154309, + "nauc_cv_recall_at_5_std": -0.039277, + "nauc_cv_recall_at_5_diff1": 0.102818, + "nauc_cv_recall_at_10_max": -0.223185, + "nauc_cv_recall_at_10_std": 0.093196, + "nauc_cv_recall_at_10_diff1": -0.175507, + "nauc_cv_recall_at_20_max": -0.050707, + "nauc_cv_recall_at_20_std": 0.423864, + "nauc_cv_recall_at_20_diff1": -0.422151, + "nauc_cv_recall_at_100_max": -0.685002, + "nauc_cv_recall_at_100_std": 0.861366, + "nauc_cv_recall_at_100_diff1": -1.727857, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.066724, + "nauc_mrr_at_1_std": 0.208172, + "nauc_mrr_at_1_diff1": 0.105542, + "nauc_mrr_at_3_max": 0.019025, + "nauc_mrr_at_3_std": 0.169981, + "nauc_mrr_at_3_diff1": 0.140516, + "nauc_mrr_at_5_max": -0.001344, + "nauc_mrr_at_5_std": 0.142897, + "nauc_mrr_at_5_diff1": 0.115395, + "nauc_mrr_at_10_max": -0.003451, + "nauc_mrr_at_10_std": 0.154988, + "nauc_mrr_at_10_diff1": 0.095035, + "nauc_mrr_at_20_max": 0.004914, + "nauc_mrr_at_20_std": 0.166481, + "nauc_mrr_at_20_diff1": 0.091146, + "nauc_mrr_at_100_max": 0.003734, + "nauc_mrr_at_100_std": 0.166369, + "nauc_mrr_at_100_diff1": 0.091385, + "nauc_mrr_at_1000_max": 0.003842, + "nauc_mrr_at_1000_std": 0.165893, + "nauc_mrr_at_1000_diff1": 0.092518, + "main_score": 0.46718, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.33333, + "ndcg_at_3": 0.39195, + "ndcg_at_5": 0.42631, + "ndcg_at_10": 0.4623, + "ndcg_at_20": 0.51048, + "ndcg_at_100": 0.57571, + "ndcg_at_1000": 0.58299, + "map_at_1": 0.14741, + "map_at_3": 0.27094, + "map_at_5": 0.30613, + "map_at_10": 0.34136, + "map_at_20": 0.36981, + "map_at_100": 0.39622, + "map_at_1000": 0.39708, + "recall_at_1": 0.14741, + "recall_at_3": 0.37231, + "recall_at_5": 0.47322, + "recall_at_10": 0.5914, + "recall_at_20": 0.74202, + "recall_at_100": 0.95334, + "recall_at_1000": 0.99561, + "cv_recall_at_1": 0.33333, + "cv_recall_at_3": 0.66667, + "cv_recall_at_5": 0.73684, + "cv_recall_at_10": 0.80702, + "cv_recall_at_20": 0.87719, + "cv_recall_at_100": 0.96491, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.33333, + "precision_at_3": 0.28655, + "precision_at_5": 0.23509, + "precision_at_10": 0.1614, + "precision_at_20": 0.11316, + "precision_at_100": 0.03737, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.333333, + "mrr_at_3": 0.48538, + "mrr_at_5": 0.50117, + "mrr_at_10": 0.51086, + "mrr_at_20": 0.515848, + "mrr_at_100": 0.518817, + "mrr_at_1000": 0.519119, + "nauc_ndcg_at_1_max": 0.040385, + "nauc_ndcg_at_1_std": -0.047547, + "nauc_ndcg_at_1_diff1": 0.280231, + "nauc_ndcg_at_3_max": 0.130008, + "nauc_ndcg_at_3_std": -0.058652, + "nauc_ndcg_at_3_diff1": 0.156798, + "nauc_ndcg_at_5_max": 0.102435, + "nauc_ndcg_at_5_std": -0.029019, + "nauc_ndcg_at_5_diff1": 0.114656, + "nauc_ndcg_at_10_max": 0.131743, + "nauc_ndcg_at_10_std": -0.023402, + "nauc_ndcg_at_10_diff1": 0.146847, + "nauc_ndcg_at_20_max": 0.157959, + "nauc_ndcg_at_20_std": 0.008538, + "nauc_ndcg_at_20_diff1": 0.14215, + "nauc_ndcg_at_100_max": 0.056904, + "nauc_ndcg_at_100_std": -0.005707, + "nauc_ndcg_at_100_diff1": 0.123882, + "nauc_ndcg_at_1000_max": 0.052032, + "nauc_ndcg_at_1000_std": -0.036885, + "nauc_ndcg_at_1000_diff1": 0.150585, + "nauc_map_at_1_max": 0.202499, + "nauc_map_at_1_std": 0.045571, + "nauc_map_at_1_diff1": 0.347533, + "nauc_map_at_3_max": 0.246965, + "nauc_map_at_3_std": 0.009046, + "nauc_map_at_3_diff1": 0.185507, + "nauc_map_at_5_max": 0.214665, + "nauc_map_at_5_std": 0.011585, + "nauc_map_at_5_diff1": 0.171464, + "nauc_map_at_10_max": 0.197605, + "nauc_map_at_10_std": 0.007534, + "nauc_map_at_10_diff1": 0.174363, + "nauc_map_at_20_max": 0.18814, + "nauc_map_at_20_std": 0.013562, + "nauc_map_at_20_diff1": 0.173076, + "nauc_map_at_100_max": 0.13798, + "nauc_map_at_100_std": -0.007614, + "nauc_map_at_100_diff1": 0.176574, + "nauc_map_at_1000_max": 0.13649, + "nauc_map_at_1000_std": -0.01043, + "nauc_map_at_1000_diff1": 0.178108, + "nauc_recall_at_1_max": 0.202499, + "nauc_recall_at_1_std": 0.045571, + "nauc_recall_at_1_diff1": 0.347533, + "nauc_recall_at_3_max": 0.299739, + "nauc_recall_at_3_std": -0.019777, + "nauc_recall_at_3_diff1": 0.121214, + "nauc_recall_at_5_max": 0.236644, + "nauc_recall_at_5_std": 0.033518, + "nauc_recall_at_5_diff1": 0.063247, + "nauc_recall_at_10_max": 0.206086, + "nauc_recall_at_10_std": -0.017959, + "nauc_recall_at_10_diff1": 0.109933, + "nauc_recall_at_20_max": 0.335137, + "nauc_recall_at_20_std": 0.122257, + "nauc_recall_at_20_diff1": 0.065947, + "nauc_recall_at_100_max": 0.105938, + "nauc_recall_at_100_std": 0.913346, + "nauc_recall_at_100_diff1": -0.662838, + "nauc_recall_at_1000_max": 0.555085, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 0.126616, + "nauc_precision_at_1_max": 0.040385, + "nauc_precision_at_1_std": -0.047547, + "nauc_precision_at_1_diff1": 0.280231, + "nauc_precision_at_3_max": -0.044544, + "nauc_precision_at_3_std": -0.110167, + "nauc_precision_at_3_diff1": 0.02202, + "nauc_precision_at_5_max": -0.204765, + "nauc_precision_at_5_std": -0.112563, + "nauc_precision_at_5_diff1": -0.100314, + "nauc_precision_at_10_max": -0.241287, + "nauc_precision_at_10_std": -0.08114, + "nauc_precision_at_10_diff1": -0.098157, + "nauc_precision_at_20_max": -0.269397, + "nauc_precision_at_20_std": -0.086526, + "nauc_precision_at_20_diff1": -0.048823, + "nauc_precision_at_100_max": -0.42309, + "nauc_precision_at_100_std": -0.163591, + "nauc_precision_at_100_diff1": 0.00018, + "nauc_precision_at_1000_max": -0.43124, + "nauc_precision_at_1000_std": -0.188034, + "nauc_precision_at_1000_diff1": 0.011875, + "nauc_cv_recall_at_1_max": 0.040385, + "nauc_cv_recall_at_1_std": -0.047547, + "nauc_cv_recall_at_1_diff1": 0.280231, + "nauc_cv_recall_at_3_max": 0.030775, + "nauc_cv_recall_at_3_std": -0.148987, + "nauc_cv_recall_at_3_diff1": 0.054273, + "nauc_cv_recall_at_5_max": 0.003005, + "nauc_cv_recall_at_5_std": -0.073907, + "nauc_cv_recall_at_5_diff1": -0.032781, + "nauc_cv_recall_at_10_max": -0.063763, + "nauc_cv_recall_at_10_std": -0.191172, + "nauc_cv_recall_at_10_diff1": -0.059729, + "nauc_cv_recall_at_20_max": 0.113335, + "nauc_cv_recall_at_20_std": -0.016414, + "nauc_cv_recall_at_20_diff1": -0.271043, + "nauc_cv_recall_at_100_max": -0.100777, + "nauc_cv_recall_at_100_std": 0.934256, + "nauc_cv_recall_at_100_diff1": -0.800621, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.040385, + "nauc_mrr_at_1_std": -0.047547, + "nauc_mrr_at_1_diff1": 0.280231, + "nauc_mrr_at_3_max": 0.031468, + "nauc_mrr_at_3_std": -0.074554, + "nauc_mrr_at_3_diff1": 0.178497, + "nauc_mrr_at_5_max": 0.026721, + "nauc_mrr_at_5_std": -0.057548, + "nauc_mrr_at_5_diff1": 0.170479, + "nauc_mrr_at_10_max": 0.019963, + "nauc_mrr_at_10_std": -0.063725, + "nauc_mrr_at_10_diff1": 0.174552, + "nauc_mrr_at_20_max": 0.027358, + "nauc_mrr_at_20_std": -0.054122, + "nauc_mrr_at_20_diff1": 0.172184, + "nauc_mrr_at_100_max": 0.026426, + "nauc_mrr_at_100_std": -0.05169, + "nauc_mrr_at_100_diff1": 0.1748, + "nauc_mrr_at_1000_max": 0.026578, + "nauc_mrr_at_1000_std": -0.052692, + "nauc_mrr_at_1000_diff1": 0.175839, + "main_score": 0.42631, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.54386, + "ndcg_at_3": 0.53189, + "ndcg_at_5": 0.55021, + "ndcg_at_10": 0.59924, + "ndcg_at_20": 0.64383, + "ndcg_at_100": 0.67851, + "ndcg_at_1000": 0.68484, + "map_at_1": 0.26466, + "map_at_3": 0.39618, + "map_at_5": 0.43, + "map_at_10": 0.47814, + "map_at_20": 0.50596, + "map_at_100": 0.52504, + "map_at_1000": 0.52626, + "recall_at_1": 0.26466, + "recall_at_3": 0.47279, + "recall_at_5": 0.55378, + "recall_at_10": 0.71815, + "recall_at_20": 0.86669, + "recall_at_100": 0.96587, + "recall_at_1000": 0.99561, + "cv_recall_at_1": 0.54386, + "cv_recall_at_3": 0.7193, + "cv_recall_at_5": 0.78947, + "cv_recall_at_10": 0.87719, + "cv_recall_at_20": 0.98246, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.54386, + "precision_at_3": 0.36842, + "precision_at_5": 0.28421, + "precision_at_10": 0.2, + "precision_at_20": 0.13158, + "precision_at_100": 0.03614, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.54386, + "mrr_at_3": 0.619883, + "mrr_at_5": 0.635673, + "mrr_at_10": 0.6467, + "mrr_at_20": 0.654291, + "mrr_at_100": 0.654291, + "mrr_at_1000": 0.654317, + "nauc_ndcg_at_1_max": 0.129144, + "nauc_ndcg_at_1_std": 0.081036, + "nauc_ndcg_at_1_diff1": 0.33581, + "nauc_ndcg_at_3_max": 0.119001, + "nauc_ndcg_at_3_std": 0.079927, + "nauc_ndcg_at_3_diff1": 0.265057, + "nauc_ndcg_at_5_max": 0.045799, + "nauc_ndcg_at_5_std": 0.080188, + "nauc_ndcg_at_5_diff1": 0.265426, + "nauc_ndcg_at_10_max": 0.093025, + "nauc_ndcg_at_10_std": 0.119, + "nauc_ndcg_at_10_diff1": 0.329409, + "nauc_ndcg_at_20_max": 0.145081, + "nauc_ndcg_at_20_std": 0.137563, + "nauc_ndcg_at_20_diff1": 0.286155, + "nauc_ndcg_at_100_max": 0.04736, + "nauc_ndcg_at_100_std": 0.111311, + "nauc_ndcg_at_100_diff1": 0.273474, + "nauc_ndcg_at_1000_max": 0.039933, + "nauc_ndcg_at_1000_std": 0.095291, + "nauc_ndcg_at_1000_diff1": 0.278859, + "nauc_map_at_1_max": 0.295599, + "nauc_map_at_1_std": 0.233744, + "nauc_map_at_1_diff1": 0.419241, + "nauc_map_at_3_max": 0.239748, + "nauc_map_at_3_std": 0.152322, + "nauc_map_at_3_diff1": 0.352765, + "nauc_map_at_5_max": 0.189225, + "nauc_map_at_5_std": 0.133053, + "nauc_map_at_5_diff1": 0.324807, + "nauc_map_at_10_max": 0.168087, + "nauc_map_at_10_std": 0.126928, + "nauc_map_at_10_diff1": 0.330034, + "nauc_map_at_20_max": 0.168182, + "nauc_map_at_20_std": 0.133236, + "nauc_map_at_20_diff1": 0.318658, + "nauc_map_at_100_max": 0.120529, + "nauc_map_at_100_std": 0.117705, + "nauc_map_at_100_diff1": 0.325817, + "nauc_map_at_1000_max": 0.117018, + "nauc_map_at_1000_std": 0.114633, + "nauc_map_at_1000_diff1": 0.323643, + "nauc_recall_at_1_max": 0.295599, + "nauc_recall_at_1_std": 0.233744, + "nauc_recall_at_1_diff1": 0.419241, + "nauc_recall_at_3_max": 0.200137, + "nauc_recall_at_3_std": 0.10735, + "nauc_recall_at_3_diff1": 0.28561, + "nauc_recall_at_5_max": 0.086573, + "nauc_recall_at_5_std": 0.107562, + "nauc_recall_at_5_diff1": 0.192232, + "nauc_recall_at_10_max": 0.127214, + "nauc_recall_at_10_std": 0.129318, + "nauc_recall_at_10_diff1": 0.290147, + "nauc_recall_at_20_max": 0.419253, + "nauc_recall_at_20_std": 0.268207, + "nauc_recall_at_20_diff1": 0.09341, + "nauc_recall_at_100_max": -0.224953, + "nauc_recall_at_100_std": 0.466171, + "nauc_recall_at_100_diff1": -0.553604, + "nauc_recall_at_1000_max": 0.555085, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 0.555085, + "nauc_precision_at_1_max": 0.129144, + "nauc_precision_at_1_std": 0.081036, + "nauc_precision_at_1_diff1": 0.33581, + "nauc_precision_at_3_max": -0.075062, + "nauc_precision_at_3_std": -0.029816, + "nauc_precision_at_3_diff1": -0.04481, + "nauc_precision_at_5_max": -0.256616, + "nauc_precision_at_5_std": -0.106219, + "nauc_precision_at_5_diff1": -0.105509, + "nauc_precision_at_10_max": -0.285228, + "nauc_precision_at_10_std": -0.096302, + "nauc_precision_at_10_diff1": -0.077725, + "nauc_precision_at_20_max": -0.282494, + "nauc_precision_at_20_std": -0.081023, + "nauc_precision_at_20_diff1": -0.135135, + "nauc_precision_at_100_max": -0.388926, + "nauc_precision_at_100_std": -0.11606, + "nauc_precision_at_100_diff1": -0.11499, + "nauc_precision_at_1000_max": -0.399401, + "nauc_precision_at_1000_std": -0.154101, + "nauc_precision_at_1000_diff1": -0.142455, + "nauc_cv_recall_at_1_max": 0.129144, + "nauc_cv_recall_at_1_std": 0.081036, + "nauc_cv_recall_at_1_diff1": 0.33581, + "nauc_cv_recall_at_3_max": 0.012531, + "nauc_cv_recall_at_3_std": 0.079766, + "nauc_cv_recall_at_3_diff1": 0.248024, + "nauc_cv_recall_at_5_max": -0.183666, + "nauc_cv_recall_at_5_std": 0.066158, + "nauc_cv_recall_at_5_diff1": 0.25005, + "nauc_cv_recall_at_10_max": -0.260368, + "nauc_cv_recall_at_10_std": 0.0213, + "nauc_cv_recall_at_10_diff1": 0.295073, + "nauc_cv_recall_at_20_max": -1.169034, + "nauc_cv_recall_at_20_std": 0.126616, + "nauc_cv_recall_at_20_diff1": -1.727857, + "nauc_cv_recall_at_100_max": -1.169034, + "nauc_cv_recall_at_100_std": 0.126616, + "nauc_cv_recall_at_100_diff1": -1.727857, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.129144, + "nauc_mrr_at_1_std": 0.081036, + "nauc_mrr_at_1_diff1": 0.33581, + "nauc_mrr_at_3_max": 0.081471, + "nauc_mrr_at_3_std": 0.078185, + "nauc_mrr_at_3_diff1": 0.305467, + "nauc_mrr_at_5_max": 0.052297, + "nauc_mrr_at_5_std": 0.07453, + "nauc_mrr_at_5_diff1": 0.310065, + "nauc_mrr_at_10_max": 0.057575, + "nauc_mrr_at_10_std": 0.072225, + "nauc_mrr_at_10_diff1": 0.317388, + "nauc_mrr_at_20_max": 0.062625, + "nauc_mrr_at_20_std": 0.074272, + "nauc_mrr_at_20_diff1": 0.307331, + "nauc_mrr_at_100_max": 0.062625, + "nauc_mrr_at_100_std": 0.074272, + "nauc_mrr_at_100_diff1": 0.307331, + "nauc_mrr_at_1000_max": 0.062745, + "nauc_mrr_at_1000_std": 0.074267, + "nauc_mrr_at_1000_diff1": 0.307529, + "main_score": 0.55021, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.40351, + "ndcg_at_3": 0.44502, + "ndcg_at_5": 0.45058, + "ndcg_at_10": 0.47855, + "ndcg_at_20": 0.53312, + "ndcg_at_100": 0.59403, + "ndcg_at_1000": 0.60299, + "map_at_1": 0.17548, + "map_at_3": 0.3098, + "map_at_5": 0.33538, + "map_at_10": 0.36375, + "map_at_20": 0.39561, + "map_at_100": 0.41995, + "map_at_1000": 0.42118, + "recall_at_1": 0.17548, + "recall_at_3": 0.40627, + "recall_at_5": 0.47746, + "recall_at_10": 0.58686, + "recall_at_20": 0.75207, + "recall_at_100": 0.94708, + "recall_at_1000": 0.99561, + "cv_recall_at_1": 0.40351, + "cv_recall_at_3": 0.68421, + "cv_recall_at_5": 0.73684, + "cv_recall_at_10": 0.82456, + "cv_recall_at_20": 0.91228, + "cv_recall_at_100": 0.96491, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.40351, + "precision_at_3": 0.32749, + "precision_at_5": 0.23509, + "precision_at_10": 0.15614, + "precision_at_20": 0.11228, + "precision_at_100": 0.03667, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.403509, + "mrr_at_3": 0.52924, + "mrr_at_5": 0.542398, + "mrr_at_10": 0.552506, + "mrr_at_20": 0.558187, + "mrr_at_100": 0.559672, + "mrr_at_1000": 0.559954, + "nauc_ndcg_at_1_max": 0.062277, + "nauc_ndcg_at_1_std": 0.032487, + "nauc_ndcg_at_1_diff1": -0.029742, + "nauc_ndcg_at_3_max": 0.062398, + "nauc_ndcg_at_3_std": 0.038499, + "nauc_ndcg_at_3_diff1": 0.027773, + "nauc_ndcg_at_5_max": 0.128605, + "nauc_ndcg_at_5_std": 0.099889, + "nauc_ndcg_at_5_diff1": 0.064164, + "nauc_ndcg_at_10_max": 0.168749, + "nauc_ndcg_at_10_std": 0.109673, + "nauc_ndcg_at_10_diff1": 0.070232, + "nauc_ndcg_at_20_max": 0.140361, + "nauc_ndcg_at_20_std": 0.096533, + "nauc_ndcg_at_20_diff1": 0.069841, + "nauc_ndcg_at_100_max": 0.084479, + "nauc_ndcg_at_100_std": 0.138194, + "nauc_ndcg_at_100_diff1": 0.022352, + "nauc_ndcg_at_1000_max": 0.082612, + "nauc_ndcg_at_1000_std": 0.114631, + "nauc_ndcg_at_1000_diff1": 0.033518, + "nauc_map_at_1_max": 0.230239, + "nauc_map_at_1_std": 0.255207, + "nauc_map_at_1_diff1": 0.155082, + "nauc_map_at_3_max": 0.14965, + "nauc_map_at_3_std": 0.093455, + "nauc_map_at_3_diff1": 0.130217, + "nauc_map_at_5_max": 0.179077, + "nauc_map_at_5_std": 0.114288, + "nauc_map_at_5_diff1": 0.13532, + "nauc_map_at_10_max": 0.197704, + "nauc_map_at_10_std": 0.131907, + "nauc_map_at_10_diff1": 0.108243, + "nauc_map_at_20_max": 0.18489, + "nauc_map_at_20_std": 0.134475, + "nauc_map_at_20_diff1": 0.105003, + "nauc_map_at_100_max": 0.15398, + "nauc_map_at_100_std": 0.144976, + "nauc_map_at_100_diff1": 0.103637, + "nauc_map_at_1000_max": 0.152526, + "nauc_map_at_1000_std": 0.142696, + "nauc_map_at_1000_diff1": 0.103433, + "nauc_recall_at_1_max": 0.230239, + "nauc_recall_at_1_std": 0.255207, + "nauc_recall_at_1_diff1": 0.155082, + "nauc_recall_at_3_max": 0.067344, + "nauc_recall_at_3_std": -0.009953, + "nauc_recall_at_3_diff1": 0.146572, + "nauc_recall_at_5_max": 0.146463, + "nauc_recall_at_5_std": 0.025479, + "nauc_recall_at_5_diff1": 0.162505, + "nauc_recall_at_10_max": 0.213795, + "nauc_recall_at_10_std": 0.037708, + "nauc_recall_at_10_diff1": 0.140349, + "nauc_recall_at_20_max": 0.178336, + "nauc_recall_at_20_std": 0.049662, + "nauc_recall_at_20_diff1": 0.139218, + "nauc_recall_at_100_max": 0.032895, + "nauc_recall_at_100_std": 0.694221, + "nauc_recall_at_100_diff1": -0.320776, + "nauc_recall_at_1000_max": 0.555085, + "nauc_recall_at_1000_std": 0.722732, + "nauc_recall_at_1000_diff1": 1.0, + "nauc_precision_at_1_max": 0.062277, + "nauc_precision_at_1_std": 0.032487, + "nauc_precision_at_1_diff1": -0.029742, + "nauc_precision_at_3_max": -0.043292, + "nauc_precision_at_3_std": -0.065532, + "nauc_precision_at_3_diff1": -0.133738, + "nauc_precision_at_5_max": 0.025816, + "nauc_precision_at_5_std": 0.044416, + "nauc_precision_at_5_diff1": -0.141259, + "nauc_precision_at_10_max": 0.024339, + "nauc_precision_at_10_std": 0.076937, + "nauc_precision_at_10_diff1": -0.183706, + "nauc_precision_at_20_max": -0.09284, + "nauc_precision_at_20_std": 0.050312, + "nauc_precision_at_20_diff1": -0.117284, + "nauc_precision_at_100_max": -0.269022, + "nauc_precision_at_100_std": 0.047139, + "nauc_precision_at_100_diff1": -0.0613, + "nauc_precision_at_1000_max": -0.279017, + "nauc_precision_at_1000_std": 0.010696, + "nauc_precision_at_1000_diff1": -0.071898, + "nauc_cv_recall_at_1_max": 0.062277, + "nauc_cv_recall_at_1_std": 0.032487, + "nauc_cv_recall_at_1_diff1": -0.029742, + "nauc_cv_recall_at_3_max": -0.096143, + "nauc_cv_recall_at_3_std": 0.005572, + "nauc_cv_recall_at_3_diff1": -0.092991, + "nauc_cv_recall_at_5_max": 0.022786, + "nauc_cv_recall_at_5_std": 0.03411, + "nauc_cv_recall_at_5_diff1": -0.106926, + "nauc_cv_recall_at_10_max": 0.050521, + "nauc_cv_recall_at_10_std": -0.07073, + "nauc_cv_recall_at_10_diff1": 0.065199, + "nauc_cv_recall_at_20_max": 0.164244, + "nauc_cv_recall_at_20_std": 0.101786, + "nauc_cv_recall_at_20_diff1": -0.355029, + "nauc_cv_recall_at_100_max": -0.150261, + "nauc_cv_recall_at_100_std": 0.777543, + "nauc_cv_recall_at_100_diff1": -0.800621, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.062277, + "nauc_mrr_at_1_std": 0.032487, + "nauc_mrr_at_1_diff1": -0.029742, + "nauc_mrr_at_3_max": 0.009212, + "nauc_mrr_at_3_std": 0.034685, + "nauc_mrr_at_3_diff1": -0.051948, + "nauc_mrr_at_5_max": 0.034195, + "nauc_mrr_at_5_std": 0.040787, + "nauc_mrr_at_5_diff1": -0.05338, + "nauc_mrr_at_10_max": 0.035412, + "nauc_mrr_at_10_std": 0.031722, + "nauc_mrr_at_10_diff1": -0.039057, + "nauc_mrr_at_20_max": 0.039147, + "nauc_mrr_at_20_std": 0.037937, + "nauc_mrr_at_20_diff1": -0.04865, + "nauc_mrr_at_100_max": 0.036641, + "nauc_mrr_at_100_std": 0.039297, + "nauc_mrr_at_100_diff1": -0.049142, + "nauc_mrr_at_1000_max": 0.036905, + "nauc_mrr_at_1000_std": 0.03862, + "nauc_mrr_at_1000_diff1": -0.048353, + "main_score": 0.45058, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 600.4107351303101, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/Vidore2EconomicsReportsRetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..7c05097f68 --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.36207, + "ndcg_at_3": 0.35822, + "ndcg_at_5": 0.36339, + "ndcg_at_10": 0.37142, + "ndcg_at_20": 0.40733, + "ndcg_at_100": 0.53886, + "ndcg_at_1000": 0.59524, + "map_at_1": 0.03448, + "map_at_3": 0.07575, + "map_at_5": 0.10925, + "map_at_10": 0.15681, + "map_at_20": 0.20443, + "map_at_100": 0.28558, + "map_at_1000": 0.30567, + "recall_at_1": 0.03448, + "recall_at_3": 0.09098, + "recall_at_5": 0.17405, + "recall_at_10": 0.29284, + "recall_at_20": 0.44224, + "recall_at_100": 0.83789, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.36207, + "cv_recall_at_3": 0.62069, + "cv_recall_at_5": 0.75862, + "cv_recall_at_10": 0.86207, + "cv_recall_at_20": 0.89655, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.36207, + "precision_at_3": 0.35057, + "precision_at_5": 0.34828, + "precision_at_10": 0.30862, + "precision_at_20": 0.25172, + "precision_at_100": 0.11569, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.362069, + "mrr_at_3": 0.482759, + "mrr_at_5": 0.513793, + "mrr_at_10": 0.529392, + "mrr_at_20": 0.531796, + "mrr_at_100": 0.535167, + "mrr_at_1000": 0.535167, + "nauc_ndcg_at_1_max": 0.111169, + "nauc_ndcg_at_1_std": 0.143892, + "nauc_ndcg_at_1_diff1": -0.173524, + "nauc_ndcg_at_3_max": 0.146295, + "nauc_ndcg_at_3_std": 0.161396, + "nauc_ndcg_at_3_diff1": -0.092344, + "nauc_ndcg_at_5_max": 0.1327, + "nauc_ndcg_at_5_std": 0.097417, + "nauc_ndcg_at_5_diff1": -0.123504, + "nauc_ndcg_at_10_max": 0.088868, + "nauc_ndcg_at_10_std": 0.033574, + "nauc_ndcg_at_10_diff1": -0.122597, + "nauc_ndcg_at_20_max": 0.154826, + "nauc_ndcg_at_20_std": 0.089641, + "nauc_ndcg_at_20_diff1": -0.045177, + "nauc_ndcg_at_100_max": 0.218179, + "nauc_ndcg_at_100_std": 0.139372, + "nauc_ndcg_at_100_diff1": 0.011592, + "nauc_ndcg_at_1000_max": 0.161215, + "nauc_ndcg_at_1000_std": 0.097816, + "nauc_ndcg_at_1000_diff1": -0.052524, + "nauc_map_at_1_max": 0.165357, + "nauc_map_at_1_std": 0.075357, + "nauc_map_at_1_diff1": -0.079896, + "nauc_map_at_3_max": 0.178751, + "nauc_map_at_3_std": 0.118733, + "nauc_map_at_3_diff1": -0.060437, + "nauc_map_at_5_max": 0.173471, + "nauc_map_at_5_std": 0.070541, + "nauc_map_at_5_diff1": -0.121966, + "nauc_map_at_10_max": 0.158249, + "nauc_map_at_10_std": 0.024907, + "nauc_map_at_10_diff1": -0.071538, + "nauc_map_at_20_max": 0.18824, + "nauc_map_at_20_std": 0.078935, + "nauc_map_at_20_diff1": -0.040987, + "nauc_map_at_100_max": 0.169966, + "nauc_map_at_100_std": 0.085141, + "nauc_map_at_100_diff1": 0.007978, + "nauc_map_at_1000_max": 0.122063, + "nauc_map_at_1000_std": 0.065403, + "nauc_map_at_1000_diff1": -0.033566, + "nauc_recall_at_1_max": 0.165357, + "nauc_recall_at_1_std": 0.075357, + "nauc_recall_at_1_diff1": -0.079896, + "nauc_recall_at_3_max": 0.181585, + "nauc_recall_at_3_std": 0.108587, + "nauc_recall_at_3_diff1": 0.002194, + "nauc_recall_at_5_max": 0.031996, + "nauc_recall_at_5_std": -0.06614, + "nauc_recall_at_5_diff1": -0.184453, + "nauc_recall_at_10_max": 0.102253, + "nauc_recall_at_10_std": -0.085273, + "nauc_recall_at_10_diff1": -0.101733, + "nauc_recall_at_20_max": 0.163138, + "nauc_recall_at_20_std": 0.044322, + "nauc_recall_at_20_diff1": 0.097681, + "nauc_recall_at_100_max": 0.192381, + "nauc_recall_at_100_std": 0.156881, + "nauc_recall_at_100_diff1": 0.222926, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.111169, + "nauc_precision_at_1_std": 0.143892, + "nauc_precision_at_1_diff1": -0.173524, + "nauc_precision_at_3_max": 0.176782, + "nauc_precision_at_3_std": 0.166966, + "nauc_precision_at_3_diff1": -0.050342, + "nauc_precision_at_5_max": 0.162818, + "nauc_precision_at_5_std": 0.093386, + "nauc_precision_at_5_diff1": -0.085671, + "nauc_precision_at_10_max": 0.060664, + "nauc_precision_at_10_std": 0.021434, + "nauc_precision_at_10_diff1": -0.048776, + "nauc_precision_at_20_max": 0.031217, + "nauc_precision_at_20_std": 0.052229, + "nauc_precision_at_20_diff1": 0.010652, + "nauc_precision_at_100_max": -0.092876, + "nauc_precision_at_100_std": -0.014716, + "nauc_precision_at_100_diff1": -0.051436, + "nauc_precision_at_1000_max": -0.153399, + "nauc_precision_at_1000_std": -0.050191, + "nauc_precision_at_1000_diff1": -0.132699, + "nauc_cv_recall_at_1_max": 0.111169, + "nauc_cv_recall_at_1_std": 0.143892, + "nauc_cv_recall_at_1_diff1": -0.173524, + "nauc_cv_recall_at_3_max": 0.147168, + "nauc_cv_recall_at_3_std": 0.121353, + "nauc_cv_recall_at_3_diff1": 0.036387, + "nauc_cv_recall_at_5_max": 0.18847, + "nauc_cv_recall_at_5_std": 0.071744, + "nauc_cv_recall_at_5_diff1": -0.21842, + "nauc_cv_recall_at_10_max": 0.493528, + "nauc_cv_recall_at_10_std": 0.121975, + "nauc_cv_recall_at_10_diff1": -0.360155, + "nauc_cv_recall_at_20_max": 0.484272, + "nauc_cv_recall_at_20_std": 0.214936, + "nauc_cv_recall_at_20_diff1": 0.106065, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.111169, + "nauc_mrr_at_1_std": 0.143892, + "nauc_mrr_at_1_diff1": -0.173524, + "nauc_mrr_at_3_max": 0.104307, + "nauc_mrr_at_3_std": 0.113692, + "nauc_mrr_at_3_diff1": -0.078649, + "nauc_mrr_at_5_max": 0.114461, + "nauc_mrr_at_5_std": 0.108458, + "nauc_mrr_at_5_diff1": -0.124003, + "nauc_mrr_at_10_max": 0.134076, + "nauc_mrr_at_10_std": 0.114886, + "nauc_mrr_at_10_diff1": -0.12707, + "nauc_mrr_at_20_max": 0.131269, + "nauc_mrr_at_20_std": 0.117072, + "nauc_mrr_at_20_diff1": -0.114236, + "nauc_mrr_at_100_max": 0.127348, + "nauc_mrr_at_100_std": 0.115979, + "nauc_mrr_at_100_diff1": -0.116948, + "nauc_mrr_at_1000_max": 0.127348, + "nauc_mrr_at_1000_std": 0.115979, + "nauc_mrr_at_1000_diff1": -0.116948, + "main_score": 0.36339, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.39655, + "ndcg_at_3": 0.39899, + "ndcg_at_5": 0.40104, + "ndcg_at_10": 0.41974, + "ndcg_at_20": 0.44859, + "ndcg_at_100": 0.57275, + "ndcg_at_1000": 0.62465, + "map_at_1": 0.03159, + "map_at_3": 0.09402, + "map_at_5": 0.13306, + "map_at_10": 0.19209, + "map_at_20": 0.23447, + "map_at_100": 0.3183, + "map_at_1000": 0.33787, + "recall_at_1": 0.03159, + "recall_at_3": 0.14416, + "recall_at_5": 0.2359, + "recall_at_10": 0.37307, + "recall_at_20": 0.50923, + "recall_at_100": 0.85258, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.39655, + "cv_recall_at_3": 0.72414, + "cv_recall_at_5": 0.89655, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 0.96552, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.39655, + "precision_at_3": 0.38506, + "precision_at_5": 0.36552, + "precision_at_10": 0.33103, + "precision_at_20": 0.25776, + "precision_at_100": 0.11897, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.396552, + "mrr_at_3": 0.537356, + "mrr_at_5": 0.577874, + "mrr_at_10": 0.584216, + "mrr_at_20": 0.585653, + "mrr_at_100": 0.586894, + "mrr_at_1000": 0.586894, + "nauc_ndcg_at_1_max": -0.006582, + "nauc_ndcg_at_1_std": -0.041227, + "nauc_ndcg_at_1_diff1": -0.003577, + "nauc_ndcg_at_3_max": -0.057211, + "nauc_ndcg_at_3_std": -0.164512, + "nauc_ndcg_at_3_diff1": 0.085101, + "nauc_ndcg_at_5_max": -0.07771, + "nauc_ndcg_at_5_std": -0.220836, + "nauc_ndcg_at_5_diff1": 0.085111, + "nauc_ndcg_at_10_max": -0.063182, + "nauc_ndcg_at_10_std": -0.293082, + "nauc_ndcg_at_10_diff1": 0.111258, + "nauc_ndcg_at_20_max": 0.020077, + "nauc_ndcg_at_20_std": -0.266636, + "nauc_ndcg_at_20_diff1": 0.114393, + "nauc_ndcg_at_100_max": 0.027315, + "nauc_ndcg_at_100_std": -0.196815, + "nauc_ndcg_at_100_diff1": -0.039862, + "nauc_ndcg_at_1000_max": -0.001735, + "nauc_ndcg_at_1000_std": -0.182399, + "nauc_ndcg_at_1000_diff1": -0.014166, + "nauc_map_at_1_max": -0.008745, + "nauc_map_at_1_std": -0.089196, + "nauc_map_at_1_diff1": -0.010047, + "nauc_map_at_3_max": -0.068317, + "nauc_map_at_3_std": -0.259164, + "nauc_map_at_3_diff1": 0.093012, + "nauc_map_at_5_max": -0.03009, + "nauc_map_at_5_std": -0.261309, + "nauc_map_at_5_diff1": 0.054235, + "nauc_map_at_10_max": 0.010889, + "nauc_map_at_10_std": -0.286662, + "nauc_map_at_10_diff1": 0.017787, + "nauc_map_at_20_max": 0.043408, + "nauc_map_at_20_std": -0.264488, + "nauc_map_at_20_diff1": 0.01336, + "nauc_map_at_100_max": 0.026868, + "nauc_map_at_100_std": -0.246619, + "nauc_map_at_100_diff1": -0.026213, + "nauc_map_at_1000_max": 0.00383, + "nauc_map_at_1000_std": -0.242546, + "nauc_map_at_1000_diff1": -0.004088, + "nauc_recall_at_1_max": -0.008745, + "nauc_recall_at_1_std": -0.089196, + "nauc_recall_at_1_diff1": -0.010047, + "nauc_recall_at_3_max": -0.143001, + "nauc_recall_at_3_std": -0.295746, + "nauc_recall_at_3_diff1": 0.128435, + "nauc_recall_at_5_max": -0.14547, + "nauc_recall_at_5_std": -0.290093, + "nauc_recall_at_5_diff1": 0.171438, + "nauc_recall_at_10_max": -0.080173, + "nauc_recall_at_10_std": -0.30886, + "nauc_recall_at_10_diff1": 0.151793, + "nauc_recall_at_20_max": 0.027078, + "nauc_recall_at_20_std": -0.230567, + "nauc_recall_at_20_diff1": 0.167132, + "nauc_recall_at_100_max": 0.145607, + "nauc_recall_at_100_std": -0.056275, + "nauc_recall_at_100_diff1": -0.012187, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.006582, + "nauc_precision_at_1_std": -0.041227, + "nauc_precision_at_1_diff1": -0.003577, + "nauc_precision_at_3_max": -0.034409, + "nauc_precision_at_3_std": -0.168745, + "nauc_precision_at_3_diff1": 0.119346, + "nauc_precision_at_5_max": -0.023672, + "nauc_precision_at_5_std": -0.195728, + "nauc_precision_at_5_diff1": 0.034085, + "nauc_precision_at_10_max": 0.009463, + "nauc_precision_at_10_std": -0.18759, + "nauc_precision_at_10_diff1": 0.025049, + "nauc_precision_at_20_max": 0.035356, + "nauc_precision_at_20_std": -0.072264, + "nauc_precision_at_20_diff1": 0.005746, + "nauc_precision_at_100_max": -0.075939, + "nauc_precision_at_100_std": -0.045275, + "nauc_precision_at_100_diff1": -0.002866, + "nauc_precision_at_1000_max": -0.097432, + "nauc_precision_at_1000_std": -0.069393, + "nauc_precision_at_1000_diff1": 0.049998, + "nauc_cv_recall_at_1_max": -0.006582, + "nauc_cv_recall_at_1_std": -0.041227, + "nauc_cv_recall_at_1_diff1": -0.003577, + "nauc_cv_recall_at_3_max": -0.266553, + "nauc_cv_recall_at_3_std": -0.475023, + "nauc_cv_recall_at_3_diff1": 0.080473, + "nauc_cv_recall_at_5_max": -0.709355, + "nauc_cv_recall_at_5_std": -1.026582, + "nauc_cv_recall_at_5_diff1": 0.155673, + "nauc_cv_recall_at_10_max": -0.39736, + "nauc_cv_recall_at_10_std": -0.953383, + "nauc_cv_recall_at_10_diff1": 0.807597, + "nauc_cv_recall_at_20_max": -0.65792, + "nauc_cv_recall_at_20_std": -1.142926, + "nauc_cv_recall_at_20_diff1": 0.711396, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.006582, + "nauc_mrr_at_1_std": -0.041227, + "nauc_mrr_at_1_diff1": -0.003577, + "nauc_mrr_at_3_max": -0.091064, + "nauc_mrr_at_3_std": -0.18296, + "nauc_mrr_at_3_diff1": 0.024196, + "nauc_mrr_at_5_max": -0.10239, + "nauc_mrr_at_5_std": -0.189439, + "nauc_mrr_at_5_diff1": 0.018286, + "nauc_mrr_at_10_max": -0.078392, + "nauc_mrr_at_10_std": -0.16597, + "nauc_mrr_at_10_diff1": 0.031312, + "nauc_mrr_at_20_max": -0.079448, + "nauc_mrr_at_20_std": -0.163837, + "nauc_mrr_at_20_diff1": 0.026252, + "nauc_mrr_at_100_max": -0.07648, + "nauc_mrr_at_100_std": -0.158994, + "nauc_mrr_at_100_diff1": 0.023035, + "nauc_mrr_at_1000_max": -0.07648, + "nauc_mrr_at_1000_std": -0.158994, + "nauc_mrr_at_1000_diff1": 0.023035, + "main_score": 0.40104, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.62069, + "ndcg_at_3": 0.59087, + "ndcg_at_5": 0.58764, + "ndcg_at_10": 0.553, + "ndcg_at_20": 0.57146, + "ndcg_at_100": 0.68226, + "ndcg_at_1000": 0.71396, + "map_at_1": 0.05781, + "map_at_3": 0.16105, + "map_at_5": 0.22452, + "map_at_10": 0.28498, + "map_at_20": 0.34148, + "map_at_100": 0.43172, + "map_at_1000": 0.44791, + "recall_at_1": 0.05781, + "recall_at_3": 0.20176, + "recall_at_5": 0.33033, + "recall_at_10": 0.43995, + "recall_at_20": 0.57592, + "recall_at_100": 0.92063, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.62069, + "cv_recall_at_3": 0.82759, + "cv_recall_at_5": 0.96552, + "cv_recall_at_10": 0.98276, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.62069, + "precision_at_3": 0.55747, + "precision_at_5": 0.52759, + "precision_at_10": 0.40862, + "precision_at_20": 0.30948, + "precision_at_100": 0.12828, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.62069, + "mrr_at_3": 0.715517, + "mrr_at_5": 0.749138, + "mrr_at_10": 0.752011, + "mrr_at_20": 0.752011, + "mrr_at_100": 0.752477, + "mrr_at_1000": 0.752477, + "nauc_ndcg_at_1_max": 0.074137, + "nauc_ndcg_at_1_std": 0.205803, + "nauc_ndcg_at_1_diff1": 0.002599, + "nauc_ndcg_at_3_max": -0.092535, + "nauc_ndcg_at_3_std": -0.040904, + "nauc_ndcg_at_3_diff1": 0.150703, + "nauc_ndcg_at_5_max": -0.04683, + "nauc_ndcg_at_5_std": -0.027077, + "nauc_ndcg_at_5_diff1": 0.176886, + "nauc_ndcg_at_10_max": -0.075235, + "nauc_ndcg_at_10_std": -0.094041, + "nauc_ndcg_at_10_diff1": 0.114243, + "nauc_ndcg_at_20_max": -0.035018, + "nauc_ndcg_at_20_std": -0.017484, + "nauc_ndcg_at_20_diff1": 0.092824, + "nauc_ndcg_at_100_max": 0.004387, + "nauc_ndcg_at_100_std": 0.120076, + "nauc_ndcg_at_100_diff1": -0.05432, + "nauc_ndcg_at_1000_max": 0.015673, + "nauc_ndcg_at_1000_std": 0.098276, + "nauc_ndcg_at_1000_diff1": 0.047266, + "nauc_map_at_1_max": 0.015488, + "nauc_map_at_1_std": 0.130594, + "nauc_map_at_1_diff1": -0.120017, + "nauc_map_at_3_max": -0.139242, + "nauc_map_at_3_std": -0.124951, + "nauc_map_at_3_diff1": 0.030887, + "nauc_map_at_5_max": -0.076174, + "nauc_map_at_5_std": -0.080818, + "nauc_map_at_5_diff1": 0.049788, + "nauc_map_at_10_max": -0.048083, + "nauc_map_at_10_std": -0.067635, + "nauc_map_at_10_diff1": -0.028932, + "nauc_map_at_20_max": -0.02671, + "nauc_map_at_20_std": -0.014196, + "nauc_map_at_20_diff1": -0.058396, + "nauc_map_at_100_max": -0.049316, + "nauc_map_at_100_std": 0.023328, + "nauc_map_at_100_diff1": -0.026572, + "nauc_map_at_1000_max": -0.058691, + "nauc_map_at_1000_std": 0.008848, + "nauc_map_at_1000_diff1": 0.027872, + "nauc_recall_at_1_max": 0.015488, + "nauc_recall_at_1_std": 0.130594, + "nauc_recall_at_1_diff1": -0.120017, + "nauc_recall_at_3_max": -0.162423, + "nauc_recall_at_3_std": -0.22111, + "nauc_recall_at_3_diff1": 0.138023, + "nauc_recall_at_5_max": -0.117653, + "nauc_recall_at_5_std": -0.206474, + "nauc_recall_at_5_diff1": 0.160531, + "nauc_recall_at_10_max": -0.099662, + "nauc_recall_at_10_std": -0.209871, + "nauc_recall_at_10_diff1": 0.038411, + "nauc_recall_at_20_max": -0.020071, + "nauc_recall_at_20_std": -0.106898, + "nauc_recall_at_20_diff1": -0.029599, + "nauc_recall_at_100_max": -0.003967, + "nauc_recall_at_100_std": 0.106207, + "nauc_recall_at_100_diff1": -0.387918, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.074137, + "nauc_precision_at_1_std": 0.205803, + "nauc_precision_at_1_diff1": 0.002599, + "nauc_precision_at_3_max": -0.087518, + "nauc_precision_at_3_std": -0.038538, + "nauc_precision_at_3_diff1": 0.122028, + "nauc_precision_at_5_max": 0.079844, + "nauc_precision_at_5_std": 0.108186, + "nauc_precision_at_5_diff1": 0.093002, + "nauc_precision_at_10_max": 0.082881, + "nauc_precision_at_10_std": 0.084026, + "nauc_precision_at_10_diff1": 0.021853, + "nauc_precision_at_20_max": 0.01902, + "nauc_precision_at_20_std": 0.076426, + "nauc_precision_at_20_diff1": 0.046773, + "nauc_precision_at_100_max": -0.08118, + "nauc_precision_at_100_std": 0.001915, + "nauc_precision_at_100_diff1": 0.149873, + "nauc_precision_at_1000_max": -0.080061, + "nauc_precision_at_1000_std": -0.026262, + "nauc_precision_at_1000_diff1": 0.214284, + "nauc_cv_recall_at_1_max": 0.074137, + "nauc_cv_recall_at_1_std": 0.205803, + "nauc_cv_recall_at_1_diff1": 0.002599, + "nauc_cv_recall_at_3_max": 0.098669, + "nauc_cv_recall_at_3_std": 0.007278, + "nauc_cv_recall_at_3_diff1": 0.355942, + "nauc_cv_recall_at_5_max": 0.212851, + "nauc_cv_recall_at_5_std": 0.212851, + "nauc_cv_recall_at_5_diff1": 0.86022, + "nauc_cv_recall_at_10_max": -0.574298, + "nauc_cv_recall_at_10_std": -0.574298, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": -0.574298, + "nauc_cv_recall_at_20_std": -0.574298, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.074137, + "nauc_mrr_at_1_std": 0.205803, + "nauc_mrr_at_1_diff1": 0.002599, + "nauc_mrr_at_3_max": 0.065605, + "nauc_mrr_at_3_std": 0.127058, + "nauc_mrr_at_3_diff1": 0.139544, + "nauc_mrr_at_5_max": 0.064233, + "nauc_mrr_at_5_std": 0.148767, + "nauc_mrr_at_5_diff1": 0.12999, + "nauc_mrr_at_10_max": 0.051187, + "nauc_mrr_at_10_std": 0.136899, + "nauc_mrr_at_10_diff1": 0.121758, + "nauc_mrr_at_20_max": 0.051187, + "nauc_mrr_at_20_std": 0.136899, + "nauc_mrr_at_20_diff1": 0.121758, + "nauc_mrr_at_100_max": 0.052604, + "nauc_mrr_at_100_std": 0.13851, + "nauc_mrr_at_100_diff1": 0.119768, + "nauc_mrr_at_1000_max": 0.052604, + "nauc_mrr_at_1000_std": 0.13851, + "nauc_mrr_at_1000_diff1": 0.119768, + "main_score": 0.58764, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.48276, + "ndcg_at_3": 0.45124, + "ndcg_at_5": 0.43005, + "ndcg_at_10": 0.42255, + "ndcg_at_20": 0.441, + "ndcg_at_100": 0.57419, + "ndcg_at_1000": 0.62603, + "map_at_1": 0.04308, + "map_at_3": 0.11089, + "map_at_5": 0.14779, + "map_at_10": 0.19471, + "map_at_20": 0.23659, + "map_at_100": 0.3207, + "map_at_1000": 0.3402, + "recall_at_1": 0.04308, + "recall_at_3": 0.14044, + "recall_at_5": 0.20458, + "recall_at_10": 0.32193, + "recall_at_20": 0.45063, + "recall_at_100": 0.85208, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.48276, + "cv_recall_at_3": 0.68966, + "cv_recall_at_5": 0.84483, + "cv_recall_at_10": 0.91379, + "cv_recall_at_20": 0.93103, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.48276, + "precision_at_3": 0.43103, + "precision_at_5": 0.39655, + "precision_at_10": 0.32931, + "precision_at_20": 0.25086, + "precision_at_100": 0.11948, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.482759, + "mrr_at_3": 0.574713, + "mrr_at_5": 0.61092, + "mrr_at_10": 0.620443, + "mrr_at_20": 0.622011, + "mrr_at_100": 0.624467, + "mrr_at_1000": 0.624467, + "nauc_ndcg_at_1_max": 0.181176, + "nauc_ndcg_at_1_std": 0.287669, + "nauc_ndcg_at_1_diff1": 0.21973, + "nauc_ndcg_at_3_max": 0.048142, + "nauc_ndcg_at_3_std": 0.180814, + "nauc_ndcg_at_3_diff1": 0.137152, + "nauc_ndcg_at_5_max": 0.015433, + "nauc_ndcg_at_5_std": 0.137601, + "nauc_ndcg_at_5_diff1": 0.155177, + "nauc_ndcg_at_10_max": 0.02795, + "nauc_ndcg_at_10_std": 0.110349, + "nauc_ndcg_at_10_diff1": 0.093715, + "nauc_ndcg_at_20_max": 0.134981, + "nauc_ndcg_at_20_std": 0.207661, + "nauc_ndcg_at_20_diff1": -0.011456, + "nauc_ndcg_at_100_max": 0.171054, + "nauc_ndcg_at_100_std": 0.299938, + "nauc_ndcg_at_100_diff1": 0.020934, + "nauc_ndcg_at_1000_max": 0.144605, + "nauc_ndcg_at_1000_std": 0.26184, + "nauc_ndcg_at_1000_diff1": 0.072486, + "nauc_map_at_1_max": 0.224711, + "nauc_map_at_1_std": 0.242682, + "nauc_map_at_1_diff1": 0.138279, + "nauc_map_at_3_max": 0.033733, + "nauc_map_at_3_std": 0.122675, + "nauc_map_at_3_diff1": 0.020488, + "nauc_map_at_5_max": 0.045007, + "nauc_map_at_5_std": 0.134367, + "nauc_map_at_5_diff1": 0.020756, + "nauc_map_at_10_max": 0.028911, + "nauc_map_at_10_std": 0.102886, + "nauc_map_at_10_diff1": -0.034843, + "nauc_map_at_20_max": 0.083523, + "nauc_map_at_20_std": 0.16606, + "nauc_map_at_20_diff1": -0.062779, + "nauc_map_at_100_max": 0.084166, + "nauc_map_at_100_std": 0.203482, + "nauc_map_at_100_diff1": -0.021276, + "nauc_map_at_1000_max": 0.068842, + "nauc_map_at_1000_std": 0.184309, + "nauc_map_at_1000_diff1": -0.019128, + "nauc_recall_at_1_max": 0.224711, + "nauc_recall_at_1_std": 0.242682, + "nauc_recall_at_1_diff1": 0.138279, + "nauc_recall_at_3_max": -0.040722, + "nauc_recall_at_3_std": 0.011434, + "nauc_recall_at_3_diff1": 0.019653, + "nauc_recall_at_5_max": -0.010713, + "nauc_recall_at_5_std": 0.038612, + "nauc_recall_at_5_diff1": 0.019318, + "nauc_recall_at_10_max": -0.032088, + "nauc_recall_at_10_std": -0.040184, + "nauc_recall_at_10_diff1": -0.103059, + "nauc_recall_at_20_max": 0.095843, + "nauc_recall_at_20_std": 0.061749, + "nauc_recall_at_20_diff1": -0.132245, + "nauc_recall_at_100_max": 0.150794, + "nauc_recall_at_100_std": 0.191828, + "nauc_recall_at_100_diff1": -0.117481, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.181176, + "nauc_precision_at_1_std": 0.287669, + "nauc_precision_at_1_diff1": 0.21973, + "nauc_precision_at_3_max": 0.038955, + "nauc_precision_at_3_std": 0.176454, + "nauc_precision_at_3_diff1": 0.114211, + "nauc_precision_at_5_max": -0.000217, + "nauc_precision_at_5_std": 0.131357, + "nauc_precision_at_5_diff1": 0.134444, + "nauc_precision_at_10_max": 0.055081, + "nauc_precision_at_10_std": 0.147752, + "nauc_precision_at_10_diff1": 0.045396, + "nauc_precision_at_20_max": 0.100947, + "nauc_precision_at_20_std": 0.200132, + "nauc_precision_at_20_diff1": -0.003605, + "nauc_precision_at_100_max": 0.000948, + "nauc_precision_at_100_std": 0.08699, + "nauc_precision_at_100_diff1": 0.026598, + "nauc_precision_at_1000_max": -0.030915, + "nauc_precision_at_1000_std": 0.032302, + "nauc_precision_at_1000_diff1": 0.026676, + "nauc_cv_recall_at_1_max": 0.181176, + "nauc_cv_recall_at_1_std": 0.287669, + "nauc_cv_recall_at_1_diff1": 0.21973, + "nauc_cv_recall_at_3_max": 0.155336, + "nauc_cv_recall_at_3_std": 0.203812, + "nauc_cv_recall_at_3_diff1": 0.222866, + "nauc_cv_recall_at_5_max": 0.088784, + "nauc_cv_recall_at_5_std": 0.284008, + "nauc_cv_recall_at_5_diff1": -0.01132, + "nauc_cv_recall_at_10_max": 0.224813, + "nauc_cv_recall_at_10_std": 0.158147, + "nauc_cv_recall_at_10_diff1": -0.134521, + "nauc_cv_recall_at_20_max": 0.566748, + "nauc_cv_recall_at_20_std": 0.625572, + "nauc_cv_recall_at_20_diff1": -0.024577, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.181176, + "nauc_mrr_at_1_std": 0.287669, + "nauc_mrr_at_1_diff1": 0.21973, + "nauc_mrr_at_3_max": 0.154181, + "nauc_mrr_at_3_std": 0.236815, + "nauc_mrr_at_3_diff1": 0.221417, + "nauc_mrr_at_5_max": 0.151148, + "nauc_mrr_at_5_std": 0.2555, + "nauc_mrr_at_5_diff1": 0.188687, + "nauc_mrr_at_10_max": 0.157969, + "nauc_mrr_at_10_std": 0.248178, + "nauc_mrr_at_10_diff1": 0.189779, + "nauc_mrr_at_20_max": 0.165307, + "nauc_mrr_at_20_std": 0.259232, + "nauc_mrr_at_20_diff1": 0.194088, + "nauc_mrr_at_100_max": 0.161676, + "nauc_mrr_at_100_std": 0.256077, + "nauc_mrr_at_100_diff1": 0.196274, + "nauc_mrr_at_1000_max": 0.161676, + "nauc_mrr_at_1000_std": 0.256077, + "nauc_mrr_at_1000_diff1": 0.196274, + "main_score": 0.43005, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 72.39851951599121, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreArxivQARetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..a15f1e67d8 --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", + "task_name": "VidoreArxivQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.73, + "ndcg_at_3": 0.78112, + "ndcg_at_5": 0.79928, + "ndcg_at_10": 0.80792, + "ndcg_at_20": 0.81879, + "ndcg_at_100": 0.82997, + "ndcg_at_1000": 0.83189, + "map_at_1": 0.73, + "map_at_3": 0.769, + "map_at_5": 0.7791, + "map_at_10": 0.78281, + "map_at_20": 0.78593, + "map_at_100": 0.78764, + "map_at_1000": 0.78773, + "recall_at_1": 0.73, + "recall_at_3": 0.816, + "recall_at_5": 0.86, + "recall_at_10": 0.886, + "recall_at_20": 0.928, + "recall_at_100": 0.986, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.73, + "cv_recall_at_3": 0.816, + "cv_recall_at_5": 0.86, + "cv_recall_at_10": 0.886, + "cv_recall_at_20": 0.928, + "cv_recall_at_100": 0.986, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.73, + "precision_at_3": 0.272, + "precision_at_5": 0.172, + "precision_at_10": 0.0886, + "precision_at_20": 0.0464, + "precision_at_100": 0.00986, + "precision_at_1000": 0.001, + "mrr_at_1": 0.73, + "mrr_at_3": 0.769, + "mrr_at_5": 0.7791, + "mrr_at_10": 0.782807, + "mrr_at_20": 0.785928, + "mrr_at_100": 0.787641, + "mrr_at_1000": 0.787733, + "nauc_ndcg_at_1_max": 0.583861, + "nauc_ndcg_at_1_std": 0.190768, + "nauc_ndcg_at_1_diff1": 0.893173, + "nauc_ndcg_at_3_max": 0.630275, + "nauc_ndcg_at_3_std": 0.246602, + "nauc_ndcg_at_3_diff1": 0.886313, + "nauc_ndcg_at_5_max": 0.627145, + "nauc_ndcg_at_5_std": 0.254454, + "nauc_ndcg_at_5_diff1": 0.882463, + "nauc_ndcg_at_10_max": 0.614246, + "nauc_ndcg_at_10_std": 0.249679, + "nauc_ndcg_at_10_diff1": 0.877593, + "nauc_ndcg_at_20_max": 0.612034, + "nauc_ndcg_at_20_std": 0.251245, + "nauc_ndcg_at_20_diff1": 0.876614, + "nauc_ndcg_at_100_max": 0.618301, + "nauc_ndcg_at_100_std": 0.245004, + "nauc_ndcg_at_100_diff1": 0.882706, + "nauc_ndcg_at_1000_max": 0.613985, + "nauc_ndcg_at_1000_std": 0.239623, + "nauc_ndcg_at_1000_diff1": 0.882694, + "nauc_map_at_1_max": 0.583861, + "nauc_map_at_1_std": 0.190768, + "nauc_map_at_1_diff1": 0.893173, + "nauc_map_at_3_max": 0.617606, + "nauc_map_at_3_std": 0.232209, + "nauc_map_at_3_diff1": 0.887876, + "nauc_map_at_5_max": 0.61536, + "nauc_map_at_5_std": 0.234787, + "nauc_map_at_5_diff1": 0.886076, + "nauc_map_at_10_max": 0.610349, + "nauc_map_at_10_std": 0.233301, + "nauc_map_at_10_diff1": 0.884389, + "nauc_map_at_20_max": 0.60964, + "nauc_map_at_20_std": 0.232894, + "nauc_map_at_20_diff1": 0.884134, + "nauc_map_at_100_max": 0.610398, + "nauc_map_at_100_std": 0.231785, + "nauc_map_at_100_diff1": 0.884963, + "nauc_map_at_1000_max": 0.610228, + "nauc_map_at_1000_std": 0.231568, + "nauc_map_at_1000_diff1": 0.884962, + "nauc_recall_at_1_max": 0.583861, + "nauc_recall_at_1_std": 0.190768, + "nauc_recall_at_1_diff1": 0.893173, + "nauc_recall_at_3_max": 0.674303, + "nauc_recall_at_3_std": 0.296194, + "nauc_recall_at_3_diff1": 0.881039, + "nauc_recall_at_5_max": 0.677013, + "nauc_recall_at_5_std": 0.341118, + "nauc_recall_at_5_diff1": 0.866226, + "nauc_recall_at_10_max": 0.624744, + "nauc_recall_at_10_std": 0.329732, + "nauc_recall_at_10_diff1": 0.838772, + "nauc_recall_at_20_max": 0.612771, + "nauc_recall_at_20_std": 0.398913, + "nauc_recall_at_20_diff1": 0.810678, + "nauc_recall_at_100_max": 0.960317, + "nauc_recall_at_100_std": 0.670535, + "nauc_recall_at_100_diff1": 0.883287, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.583861, + "nauc_precision_at_1_std": 0.190768, + "nauc_precision_at_1_diff1": 0.893173, + "nauc_precision_at_3_max": 0.674303, + "nauc_precision_at_3_std": 0.296194, + "nauc_precision_at_3_diff1": 0.881039, + "nauc_precision_at_5_max": 0.677013, + "nauc_precision_at_5_std": 0.341118, + "nauc_precision_at_5_diff1": 0.866226, + "nauc_precision_at_10_max": 0.624744, + "nauc_precision_at_10_std": 0.329732, + "nauc_precision_at_10_diff1": 0.838772, + "nauc_precision_at_20_max": 0.612771, + "nauc_precision_at_20_std": 0.398913, + "nauc_precision_at_20_diff1": 0.810678, + "nauc_precision_at_100_max": 0.960317, + "nauc_precision_at_100_std": 0.670535, + "nauc_precision_at_100_diff1": 0.883287, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_cv_recall_at_1_max": 0.583861, + "nauc_cv_recall_at_1_std": 0.190768, + "nauc_cv_recall_at_1_diff1": 0.893173, + "nauc_cv_recall_at_3_max": 0.674303, + "nauc_cv_recall_at_3_std": 0.296194, + "nauc_cv_recall_at_3_diff1": 0.881039, + "nauc_cv_recall_at_5_max": 0.677013, + "nauc_cv_recall_at_5_std": 0.341118, + "nauc_cv_recall_at_5_diff1": 0.866226, + "nauc_cv_recall_at_10_max": 0.624744, + "nauc_cv_recall_at_10_std": 0.329732, + "nauc_cv_recall_at_10_diff1": 0.838772, + "nauc_cv_recall_at_20_max": 0.612771, + "nauc_cv_recall_at_20_std": 0.398913, + "nauc_cv_recall_at_20_diff1": 0.810678, + "nauc_cv_recall_at_100_max": 0.960317, + "nauc_cv_recall_at_100_std": 0.670535, + "nauc_cv_recall_at_100_diff1": 0.883287, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.583861, + "nauc_mrr_at_1_std": 0.190768, + "nauc_mrr_at_1_diff1": 0.893173, + "nauc_mrr_at_3_max": 0.617606, + "nauc_mrr_at_3_std": 0.232209, + "nauc_mrr_at_3_diff1": 0.887876, + "nauc_mrr_at_5_max": 0.61536, + "nauc_mrr_at_5_std": 0.234787, + "nauc_mrr_at_5_diff1": 0.886076, + "nauc_mrr_at_10_max": 0.610349, + "nauc_mrr_at_10_std": 0.233301, + "nauc_mrr_at_10_diff1": 0.884389, + "nauc_mrr_at_20_max": 0.60964, + "nauc_mrr_at_20_std": 0.232894, + "nauc_mrr_at_20_diff1": 0.884134, + "nauc_mrr_at_100_max": 0.610398, + "nauc_mrr_at_100_std": 0.231785, + "nauc_mrr_at_100_diff1": 0.884963, + "nauc_mrr_at_1000_max": 0.610228, + "nauc_mrr_at_1000_std": 0.231568, + "nauc_mrr_at_1000_diff1": 0.884962, + "main_score": 0.79928, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 32.9813072681427, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreDocVQARetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..73980de9bf --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", + "task_name": "VidoreDocVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.48337, + "ndcg_at_3": 0.56782, + "ndcg_at_5": 0.5896, + "ndcg_at_10": 0.60897, + "ndcg_at_20": 0.61979, + "ndcg_at_100": 0.64255, + "ndcg_at_1000": 0.65654, + "map_at_1": 0.48023, + "map_at_3": 0.545, + "map_at_5": 0.55786, + "map_at_10": 0.56648, + "map_at_20": 0.56958, + "map_at_100": 0.57287, + "map_at_1000": 0.57344, + "recall_at_1": 0.48023, + "recall_at_3": 0.62201, + "recall_at_5": 0.67585, + "recall_at_10": 0.73609, + "recall_at_20": 0.77785, + "recall_at_100": 0.89632, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.48337, + "cv_recall_at_3": 0.63193, + "cv_recall_at_5": 0.68514, + "cv_recall_at_10": 0.74279, + "cv_recall_at_20": 0.78492, + "cv_recall_at_100": 0.90244, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.48337, + "precision_at_3": 0.2136, + "precision_at_5": 0.14013, + "precision_at_10": 0.07672, + "precision_at_20": 0.04069, + "precision_at_100": 0.00965, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.48337, + "mrr_at_3": 0.550628, + "mrr_at_5": 0.562712, + "mrr_at_10": 0.570581, + "mrr_at_20": 0.573631, + "mrr_at_100": 0.576589, + "mrr_at_1000": 0.57704, + "nauc_ndcg_at_1_max": 0.114409, + "nauc_ndcg_at_1_std": 0.440719, + "nauc_ndcg_at_1_diff1": 0.8728, + "nauc_ndcg_at_3_max": 0.059588, + "nauc_ndcg_at_3_std": 0.490311, + "nauc_ndcg_at_3_diff1": 0.834934, + "nauc_ndcg_at_5_max": 0.037991, + "nauc_ndcg_at_5_std": 0.50263, + "nauc_ndcg_at_5_diff1": 0.821942, + "nauc_ndcg_at_10_max": 0.010986, + "nauc_ndcg_at_10_std": 0.518521, + "nauc_ndcg_at_10_diff1": 0.826791, + "nauc_ndcg_at_20_max": 0.002815, + "nauc_ndcg_at_20_std": 0.524606, + "nauc_ndcg_at_20_diff1": 0.833333, + "nauc_ndcg_at_100_max": 0.008045, + "nauc_ndcg_at_100_std": 0.529714, + "nauc_ndcg_at_100_diff1": 0.833939, + "nauc_ndcg_at_1000_max": 0.026839, + "nauc_ndcg_at_1000_std": 0.50887, + "nauc_ndcg_at_1000_diff1": 0.835623, + "nauc_map_at_1_max": 0.11922, + "nauc_map_at_1_std": 0.447741, + "nauc_map_at_1_diff1": 0.878596, + "nauc_map_at_3_max": 0.081017, + "nauc_map_at_3_std": 0.484918, + "nauc_map_at_3_diff1": 0.847915, + "nauc_map_at_5_max": 0.06827, + "nauc_map_at_5_std": 0.491715, + "nauc_map_at_5_diff1": 0.840585, + "nauc_map_at_10_max": 0.057501, + "nauc_map_at_10_std": 0.497753, + "nauc_map_at_10_diff1": 0.84199, + "nauc_map_at_20_max": 0.055469, + "nauc_map_at_20_std": 0.498992, + "nauc_map_at_20_diff1": 0.843778, + "nauc_map_at_100_max": 0.055693, + "nauc_map_at_100_std": 0.499058, + "nauc_map_at_100_diff1": 0.844036, + "nauc_map_at_1000_max": 0.056197, + "nauc_map_at_1000_std": 0.498296, + "nauc_map_at_1000_diff1": 0.84408, + "nauc_recall_at_1_max": 0.11922, + "nauc_recall_at_1_std": 0.447741, + "nauc_recall_at_1_diff1": 0.878596, + "nauc_recall_at_3_max": 0.016474, + "nauc_recall_at_3_std": 0.525796, + "nauc_recall_at_3_diff1": 0.806334, + "nauc_recall_at_5_max": -0.050917, + "nauc_recall_at_5_std": 0.555909, + "nauc_recall_at_5_diff1": 0.765433, + "nauc_recall_at_10_max": -0.174764, + "nauc_recall_at_10_std": 0.61605, + "nauc_recall_at_10_diff1": 0.770143, + "nauc_recall_at_20_max": -0.24564, + "nauc_recall_at_20_std": 0.665467, + "nauc_recall_at_20_diff1": 0.797237, + "nauc_recall_at_100_max": -0.375265, + "nauc_recall_at_100_std": 0.900193, + "nauc_recall_at_100_diff1": 0.780878, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.114409, + "nauc_precision_at_1_std": 0.440719, + "nauc_precision_at_1_diff1": 0.8728, + "nauc_precision_at_3_max": -0.022863, + "nauc_precision_at_3_std": 0.477409, + "nauc_precision_at_3_diff1": 0.739515, + "nauc_precision_at_5_max": -0.103363, + "nauc_precision_at_5_std": 0.471897, + "nauc_precision_at_5_diff1": 0.652064, + "nauc_precision_at_10_max": -0.226095, + "nauc_precision_at_10_std": 0.480043, + "nauc_precision_at_10_diff1": 0.588703, + "nauc_precision_at_20_max": -0.293075, + "nauc_precision_at_20_std": 0.476412, + "nauc_precision_at_20_diff1": 0.560758, + "nauc_precision_at_100_max": -0.369625, + "nauc_precision_at_100_std": 0.22479, + "nauc_precision_at_100_diff1": 0.157804, + "nauc_precision_at_1000_max": -0.233207, + "nauc_precision_at_1000_std": -0.456812, + "nauc_precision_at_1000_diff1": -0.451429, + "nauc_cv_recall_at_1_max": 0.114409, + "nauc_cv_recall_at_1_std": 0.440719, + "nauc_cv_recall_at_1_diff1": 0.8728, + "nauc_cv_recall_at_3_max": -0.007099, + "nauc_cv_recall_at_3_std": 0.522307, + "nauc_cv_recall_at_3_diff1": 0.797886, + "nauc_cv_recall_at_5_max": -0.075978, + "nauc_cv_recall_at_5_std": 0.544046, + "nauc_cv_recall_at_5_diff1": 0.756546, + "nauc_cv_recall_at_10_max": -0.200205, + "nauc_cv_recall_at_10_std": 0.602534, + "nauc_cv_recall_at_10_diff1": 0.767322, + "nauc_cv_recall_at_20_max": -0.273109, + "nauc_cv_recall_at_20_std": 0.649379, + "nauc_cv_recall_at_20_diff1": 0.792266, + "nauc_cv_recall_at_100_max": -0.398829, + "nauc_cv_recall_at_100_std": 0.890186, + "nauc_cv_recall_at_100_diff1": 0.769317, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.114409, + "nauc_mrr_at_1_std": 0.440719, + "nauc_mrr_at_1_diff1": 0.8728, + "nauc_mrr_at_3_max": 0.06955, + "nauc_mrr_at_3_std": 0.477196, + "nauc_mrr_at_3_diff1": 0.840822, + "nauc_mrr_at_5_max": 0.058496, + "nauc_mrr_at_5_std": 0.481178, + "nauc_mrr_at_5_diff1": 0.833906, + "nauc_mrr_at_10_max": 0.048749, + "nauc_mrr_at_10_std": 0.486374, + "nauc_mrr_at_10_diff1": 0.836017, + "nauc_mrr_at_20_max": 0.047, + "nauc_mrr_at_20_std": 0.487495, + "nauc_mrr_at_20_diff1": 0.837626, + "nauc_mrr_at_100_max": 0.048094, + "nauc_mrr_at_100_std": 0.488182, + "nauc_mrr_at_100_diff1": 0.838148, + "nauc_mrr_at_1000_max": 0.04867, + "nauc_mrr_at_1000_std": 0.487615, + "nauc_mrr_at_1000_diff1": 0.838267, + "main_score": 0.5896, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 36.83208656311035, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreInfoVQARetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..1dafb3b9ec --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", + "task_name": "VidoreInfoVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.73482, + "ndcg_at_3": 0.80779, + "ndcg_at_5": 0.82187, + "ndcg_at_10": 0.83472, + "ndcg_at_20": 0.83728, + "ndcg_at_100": 0.84322, + "ndcg_at_1000": 0.84701, + "map_at_1": 0.73381, + "map_at_3": 0.78981, + "map_at_5": 0.79791, + "map_at_10": 0.80345, + "map_at_20": 0.80415, + "map_at_100": 0.80495, + "map_at_1000": 0.80511, + "recall_at_1": 0.73381, + "recall_at_3": 0.8583, + "recall_at_5": 0.8917, + "recall_at_10": 0.93016, + "recall_at_20": 0.94028, + "recall_at_100": 0.97206, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.73482, + "cv_recall_at_3": 0.86032, + "cv_recall_at_5": 0.89271, + "cv_recall_at_10": 0.93117, + "cv_recall_at_20": 0.9413, + "cv_recall_at_100": 0.97368, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.73482, + "precision_at_3": 0.28677, + "precision_at_5": 0.17895, + "precision_at_10": 0.09332, + "precision_at_20": 0.04717, + "precision_at_100": 0.00978, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.734818, + "mrr_at_3": 0.791161, + "mrr_at_5": 0.798853, + "mrr_at_10": 0.804396, + "mrr_at_20": 0.805097, + "mrr_at_100": 0.805908, + "mrr_at_1000": 0.806052, + "nauc_ndcg_at_1_max": 0.506186, + "nauc_ndcg_at_1_std": 0.049529, + "nauc_ndcg_at_1_diff1": 0.859422, + "nauc_ndcg_at_3_max": 0.562249, + "nauc_ndcg_at_3_std": 0.09538, + "nauc_ndcg_at_3_diff1": 0.835439, + "nauc_ndcg_at_5_max": 0.550315, + "nauc_ndcg_at_5_std": 0.081679, + "nauc_ndcg_at_5_diff1": 0.839724, + "nauc_ndcg_at_10_max": 0.559161, + "nauc_ndcg_at_10_std": 0.103896, + "nauc_ndcg_at_10_diff1": 0.839734, + "nauc_ndcg_at_20_max": 0.559757, + "nauc_ndcg_at_20_std": 0.11542, + "nauc_ndcg_at_20_diff1": 0.838318, + "nauc_ndcg_at_100_max": 0.560957, + "nauc_ndcg_at_100_std": 0.11758, + "nauc_ndcg_at_100_diff1": 0.841234, + "nauc_ndcg_at_1000_max": 0.551093, + "nauc_ndcg_at_1000_std": 0.09864, + "nauc_ndcg_at_1000_diff1": 0.842151, + "nauc_map_at_1_max": 0.508358, + "nauc_map_at_1_std": 0.051765, + "nauc_map_at_1_diff1": 0.860495, + "nauc_map_at_3_max": 0.547072, + "nauc_map_at_3_std": 0.083424, + "nauc_map_at_3_diff1": 0.842402, + "nauc_map_at_5_max": 0.540497, + "nauc_map_at_5_std": 0.07661, + "nauc_map_at_5_diff1": 0.844739, + "nauc_map_at_10_max": 0.543188, + "nauc_map_at_10_std": 0.083976, + "nauc_map_at_10_diff1": 0.845033, + "nauc_map_at_20_max": 0.5433, + "nauc_map_at_20_std": 0.086642, + "nauc_map_at_20_diff1": 0.8447, + "nauc_map_at_100_max": 0.543412, + "nauc_map_at_100_std": 0.08671, + "nauc_map_at_100_diff1": 0.844952, + "nauc_map_at_1000_max": 0.543061, + "nauc_map_at_1000_std": 0.086063, + "nauc_map_at_1000_diff1": 0.845012, + "nauc_recall_at_1_max": 0.508358, + "nauc_recall_at_1_std": 0.051765, + "nauc_recall_at_1_diff1": 0.860495, + "nauc_recall_at_3_max": 0.625437, + "nauc_recall_at_3_std": 0.140046, + "nauc_recall_at_3_diff1": 0.809626, + "nauc_recall_at_5_max": 0.599205, + "nauc_recall_at_5_std": 0.105212, + "nauc_recall_at_5_diff1": 0.817891, + "nauc_recall_at_10_max": 0.685053, + "nauc_recall_at_10_std": 0.270108, + "nauc_recall_at_10_diff1": 0.802882, + "nauc_recall_at_20_max": 0.711396, + "nauc_recall_at_20_std": 0.409736, + "nauc_recall_at_20_diff1": 0.783313, + "nauc_recall_at_100_max": 0.923091, + "nauc_recall_at_100_std": 0.818548, + "nauc_recall_at_100_diff1": 0.817506, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.506186, + "nauc_precision_at_1_std": 0.049529, + "nauc_precision_at_1_diff1": 0.859422, + "nauc_precision_at_3_max": 0.624383, + "nauc_precision_at_3_std": 0.150837, + "nauc_precision_at_3_diff1": 0.806149, + "nauc_precision_at_5_max": 0.570121, + "nauc_precision_at_5_std": 0.129719, + "nauc_precision_at_5_diff1": 0.771686, + "nauc_precision_at_10_max": 0.639042, + "nauc_precision_at_10_std": 0.301738, + "nauc_precision_at_10_diff1": 0.734808, + "nauc_precision_at_20_max": 0.656836, + "nauc_precision_at_20_std": 0.440862, + "nauc_precision_at_20_diff1": 0.70496, + "nauc_precision_at_100_max": 0.649671, + "nauc_precision_at_100_std": 0.708747, + "nauc_precision_at_100_diff1": 0.53583, + "nauc_precision_at_1000_max": -0.49966, + "nauc_precision_at_1000_std": -0.268659, + "nauc_precision_at_1000_diff1": -0.45484, + "nauc_cv_recall_at_1_max": 0.506186, + "nauc_cv_recall_at_1_std": 0.049529, + "nauc_cv_recall_at_1_diff1": 0.859422, + "nauc_cv_recall_at_3_max": 0.624383, + "nauc_cv_recall_at_3_std": 0.150837, + "nauc_cv_recall_at_3_diff1": 0.806149, + "nauc_cv_recall_at_5_max": 0.595885, + "nauc_cv_recall_at_5_std": 0.100802, + "nauc_cv_recall_at_5_diff1": 0.816382, + "nauc_cv_recall_at_10_max": 0.682342, + "nauc_cv_recall_at_10_std": 0.265929, + "nauc_cv_recall_at_10_diff1": 0.801903, + "nauc_cv_recall_at_20_max": 0.708672, + "nauc_cv_recall_at_20_std": 0.407243, + "nauc_cv_recall_at_20_diff1": 0.781829, + "nauc_cv_recall_at_100_max": 0.918358, + "nauc_cv_recall_at_100_std": 0.807382, + "nauc_cv_recall_at_100_diff1": 0.823367, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.506186, + "nauc_mrr_at_1_std": 0.049529, + "nauc_mrr_at_1_diff1": 0.859422, + "nauc_mrr_at_3_max": 0.544752, + "nauc_mrr_at_3_std": 0.084018, + "nauc_mrr_at_3_diff1": 0.840534, + "nauc_mrr_at_5_max": 0.537807, + "nauc_mrr_at_5_std": 0.073184, + "nauc_mrr_at_5_diff1": 0.843403, + "nauc_mrr_at_10_max": 0.540428, + "nauc_mrr_at_10_std": 0.080475, + "nauc_mrr_at_10_diff1": 0.843658, + "nauc_mrr_at_20_max": 0.540529, + "nauc_mrr_at_20_std": 0.083137, + "nauc_mrr_at_20_diff1": 0.843319, + "nauc_mrr_at_100_max": 0.540588, + "nauc_mrr_at_100_std": 0.08304, + "nauc_mrr_at_100_diff1": 0.843632, + "nauc_mrr_at_1000_max": 0.540287, + "nauc_mrr_at_1000_std": 0.082495, + "nauc_mrr_at_1000_diff1": 0.843677, + "main_score": 0.82187, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 36.087005853652954, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreShiftProjectRetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..1071061e36 --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "84a382e05c4473fed9cff2bbae95fe2379416117", + "task_name": "VidoreShiftProjectRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.56, + "ndcg_at_3": 0.68702, + "ndcg_at_5": 0.7206, + "ndcg_at_10": 0.73926, + "ndcg_at_20": 0.74171, + "ndcg_at_100": 0.74961, + "ndcg_at_1000": 0.75368, + "map_at_1": 0.56, + "map_at_3": 0.655, + "map_at_5": 0.674, + "map_at_10": 0.68128, + "map_at_20": 0.6819, + "map_at_100": 0.6832, + "map_at_1000": 0.68338, + "recall_at_1": 0.56, + "recall_at_3": 0.78, + "recall_at_5": 0.86, + "recall_at_10": 0.92, + "recall_at_20": 0.93, + "recall_at_100": 0.97, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.56, + "cv_recall_at_3": 0.78, + "cv_recall_at_5": 0.86, + "cv_recall_at_10": 0.92, + "cv_recall_at_20": 0.93, + "cv_recall_at_100": 0.97, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.56, + "precision_at_3": 0.26, + "precision_at_5": 0.172, + "precision_at_10": 0.092, + "precision_at_20": 0.0465, + "precision_at_100": 0.0097, + "precision_at_1000": 0.001, + "mrr_at_1": 0.56, + "mrr_at_3": 0.655, + "mrr_at_5": 0.674, + "mrr_at_10": 0.681278, + "mrr_at_20": 0.681903, + "mrr_at_100": 0.683197, + "mrr_at_1000": 0.683384, + "nauc_ndcg_at_1_max": 0.028475, + "nauc_ndcg_at_1_std": -0.087923, + "nauc_ndcg_at_1_diff1": 0.689189, + "nauc_ndcg_at_3_max": -0.053876, + "nauc_ndcg_at_3_std": -0.249542, + "nauc_ndcg_at_3_diff1": 0.668128, + "nauc_ndcg_at_5_max": -0.05928, + "nauc_ndcg_at_5_std": -0.21949, + "nauc_ndcg_at_5_diff1": 0.65171, + "nauc_ndcg_at_10_max": -0.066733, + "nauc_ndcg_at_10_std": -0.245544, + "nauc_ndcg_at_10_diff1": 0.656193, + "nauc_ndcg_at_20_max": -0.068948, + "nauc_ndcg_at_20_std": -0.234956, + "nauc_ndcg_at_20_diff1": 0.655421, + "nauc_ndcg_at_100_max": -0.059663, + "nauc_ndcg_at_100_std": -0.214058, + "nauc_ndcg_at_100_diff1": 0.668043, + "nauc_ndcg_at_1000_max": -0.043852, + "nauc_ndcg_at_1000_std": -0.201434, + "nauc_ndcg_at_1000_diff1": 0.665125, + "nauc_map_at_1_max": 0.028475, + "nauc_map_at_1_std": -0.087923, + "nauc_map_at_1_diff1": 0.689189, + "nauc_map_at_3_max": -0.028255, + "nauc_map_at_3_std": -0.20166, + "nauc_map_at_3_diff1": 0.671741, + "nauc_map_at_5_max": -0.030334, + "nauc_map_at_5_std": -0.184607, + "nauc_map_at_5_diff1": 0.663818, + "nauc_map_at_10_max": -0.031042, + "nauc_map_at_10_std": -0.190535, + "nauc_map_at_10_diff1": 0.665456, + "nauc_map_at_20_max": -0.031423, + "nauc_map_at_20_std": -0.188158, + "nauc_map_at_20_diff1": 0.665315, + "nauc_map_at_100_max": -0.029594, + "nauc_map_at_100_std": -0.184361, + "nauc_map_at_100_diff1": 0.66728, + "nauc_map_at_1000_max": -0.028861, + "nauc_map_at_1000_std": -0.183743, + "nauc_map_at_1000_diff1": 0.66719, + "nauc_recall_at_1_max": 0.028475, + "nauc_recall_at_1_std": -0.087923, + "nauc_recall_at_1_diff1": 0.689189, + "nauc_recall_at_3_max": -0.156188, + "nauc_recall_at_3_std": -0.439491, + "nauc_recall_at_3_diff1": 0.655237, + "nauc_recall_at_5_max": -0.221075, + "nauc_recall_at_5_std": -0.405224, + "nauc_recall_at_5_diff1": 0.582167, + "nauc_recall_at_10_max": -0.411531, + "nauc_recall_at_10_std": -0.781629, + "nauc_recall_at_10_diff1": 0.575222, + "nauc_recall_at_20_max": -0.487862, + "nauc_recall_at_20_std": -0.728758, + "nauc_recall_at_20_diff1": 0.554222, + "nauc_recall_at_100_max": -0.779178, + "nauc_recall_at_100_std": -0.779178, + "nauc_recall_at_100_diff1": 0.814815, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.028475, + "nauc_precision_at_1_std": -0.087923, + "nauc_precision_at_1_diff1": 0.689189, + "nauc_precision_at_3_max": -0.156188, + "nauc_precision_at_3_std": -0.439491, + "nauc_precision_at_3_diff1": 0.655237, + "nauc_precision_at_5_max": -0.221075, + "nauc_precision_at_5_std": -0.405224, + "nauc_precision_at_5_diff1": 0.582167, + "nauc_precision_at_10_max": -0.411531, + "nauc_precision_at_10_std": -0.781629, + "nauc_precision_at_10_diff1": 0.575222, + "nauc_precision_at_20_max": -0.487862, + "nauc_precision_at_20_std": -0.728758, + "nauc_precision_at_20_diff1": 0.554222, + "nauc_precision_at_100_max": -0.779178, + "nauc_precision_at_100_std": -0.779178, + "nauc_precision_at_100_diff1": 0.814815, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.028475, + "nauc_cv_recall_at_1_std": -0.087923, + "nauc_cv_recall_at_1_diff1": 0.689189, + "nauc_cv_recall_at_3_max": -0.156188, + "nauc_cv_recall_at_3_std": -0.439491, + "nauc_cv_recall_at_3_diff1": 0.655237, + "nauc_cv_recall_at_5_max": -0.221075, + "nauc_cv_recall_at_5_std": -0.405224, + "nauc_cv_recall_at_5_diff1": 0.582167, + "nauc_cv_recall_at_10_max": -0.411531, + "nauc_cv_recall_at_10_std": -0.781629, + "nauc_cv_recall_at_10_diff1": 0.575222, + "nauc_cv_recall_at_20_max": -0.487862, + "nauc_cv_recall_at_20_std": -0.728758, + "nauc_cv_recall_at_20_diff1": 0.554222, + "nauc_cv_recall_at_100_max": -0.779178, + "nauc_cv_recall_at_100_std": -0.779178, + "nauc_cv_recall_at_100_diff1": 0.814815, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.028475, + "nauc_mrr_at_1_std": -0.087923, + "nauc_mrr_at_1_diff1": 0.689189, + "nauc_mrr_at_3_max": -0.028255, + "nauc_mrr_at_3_std": -0.20166, + "nauc_mrr_at_3_diff1": 0.671741, + "nauc_mrr_at_5_max": -0.030334, + "nauc_mrr_at_5_std": -0.184607, + "nauc_mrr_at_5_diff1": 0.663818, + "nauc_mrr_at_10_max": -0.031042, + "nauc_mrr_at_10_std": -0.190535, + "nauc_mrr_at_10_diff1": 0.665456, + "nauc_mrr_at_20_max": -0.031423, + "nauc_mrr_at_20_std": -0.188158, + "nauc_mrr_at_20_diff1": 0.665315, + "nauc_mrr_at_100_max": -0.029594, + "nauc_mrr_at_100_std": -0.184361, + "nauc_mrr_at_100_diff1": 0.66728, + "nauc_mrr_at_1000_max": -0.028861, + "nauc_mrr_at_1000_std": -0.183743, + "nauc_mrr_at_1000_diff1": 0.66719, + "main_score": 0.7206, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 75.00767540931702, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreSyntheticDocQAAIRetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..e71b2975e9 --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", + "task_name": "VidoreSyntheticDocQAAIRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.95, + "ndcg_at_3": 0.96893, + "ndcg_at_5": 0.9728, + "ndcg_at_10": 0.9728, + "ndcg_at_20": 0.9728, + "ndcg_at_100": 0.9728, + "ndcg_at_1000": 0.97423, + "map_at_1": 0.95, + "map_at_3": 0.965, + "map_at_5": 0.967, + "map_at_10": 0.967, + "map_at_20": 0.967, + "map_at_100": 0.967, + "map_at_1000": 0.96708, + "recall_at_1": 0.95, + "recall_at_3": 0.98, + "recall_at_5": 0.99, + "recall_at_10": 0.99, + "recall_at_20": 0.99, + "recall_at_100": 0.99, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.95, + "cv_recall_at_3": 0.98, + "cv_recall_at_5": 0.99, + "cv_recall_at_10": 0.99, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 0.99, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.95, + "precision_at_3": 0.32667, + "precision_at_5": 0.198, + "precision_at_10": 0.099, + "precision_at_20": 0.0495, + "precision_at_100": 0.0099, + "precision_at_1000": 0.001, + "mrr_at_1": 0.95, + "mrr_at_3": 0.965, + "mrr_at_5": 0.967, + "mrr_at_10": 0.967, + "mrr_at_20": 0.967, + "mrr_at_100": 0.967, + "mrr_at_1000": 0.967079, + "nauc_ndcg_at_1_max": -0.070775, + "nauc_ndcg_at_1_std": -1.085061, + "nauc_ndcg_at_1_diff1": 0.944444, + "nauc_ndcg_at_3_max": -0.248688, + "nauc_ndcg_at_3_std": -1.351021, + "nauc_ndcg_at_3_diff1": 0.967006, + "nauc_ndcg_at_5_max": -0.386758, + "nauc_ndcg_at_5_std": -1.295711, + "nauc_ndcg_at_5_diff1": 0.962314, + "nauc_ndcg_at_10_max": -0.386758, + "nauc_ndcg_at_10_std": -1.295711, + "nauc_ndcg_at_10_diff1": 0.962314, + "nauc_ndcg_at_20_max": -0.386758, + "nauc_ndcg_at_20_std": -1.295711, + "nauc_ndcg_at_20_diff1": 0.962314, + "nauc_ndcg_at_100_max": -0.386758, + "nauc_ndcg_at_100_std": -1.295711, + "nauc_ndcg_at_100_diff1": 0.962314, + "nauc_ndcg_at_1000_max": -0.311629, + "nauc_ndcg_at_1000_std": -1.271046, + "nauc_ndcg_at_1000_diff1": 0.960222, + "nauc_map_at_1_max": -0.070775, + "nauc_map_at_1_std": -1.085061, + "nauc_map_at_1_diff1": 0.944444, + "nauc_map_at_3_max": -0.195945, + "nauc_map_at_3_std": -1.272176, + "nauc_map_at_3_diff1": 0.960317, + "nauc_map_at_5_max": -0.251592, + "nauc_map_at_5_std": -1.243825, + "nauc_map_at_5_diff1": 0.957912, + "nauc_map_at_10_max": -0.251592, + "nauc_map_at_10_std": -1.243825, + "nauc_map_at_10_diff1": 0.957912, + "nauc_map_at_20_max": -0.251592, + "nauc_map_at_20_std": -1.243825, + "nauc_map_at_20_diff1": 0.957912, + "nauc_map_at_100_max": -0.251592, + "nauc_map_at_100_std": -1.243825, + "nauc_map_at_100_diff1": 0.957912, + "nauc_map_at_1000_max": -0.248003, + "nauc_map_at_1000_std": -1.242629, + "nauc_map_at_1000_diff1": 0.957811, + "nauc_recall_at_1_max": -0.070775, + "nauc_recall_at_1_std": -1.085061, + "nauc_recall_at_1_diff1": 0.944444, + "nauc_recall_at_3_max": -0.50887, + "nauc_recall_at_3_std": -1.739963, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": -1.739963, + "nauc_recall_at_5_std": -1.739963, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": -1.739963, + "nauc_recall_at_10_std": -1.739963, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": -1.739963, + "nauc_recall_at_20_std": -1.739963, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": -1.739963, + "nauc_recall_at_100_std": -1.739963, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.070775, + "nauc_precision_at_1_std": -1.085061, + "nauc_precision_at_1_diff1": 0.944444, + "nauc_precision_at_3_max": -0.50887, + "nauc_precision_at_3_std": -1.739963, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": -1.739963, + "nauc_precision_at_5_std": -1.739963, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": -1.739963, + "nauc_precision_at_10_std": -1.739963, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": -1.739963, + "nauc_precision_at_20_std": -1.739963, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": -1.739963, + "nauc_precision_at_100_std": -1.739963, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": -0.070775, + "nauc_cv_recall_at_1_std": -1.085061, + "nauc_cv_recall_at_1_diff1": 0.944444, + "nauc_cv_recall_at_3_max": -0.50887, + "nauc_cv_recall_at_3_std": -1.739963, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": -1.739963, + "nauc_cv_recall_at_5_std": -1.739963, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": -1.739963, + "nauc_cv_recall_at_10_std": -1.739963, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": -1.739963, + "nauc_cv_recall_at_20_std": -1.739963, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": -1.739963, + "nauc_cv_recall_at_100_std": -1.739963, + "nauc_cv_recall_at_100_diff1": 1.0, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.070775, + "nauc_mrr_at_1_std": -1.085061, + "nauc_mrr_at_1_diff1": 0.944444, + "nauc_mrr_at_3_max": -0.195945, + "nauc_mrr_at_3_std": -1.272176, + "nauc_mrr_at_3_diff1": 0.960317, + "nauc_mrr_at_5_max": -0.251592, + "nauc_mrr_at_5_std": -1.243825, + "nauc_mrr_at_5_diff1": 0.957912, + "nauc_mrr_at_10_max": -0.251592, + "nauc_mrr_at_10_std": -1.243825, + "nauc_mrr_at_10_diff1": 0.957912, + "nauc_mrr_at_20_max": -0.251592, + "nauc_mrr_at_20_std": -1.243825, + "nauc_mrr_at_20_diff1": 0.957912, + "nauc_mrr_at_100_max": -0.251592, + "nauc_mrr_at_100_std": -1.243825, + "nauc_mrr_at_100_diff1": 0.957912, + "nauc_mrr_at_1000_max": -0.248003, + "nauc_mrr_at_1000_std": -1.242629, + "nauc_mrr_at_1000_diff1": 0.957811, + "main_score": 0.9728, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 68.45232725143433, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreSyntheticDocQAEnergyRetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..991d088976 --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", + "task_name": "VidoreSyntheticDocQAEnergyRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.9, + "ndcg_at_3": 0.92393, + "ndcg_at_5": 0.92823, + "ndcg_at_10": 0.9344, + "ndcg_at_20": 0.9344, + "ndcg_at_100": 0.93841, + "ndcg_at_1000": 0.93981, + "map_at_1": 0.9, + "map_at_3": 0.91833, + "map_at_5": 0.92083, + "map_at_10": 0.92319, + "map_at_20": 0.92319, + "map_at_100": 0.92385, + "map_at_1000": 0.92392, + "recall_at_1": 0.9, + "recall_at_3": 0.94, + "recall_at_5": 0.95, + "recall_at_10": 0.97, + "recall_at_20": 0.97, + "recall_at_100": 0.99, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.9, + "cv_recall_at_3": 0.94, + "cv_recall_at_5": 0.95, + "cv_recall_at_10": 0.97, + "cv_recall_at_20": 0.97, + "cv_recall_at_100": 0.99, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.9, + "precision_at_3": 0.31333, + "precision_at_5": 0.19, + "precision_at_10": 0.097, + "precision_at_20": 0.0485, + "precision_at_100": 0.0099, + "precision_at_1000": 0.001, + "mrr_at_1": 0.9, + "mrr_at_3": 0.918333, + "mrr_at_5": 0.920833, + "mrr_at_10": 0.923194, + "mrr_at_20": 0.923194, + "mrr_at_100": 0.923851, + "mrr_at_1000": 0.923922, + "nauc_ndcg_at_1_max": 0.411391, + "nauc_ndcg_at_1_std": 0.104062, + "nauc_ndcg_at_1_diff1": 0.903595, + "nauc_ndcg_at_3_max": 0.543715, + "nauc_ndcg_at_3_std": 0.101416, + "nauc_ndcg_at_3_diff1": 0.907151, + "nauc_ndcg_at_5_max": 0.533002, + "nauc_ndcg_at_5_std": 0.06416, + "nauc_ndcg_at_5_diff1": 0.901579, + "nauc_ndcg_at_10_max": 0.537298, + "nauc_ndcg_at_10_std": 0.080856, + "nauc_ndcg_at_10_diff1": 0.905077, + "nauc_ndcg_at_20_max": 0.537298, + "nauc_ndcg_at_20_std": 0.080856, + "nauc_ndcg_at_20_diff1": 0.905077, + "nauc_ndcg_at_100_max": 0.527298, + "nauc_ndcg_at_100_std": 0.11638, + "nauc_ndcg_at_100_diff1": 0.898889, + "nauc_ndcg_at_1000_max": 0.51934, + "nauc_ndcg_at_1000_std": 0.095816, + "nauc_ndcg_at_1000_diff1": 0.903001, + "nauc_map_at_1_max": 0.411391, + "nauc_map_at_1_std": 0.104062, + "nauc_map_at_1_diff1": 0.903595, + "nauc_map_at_3_max": 0.513605, + "nauc_map_at_3_std": 0.105576, + "nauc_map_at_3_diff1": 0.906963, + "nauc_map_at_5_max": 0.507018, + "nauc_map_at_5_std": 0.086103, + "nauc_map_at_5_diff1": 0.904025, + "nauc_map_at_10_max": 0.50803, + "nauc_map_at_10_std": 0.09274, + "nauc_map_at_10_diff1": 0.905093, + "nauc_map_at_20_max": 0.50803, + "nauc_map_at_20_std": 0.09274, + "nauc_map_at_20_diff1": 0.905093, + "nauc_map_at_100_max": 0.5062, + "nauc_map_at_100_std": 0.096555, + "nauc_map_at_100_diff1": 0.904275, + "nauc_map_at_1000_max": 0.505859, + "nauc_map_at_1000_std": 0.095707, + "nauc_map_at_1000_diff1": 0.904446, + "nauc_recall_at_1_max": 0.411391, + "nauc_recall_at_1_std": 0.104062, + "nauc_recall_at_1_diff1": 0.903595, + "nauc_recall_at_3_max": 0.656941, + "nauc_recall_at_3_std": 0.083178, + "nauc_recall_at_3_diff1": 0.907407, + "nauc_recall_at_5_max": 0.643884, + "nauc_recall_at_5_std": -0.044631, + "nauc_recall_at_5_diff1": 0.888889, + "nauc_recall_at_10_max": 0.742453, + "nauc_recall_at_10_std": -0.005913, + "nauc_recall_at_10_diff1": 0.907407, + "nauc_recall_at_20_max": 0.742453, + "nauc_recall_at_20_std": -0.005913, + "nauc_recall_at_20_diff1": 0.907407, + "nauc_recall_at_100_max": 0.869281, + "nauc_recall_at_100_std": 1.0, + "nauc_recall_at_100_diff1": 0.722222, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.411391, + "nauc_precision_at_1_std": 0.104062, + "nauc_precision_at_1_diff1": 0.903595, + "nauc_precision_at_3_max": 0.656941, + "nauc_precision_at_3_std": 0.083178, + "nauc_precision_at_3_diff1": 0.907407, + "nauc_precision_at_5_max": 0.643884, + "nauc_precision_at_5_std": -0.044631, + "nauc_precision_at_5_diff1": 0.888889, + "nauc_precision_at_10_max": 0.742453, + "nauc_precision_at_10_std": -0.005913, + "nauc_precision_at_10_diff1": 0.907407, + "nauc_precision_at_20_max": 0.742453, + "nauc_precision_at_20_std": -0.005913, + "nauc_precision_at_20_diff1": 0.907407, + "nauc_precision_at_100_max": 0.869281, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 0.722222, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.411391, + "nauc_cv_recall_at_1_std": 0.104062, + "nauc_cv_recall_at_1_diff1": 0.903595, + "nauc_cv_recall_at_3_max": 0.656941, + "nauc_cv_recall_at_3_std": 0.083178, + "nauc_cv_recall_at_3_diff1": 0.907407, + "nauc_cv_recall_at_5_max": 0.643884, + "nauc_cv_recall_at_5_std": -0.044631, + "nauc_cv_recall_at_5_diff1": 0.888889, + "nauc_cv_recall_at_10_max": 0.742453, + "nauc_cv_recall_at_10_std": -0.005913, + "nauc_cv_recall_at_10_diff1": 0.907407, + "nauc_cv_recall_at_20_max": 0.742453, + "nauc_cv_recall_at_20_std": -0.005913, + "nauc_cv_recall_at_20_diff1": 0.907407, + "nauc_cv_recall_at_100_max": 0.869281, + "nauc_cv_recall_at_100_std": 1.0, + "nauc_cv_recall_at_100_diff1": 0.722222, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.411391, + "nauc_mrr_at_1_std": 0.104062, + "nauc_mrr_at_1_diff1": 0.903595, + "nauc_mrr_at_3_max": 0.513605, + "nauc_mrr_at_3_std": 0.105576, + "nauc_mrr_at_3_diff1": 0.906963, + "nauc_mrr_at_5_max": 0.507018, + "nauc_mrr_at_5_std": 0.086103, + "nauc_mrr_at_5_diff1": 0.904025, + "nauc_mrr_at_10_max": 0.50803, + "nauc_mrr_at_10_std": 0.09274, + "nauc_mrr_at_10_diff1": 0.905093, + "nauc_mrr_at_20_max": 0.50803, + "nauc_mrr_at_20_std": 0.09274, + "nauc_mrr_at_20_diff1": 0.905093, + "nauc_mrr_at_100_max": 0.5062, + "nauc_mrr_at_100_std": 0.096555, + "nauc_mrr_at_100_diff1": 0.904275, + "nauc_mrr_at_1000_max": 0.505859, + "nauc_mrr_at_1000_std": 0.095707, + "nauc_mrr_at_1000_diff1": 0.904446, + "main_score": 0.92823, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 66.4869372844696, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..07284a7462 --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b4909afa930f81282fd20601e860668073ad02aa", + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.86, + "ndcg_at_3": 0.91047, + "ndcg_at_5": 0.92339, + "ndcg_at_10": 0.92997, + "ndcg_at_20": 0.92997, + "ndcg_at_100": 0.93205, + "ndcg_at_1000": 0.93205, + "map_at_1": 0.86, + "map_at_3": 0.9, + "map_at_5": 0.9075, + "map_at_10": 0.91028, + "map_at_20": 0.91028, + "map_at_100": 0.91065, + "map_at_1000": 0.91065, + "recall_at_1": 0.86, + "recall_at_3": 0.94, + "recall_at_5": 0.97, + "recall_at_10": 0.99, + "recall_at_20": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.86, + "cv_recall_at_3": 0.94, + "cv_recall_at_5": 0.97, + "cv_recall_at_10": 0.99, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.86, + "precision_at_3": 0.31333, + "precision_at_5": 0.194, + "precision_at_10": 0.099, + "precision_at_20": 0.0495, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.86, + "mrr_at_3": 0.9, + "mrr_at_5": 0.9075, + "mrr_at_10": 0.910278, + "mrr_at_20": 0.910278, + "mrr_at_100": 0.910648, + "mrr_at_1000": 0.910648, + "nauc_ndcg_at_1_max": 0.500104, + "nauc_ndcg_at_1_std": 0.369891, + "nauc_ndcg_at_1_diff1": 0.912013, + "nauc_ndcg_at_3_max": 0.606721, + "nauc_ndcg_at_3_std": 0.520172, + "nauc_ndcg_at_3_diff1": 0.950038, + "nauc_ndcg_at_5_max": 0.62629, + "nauc_ndcg_at_5_std": 0.534863, + "nauc_ndcg_at_5_diff1": 0.941394, + "nauc_ndcg_at_10_max": 0.590225, + "nauc_ndcg_at_10_std": 0.489974, + "nauc_ndcg_at_10_diff1": 0.935738, + "nauc_ndcg_at_20_max": 0.590225, + "nauc_ndcg_at_20_std": 0.489974, + "nauc_ndcg_at_20_diff1": 0.935738, + "nauc_ndcg_at_100_max": 0.577314, + "nauc_ndcg_at_100_std": 0.473905, + "nauc_ndcg_at_100_diff1": 0.933713, + "nauc_ndcg_at_1000_max": 0.577314, + "nauc_ndcg_at_1000_std": 0.473905, + "nauc_ndcg_at_1000_diff1": 0.933713, + "nauc_map_at_1_max": 0.500104, + "nauc_map_at_1_std": 0.369891, + "nauc_map_at_1_diff1": 0.912013, + "nauc_map_at_3_max": 0.576103, + "nauc_map_at_3_std": 0.477013, + "nauc_map_at_3_diff1": 0.939118, + "nauc_map_at_5_max": 0.582985, + "nauc_map_at_5_std": 0.480497, + "nauc_map_at_5_diff1": 0.934038, + "nauc_map_at_10_max": 0.569687, + "nauc_map_at_10_std": 0.463931, + "nauc_map_at_10_diff1": 0.931935, + "nauc_map_at_20_max": 0.569687, + "nauc_map_at_20_std": 0.463931, + "nauc_map_at_20_diff1": 0.931935, + "nauc_map_at_100_max": 0.567849, + "nauc_map_at_100_std": 0.461642, + "nauc_map_at_100_diff1": 0.931644, + "nauc_map_at_1000_max": 0.567849, + "nauc_map_at_1000_std": 0.461642, + "nauc_map_at_1000_diff1": 0.931644, + "nauc_recall_at_1_max": 0.500104, + "nauc_recall_at_1_std": 0.369891, + "nauc_recall_at_1_diff1": 0.912013, + "nauc_recall_at_3_max": 0.74681, + "nauc_recall_at_3_std": 0.717631, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": 1.0, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": 1.0, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 1.0, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.500104, + "nauc_precision_at_1_std": 0.369891, + "nauc_precision_at_1_diff1": 0.912013, + "nauc_precision_at_3_max": 0.74681, + "nauc_precision_at_3_std": 0.717631, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.500104, + "nauc_cv_recall_at_1_std": 0.369891, + "nauc_cv_recall_at_1_diff1": 0.912013, + "nauc_cv_recall_at_3_max": 0.74681, + "nauc_cv_recall_at_3_std": 0.717631, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 1.0, + "nauc_cv_recall_at_5_std": 1.0, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": 1.0, + "nauc_cv_recall_at_10_std": 1.0, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": 1.0, + "nauc_cv_recall_at_20_std": 1.0, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.500104, + "nauc_mrr_at_1_std": 0.369891, + "nauc_mrr_at_1_diff1": 0.912013, + "nauc_mrr_at_3_max": 0.576103, + "nauc_mrr_at_3_std": 0.477013, + "nauc_mrr_at_3_diff1": 0.939118, + "nauc_mrr_at_5_max": 0.582985, + "nauc_mrr_at_5_std": 0.480497, + "nauc_mrr_at_5_diff1": 0.934038, + "nauc_mrr_at_10_max": 0.569687, + "nauc_mrr_at_10_std": 0.463931, + "nauc_mrr_at_10_diff1": 0.931935, + "nauc_mrr_at_20_max": 0.569687, + "nauc_mrr_at_20_std": 0.463931, + "nauc_mrr_at_20_diff1": 0.931935, + "nauc_mrr_at_100_max": 0.567849, + "nauc_mrr_at_100_std": 0.461642, + "nauc_mrr_at_100_diff1": 0.931644, + "nauc_mrr_at_1000_max": 0.567849, + "nauc_mrr_at_1000_std": 0.461642, + "nauc_mrr_at_1000_diff1": 0.931644, + "main_score": 0.92339, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 75.20986652374268, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..71bb7c763b --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.89, + "ndcg_at_3": 0.94917, + "ndcg_at_5": 0.94917, + "ndcg_at_10": 0.94917, + "ndcg_at_20": 0.94917, + "ndcg_at_100": 0.95089, + "ndcg_at_1000": 0.95089, + "map_at_1": 0.89, + "map_at_3": 0.935, + "map_at_5": 0.935, + "map_at_10": 0.935, + "map_at_20": 0.935, + "map_at_100": 0.93518, + "map_at_1000": 0.93518, + "recall_at_1": 0.89, + "recall_at_3": 0.99, + "recall_at_5": 0.99, + "recall_at_10": 0.99, + "recall_at_20": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.89, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 0.99, + "cv_recall_at_10": 0.99, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.89, + "precision_at_3": 0.33, + "precision_at_5": 0.198, + "precision_at_10": 0.099, + "precision_at_20": 0.0495, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.89, + "mrr_at_3": 0.935, + "mrr_at_5": 0.935, + "mrr_at_10": 0.935, + "mrr_at_20": 0.935, + "mrr_at_100": 0.935182, + "mrr_at_1000": 0.935182, + "nauc_ndcg_at_1_max": 0.335238, + "nauc_ndcg_at_1_std": -0.065501, + "nauc_ndcg_at_1_diff1": 0.924835, + "nauc_ndcg_at_3_max": 0.080342, + "nauc_ndcg_at_3_std": -0.223594, + "nauc_ndcg_at_3_diff1": 0.929495, + "nauc_ndcg_at_5_max": 0.080342, + "nauc_ndcg_at_5_std": -0.223594, + "nauc_ndcg_at_5_diff1": 0.929495, + "nauc_ndcg_at_10_max": 0.080342, + "nauc_ndcg_at_10_std": -0.223594, + "nauc_ndcg_at_10_diff1": 0.929495, + "nauc_ndcg_at_20_max": 0.080342, + "nauc_ndcg_at_20_std": -0.223594, + "nauc_ndcg_at_20_diff1": 0.929495, + "nauc_ndcg_at_100_max": 0.144797, + "nauc_ndcg_at_100_std": -0.19073, + "nauc_ndcg_at_100_diff1": 0.926998, + "nauc_ndcg_at_1000_max": 0.144797, + "nauc_ndcg_at_1000_std": -0.19073, + "nauc_ndcg_at_1000_diff1": 0.926998, + "nauc_map_at_1_max": 0.335238, + "nauc_map_at_1_std": -0.065501, + "nauc_map_at_1_diff1": 0.924835, + "nauc_map_at_3_max": 0.179097, + "nauc_map_at_3_std": -0.171806, + "nauc_map_at_3_diff1": 0.925935, + "nauc_map_at_5_max": 0.179097, + "nauc_map_at_5_std": -0.171806, + "nauc_map_at_5_diff1": 0.925935, + "nauc_map_at_10_max": 0.179097, + "nauc_map_at_10_std": -0.171806, + "nauc_map_at_10_diff1": 0.925935, + "nauc_map_at_20_max": 0.179097, + "nauc_map_at_20_std": -0.171806, + "nauc_map_at_20_diff1": 0.925935, + "nauc_map_at_100_max": 0.184535, + "nauc_map_at_100_std": -0.169029, + "nauc_map_at_100_diff1": 0.925725, + "nauc_map_at_1000_max": 0.184535, + "nauc_map_at_1000_std": -0.169029, + "nauc_map_at_1000_diff1": 0.925725, + "nauc_recall_at_1_max": 0.335238, + "nauc_recall_at_1_std": -0.065501, + "nauc_recall_at_1_diff1": 0.924835, + "nauc_recall_at_3_max": -1.739963, + "nauc_recall_at_3_std": -1.151727, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": -1.739963, + "nauc_recall_at_5_std": -1.151727, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": -1.739963, + "nauc_recall_at_10_std": -1.151727, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": -1.739963, + "nauc_recall_at_20_std": -1.151727, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.335238, + "nauc_precision_at_1_std": -0.065501, + "nauc_precision_at_1_diff1": 0.924835, + "nauc_precision_at_3_max": -1.739963, + "nauc_precision_at_3_std": -1.151727, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": -1.739963, + "nauc_precision_at_5_std": -1.151727, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": -1.739963, + "nauc_precision_at_10_std": -1.151727, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": -1.739963, + "nauc_precision_at_20_std": -1.151727, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.335238, + "nauc_cv_recall_at_1_std": -0.065501, + "nauc_cv_recall_at_1_diff1": 0.924835, + "nauc_cv_recall_at_3_max": -1.739963, + "nauc_cv_recall_at_3_std": -1.151727, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": -1.739963, + "nauc_cv_recall_at_5_std": -1.151727, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": -1.739963, + "nauc_cv_recall_at_10_std": -1.151727, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": -1.739963, + "nauc_cv_recall_at_20_std": -1.151727, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.335238, + "nauc_mrr_at_1_std": -0.065501, + "nauc_mrr_at_1_diff1": 0.924835, + "nauc_mrr_at_3_max": 0.179097, + "nauc_mrr_at_3_std": -0.171806, + "nauc_mrr_at_3_diff1": 0.925935, + "nauc_mrr_at_5_max": 0.179097, + "nauc_mrr_at_5_std": -0.171806, + "nauc_mrr_at_5_diff1": 0.925935, + "nauc_mrr_at_10_max": 0.179097, + "nauc_mrr_at_10_std": -0.171806, + "nauc_mrr_at_10_diff1": 0.925935, + "nauc_mrr_at_20_max": 0.179097, + "nauc_mrr_at_20_std": -0.171806, + "nauc_mrr_at_20_diff1": 0.925935, + "nauc_mrr_at_100_max": 0.184535, + "nauc_mrr_at_100_std": -0.169029, + "nauc_mrr_at_100_diff1": 0.925725, + "nauc_mrr_at_1000_max": 0.184535, + "nauc_mrr_at_1000_std": -0.169029, + "nauc_mrr_at_1000_diff1": 0.925725, + "main_score": 0.94917, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 66.67029047012329, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreTabfquadRetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..a5640de493 --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", + "task_name": "VidoreTabfquadRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.75714, + "ndcg_at_3": 0.8088, + "ndcg_at_5": 0.82032, + "ndcg_at_10": 0.83448, + "ndcg_at_20": 0.84684, + "ndcg_at_100": 0.85405, + "ndcg_at_1000": 0.85405, + "map_at_1": 0.75714, + "map_at_3": 0.79583, + "map_at_5": 0.80208, + "map_at_10": 0.80811, + "map_at_20": 0.81183, + "map_at_100": 0.81304, + "map_at_1000": 0.81304, + "recall_at_1": 0.75714, + "recall_at_3": 0.84643, + "recall_at_5": 0.875, + "recall_at_10": 0.91786, + "recall_at_20": 0.96429, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.75714, + "cv_recall_at_3": 0.84643, + "cv_recall_at_5": 0.875, + "cv_recall_at_10": 0.91786, + "cv_recall_at_20": 0.96429, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.75714, + "precision_at_3": 0.28214, + "precision_at_5": 0.175, + "precision_at_10": 0.09179, + "precision_at_20": 0.04821, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.757143, + "mrr_at_3": 0.795833, + "mrr_at_5": 0.802083, + "mrr_at_10": 0.808108, + "mrr_at_20": 0.811831, + "mrr_at_100": 0.81304, + "mrr_at_1000": 0.81304, + "nauc_ndcg_at_1_max": 0.407144, + "nauc_ndcg_at_1_std": 0.27699, + "nauc_ndcg_at_1_diff1": 0.807761, + "nauc_ndcg_at_3_max": 0.419799, + "nauc_ndcg_at_3_std": 0.297209, + "nauc_ndcg_at_3_diff1": 0.792568, + "nauc_ndcg_at_5_max": 0.398595, + "nauc_ndcg_at_5_std": 0.295661, + "nauc_ndcg_at_5_diff1": 0.785833, + "nauc_ndcg_at_10_max": 0.407161, + "nauc_ndcg_at_10_std": 0.307375, + "nauc_ndcg_at_10_diff1": 0.786815, + "nauc_ndcg_at_20_max": 0.393778, + "nauc_ndcg_at_20_std": 0.278234, + "nauc_ndcg_at_20_diff1": 0.793389, + "nauc_ndcg_at_100_max": 0.405571, + "nauc_ndcg_at_100_std": 0.290229, + "nauc_ndcg_at_100_diff1": 0.792331, + "nauc_ndcg_at_1000_max": 0.405571, + "nauc_ndcg_at_1000_std": 0.290229, + "nauc_ndcg_at_1000_diff1": 0.792331, + "nauc_map_at_1_max": 0.407144, + "nauc_map_at_1_std": 0.27699, + "nauc_map_at_1_diff1": 0.807761, + "nauc_map_at_3_max": 0.417577, + "nauc_map_at_3_std": 0.29383, + "nauc_map_at_3_diff1": 0.796118, + "nauc_map_at_5_max": 0.406925, + "nauc_map_at_5_std": 0.293028, + "nauc_map_at_5_diff1": 0.792768, + "nauc_map_at_10_max": 0.409397, + "nauc_map_at_10_std": 0.29627, + "nauc_map_at_10_diff1": 0.793423, + "nauc_map_at_20_max": 0.406118, + "nauc_map_at_20_std": 0.288518, + "nauc_map_at_20_diff1": 0.794603, + "nauc_map_at_100_max": 0.407962, + "nauc_map_at_100_std": 0.290454, + "nauc_map_at_100_diff1": 0.794406, + "nauc_map_at_1000_max": 0.407962, + "nauc_map_at_1000_std": 0.290454, + "nauc_map_at_1000_diff1": 0.794406, + "nauc_recall_at_1_max": 0.407144, + "nauc_recall_at_1_std": 0.27699, + "nauc_recall_at_1_diff1": 0.807761, + "nauc_recall_at_3_max": 0.427297, + "nauc_recall_at_3_std": 0.308466, + "nauc_recall_at_3_diff1": 0.779844, + "nauc_recall_at_5_max": 0.356937, + "nauc_recall_at_5_std": 0.305068, + "nauc_recall_at_5_diff1": 0.754992, + "nauc_recall_at_10_max": 0.396115, + "nauc_recall_at_10_std": 0.38757, + "nauc_recall_at_10_diff1": 0.744875, + "nauc_recall_at_20_max": 0.183473, + "nauc_recall_at_20_std": 0.069094, + "nauc_recall_at_20_diff1": 0.809524, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.407144, + "nauc_precision_at_1_std": 0.27699, + "nauc_precision_at_1_diff1": 0.807761, + "nauc_precision_at_3_max": 0.427297, + "nauc_precision_at_3_std": 0.308466, + "nauc_precision_at_3_diff1": 0.779844, + "nauc_precision_at_5_max": 0.356937, + "nauc_precision_at_5_std": 0.305068, + "nauc_precision_at_5_diff1": 0.754992, + "nauc_precision_at_10_max": 0.396115, + "nauc_precision_at_10_std": 0.38757, + "nauc_precision_at_10_diff1": 0.744875, + "nauc_precision_at_20_max": 0.183473, + "nauc_precision_at_20_std": 0.069094, + "nauc_precision_at_20_diff1": 0.809524, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.407144, + "nauc_cv_recall_at_1_std": 0.27699, + "nauc_cv_recall_at_1_diff1": 0.807761, + "nauc_cv_recall_at_3_max": 0.427297, + "nauc_cv_recall_at_3_std": 0.308466, + "nauc_cv_recall_at_3_diff1": 0.779844, + "nauc_cv_recall_at_5_max": 0.356937, + "nauc_cv_recall_at_5_std": 0.305068, + "nauc_cv_recall_at_5_diff1": 0.754992, + "nauc_cv_recall_at_10_max": 0.396115, + "nauc_cv_recall_at_10_std": 0.38757, + "nauc_cv_recall_at_10_diff1": 0.744875, + "nauc_cv_recall_at_20_max": 0.183473, + "nauc_cv_recall_at_20_std": 0.069094, + "nauc_cv_recall_at_20_diff1": 0.809524, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.407144, + "nauc_mrr_at_1_std": 0.27699, + "nauc_mrr_at_1_diff1": 0.807761, + "nauc_mrr_at_3_max": 0.417577, + "nauc_mrr_at_3_std": 0.29383, + "nauc_mrr_at_3_diff1": 0.796118, + "nauc_mrr_at_5_max": 0.406925, + "nauc_mrr_at_5_std": 0.293028, + "nauc_mrr_at_5_diff1": 0.792768, + "nauc_mrr_at_10_max": 0.409397, + "nauc_mrr_at_10_std": 0.29627, + "nauc_mrr_at_10_diff1": 0.793423, + "nauc_mrr_at_20_max": 0.406118, + "nauc_mrr_at_20_std": 0.288518, + "nauc_mrr_at_20_diff1": 0.794603, + "nauc_mrr_at_100_max": 0.407962, + "nauc_mrr_at_100_std": 0.290454, + "nauc_mrr_at_100_diff1": 0.794406, + "nauc_mrr_at_1000_max": 0.407962, + "nauc_mrr_at_1000_std": 0.290454, + "nauc_mrr_at_1000_diff1": 0.794406, + "main_score": 0.82032, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.979940176010132, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreTatdqaRetrieval.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..1b7de7aac0 --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", + "task_name": "VidoreTatdqaRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.53098, + "ndcg_at_3": 0.638, + "ndcg_at_5": 0.66434, + "ndcg_at_10": 0.69083, + "ndcg_at_20": 0.70259, + "ndcg_at_100": 0.71691, + "ndcg_at_1000": 0.71947, + "map_at_1": 0.52977, + "map_at_3": 0.61158, + "map_at_5": 0.62613, + "map_at_10": 0.63723, + "map_at_20": 0.64054, + "map_at_100": 0.64251, + "map_at_1000": 0.64266, + "recall_at_1": 0.52977, + "recall_at_3": 0.71416, + "recall_at_5": 0.77825, + "recall_at_10": 0.85936, + "recall_at_20": 0.90529, + "recall_at_100": 0.98244, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.53098, + "cv_recall_at_3": 0.71446, + "cv_recall_at_5": 0.77886, + "cv_recall_at_10": 0.85966, + "cv_recall_at_20": 0.90583, + "cv_recall_at_100": 0.9836, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.53098, + "precision_at_3": 0.23876, + "precision_at_5": 0.15614, + "precision_at_10": 0.08627, + "precision_at_20": 0.04547, + "precision_at_100": 0.00987, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.530984, + "mrr_at_3": 0.611887, + "mrr_at_5": 0.626499, + "mrr_at_10": 0.63748, + "mrr_at_20": 0.640768, + "mrr_at_100": 0.642751, + "mrr_at_1000": 0.64287, + "nauc_ndcg_at_1_max": 0.160093, + "nauc_ndcg_at_1_std": -0.098111, + "nauc_ndcg_at_1_diff1": 0.714243, + "nauc_ndcg_at_3_max": 0.167679, + "nauc_ndcg_at_3_std": -0.093349, + "nauc_ndcg_at_3_diff1": 0.657055, + "nauc_ndcg_at_5_max": 0.166189, + "nauc_ndcg_at_5_std": -0.083812, + "nauc_ndcg_at_5_diff1": 0.64824, + "nauc_ndcg_at_10_max": 0.171617, + "nauc_ndcg_at_10_std": -0.067038, + "nauc_ndcg_at_10_diff1": 0.643772, + "nauc_ndcg_at_20_max": 0.169354, + "nauc_ndcg_at_20_std": -0.054339, + "nauc_ndcg_at_20_diff1": 0.643543, + "nauc_ndcg_at_100_max": 0.169257, + "nauc_ndcg_at_100_std": -0.062875, + "nauc_ndcg_at_100_diff1": 0.657201, + "nauc_ndcg_at_1000_max": 0.166233, + "nauc_ndcg_at_1000_std": -0.071036, + "nauc_ndcg_at_1000_diff1": 0.658378, + "nauc_map_at_1_max": 0.158069, + "nauc_map_at_1_std": -0.099016, + "nauc_map_at_1_diff1": 0.713504, + "nauc_map_at_3_max": 0.16299, + "nauc_map_at_3_std": -0.095535, + "nauc_map_at_3_diff1": 0.67179, + "nauc_map_at_5_max": 0.16211, + "nauc_map_at_5_std": -0.090489, + "nauc_map_at_5_diff1": 0.667895, + "nauc_map_at_10_max": 0.164507, + "nauc_map_at_10_std": -0.083951, + "nauc_map_at_10_diff1": 0.666784, + "nauc_map_at_20_max": 0.163956, + "nauc_map_at_20_std": -0.08072, + "nauc_map_at_20_diff1": 0.66677, + "nauc_map_at_100_max": 0.163848, + "nauc_map_at_100_std": -0.081049, + "nauc_map_at_100_diff1": 0.668231, + "nauc_map_at_1000_max": 0.163681, + "nauc_map_at_1000_std": -0.081445, + "nauc_map_at_1000_diff1": 0.668271, + "nauc_recall_at_1_max": 0.158069, + "nauc_recall_at_1_std": -0.099016, + "nauc_recall_at_1_diff1": 0.713504, + "nauc_recall_at_3_max": 0.182947, + "nauc_recall_at_3_std": -0.087436, + "nauc_recall_at_3_diff1": 0.60573, + "nauc_recall_at_5_max": 0.181695, + "nauc_recall_at_5_std": -0.057109, + "nauc_recall_at_5_diff1": 0.565155, + "nauc_recall_at_10_max": 0.214898, + "nauc_recall_at_10_std": 0.042988, + "nauc_recall_at_10_diff1": 0.500665, + "nauc_recall_at_20_max": 0.210979, + "nauc_recall_at_20_std": 0.208649, + "nauc_recall_at_20_diff1": 0.441769, + "nauc_recall_at_100_max": 0.400749, + "nauc_recall_at_100_std": 0.616924, + "nauc_recall_at_100_diff1": 0.536133, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.160093, + "nauc_precision_at_1_std": -0.098111, + "nauc_precision_at_1_diff1": 0.714243, + "nauc_precision_at_3_max": 0.183612, + "nauc_precision_at_3_std": -0.085729, + "nauc_precision_at_3_diff1": 0.598309, + "nauc_precision_at_5_max": 0.181654, + "nauc_precision_at_5_std": -0.05468, + "nauc_precision_at_5_diff1": 0.55642, + "nauc_precision_at_10_max": 0.212115, + "nauc_precision_at_10_std": 0.05264, + "nauc_precision_at_10_diff1": 0.478319, + "nauc_precision_at_20_max": 0.199142, + "nauc_precision_at_20_std": 0.212068, + "nauc_precision_at_20_diff1": 0.402982, + "nauc_precision_at_100_max": 0.284096, + "nauc_precision_at_100_std": 0.517826, + "nauc_precision_at_100_diff1": 0.328292, + "nauc_precision_at_1000_max": -0.339874, + "nauc_precision_at_1000_std": -0.255242, + "nauc_precision_at_1000_diff1": -0.416266, + "nauc_cv_recall_at_1_max": 0.160093, + "nauc_cv_recall_at_1_std": -0.098111, + "nauc_cv_recall_at_1_diff1": 0.714243, + "nauc_cv_recall_at_3_max": 0.185232, + "nauc_cv_recall_at_3_std": -0.085389, + "nauc_cv_recall_at_3_diff1": 0.606114, + "nauc_cv_recall_at_5_max": 0.183617, + "nauc_cv_recall_at_5_std": -0.053775, + "nauc_cv_recall_at_5_diff1": 0.565364, + "nauc_cv_recall_at_10_max": 0.214513, + "nauc_cv_recall_at_10_std": 0.046982, + "nauc_cv_recall_at_10_diff1": 0.500398, + "nauc_cv_recall_at_20_max": 0.206398, + "nauc_cv_recall_at_20_std": 0.204054, + "nauc_cv_recall_at_20_diff1": 0.438528, + "nauc_cv_recall_at_100_max": 0.358579, + "nauc_cv_recall_at_100_std": 0.589967, + "nauc_cv_recall_at_100_diff1": 0.50349, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.160093, + "nauc_mrr_at_1_std": -0.098111, + "nauc_mrr_at_1_diff1": 0.714243, + "nauc_mrr_at_3_max": 0.164807, + "nauc_mrr_at_3_std": -0.093811, + "nauc_mrr_at_3_diff1": 0.671909, + "nauc_mrr_at_5_max": 0.163867, + "nauc_mrr_at_5_std": -0.088597, + "nauc_mrr_at_5_diff1": 0.668002, + "nauc_mrr_at_10_max": 0.165972, + "nauc_mrr_at_10_std": -0.082391, + "nauc_mrr_at_10_diff1": 0.666885, + "nauc_mrr_at_20_max": 0.165369, + "nauc_mrr_at_20_std": -0.079616, + "nauc_mrr_at_20_diff1": 0.66682, + "nauc_mrr_at_100_max": 0.165237, + "nauc_mrr_at_100_std": -0.079981, + "nauc_mrr_at_100_diff1": 0.668269, + "nauc_mrr_at_1000_max": 0.165146, + "nauc_mrr_at_1000_std": -0.080279, + "nauc_mrr_at_1000_diff1": 0.668339, + "main_score": 0.66434, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 24.504592180252075, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/model_meta.json b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/model_meta.json new file mode 100644 index 0000000000..36aeba31ca --- /dev/null +++ b/results/vidore__colpali-v1.1/a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a/model_meta.json @@ -0,0 +1 @@ +{"name": "vidore/colpali-v1.1", "revision": "a0f15e3bcf97110e7ac1bb4be4bcd30eeb31992a", "release_date": "2024-08-21", "languages": ["eng-Latn"], "n_parameters": 2920000000, "memory_usage_mb": 4700.0, "max_tokens": 16384.0, "embed_dim": 128, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/illuin-tech/colpali", "public_training_data": "https://huggingface.co/datasets/vidore/colpali_train_set", "framework": ["ColPali"], "reference": "https://huggingface.co/vidore/colpali-v1.1", "similarity_fn_name": "max_sim", "use_instructions": true, "training_datasets": {"DocVQA": ["train"], "InfoVQA": ["train"], "TATDQA": ["train"], "arXivQA": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "ColPaliWrapper"} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/Vidore2BioMedicalLecturesRetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..f60be3b586 --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.53125, + "ndcg_at_3": 0.5249, + "ndcg_at_5": 0.5482, + "ndcg_at_10": 0.58951, + "ndcg_at_20": 0.61834, + "ndcg_at_100": 0.65288, + "ndcg_at_1000": 0.67559, + "map_at_1": 0.32766, + "map_at_3": 0.42664, + "map_at_5": 0.46046, + "map_at_10": 0.49417, + "map_at_20": 0.50943, + "map_at_100": 0.51992, + "map_at_1000": 0.52224, + "recall_at_1": 0.32766, + "recall_at_3": 0.49668, + "recall_at_5": 0.57876, + "recall_at_10": 0.69387, + "recall_at_20": 0.77603, + "recall_at_100": 0.894, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.53125, + "cv_recall_at_3": 0.725, + "cv_recall_at_5": 0.8, + "cv_recall_at_10": 0.90625, + "cv_recall_at_20": 0.9375, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.53125, + "precision_at_3": 0.31458, + "precision_at_5": 0.2425, + "precision_at_10": 0.1625, + "precision_at_20": 0.09813, + "precision_at_100": 0.02519, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.53125, + "mrr_at_3": 0.616667, + "mrr_at_5": 0.634792, + "mrr_at_10": 0.649573, + "mrr_at_20": 0.651686, + "mrr_at_100": 0.652594, + "mrr_at_1000": 0.652665, + "nauc_ndcg_at_1_max": 0.341296, + "nauc_ndcg_at_1_std": 0.0442, + "nauc_ndcg_at_1_diff1": 0.549443, + "nauc_ndcg_at_3_max": 0.37812, + "nauc_ndcg_at_3_std": 0.024258, + "nauc_ndcg_at_3_diff1": 0.442021, + "nauc_ndcg_at_5_max": 0.399546, + "nauc_ndcg_at_5_std": 0.047033, + "nauc_ndcg_at_5_diff1": 0.439947, + "nauc_ndcg_at_10_max": 0.377128, + "nauc_ndcg_at_10_std": 0.026868, + "nauc_ndcg_at_10_diff1": 0.443543, + "nauc_ndcg_at_20_max": 0.390729, + "nauc_ndcg_at_20_std": 0.035124, + "nauc_ndcg_at_20_diff1": 0.445888, + "nauc_ndcg_at_100_max": 0.372355, + "nauc_ndcg_at_100_std": 0.041654, + "nauc_ndcg_at_100_diff1": 0.430662, + "nauc_ndcg_at_1000_max": 0.351592, + "nauc_ndcg_at_1000_std": 0.019852, + "nauc_ndcg_at_1000_diff1": 0.442071, + "nauc_map_at_1_max": 0.333184, + "nauc_map_at_1_std": 0.056548, + "nauc_map_at_1_diff1": 0.540249, + "nauc_map_at_3_max": 0.367773, + "nauc_map_at_3_std": 0.037248, + "nauc_map_at_3_diff1": 0.45938, + "nauc_map_at_5_max": 0.379935, + "nauc_map_at_5_std": 0.040439, + "nauc_map_at_5_diff1": 0.450103, + "nauc_map_at_10_max": 0.367611, + "nauc_map_at_10_std": 0.028361, + "nauc_map_at_10_diff1": 0.448524, + "nauc_map_at_20_max": 0.370765, + "nauc_map_at_20_std": 0.031902, + "nauc_map_at_20_diff1": 0.446768, + "nauc_map_at_100_max": 0.366891, + "nauc_map_at_100_std": 0.036217, + "nauc_map_at_100_diff1": 0.442972, + "nauc_map_at_1000_max": 0.364821, + "nauc_map_at_1000_std": 0.034311, + "nauc_map_at_1000_diff1": 0.443236, + "nauc_recall_at_1_max": 0.333184, + "nauc_recall_at_1_std": 0.056548, + "nauc_recall_at_1_diff1": 0.540249, + "nauc_recall_at_3_max": 0.413295, + "nauc_recall_at_3_std": 0.027846, + "nauc_recall_at_3_diff1": 0.360178, + "nauc_recall_at_5_max": 0.426079, + "nauc_recall_at_5_std": 0.062609, + "nauc_recall_at_5_diff1": 0.322281, + "nauc_recall_at_10_max": 0.345806, + "nauc_recall_at_10_std": 0.009229, + "nauc_recall_at_10_diff1": 0.291795, + "nauc_recall_at_20_max": 0.434729, + "nauc_recall_at_20_std": 0.067958, + "nauc_recall_at_20_diff1": 0.288996, + "nauc_recall_at_100_max": 0.395005, + "nauc_recall_at_100_std": 0.1464, + "nauc_recall_at_100_diff1": 0.094263, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.341296, + "nauc_precision_at_1_std": 0.0442, + "nauc_precision_at_1_diff1": 0.549443, + "nauc_precision_at_3_max": 0.19932, + "nauc_precision_at_3_std": -0.011705, + "nauc_precision_at_3_diff1": 0.133496, + "nauc_precision_at_5_max": 0.109106, + "nauc_precision_at_5_std": -0.037074, + "nauc_precision_at_5_diff1": 0.008465, + "nauc_precision_at_10_max": -0.02888, + "nauc_precision_at_10_std": -0.060327, + "nauc_precision_at_10_diff1": -0.104853, + "nauc_precision_at_20_max": -0.080702, + "nauc_precision_at_20_std": -0.036104, + "nauc_precision_at_20_diff1": -0.154683, + "nauc_precision_at_100_max": -0.170898, + "nauc_precision_at_100_std": -0.024299, + "nauc_precision_at_100_diff1": -0.226891, + "nauc_precision_at_1000_max": -0.296302, + "nauc_precision_at_1000_std": -0.122335, + "nauc_precision_at_1000_diff1": -0.286821, + "nauc_cv_recall_at_1_max": 0.341296, + "nauc_cv_recall_at_1_std": 0.0442, + "nauc_cv_recall_at_1_diff1": 0.549443, + "nauc_cv_recall_at_3_max": 0.498693, + "nauc_cv_recall_at_3_std": 0.024919, + "nauc_cv_recall_at_3_diff1": 0.456722, + "nauc_cv_recall_at_5_max": 0.509288, + "nauc_cv_recall_at_5_std": 0.061782, + "nauc_cv_recall_at_5_diff1": 0.420907, + "nauc_cv_recall_at_10_max": 0.340336, + "nauc_cv_recall_at_10_std": -0.052225, + "nauc_cv_recall_at_10_diff1": 0.318425, + "nauc_cv_recall_at_20_max": 0.502894, + "nauc_cv_recall_at_20_std": 0.097526, + "nauc_cv_recall_at_20_diff1": 0.310878, + "nauc_cv_recall_at_100_max": 0.268908, + "nauc_cv_recall_at_100_std": -0.148109, + "nauc_cv_recall_at_100_diff1": -0.295168, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.341296, + "nauc_mrr_at_1_std": 0.0442, + "nauc_mrr_at_1_diff1": 0.549443, + "nauc_mrr_at_3_max": 0.387591, + "nauc_mrr_at_3_std": 0.029618, + "nauc_mrr_at_3_diff1": 0.527505, + "nauc_mrr_at_5_max": 0.382937, + "nauc_mrr_at_5_std": 0.034403, + "nauc_mrr_at_5_diff1": 0.525385, + "nauc_mrr_at_10_max": 0.367401, + "nauc_mrr_at_10_std": 0.026732, + "nauc_mrr_at_10_diff1": 0.525235, + "nauc_mrr_at_20_max": 0.37055, + "nauc_mrr_at_20_std": 0.030053, + "nauc_mrr_at_20_diff1": 0.526269, + "nauc_mrr_at_100_max": 0.37003, + "nauc_mrr_at_100_std": 0.0304, + "nauc_mrr_at_100_diff1": 0.525921, + "nauc_mrr_at_1000_max": 0.36997, + "nauc_mrr_at_1000_std": 0.03038, + "nauc_mrr_at_1000_diff1": 0.526138, + "main_score": 0.5482, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.53125, + "ndcg_at_3": 0.53959, + "ndcg_at_5": 0.55301, + "ndcg_at_10": 0.59259, + "ndcg_at_20": 0.62264, + "ndcg_at_100": 0.66187, + "ndcg_at_1000": 0.6816, + "map_at_1": 0.33453, + "map_at_3": 0.43969, + "map_at_5": 0.46988, + "map_at_10": 0.50358, + "map_at_20": 0.52036, + "map_at_100": 0.53129, + "map_at_1000": 0.5333, + "recall_at_1": 0.33453, + "recall_at_3": 0.51027, + "recall_at_5": 0.571, + "recall_at_10": 0.68934, + "recall_at_20": 0.76951, + "recall_at_100": 0.90637, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.53125, + "cv_recall_at_3": 0.75, + "cv_recall_at_5": 0.7875, + "cv_recall_at_10": 0.88125, + "cv_recall_at_20": 0.93125, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.53125, + "precision_at_3": 0.32292, + "precision_at_5": 0.24125, + "precision_at_10": 0.15875, + "precision_at_20": 0.09844, + "precision_at_100": 0.02612, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.53125, + "mrr_at_3": 0.625, + "mrr_at_5": 0.633125, + "mrr_at_10": 0.646404, + "mrr_at_20": 0.650069, + "mrr_at_100": 0.651305, + "mrr_at_1000": 0.651393, + "nauc_ndcg_at_1_max": 0.354587, + "nauc_ndcg_at_1_std": 0.030376, + "nauc_ndcg_at_1_diff1": 0.531077, + "nauc_ndcg_at_3_max": 0.291708, + "nauc_ndcg_at_3_std": -0.020069, + "nauc_ndcg_at_3_diff1": 0.37952, + "nauc_ndcg_at_5_max": 0.330496, + "nauc_ndcg_at_5_std": -0.022644, + "nauc_ndcg_at_5_diff1": 0.39077, + "nauc_ndcg_at_10_max": 0.332978, + "nauc_ndcg_at_10_std": -0.031511, + "nauc_ndcg_at_10_diff1": 0.396104, + "nauc_ndcg_at_20_max": 0.31406, + "nauc_ndcg_at_20_std": -0.052241, + "nauc_ndcg_at_20_diff1": 0.390274, + "nauc_ndcg_at_100_max": 0.308134, + "nauc_ndcg_at_100_std": -0.036739, + "nauc_ndcg_at_100_diff1": 0.389623, + "nauc_ndcg_at_1000_max": 0.292011, + "nauc_ndcg_at_1000_std": -0.050667, + "nauc_ndcg_at_1000_diff1": 0.390695, + "nauc_map_at_1_max": 0.286242, + "nauc_map_at_1_std": 0.006317, + "nauc_map_at_1_diff1": 0.50228, + "nauc_map_at_3_max": 0.292813, + "nauc_map_at_3_std": 0.001961, + "nauc_map_at_3_diff1": 0.402344, + "nauc_map_at_5_max": 0.311448, + "nauc_map_at_5_std": -0.017441, + "nauc_map_at_5_diff1": 0.405043, + "nauc_map_at_10_max": 0.300073, + "nauc_map_at_10_std": -0.041035, + "nauc_map_at_10_diff1": 0.40411, + "nauc_map_at_20_max": 0.290679, + "nauc_map_at_20_std": -0.049362, + "nauc_map_at_20_diff1": 0.39872, + "nauc_map_at_100_max": 0.28999, + "nauc_map_at_100_std": -0.045237, + "nauc_map_at_100_diff1": 0.396685, + "nauc_map_at_1000_max": 0.287974, + "nauc_map_at_1000_std": -0.046337, + "nauc_map_at_1000_diff1": 0.396217, + "nauc_recall_at_1_max": 0.286242, + "nauc_recall_at_1_std": 0.006317, + "nauc_recall_at_1_diff1": 0.50228, + "nauc_recall_at_3_max": 0.297816, + "nauc_recall_at_3_std": -0.001913, + "nauc_recall_at_3_diff1": 0.316051, + "nauc_recall_at_5_max": 0.35131, + "nauc_recall_at_5_std": -0.021632, + "nauc_recall_at_5_diff1": 0.309505, + "nauc_recall_at_10_max": 0.326676, + "nauc_recall_at_10_std": -0.037474, + "nauc_recall_at_10_diff1": 0.272661, + "nauc_recall_at_20_max": 0.292896, + "nauc_recall_at_20_std": -0.09153, + "nauc_recall_at_20_diff1": 0.249628, + "nauc_recall_at_100_max": 0.324322, + "nauc_recall_at_100_std": 0.055932, + "nauc_recall_at_100_diff1": 0.19967, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.354587, + "nauc_precision_at_1_std": 0.030376, + "nauc_precision_at_1_diff1": 0.531077, + "nauc_precision_at_3_max": 0.115928, + "nauc_precision_at_3_std": -0.063199, + "nauc_precision_at_3_diff1": 0.071017, + "nauc_precision_at_5_max": 0.073731, + "nauc_precision_at_5_std": -0.114701, + "nauc_precision_at_5_diff1": 0.003641, + "nauc_precision_at_10_max": -0.028242, + "nauc_precision_at_10_std": -0.132699, + "nauc_precision_at_10_diff1": -0.079687, + "nauc_precision_at_20_max": -0.125072, + "nauc_precision_at_20_std": -0.134074, + "nauc_precision_at_20_diff1": -0.157404, + "nauc_precision_at_100_max": -0.217222, + "nauc_precision_at_100_std": -0.091079, + "nauc_precision_at_100_diff1": -0.232846, + "nauc_precision_at_1000_max": -0.295348, + "nauc_precision_at_1000_std": -0.13499, + "nauc_precision_at_1000_diff1": -0.276487, + "nauc_cv_recall_at_1_max": 0.354587, + "nauc_cv_recall_at_1_std": 0.030376, + "nauc_cv_recall_at_1_diff1": 0.531077, + "nauc_cv_recall_at_3_max": 0.419139, + "nauc_cv_recall_at_3_std": -0.012807, + "nauc_cv_recall_at_3_diff1": 0.403284, + "nauc_cv_recall_at_5_max": 0.407412, + "nauc_cv_recall_at_5_std": -0.043896, + "nauc_cv_recall_at_5_diff1": 0.370604, + "nauc_cv_recall_at_10_max": 0.389934, + "nauc_cv_recall_at_10_std": -0.015355, + "nauc_cv_recall_at_10_diff1": 0.283256, + "nauc_cv_recall_at_20_max": 0.283974, + "nauc_cv_recall_at_20_std": -0.162635, + "nauc_cv_recall_at_20_diff1": 0.119133, + "nauc_cv_recall_at_100_max": 0.14881, + "nauc_cv_recall_at_100_std": -0.048436, + "nauc_cv_recall_at_100_diff1": -0.305789, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.354587, + "nauc_mrr_at_1_std": 0.030376, + "nauc_mrr_at_1_diff1": 0.531077, + "nauc_mrr_at_3_max": 0.389072, + "nauc_mrr_at_3_std": 0.028386, + "nauc_mrr_at_3_diff1": 0.48393, + "nauc_mrr_at_5_max": 0.385804, + "nauc_mrr_at_5_std": 0.02324, + "nauc_mrr_at_5_diff1": 0.480393, + "nauc_mrr_at_10_max": 0.38174, + "nauc_mrr_at_10_std": 0.025964, + "nauc_mrr_at_10_diff1": 0.481354, + "nauc_mrr_at_20_max": 0.379626, + "nauc_mrr_at_20_std": 0.023856, + "nauc_mrr_at_20_diff1": 0.480872, + "nauc_mrr_at_100_max": 0.380702, + "nauc_mrr_at_100_std": 0.026534, + "nauc_mrr_at_100_diff1": 0.481358, + "nauc_mrr_at_1000_max": 0.380677, + "nauc_mrr_at_1000_std": 0.026527, + "nauc_mrr_at_1000_diff1": 0.481539, + "main_score": 0.55301, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.54375, + "ndcg_at_3": 0.56344, + "ndcg_at_5": 0.58778, + "ndcg_at_10": 0.61953, + "ndcg_at_20": 0.65322, + "ndcg_at_100": 0.68763, + "ndcg_at_1000": 0.70495, + "map_at_1": 0.33981, + "map_at_3": 0.45755, + "map_at_5": 0.49311, + "map_at_10": 0.52669, + "map_at_20": 0.54503, + "map_at_100": 0.55597, + "map_at_1000": 0.5578, + "recall_at_1": 0.33981, + "recall_at_3": 0.5359, + "recall_at_5": 0.61862, + "recall_at_10": 0.71708, + "recall_at_20": 0.8068, + "recall_at_100": 0.92225, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.54375, + "cv_recall_at_3": 0.7875, + "cv_recall_at_5": 0.8625, + "cv_recall_at_10": 0.90625, + "cv_recall_at_20": 0.9625, + "cv_recall_at_100": 0.9875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.54375, + "precision_at_3": 0.34167, + "precision_at_5": 0.265, + "precision_at_10": 0.1675, + "precision_at_20": 0.10375, + "precision_at_100": 0.02662, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.54375, + "mrr_at_3": 0.654167, + "mrr_at_5": 0.671667, + "mrr_at_10": 0.677984, + "mrr_at_20": 0.681803, + "mrr_at_100": 0.682387, + "mrr_at_1000": 0.682421, + "nauc_ndcg_at_1_max": 0.353628, + "nauc_ndcg_at_1_std": -0.051538, + "nauc_ndcg_at_1_diff1": 0.470234, + "nauc_ndcg_at_3_max": 0.397489, + "nauc_ndcg_at_3_std": -0.020683, + "nauc_ndcg_at_3_diff1": 0.32175, + "nauc_ndcg_at_5_max": 0.391685, + "nauc_ndcg_at_5_std": -0.011383, + "nauc_ndcg_at_5_diff1": 0.346564, + "nauc_ndcg_at_10_max": 0.378728, + "nauc_ndcg_at_10_std": -0.021988, + "nauc_ndcg_at_10_diff1": 0.34024, + "nauc_ndcg_at_20_max": 0.372967, + "nauc_ndcg_at_20_std": -0.02563, + "nauc_ndcg_at_20_diff1": 0.309466, + "nauc_ndcg_at_100_max": 0.347858, + "nauc_ndcg_at_100_std": -0.044985, + "nauc_ndcg_at_100_diff1": 0.312006, + "nauc_ndcg_at_1000_max": 0.337307, + "nauc_ndcg_at_1000_std": -0.050045, + "nauc_ndcg_at_1000_diff1": 0.295746, + "nauc_map_at_1_max": 0.366909, + "nauc_map_at_1_std": 0.074482, + "nauc_map_at_1_diff1": 0.474375, + "nauc_map_at_3_max": 0.416954, + "nauc_map_at_3_std": 0.05302, + "nauc_map_at_3_diff1": 0.389736, + "nauc_map_at_5_max": 0.407026, + "nauc_map_at_5_std": 0.025216, + "nauc_map_at_5_diff1": 0.385568, + "nauc_map_at_10_max": 0.39227, + "nauc_map_at_10_std": 0.007064, + "nauc_map_at_10_diff1": 0.356552, + "nauc_map_at_20_max": 0.385479, + "nauc_map_at_20_std": -0.001644, + "nauc_map_at_20_diff1": 0.338355, + "nauc_map_at_100_max": 0.379824, + "nauc_map_at_100_std": -0.006166, + "nauc_map_at_100_diff1": 0.33299, + "nauc_map_at_1000_max": 0.377813, + "nauc_map_at_1000_std": -0.006469, + "nauc_map_at_1000_diff1": 0.330494, + "nauc_recall_at_1_max": 0.366909, + "nauc_recall_at_1_std": 0.074482, + "nauc_recall_at_1_diff1": 0.474375, + "nauc_recall_at_3_max": 0.417373, + "nauc_recall_at_3_std": 0.047363, + "nauc_recall_at_3_diff1": 0.321135, + "nauc_recall_at_5_max": 0.363095, + "nauc_recall_at_5_std": 0.00472, + "nauc_recall_at_5_diff1": 0.327677, + "nauc_recall_at_10_max": 0.308721, + "nauc_recall_at_10_std": -0.0175, + "nauc_recall_at_10_diff1": 0.243347, + "nauc_recall_at_20_max": 0.323347, + "nauc_recall_at_20_std": 0.033176, + "nauc_recall_at_20_diff1": 0.116055, + "nauc_recall_at_100_max": 0.163251, + "nauc_recall_at_100_std": -0.067773, + "nauc_recall_at_100_diff1": 0.241164, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.353628, + "nauc_precision_at_1_std": -0.051538, + "nauc_precision_at_1_diff1": 0.470234, + "nauc_precision_at_3_max": 0.190063, + "nauc_precision_at_3_std": -0.135541, + "nauc_precision_at_3_diff1": -0.045449, + "nauc_precision_at_5_max": 0.024996, + "nauc_precision_at_5_std": -0.179183, + "nauc_precision_at_5_diff1": -0.162163, + "nauc_precision_at_10_max": -0.062273, + "nauc_precision_at_10_std": -0.174246, + "nauc_precision_at_10_diff1": -0.24931, + "nauc_precision_at_20_max": -0.162142, + "nauc_precision_at_20_std": -0.179585, + "nauc_precision_at_20_diff1": -0.349126, + "nauc_precision_at_100_max": -0.262105, + "nauc_precision_at_100_std": -0.177174, + "nauc_precision_at_100_diff1": -0.395272, + "nauc_precision_at_1000_max": -0.331089, + "nauc_precision_at_1000_std": -0.173273, + "nauc_precision_at_1000_diff1": -0.431738, + "nauc_cv_recall_at_1_max": 0.353628, + "nauc_cv_recall_at_1_std": -0.051538, + "nauc_cv_recall_at_1_diff1": 0.470234, + "nauc_cv_recall_at_3_max": 0.433872, + "nauc_cv_recall_at_3_std": -0.131451, + "nauc_cv_recall_at_3_diff1": 0.300426, + "nauc_cv_recall_at_5_max": 0.274228, + "nauc_cv_recall_at_5_std": -0.180486, + "nauc_cv_recall_at_5_diff1": 0.256844, + "nauc_cv_recall_at_10_max": 0.19281, + "nauc_cv_recall_at_10_std": -0.213974, + "nauc_cv_recall_at_10_diff1": 0.132929, + "nauc_cv_recall_at_20_max": 0.141145, + "nauc_cv_recall_at_20_std": 0.007781, + "nauc_cv_recall_at_20_diff1": -0.502568, + "nauc_cv_recall_at_100_max": -1.445845, + "nauc_cv_recall_at_100_std": -0.80859, + "nauc_cv_recall_at_100_diff1": -1.151727, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.353628, + "nauc_mrr_at_1_std": -0.051538, + "nauc_mrr_at_1_diff1": 0.470234, + "nauc_mrr_at_3_max": 0.379227, + "nauc_mrr_at_3_std": -0.071572, + "nauc_mrr_at_3_diff1": 0.406725, + "nauc_mrr_at_5_max": 0.357309, + "nauc_mrr_at_5_std": -0.071148, + "nauc_mrr_at_5_diff1": 0.408882, + "nauc_mrr_at_10_max": 0.355018, + "nauc_mrr_at_10_std": -0.069266, + "nauc_mrr_at_10_diff1": 0.405039, + "nauc_mrr_at_20_max": 0.357011, + "nauc_mrr_at_20_std": -0.064826, + "nauc_mrr_at_20_diff1": 0.402509, + "nauc_mrr_at_100_max": 0.355622, + "nauc_mrr_at_100_std": -0.066003, + "nauc_mrr_at_100_diff1": 0.403571, + "nauc_mrr_at_1000_max": 0.355841, + "nauc_mrr_at_1000_std": -0.065829, + "nauc_mrr_at_1000_diff1": 0.40378, + "main_score": 0.58778, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.5125, + "ndcg_at_3": 0.50482, + "ndcg_at_5": 0.52965, + "ndcg_at_10": 0.5607, + "ndcg_at_20": 0.5867, + "ndcg_at_100": 0.63032, + "ndcg_at_1000": 0.6548, + "map_at_1": 0.31324, + "map_at_3": 0.41106, + "map_at_5": 0.44214, + "map_at_10": 0.47033, + "map_at_20": 0.48533, + "map_at_100": 0.49735, + "map_at_1000": 0.49967, + "recall_at_1": 0.31324, + "recall_at_3": 0.47064, + "recall_at_5": 0.5583, + "recall_at_10": 0.6541, + "recall_at_20": 0.72431, + "recall_at_100": 0.87795, + "recall_at_1000": 0.99978, + "cv_recall_at_1": 0.5125, + "cv_recall_at_3": 0.6875, + "cv_recall_at_5": 0.7875, + "cv_recall_at_10": 0.85625, + "cv_recall_at_20": 0.90625, + "cv_recall_at_100": 0.9625, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.5125, + "precision_at_3": 0.29792, + "precision_at_5": 0.23, + "precision_at_10": 0.15125, + "precision_at_20": 0.09188, + "precision_at_100": 0.02562, + "precision_at_1000": 0.00321, + "mrr_at_1": 0.5125, + "mrr_at_3": 0.59375, + "mrr_at_5": 0.615938, + "mrr_at_10": 0.624546, + "mrr_at_20": 0.62821, + "mrr_at_100": 0.629665, + "mrr_at_1000": 0.62987, + "nauc_ndcg_at_1_max": 0.422899, + "nauc_ndcg_at_1_std": 0.086549, + "nauc_ndcg_at_1_diff1": 0.388668, + "nauc_ndcg_at_3_max": 0.440487, + "nauc_ndcg_at_3_std": 0.116383, + "nauc_ndcg_at_3_diff1": 0.314215, + "nauc_ndcg_at_5_max": 0.438286, + "nauc_ndcg_at_5_std": 0.088782, + "nauc_ndcg_at_5_diff1": 0.30299, + "nauc_ndcg_at_10_max": 0.415931, + "nauc_ndcg_at_10_std": 0.061114, + "nauc_ndcg_at_10_diff1": 0.286853, + "nauc_ndcg_at_20_max": 0.416175, + "nauc_ndcg_at_20_std": 0.056266, + "nauc_ndcg_at_20_diff1": 0.271643, + "nauc_ndcg_at_100_max": 0.409007, + "nauc_ndcg_at_100_std": 0.054537, + "nauc_ndcg_at_100_diff1": 0.273702, + "nauc_ndcg_at_1000_max": 0.406504, + "nauc_ndcg_at_1000_std": 0.061305, + "nauc_ndcg_at_1000_diff1": 0.282223, + "nauc_map_at_1_max": 0.421648, + "nauc_map_at_1_std": 0.113462, + "nauc_map_at_1_diff1": 0.472717, + "nauc_map_at_3_max": 0.453123, + "nauc_map_at_3_std": 0.124829, + "nauc_map_at_3_diff1": 0.376039, + "nauc_map_at_5_max": 0.448241, + "nauc_map_at_5_std": 0.098127, + "nauc_map_at_5_diff1": 0.354522, + "nauc_map_at_10_max": 0.428168, + "nauc_map_at_10_std": 0.075214, + "nauc_map_at_10_diff1": 0.322735, + "nauc_map_at_20_max": 0.425906, + "nauc_map_at_20_std": 0.069956, + "nauc_map_at_20_diff1": 0.309479, + "nauc_map_at_100_max": 0.425976, + "nauc_map_at_100_std": 0.072918, + "nauc_map_at_100_diff1": 0.307571, + "nauc_map_at_1000_max": 0.425109, + "nauc_map_at_1000_std": 0.072929, + "nauc_map_at_1000_diff1": 0.307278, + "nauc_recall_at_1_max": 0.421648, + "nauc_recall_at_1_std": 0.113462, + "nauc_recall_at_1_diff1": 0.472717, + "nauc_recall_at_3_max": 0.427171, + "nauc_recall_at_3_std": 0.136587, + "nauc_recall_at_3_diff1": 0.319245, + "nauc_recall_at_5_max": 0.393335, + "nauc_recall_at_5_std": 0.062034, + "nauc_recall_at_5_diff1": 0.246769, + "nauc_recall_at_10_max": 0.316433, + "nauc_recall_at_10_std": -0.01103, + "nauc_recall_at_10_diff1": 0.15879, + "nauc_recall_at_20_max": 0.328733, + "nauc_recall_at_20_std": -0.004927, + "nauc_recall_at_20_diff1": 0.090237, + "nauc_recall_at_100_max": 0.279077, + "nauc_recall_at_100_std": -0.059458, + "nauc_recall_at_100_diff1": 0.033127, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.722222, + "nauc_recall_at_1000_diff1": 0.869281, + "nauc_precision_at_1_max": 0.422899, + "nauc_precision_at_1_std": 0.086549, + "nauc_precision_at_1_diff1": 0.388668, + "nauc_precision_at_3_max": 0.270273, + "nauc_precision_at_3_std": 0.047023, + "nauc_precision_at_3_diff1": 0.019655, + "nauc_precision_at_5_max": 0.121575, + "nauc_precision_at_5_std": -0.085041, + "nauc_precision_at_5_diff1": -0.130001, + "nauc_precision_at_10_max": -0.03275, + "nauc_precision_at_10_std": -0.137495, + "nauc_precision_at_10_diff1": -0.259431, + "nauc_precision_at_20_max": -0.082519, + "nauc_precision_at_20_std": -0.126235, + "nauc_precision_at_20_diff1": -0.29521, + "nauc_precision_at_100_max": -0.178395, + "nauc_precision_at_100_std": -0.109942, + "nauc_precision_at_100_diff1": -0.324267, + "nauc_precision_at_1000_max": -0.253098, + "nauc_precision_at_1000_std": -0.100373, + "nauc_precision_at_1000_diff1": -0.320884, + "nauc_cv_recall_at_1_max": 0.422899, + "nauc_cv_recall_at_1_std": 0.086549, + "nauc_cv_recall_at_1_diff1": 0.388668, + "nauc_cv_recall_at_3_max": 0.451475, + "nauc_cv_recall_at_3_std": 0.179757, + "nauc_cv_recall_at_3_diff1": 0.312183, + "nauc_cv_recall_at_5_max": 0.366062, + "nauc_cv_recall_at_5_std": 0.020191, + "nauc_cv_recall_at_5_diff1": 0.170017, + "nauc_cv_recall_at_10_max": 0.310806, + "nauc_cv_recall_at_10_std": -0.125671, + "nauc_cv_recall_at_10_diff1": 0.188369, + "nauc_cv_recall_at_20_max": 0.477996, + "nauc_cv_recall_at_20_std": 0.132835, + "nauc_cv_recall_at_20_diff1": 0.073358, + "nauc_cv_recall_at_100_max": 0.140367, + "nauc_cv_recall_at_100_std": -0.06108, + "nauc_cv_recall_at_100_diff1": -0.061936, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.422899, + "nauc_mrr_at_1_std": 0.086549, + "nauc_mrr_at_1_diff1": 0.388668, + "nauc_mrr_at_3_max": 0.438364, + "nauc_mrr_at_3_std": 0.128246, + "nauc_mrr_at_3_diff1": 0.355773, + "nauc_mrr_at_5_max": 0.423222, + "nauc_mrr_at_5_std": 0.100721, + "nauc_mrr_at_5_diff1": 0.336008, + "nauc_mrr_at_10_max": 0.421423, + "nauc_mrr_at_10_std": 0.09449, + "nauc_mrr_at_10_diff1": 0.342001, + "nauc_mrr_at_20_max": 0.427318, + "nauc_mrr_at_20_std": 0.104141, + "nauc_mrr_at_20_diff1": 0.341095, + "nauc_mrr_at_100_max": 0.425449, + "nauc_mrr_at_100_std": 0.102509, + "nauc_mrr_at_100_diff1": 0.342982, + "nauc_mrr_at_1000_max": 0.425386, + "nauc_mrr_at_1000_std": 0.102566, + "nauc_mrr_at_1000_diff1": 0.342885, + "main_score": 0.52965, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 152.9097638130188, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/Vidore2ESGReportsHLRetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..20633f1c54 --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.52564, + "ndcg_at_3": 0.55579, + "ndcg_at_5": 0.5676, + "ndcg_at_10": 0.60197, + "ndcg_at_20": 0.64422, + "ndcg_at_100": 0.67421, + "ndcg_at_1000": 0.68739, + "map_at_1": 0.40256, + "map_at_3": 0.49135, + "map_at_5": 0.51199, + "map_at_10": 0.53452, + "map_at_20": 0.55338, + "map_at_100": 0.56178, + "map_at_1000": 0.56292, + "recall_at_1": 0.40256, + "recall_at_3": 0.54231, + "recall_at_5": 0.60449, + "recall_at_10": 0.68027, + "recall_at_20": 0.81017, + "recall_at_100": 0.93409, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.53846, + "cv_recall_at_3": 0.69231, + "cv_recall_at_5": 0.75, + "cv_recall_at_10": 0.82692, + "cv_recall_at_20": 0.92308, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.53846, + "precision_at_3": 0.28846, + "precision_at_5": 0.20385, + "precision_at_10": 0.12885, + "precision_at_20": 0.08558, + "precision_at_100": 0.02115, + "precision_at_1000": 0.00246, + "mrr_at_1": 0.538462, + "mrr_at_3": 0.615385, + "mrr_at_5": 0.628846, + "mrr_at_10": 0.639125, + "mrr_at_20": 0.646112, + "mrr_at_100": 0.64799, + "mrr_at_1000": 0.64799, + "nauc_ndcg_at_1_max": 0.191296, + "nauc_ndcg_at_1_std": 0.202101, + "nauc_ndcg_at_1_diff1": 0.708869, + "nauc_ndcg_at_3_max": 0.184355, + "nauc_ndcg_at_3_std": 0.275827, + "nauc_ndcg_at_3_diff1": 0.62749, + "nauc_ndcg_at_5_max": 0.278656, + "nauc_ndcg_at_5_std": 0.314058, + "nauc_ndcg_at_5_diff1": 0.60445, + "nauc_ndcg_at_10_max": 0.237617, + "nauc_ndcg_at_10_std": 0.310481, + "nauc_ndcg_at_10_diff1": 0.627535, + "nauc_ndcg_at_20_max": 0.241543, + "nauc_ndcg_at_20_std": 0.328035, + "nauc_ndcg_at_20_diff1": 0.620167, + "nauc_ndcg_at_100_max": 0.225451, + "nauc_ndcg_at_100_std": 0.315079, + "nauc_ndcg_at_100_diff1": 0.617345, + "nauc_ndcg_at_1000_max": 0.205185, + "nauc_ndcg_at_1000_std": 0.286609, + "nauc_ndcg_at_1000_diff1": 0.618185, + "nauc_map_at_1_max": 0.312774, + "nauc_map_at_1_std": 0.25916, + "nauc_map_at_1_diff1": 0.709315, + "nauc_map_at_3_max": 0.252852, + "nauc_map_at_3_std": 0.280937, + "nauc_map_at_3_diff1": 0.631839, + "nauc_map_at_5_max": 0.290813, + "nauc_map_at_5_std": 0.300378, + "nauc_map_at_5_diff1": 0.623413, + "nauc_map_at_10_max": 0.264933, + "nauc_map_at_10_std": 0.297331, + "nauc_map_at_10_diff1": 0.631721, + "nauc_map_at_20_max": 0.257572, + "nauc_map_at_20_std": 0.303736, + "nauc_map_at_20_diff1": 0.628999, + "nauc_map_at_100_max": 0.256937, + "nauc_map_at_100_std": 0.308463, + "nauc_map_at_100_diff1": 0.627976, + "nauc_map_at_1000_max": 0.255312, + "nauc_map_at_1000_std": 0.306708, + "nauc_map_at_1000_diff1": 0.627934, + "nauc_recall_at_1_max": 0.312774, + "nauc_recall_at_1_std": 0.25916, + "nauc_recall_at_1_diff1": 0.709315, + "nauc_recall_at_3_max": 0.215479, + "nauc_recall_at_3_std": 0.31855, + "nauc_recall_at_3_diff1": 0.577256, + "nauc_recall_at_5_max": 0.423126, + "nauc_recall_at_5_std": 0.433709, + "nauc_recall_at_5_diff1": 0.53104, + "nauc_recall_at_10_max": 0.33231, + "nauc_recall_at_10_std": 0.406314, + "nauc_recall_at_10_diff1": 0.541901, + "nauc_recall_at_20_max": 0.429784, + "nauc_recall_at_20_std": 0.549037, + "nauc_recall_at_20_diff1": 0.461582, + "nauc_recall_at_100_max": 0.450928, + "nauc_recall_at_100_std": 0.723879, + "nauc_recall_at_100_diff1": 0.380381, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.172821, + "nauc_precision_at_1_std": 0.249088, + "nauc_precision_at_1_diff1": 0.739475, + "nauc_precision_at_3_max": -0.120015, + "nauc_precision_at_3_std": 0.101763, + "nauc_precision_at_3_diff1": 0.201369, + "nauc_precision_at_5_max": -0.085408, + "nauc_precision_at_5_std": 0.097243, + "nauc_precision_at_5_diff1": 0.077214, + "nauc_precision_at_10_max": -0.218604, + "nauc_precision_at_10_std": 0.00699, + "nauc_precision_at_10_diff1": -0.008897, + "nauc_precision_at_20_max": -0.238433, + "nauc_precision_at_20_std": -0.013107, + "nauc_precision_at_20_diff1": -0.206697, + "nauc_precision_at_100_max": -0.274575, + "nauc_precision_at_100_std": -0.077173, + "nauc_precision_at_100_diff1": -0.310203, + "nauc_precision_at_1000_max": -0.33175, + "nauc_precision_at_1000_std": -0.200949, + "nauc_precision_at_1000_diff1": -0.340703, + "nauc_cv_recall_at_1_max": 0.172821, + "nauc_cv_recall_at_1_std": 0.249088, + "nauc_cv_recall_at_1_diff1": 0.739475, + "nauc_cv_recall_at_3_max": -0.048369, + "nauc_cv_recall_at_3_std": 0.276299, + "nauc_cv_recall_at_3_diff1": 0.63097, + "nauc_cv_recall_at_5_max": 0.202623, + "nauc_cv_recall_at_5_std": 0.414471, + "nauc_cv_recall_at_5_diff1": 0.590884, + "nauc_cv_recall_at_10_max": 0.091306, + "nauc_cv_recall_at_10_std": 0.322367, + "nauc_cv_recall_at_10_diff1": 0.544669, + "nauc_cv_recall_at_20_max": 0.3272, + "nauc_cv_recall_at_20_std": 0.567012, + "nauc_cv_recall_at_20_diff1": 0.455434, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.172821, + "nauc_mrr_at_1_std": 0.249088, + "nauc_mrr_at_1_diff1": 0.739475, + "nauc_mrr_at_3_max": 0.075832, + "nauc_mrr_at_3_std": 0.26102, + "nauc_mrr_at_3_diff1": 0.691897, + "nauc_mrr_at_5_max": 0.122091, + "nauc_mrr_at_5_std": 0.282336, + "nauc_mrr_at_5_diff1": 0.686621, + "nauc_mrr_at_10_max": 0.110597, + "nauc_mrr_at_10_std": 0.270904, + "nauc_mrr_at_10_diff1": 0.68599, + "nauc_mrr_at_20_max": 0.115714, + "nauc_mrr_at_20_std": 0.274046, + "nauc_mrr_at_20_diff1": 0.686542, + "nauc_mrr_at_100_max": 0.114579, + "nauc_mrr_at_100_std": 0.2724, + "nauc_mrr_at_100_diff1": 0.687121, + "nauc_mrr_at_1000_max": 0.114579, + "nauc_mrr_at_1000_std": 0.2724, + "nauc_mrr_at_1000_diff1": 0.687121, + "main_score": 0.5676, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 146.22701144218445, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/Vidore2ESGReportsRetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..9f30d19be8 --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.45614, + "ndcg_at_3": 0.47835, + "ndcg_at_5": 0.51189, + "ndcg_at_10": 0.54993, + "ndcg_at_20": 0.58541, + "ndcg_at_100": 0.63348, + "ndcg_at_1000": 0.64203, + "map_at_1": 0.24069, + "map_at_3": 0.3403, + "map_at_5": 0.38779, + "map_at_10": 0.42813, + "map_at_20": 0.45067, + "map_at_100": 0.47462, + "map_at_1000": 0.47533, + "recall_at_1": 0.24069, + "recall_at_3": 0.42552, + "recall_at_5": 0.53229, + "recall_at_10": 0.67126, + "recall_at_20": 0.7916, + "recall_at_100": 0.94269, + "recall_at_1000": 0.99415, + "cv_recall_at_1": 0.45614, + "cv_recall_at_3": 0.70175, + "cv_recall_at_5": 0.78947, + "cv_recall_at_10": 0.85965, + "cv_recall_at_20": 0.92982, + "cv_recall_at_100": 0.96491, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.45614, + "precision_at_3": 0.32749, + "precision_at_5": 0.27368, + "precision_at_10": 0.18596, + "precision_at_20": 0.12281, + "precision_at_100": 0.03737, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.45614, + "mrr_at_3": 0.561404, + "mrr_at_5": 0.583333, + "mrr_at_10": 0.593567, + "mrr_at_20": 0.598645, + "mrr_at_100": 0.599239, + "mrr_at_1000": 0.599513, + "nauc_ndcg_at_1_max": -0.140462, + "nauc_ndcg_at_1_std": -0.160068, + "nauc_ndcg_at_1_diff1": 0.004066, + "nauc_ndcg_at_3_max": -0.121493, + "nauc_ndcg_at_3_std": -0.167259, + "nauc_ndcg_at_3_diff1": 0.046004, + "nauc_ndcg_at_5_max": -0.097818, + "nauc_ndcg_at_5_std": -0.136188, + "nauc_ndcg_at_5_diff1": 0.031252, + "nauc_ndcg_at_10_max": -0.025942, + "nauc_ndcg_at_10_std": -0.052504, + "nauc_ndcg_at_10_diff1": 0.083309, + "nauc_ndcg_at_20_max": -0.016414, + "nauc_ndcg_at_20_std": -0.043537, + "nauc_ndcg_at_20_diff1": 0.034528, + "nauc_ndcg_at_100_max": -0.091915, + "nauc_ndcg_at_100_std": -0.052635, + "nauc_ndcg_at_100_diff1": -0.042233, + "nauc_ndcg_at_1000_max": -0.103362, + "nauc_ndcg_at_1000_std": -0.082145, + "nauc_ndcg_at_1000_diff1": -0.037572, + "nauc_map_at_1_max": 0.094219, + "nauc_map_at_1_std": -0.007241, + "nauc_map_at_1_diff1": 0.180016, + "nauc_map_at_3_max": 0.038058, + "nauc_map_at_3_std": -0.05493, + "nauc_map_at_3_diff1": 0.185733, + "nauc_map_at_5_max": 0.049312, + "nauc_map_at_5_std": -0.038382, + "nauc_map_at_5_diff1": 0.137353, + "nauc_map_at_10_max": 0.061929, + "nauc_map_at_10_std": -0.021691, + "nauc_map_at_10_diff1": 0.140091, + "nauc_map_at_20_max": 0.035185, + "nauc_map_at_20_std": -0.037175, + "nauc_map_at_20_diff1": 0.10334, + "nauc_map_at_100_max": -0.011677, + "nauc_map_at_100_std": -0.052368, + "nauc_map_at_100_diff1": 0.055974, + "nauc_map_at_1000_max": -0.012942, + "nauc_map_at_1000_std": -0.054522, + "nauc_map_at_1000_diff1": 0.055854, + "nauc_recall_at_1_max": 0.094219, + "nauc_recall_at_1_std": -0.007241, + "nauc_recall_at_1_diff1": 0.180016, + "nauc_recall_at_3_max": 0.013553, + "nauc_recall_at_3_std": -0.091165, + "nauc_recall_at_3_diff1": 0.254075, + "nauc_recall_at_5_max": 0.064194, + "nauc_recall_at_5_std": -0.046099, + "nauc_recall_at_5_diff1": 0.225211, + "nauc_recall_at_10_max": 0.113489, + "nauc_recall_at_10_std": 0.054074, + "nauc_recall_at_10_diff1": 0.295176, + "nauc_recall_at_20_max": 0.239011, + "nauc_recall_at_20_std": 0.175975, + "nauc_recall_at_20_diff1": 0.138356, + "nauc_recall_at_100_max": 0.144835, + "nauc_recall_at_100_std": 0.681254, + "nauc_recall_at_100_diff1": -0.355738, + "nauc_recall_at_1000_max": 0.555085, + "nauc_recall_at_1000_std": 0.868512, + "nauc_recall_at_1000_diff1": -1.727857, + "nauc_precision_at_1_max": -0.140462, + "nauc_precision_at_1_std": -0.160068, + "nauc_precision_at_1_diff1": 0.004066, + "nauc_precision_at_3_max": -0.298114, + "nauc_precision_at_3_std": -0.21458, + "nauc_precision_at_3_diff1": -0.160828, + "nauc_precision_at_5_max": -0.260934, + "nauc_precision_at_5_std": -0.157112, + "nauc_precision_at_5_diff1": -0.288978, + "nauc_precision_at_10_max": -0.240808, + "nauc_precision_at_10_std": -0.110078, + "nauc_precision_at_10_diff1": -0.290112, + "nauc_precision_at_20_max": -0.317889, + "nauc_precision_at_20_std": -0.145007, + "nauc_precision_at_20_diff1": -0.371512, + "nauc_precision_at_100_max": -0.375941, + "nauc_precision_at_100_std": -0.137885, + "nauc_precision_at_100_diff1": -0.410327, + "nauc_precision_at_1000_max": -0.393133, + "nauc_precision_at_1000_std": -0.168562, + "nauc_precision_at_1000_diff1": -0.402714, + "nauc_cv_recall_at_1_max": -0.140462, + "nauc_cv_recall_at_1_std": -0.160068, + "nauc_cv_recall_at_1_diff1": 0.004066, + "nauc_cv_recall_at_3_max": -0.346884, + "nauc_cv_recall_at_3_std": -0.308936, + "nauc_cv_recall_at_3_diff1": -0.08696, + "nauc_cv_recall_at_5_max": -0.394838, + "nauc_cv_recall_at_5_std": -0.290346, + "nauc_cv_recall_at_5_diff1": -0.163329, + "nauc_cv_recall_at_10_max": -0.382662, + "nauc_cv_recall_at_10_std": -0.040693, + "nauc_cv_recall_at_10_diff1": -0.123791, + "nauc_cv_recall_at_20_max": -0.154568, + "nauc_cv_recall_at_20_std": 0.326058, + "nauc_cv_recall_at_20_diff1": -0.359479, + "nauc_cv_recall_at_100_max": -0.306974, + "nauc_cv_recall_at_100_std": 0.497564, + "nauc_cv_recall_at_100_diff1": -0.216396, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.140462, + "nauc_mrr_at_1_std": -0.160068, + "nauc_mrr_at_1_diff1": 0.004066, + "nauc_mrr_at_3_max": -0.209675, + "nauc_mrr_at_3_std": -0.20785, + "nauc_mrr_at_3_diff1": -0.057231, + "nauc_mrr_at_5_max": -0.210184, + "nauc_mrr_at_5_std": -0.199492, + "nauc_mrr_at_5_diff1": -0.068028, + "nauc_mrr_at_10_max": -0.200934, + "nauc_mrr_at_10_std": -0.17479, + "nauc_mrr_at_10_diff1": -0.063511, + "nauc_mrr_at_20_max": -0.193226, + "nauc_mrr_at_20_std": -0.170563, + "nauc_mrr_at_20_diff1": -0.067449, + "nauc_mrr_at_100_max": -0.193442, + "nauc_mrr_at_100_std": -0.171006, + "nauc_mrr_at_100_diff1": -0.066944, + "nauc_mrr_at_1000_max": -0.193304, + "nauc_mrr_at_1000_std": -0.171634, + "nauc_mrr_at_1000_diff1": -0.066812, + "main_score": 0.51189, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.42105, + "ndcg_at_3": 0.46836, + "ndcg_at_5": 0.53166, + "ndcg_at_10": 0.55421, + "ndcg_at_20": 0.58527, + "ndcg_at_100": 0.63373, + "ndcg_at_1000": 0.64079, + "map_at_1": 0.18264, + "map_at_3": 0.31713, + "map_at_5": 0.38759, + "map_at_10": 0.41959, + "map_at_20": 0.4426, + "map_at_100": 0.46716, + "map_at_1000": 0.46781, + "recall_at_1": 0.18264, + "recall_at_3": 0.43356, + "recall_at_5": 0.61533, + "recall_at_10": 0.69806, + "recall_at_20": 0.80641, + "recall_at_100": 0.95741, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.42105, + "cv_recall_at_3": 0.75439, + "cv_recall_at_5": 0.85965, + "cv_recall_at_10": 0.89474, + "cv_recall_at_20": 0.92982, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.42105, + "precision_at_3": 0.33333, + "precision_at_5": 0.29474, + "precision_at_10": 0.19474, + "precision_at_20": 0.12544, + "precision_at_100": 0.03754, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.421053, + "mrr_at_3": 0.567251, + "mrr_at_5": 0.590936, + "mrr_at_10": 0.595809, + "mrr_at_20": 0.598733, + "mrr_at_100": 0.600282, + "mrr_at_1000": 0.600333, + "nauc_ndcg_at_1_max": -0.426026, + "nauc_ndcg_at_1_std": -0.275906, + "nauc_ndcg_at_1_diff1": -0.047302, + "nauc_ndcg_at_3_max": -0.412754, + "nauc_ndcg_at_3_std": -0.359165, + "nauc_ndcg_at_3_diff1": -0.022225, + "nauc_ndcg_at_5_max": -0.208761, + "nauc_ndcg_at_5_std": -0.208233, + "nauc_ndcg_at_5_diff1": 0.013946, + "nauc_ndcg_at_10_max": -0.22026, + "nauc_ndcg_at_10_std": -0.174175, + "nauc_ndcg_at_10_diff1": -0.00523, + "nauc_ndcg_at_20_max": -0.218572, + "nauc_ndcg_at_20_std": -0.18042, + "nauc_ndcg_at_20_diff1": -0.058267, + "nauc_ndcg_at_100_max": -0.315095, + "nauc_ndcg_at_100_std": -0.213178, + "nauc_ndcg_at_100_diff1": -0.097058, + "nauc_ndcg_at_1000_max": -0.33869, + "nauc_ndcg_at_1000_std": -0.248351, + "nauc_ndcg_at_1000_diff1": -0.063262, + "nauc_map_at_1_max": -0.167908, + "nauc_map_at_1_std": -0.201276, + "nauc_map_at_1_diff1": 0.229385, + "nauc_map_at_3_max": -0.208447, + "nauc_map_at_3_std": -0.261735, + "nauc_map_at_3_diff1": 0.176824, + "nauc_map_at_5_max": -0.093616, + "nauc_map_at_5_std": -0.166291, + "nauc_map_at_5_diff1": 0.152946, + "nauc_map_at_10_max": -0.149667, + "nauc_map_at_10_std": -0.185838, + "nauc_map_at_10_diff1": 0.112637, + "nauc_map_at_20_max": -0.189829, + "nauc_map_at_20_std": -0.20837, + "nauc_map_at_20_diff1": 0.068526, + "nauc_map_at_100_max": -0.250758, + "nauc_map_at_100_std": -0.232603, + "nauc_map_at_100_diff1": 0.029046, + "nauc_map_at_1000_max": -0.252933, + "nauc_map_at_1000_std": -0.235094, + "nauc_map_at_1000_diff1": 0.030292, + "nauc_recall_at_1_max": -0.167908, + "nauc_recall_at_1_std": -0.201276, + "nauc_recall_at_1_diff1": 0.229385, + "nauc_recall_at_3_max": -0.182094, + "nauc_recall_at_3_std": -0.228463, + "nauc_recall_at_3_diff1": 0.142142, + "nauc_recall_at_5_max": 0.214775, + "nauc_recall_at_5_std": 0.063432, + "nauc_recall_at_5_diff1": 0.119874, + "nauc_recall_at_10_max": 0.168725, + "nauc_recall_at_10_std": 0.149959, + "nauc_recall_at_10_diff1": 0.004185, + "nauc_recall_at_20_max": 0.278292, + "nauc_recall_at_20_std": 0.22353, + "nauc_recall_at_20_diff1": -0.230091, + "nauc_recall_at_100_max": 0.317343, + "nauc_recall_at_100_std": 0.79033, + "nauc_recall_at_100_diff1": -1.143332, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.426026, + "nauc_precision_at_1_std": -0.275906, + "nauc_precision_at_1_diff1": -0.047302, + "nauc_precision_at_3_max": -0.448507, + "nauc_precision_at_3_std": -0.292517, + "nauc_precision_at_3_diff1": -0.23062, + "nauc_precision_at_5_max": -0.172908, + "nauc_precision_at_5_std": -0.060699, + "nauc_precision_at_5_diff1": -0.256632, + "nauc_precision_at_10_max": -0.300522, + "nauc_precision_at_10_std": -0.117822, + "nauc_precision_at_10_diff1": -0.316156, + "nauc_precision_at_20_max": -0.359863, + "nauc_precision_at_20_std": -0.145512, + "nauc_precision_at_20_diff1": -0.381759, + "nauc_precision_at_100_max": -0.397974, + "nauc_precision_at_100_std": -0.136318, + "nauc_precision_at_100_diff1": -0.342161, + "nauc_precision_at_1000_max": -0.412851, + "nauc_precision_at_1000_std": -0.16444, + "nauc_precision_at_1000_diff1": -0.312416, + "nauc_cv_recall_at_1_max": -0.426026, + "nauc_cv_recall_at_1_std": -0.275906, + "nauc_cv_recall_at_1_diff1": -0.047302, + "nauc_cv_recall_at_3_max": -0.746651, + "nauc_cv_recall_at_3_std": -0.54991, + "nauc_cv_recall_at_3_diff1": -0.028767, + "nauc_cv_recall_at_5_max": -0.241662, + "nauc_cv_recall_at_5_std": -0.139806, + "nauc_cv_recall_at_5_diff1": -0.257715, + "nauc_cv_recall_at_10_max": -0.190899, + "nauc_cv_recall_at_10_std": 0.20093, + "nauc_cv_recall_at_10_diff1": -0.569591, + "nauc_cv_recall_at_20_max": 0.145762, + "nauc_cv_recall_at_20_std": 0.351784, + "nauc_cv_recall_at_20_diff1": -1.066724, + "nauc_cv_recall_at_100_max": -0.164944, + "nauc_cv_recall_at_100_std": 0.722732, + "nauc_cv_recall_at_100_diff1": -1.727857, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.426026, + "nauc_mrr_at_1_std": -0.275906, + "nauc_mrr_at_1_diff1": -0.047302, + "nauc_mrr_at_3_max": -0.510736, + "nauc_mrr_at_3_std": -0.367109, + "nauc_mrr_at_3_diff1": -0.05889, + "nauc_mrr_at_5_max": -0.439355, + "nauc_mrr_at_5_std": -0.306846, + "nauc_mrr_at_5_diff1": -0.086985, + "nauc_mrr_at_10_max": -0.439005, + "nauc_mrr_at_10_std": -0.289479, + "nauc_mrr_at_10_diff1": -0.101289, + "nauc_mrr_at_20_max": -0.4345, + "nauc_mrr_at_20_std": -0.291478, + "nauc_mrr_at_20_diff1": -0.106861, + "nauc_mrr_at_100_max": -0.437814, + "nauc_mrr_at_100_std": -0.29385, + "nauc_mrr_at_100_diff1": -0.103144, + "nauc_mrr_at_1000_max": -0.437865, + "nauc_mrr_at_1000_std": -0.294038, + "nauc_mrr_at_1000_diff1": -0.102843, + "main_score": 0.53166, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.50877, + "ndcg_at_3": 0.49981, + "ndcg_at_5": 0.54229, + "ndcg_at_10": 0.57116, + "ndcg_at_20": 0.60326, + "ndcg_at_100": 0.65948, + "ndcg_at_1000": 0.66541, + "map_at_1": 0.25541, + "map_at_3": 0.36388, + "map_at_5": 0.41489, + "map_at_10": 0.44819, + "map_at_20": 0.46958, + "map_at_100": 0.49714, + "map_at_1000": 0.49775, + "recall_at_1": 0.25541, + "recall_at_3": 0.44361, + "recall_at_5": 0.56719, + "recall_at_10": 0.67711, + "recall_at_20": 0.79244, + "recall_at_100": 0.96023, + "recall_at_1000": 0.99561, + "cv_recall_at_1": 0.50877, + "cv_recall_at_3": 0.73684, + "cv_recall_at_5": 0.82456, + "cv_recall_at_10": 0.85965, + "cv_recall_at_20": 0.92982, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.50877, + "precision_at_3": 0.32749, + "precision_at_5": 0.2807, + "precision_at_10": 0.18772, + "precision_at_20": 0.11842, + "precision_at_100": 0.03754, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.508772, + "mrr_at_3": 0.611111, + "mrr_at_5": 0.630409, + "mrr_at_10": 0.636257, + "mrr_at_20": 0.641418, + "mrr_at_100": 0.643392, + "mrr_at_1000": 0.643466, + "nauc_ndcg_at_1_max": -0.32813, + "nauc_ndcg_at_1_std": -0.154047, + "nauc_ndcg_at_1_diff1": 0.044696, + "nauc_ndcg_at_3_max": -0.130344, + "nauc_ndcg_at_3_std": -0.056818, + "nauc_ndcg_at_3_diff1": 0.021022, + "nauc_ndcg_at_5_max": -0.184323, + "nauc_ndcg_at_5_std": -0.075723, + "nauc_ndcg_at_5_diff1": 0.052126, + "nauc_ndcg_at_10_max": -0.143512, + "nauc_ndcg_at_10_std": -0.057811, + "nauc_ndcg_at_10_diff1": 0.126006, + "nauc_ndcg_at_20_max": -0.088336, + "nauc_ndcg_at_20_std": -0.050453, + "nauc_ndcg_at_20_diff1": 0.130549, + "nauc_ndcg_at_100_max": -0.199442, + "nauc_ndcg_at_100_std": -0.05349, + "nauc_ndcg_at_100_diff1": 0.034663, + "nauc_ndcg_at_1000_max": -0.207519, + "nauc_ndcg_at_1000_std": -0.071101, + "nauc_ndcg_at_1000_diff1": 0.045517, + "nauc_map_at_1_max": -0.128475, + "nauc_map_at_1_std": -0.13357, + "nauc_map_at_1_diff1": 0.174662, + "nauc_map_at_3_max": -0.026657, + "nauc_map_at_3_std": -0.040896, + "nauc_map_at_3_diff1": 0.136065, + "nauc_map_at_5_max": -0.069782, + "nauc_map_at_5_std": -0.061177, + "nauc_map_at_5_diff1": 0.11084, + "nauc_map_at_10_max": -0.078721, + "nauc_map_at_10_std": -0.062536, + "nauc_map_at_10_diff1": 0.123128, + "nauc_map_at_20_max": -0.075814, + "nauc_map_at_20_std": -0.060908, + "nauc_map_at_20_diff1": 0.109743, + "nauc_map_at_100_max": -0.129667, + "nauc_map_at_100_std": -0.069644, + "nauc_map_at_100_diff1": 0.069389, + "nauc_map_at_1000_max": -0.131142, + "nauc_map_at_1000_std": -0.070748, + "nauc_map_at_1000_diff1": 0.070108, + "nauc_recall_at_1_max": -0.128475, + "nauc_recall_at_1_std": -0.13357, + "nauc_recall_at_1_diff1": 0.174662, + "nauc_recall_at_3_max": 0.056717, + "nauc_recall_at_3_std": 0.021144, + "nauc_recall_at_3_diff1": 0.153269, + "nauc_recall_at_5_max": 0.005447, + "nauc_recall_at_5_std": -0.003554, + "nauc_recall_at_5_diff1": 0.170055, + "nauc_recall_at_10_max": 0.019718, + "nauc_recall_at_10_std": 0.003335, + "nauc_recall_at_10_diff1": 0.264209, + "nauc_recall_at_20_max": 0.276549, + "nauc_recall_at_20_std": 0.074249, + "nauc_recall_at_20_diff1": 0.323241, + "nauc_recall_at_100_max": -0.023201, + "nauc_recall_at_100_std": 0.571855, + "nauc_recall_at_100_diff1": -0.292289, + "nauc_recall_at_1000_max": 0.722732, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": -0.164944, + "nauc_precision_at_1_max": -0.32813, + "nauc_precision_at_1_std": -0.154047, + "nauc_precision_at_1_diff1": 0.044696, + "nauc_precision_at_3_max": -0.160173, + "nauc_precision_at_3_std": 0.016423, + "nauc_precision_at_3_diff1": -0.1218, + "nauc_precision_at_5_max": -0.284876, + "nauc_precision_at_5_std": -0.043366, + "nauc_precision_at_5_diff1": -0.155233, + "nauc_precision_at_10_max": -0.282837, + "nauc_precision_at_10_std": -0.041935, + "nauc_precision_at_10_diff1": -0.159994, + "nauc_precision_at_20_max": -0.265507, + "nauc_precision_at_20_std": -0.045363, + "nauc_precision_at_20_diff1": -0.188737, + "nauc_precision_at_100_max": -0.32847, + "nauc_precision_at_100_std": -0.03988, + "nauc_precision_at_100_diff1": -0.245099, + "nauc_precision_at_1000_max": -0.338797, + "nauc_precision_at_1000_std": -0.057198, + "nauc_precision_at_1000_diff1": -0.226057, + "nauc_cv_recall_at_1_max": -0.32813, + "nauc_cv_recall_at_1_std": -0.154047, + "nauc_cv_recall_at_1_diff1": 0.044696, + "nauc_cv_recall_at_3_max": -0.266403, + "nauc_cv_recall_at_3_std": -0.06492, + "nauc_cv_recall_at_3_diff1": 0.027332, + "nauc_cv_recall_at_5_max": -0.355529, + "nauc_cv_recall_at_5_std": -0.067343, + "nauc_cv_recall_at_5_diff1": 0.26812, + "nauc_cv_recall_at_10_max": -0.341727, + "nauc_cv_recall_at_10_std": -0.115192, + "nauc_cv_recall_at_10_diff1": 0.384253, + "nauc_cv_recall_at_20_max": -0.034299, + "nauc_cv_recall_at_20_std": -0.060847, + "nauc_cv_recall_at_20_diff1": 0.515355, + "nauc_cv_recall_at_100_max": -1.169034, + "nauc_cv_recall_at_100_std": 0.126616, + "nauc_cv_recall_at_100_diff1": -0.164944, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.32813, + "nauc_mrr_at_1_std": -0.154047, + "nauc_mrr_at_1_diff1": 0.044696, + "nauc_mrr_at_3_max": -0.275353, + "nauc_mrr_at_3_std": -0.099248, + "nauc_mrr_at_3_diff1": 0.025211, + "nauc_mrr_at_5_max": -0.28614, + "nauc_mrr_at_5_std": -0.10279, + "nauc_mrr_at_5_diff1": 0.053546, + "nauc_mrr_at_10_max": -0.283747, + "nauc_mrr_at_10_std": -0.107336, + "nauc_mrr_at_10_diff1": 0.059082, + "nauc_mrr_at_20_max": -0.275394, + "nauc_mrr_at_20_std": -0.10618, + "nauc_mrr_at_20_diff1": 0.056779, + "nauc_mrr_at_100_max": -0.280656, + "nauc_mrr_at_100_std": -0.107301, + "nauc_mrr_at_100_diff1": 0.051894, + "nauc_mrr_at_1000_max": -0.280412, + "nauc_mrr_at_1000_std": -0.107365, + "nauc_mrr_at_1000_diff1": 0.051954, + "main_score": 0.54229, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.4386, + "ndcg_at_3": 0.48409, + "ndcg_at_5": 0.4955, + "ndcg_at_10": 0.54997, + "ndcg_at_20": 0.58054, + "ndcg_at_100": 0.63182, + "ndcg_at_1000": 0.64067, + "map_at_1": 0.20556, + "map_at_3": 0.3418, + "map_at_5": 0.37025, + "map_at_10": 0.41941, + "map_at_20": 0.44471, + "map_at_100": 0.47048, + "map_at_1000": 0.47085, + "recall_at_1": 0.20556, + "recall_at_3": 0.45794, + "recall_at_5": 0.515, + "recall_at_10": 0.67112, + "recall_at_20": 0.77279, + "recall_at_100": 0.93977, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.4386, + "cv_recall_at_3": 0.75439, + "cv_recall_at_5": 0.78947, + "cv_recall_at_10": 0.85965, + "cv_recall_at_20": 0.89474, + "cv_recall_at_100": 0.96491, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.4386, + "precision_at_3": 0.33333, + "precision_at_5": 0.25614, + "precision_at_10": 0.19298, + "precision_at_20": 0.12632, + "precision_at_100": 0.03754, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.438596, + "mrr_at_3": 0.584795, + "mrr_at_5": 0.59269, + "mrr_at_10": 0.60282, + "mrr_at_20": 0.605144, + "mrr_at_100": 0.607425, + "mrr_at_1000": 0.607597, + "nauc_ndcg_at_1_max": -0.145654, + "nauc_ndcg_at_1_std": -0.280818, + "nauc_ndcg_at_1_diff1": 0.171689, + "nauc_ndcg_at_3_max": 0.029317, + "nauc_ndcg_at_3_std": -0.180006, + "nauc_ndcg_at_3_diff1": 0.206655, + "nauc_ndcg_at_5_max": 0.003118, + "nauc_ndcg_at_5_std": -0.138442, + "nauc_ndcg_at_5_diff1": 0.198075, + "nauc_ndcg_at_10_max": 0.015323, + "nauc_ndcg_at_10_std": -0.118088, + "nauc_ndcg_at_10_diff1": 0.214188, + "nauc_ndcg_at_20_max": 0.006176, + "nauc_ndcg_at_20_std": -0.154549, + "nauc_ndcg_at_20_diff1": 0.220886, + "nauc_ndcg_at_100_max": -0.042965, + "nauc_ndcg_at_100_std": -0.131176, + "nauc_ndcg_at_100_diff1": 0.129661, + "nauc_ndcg_at_1000_max": -0.053931, + "nauc_ndcg_at_1000_std": -0.168201, + "nauc_ndcg_at_1000_diff1": 0.13618, + "nauc_map_at_1_max": 0.010086, + "nauc_map_at_1_std": -0.207289, + "nauc_map_at_1_diff1": 0.304011, + "nauc_map_at_3_max": 0.035273, + "nauc_map_at_3_std": -0.214456, + "nauc_map_at_3_diff1": 0.30199, + "nauc_map_at_5_max": 0.031579, + "nauc_map_at_5_std": -0.180982, + "nauc_map_at_5_diff1": 0.274742, + "nauc_map_at_10_max": 0.035563, + "nauc_map_at_10_std": -0.165051, + "nauc_map_at_10_diff1": 0.280655, + "nauc_map_at_20_max": 0.022834, + "nauc_map_at_20_std": -0.174932, + "nauc_map_at_20_diff1": 0.258082, + "nauc_map_at_100_max": -0.015837, + "nauc_map_at_100_std": -0.184568, + "nauc_map_at_100_diff1": 0.203666, + "nauc_map_at_1000_max": -0.016445, + "nauc_map_at_1000_std": -0.185947, + "nauc_map_at_1000_diff1": 0.204115, + "nauc_recall_at_1_max": 0.010086, + "nauc_recall_at_1_std": -0.207289, + "nauc_recall_at_1_diff1": 0.304011, + "nauc_recall_at_3_max": 0.113203, + "nauc_recall_at_3_std": -0.126186, + "nauc_recall_at_3_diff1": 0.303922, + "nauc_recall_at_5_max": 0.107533, + "nauc_recall_at_5_std": -0.025763, + "nauc_recall_at_5_diff1": 0.316932, + "nauc_recall_at_10_max": 0.18731, + "nauc_recall_at_10_std": 0.08851, + "nauc_recall_at_10_diff1": 0.346215, + "nauc_recall_at_20_max": 0.175617, + "nauc_recall_at_20_std": 0.020313, + "nauc_recall_at_20_diff1": 0.364308, + "nauc_recall_at_100_max": 0.189643, + "nauc_recall_at_100_std": 0.730027, + "nauc_recall_at_100_diff1": -0.055567, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.145654, + "nauc_precision_at_1_std": -0.280818, + "nauc_precision_at_1_diff1": 0.171689, + "nauc_precision_at_3_max": -0.048481, + "nauc_precision_at_3_std": -0.062543, + "nauc_precision_at_3_diff1": -0.080619, + "nauc_precision_at_5_max": -0.082544, + "nauc_precision_at_5_std": 0.021134, + "nauc_precision_at_5_diff1": -0.178469, + "nauc_precision_at_10_max": -0.101426, + "nauc_precision_at_10_std": 0.056066, + "nauc_precision_at_10_diff1": -0.217664, + "nauc_precision_at_20_max": -0.146666, + "nauc_precision_at_20_std": 0.002861, + "nauc_precision_at_20_diff1": -0.255486, + "nauc_precision_at_100_max": -0.23711, + "nauc_precision_at_100_std": 0.003964, + "nauc_precision_at_100_diff1": -0.368852, + "nauc_precision_at_1000_max": -0.246455, + "nauc_precision_at_1000_std": -0.023799, + "nauc_precision_at_1000_diff1": -0.367675, + "nauc_cv_recall_at_1_max": -0.145654, + "nauc_cv_recall_at_1_std": -0.280818, + "nauc_cv_recall_at_1_diff1": 0.171689, + "nauc_cv_recall_at_3_max": 0.036351, + "nauc_cv_recall_at_3_std": -0.156684, + "nauc_cv_recall_at_3_diff1": 0.060225, + "nauc_cv_recall_at_5_max": 0.005842, + "nauc_cv_recall_at_5_std": -0.071165, + "nauc_cv_recall_at_5_diff1": 0.141401, + "nauc_cv_recall_at_10_max": 0.155836, + "nauc_cv_recall_at_10_std": 0.137796, + "nauc_cv_recall_at_10_diff1": 0.033806, + "nauc_cv_recall_at_20_max": 0.158746, + "nauc_cv_recall_at_20_std": 0.084724, + "nauc_cv_recall_at_20_diff1": 0.115726, + "nauc_cv_recall_at_100_max": 0.154552, + "nauc_cv_recall_at_100_std": 0.777543, + "nauc_cv_recall_at_100_diff1": -0.084517, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.145654, + "nauc_mrr_at_1_std": -0.280818, + "nauc_mrr_at_1_diff1": 0.171689, + "nauc_mrr_at_3_max": -0.088776, + "nauc_mrr_at_3_std": -0.247313, + "nauc_mrr_at_3_diff1": 0.116844, + "nauc_mrr_at_5_max": -0.096919, + "nauc_mrr_at_5_std": -0.239671, + "nauc_mrr_at_5_diff1": 0.12787, + "nauc_mrr_at_10_max": -0.088098, + "nauc_mrr_at_10_std": -0.228173, + "nauc_mrr_at_10_diff1": 0.119072, + "nauc_mrr_at_20_max": -0.089786, + "nauc_mrr_at_20_std": -0.232731, + "nauc_mrr_at_20_diff1": 0.121269, + "nauc_mrr_at_100_max": -0.091795, + "nauc_mrr_at_100_std": -0.231591, + "nauc_mrr_at_100_diff1": 0.118707, + "nauc_mrr_at_1000_max": -0.091975, + "nauc_mrr_at_1000_std": -0.232225, + "nauc_mrr_at_1000_diff1": 0.118791, + "main_score": 0.4955, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 582.0948212146759, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/Vidore2EconomicsReportsRetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..67818489ce --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.44828, + "ndcg_at_3": 0.45499, + "ndcg_at_5": 0.45914, + "ndcg_at_10": 0.44688, + "ndcg_at_20": 0.48517, + "ndcg_at_100": 0.60654, + "ndcg_at_1000": 0.65355, + "map_at_1": 0.09296, + "map_at_3": 0.13957, + "map_at_5": 0.17923, + "map_at_10": 0.23061, + "map_at_20": 0.28538, + "map_at_100": 0.37261, + "map_at_1000": 0.39061, + "recall_at_1": 0.09296, + "recall_at_3": 0.14975, + "recall_at_5": 0.23502, + "recall_at_10": 0.33744, + "recall_at_20": 0.49741, + "recall_at_100": 0.85931, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.44828, + "cv_recall_at_3": 0.67241, + "cv_recall_at_5": 0.81034, + "cv_recall_at_10": 0.84483, + "cv_recall_at_20": 0.93103, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.44828, + "precision_at_3": 0.41379, + "precision_at_5": 0.40345, + "precision_at_10": 0.33276, + "precision_at_20": 0.27759, + "precision_at_100": 0.12293, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.448276, + "mrr_at_3": 0.557471, + "mrr_at_5": 0.59023, + "mrr_at_10": 0.595977, + "mrr_at_20": 0.602506, + "mrr_at_100": 0.60441, + "mrr_at_1000": 0.60441, + "nauc_ndcg_at_1_max": 0.410664, + "nauc_ndcg_at_1_std": 0.37649, + "nauc_ndcg_at_1_diff1": 0.133475, + "nauc_ndcg_at_3_max": 0.194167, + "nauc_ndcg_at_3_std": 0.231006, + "nauc_ndcg_at_3_diff1": 0.099237, + "nauc_ndcg_at_5_max": 0.154885, + "nauc_ndcg_at_5_std": 0.167293, + "nauc_ndcg_at_5_diff1": 0.087179, + "nauc_ndcg_at_10_max": 0.146705, + "nauc_ndcg_at_10_std": 0.133126, + "nauc_ndcg_at_10_diff1": 0.134057, + "nauc_ndcg_at_20_max": 0.211349, + "nauc_ndcg_at_20_std": 0.167182, + "nauc_ndcg_at_20_diff1": 0.094132, + "nauc_ndcg_at_100_max": 0.274148, + "nauc_ndcg_at_100_std": 0.244253, + "nauc_ndcg_at_100_diff1": 0.109005, + "nauc_ndcg_at_1000_max": 0.222586, + "nauc_ndcg_at_1000_std": 0.228915, + "nauc_ndcg_at_1000_diff1": 0.10224, + "nauc_map_at_1_max": -0.004573, + "nauc_map_at_1_std": -0.179939, + "nauc_map_at_1_diff1": 0.022629, + "nauc_map_at_3_max": 0.048144, + "nauc_map_at_3_std": -0.08246, + "nauc_map_at_3_diff1": 0.07101, + "nauc_map_at_5_max": 0.078583, + "nauc_map_at_5_std": -0.064632, + "nauc_map_at_5_diff1": 0.111377, + "nauc_map_at_10_max": 0.143799, + "nauc_map_at_10_std": -0.019919, + "nauc_map_at_10_diff1": 0.186594, + "nauc_map_at_20_max": 0.168446, + "nauc_map_at_20_std": 0.023288, + "nauc_map_at_20_diff1": 0.161447, + "nauc_map_at_100_max": 0.144273, + "nauc_map_at_100_std": 0.064801, + "nauc_map_at_100_diff1": 0.098792, + "nauc_map_at_1000_max": 0.116826, + "nauc_map_at_1000_std": 0.06335, + "nauc_map_at_1000_diff1": 0.080132, + "nauc_recall_at_1_max": -0.004573, + "nauc_recall_at_1_std": -0.179939, + "nauc_recall_at_1_diff1": 0.022629, + "nauc_recall_at_3_max": 0.053356, + "nauc_recall_at_3_std": -0.062291, + "nauc_recall_at_3_diff1": 0.072793, + "nauc_recall_at_5_max": 0.050293, + "nauc_recall_at_5_std": -0.07223, + "nauc_recall_at_5_diff1": 0.083597, + "nauc_recall_at_10_max": 0.138122, + "nauc_recall_at_10_std": -0.007141, + "nauc_recall_at_10_diff1": 0.197256, + "nauc_recall_at_20_max": 0.208241, + "nauc_recall_at_20_std": 0.071212, + "nauc_recall_at_20_diff1": 0.138479, + "nauc_recall_at_100_max": 0.364607, + "nauc_recall_at_100_std": 0.250045, + "nauc_recall_at_100_diff1": 0.022367, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.410664, + "nauc_precision_at_1_std": 0.37649, + "nauc_precision_at_1_diff1": 0.133475, + "nauc_precision_at_3_max": 0.18457, + "nauc_precision_at_3_std": 0.295093, + "nauc_precision_at_3_diff1": 0.100854, + "nauc_precision_at_5_max": 0.167793, + "nauc_precision_at_5_std": 0.245898, + "nauc_precision_at_5_diff1": 0.094136, + "nauc_precision_at_10_max": 0.168178, + "nauc_precision_at_10_std": 0.25777, + "nauc_precision_at_10_diff1": 0.133817, + "nauc_precision_at_20_max": 0.075552, + "nauc_precision_at_20_std": 0.211735, + "nauc_precision_at_20_diff1": -0.065875, + "nauc_precision_at_100_max": -0.109276, + "nauc_precision_at_100_std": 0.080383, + "nauc_precision_at_100_diff1": -0.177289, + "nauc_precision_at_1000_max": -0.149277, + "nauc_precision_at_1000_std": 0.057784, + "nauc_precision_at_1000_diff1": -0.169962, + "nauc_cv_recall_at_1_max": 0.410664, + "nauc_cv_recall_at_1_std": 0.37649, + "nauc_cv_recall_at_1_diff1": 0.133475, + "nauc_cv_recall_at_3_max": 0.427757, + "nauc_cv_recall_at_3_std": 0.51808, + "nauc_cv_recall_at_3_diff1": 0.172572, + "nauc_cv_recall_at_5_max": 0.384103, + "nauc_cv_recall_at_5_std": 0.463259, + "nauc_cv_recall_at_5_diff1": 0.314511, + "nauc_cv_recall_at_10_max": 0.427434, + "nauc_cv_recall_at_10_std": 0.507324, + "nauc_cv_recall_at_10_diff1": 0.343792, + "nauc_cv_recall_at_20_max": 0.480924, + "nauc_cv_recall_at_20_std": 0.636638, + "nauc_cv_recall_at_20_diff1": -0.061118, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.410664, + "nauc_mrr_at_1_std": 0.37649, + "nauc_mrr_at_1_diff1": 0.133475, + "nauc_mrr_at_3_max": 0.414436, + "nauc_mrr_at_3_std": 0.425549, + "nauc_mrr_at_3_diff1": 0.15761, + "nauc_mrr_at_5_max": 0.40946, + "nauc_mrr_at_5_std": 0.410949, + "nauc_mrr_at_5_diff1": 0.181707, + "nauc_mrr_at_10_max": 0.413601, + "nauc_mrr_at_10_std": 0.413813, + "nauc_mrr_at_10_diff1": 0.181918, + "nauc_mrr_at_20_max": 0.414239, + "nauc_mrr_at_20_std": 0.414273, + "nauc_mrr_at_20_diff1": 0.170267, + "nauc_mrr_at_100_max": 0.414465, + "nauc_mrr_at_100_std": 0.413338, + "nauc_mrr_at_100_diff1": 0.169802, + "nauc_mrr_at_1000_max": 0.414465, + "nauc_mrr_at_1000_std": 0.413338, + "nauc_mrr_at_1000_diff1": 0.169802, + "main_score": 0.45914, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.48276, + "ndcg_at_3": 0.46992, + "ndcg_at_5": 0.461, + "ndcg_at_10": 0.47127, + "ndcg_at_20": 0.50427, + "ndcg_at_100": 0.62715, + "ndcg_at_1000": 0.66852, + "map_at_1": 0.09797, + "map_at_3": 0.14225, + "map_at_5": 0.18082, + "map_at_10": 0.2379, + "map_at_20": 0.29012, + "map_at_100": 0.38085, + "map_at_1000": 0.39848, + "recall_at_1": 0.09797, + "recall_at_3": 0.16502, + "recall_at_5": 0.2316, + "recall_at_10": 0.36553, + "recall_at_20": 0.52334, + "recall_at_100": 0.88406, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.48276, + "cv_recall_at_3": 0.74138, + "cv_recall_at_5": 0.82759, + "cv_recall_at_10": 0.91379, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.48276, + "precision_at_3": 0.43103, + "precision_at_5": 0.40345, + "precision_at_10": 0.3569, + "precision_at_20": 0.2819, + "precision_at_100": 0.12414, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.482759, + "mrr_at_3": 0.603448, + "mrr_at_5": 0.623276, + "mrr_at_10": 0.635365, + "mrr_at_20": 0.639798, + "mrr_at_100": 0.640619, + "mrr_at_1000": 0.640619, + "nauc_ndcg_at_1_max": 0.348887, + "nauc_ndcg_at_1_std": 0.297446, + "nauc_ndcg_at_1_diff1": 0.043053, + "nauc_ndcg_at_3_max": 0.191868, + "nauc_ndcg_at_3_std": 0.132168, + "nauc_ndcg_at_3_diff1": 0.024684, + "nauc_ndcg_at_5_max": 0.191939, + "nauc_ndcg_at_5_std": 0.092787, + "nauc_ndcg_at_5_diff1": 0.067378, + "nauc_ndcg_at_10_max": 0.11067, + "nauc_ndcg_at_10_std": 0.029782, + "nauc_ndcg_at_10_diff1": 0.076161, + "nauc_ndcg_at_20_max": 0.171211, + "nauc_ndcg_at_20_std": 0.078123, + "nauc_ndcg_at_20_diff1": 0.115117, + "nauc_ndcg_at_100_max": 0.223298, + "nauc_ndcg_at_100_std": 0.173287, + "nauc_ndcg_at_100_diff1": 0.029292, + "nauc_ndcg_at_1000_max": 0.178621, + "nauc_ndcg_at_1000_std": 0.154987, + "nauc_ndcg_at_1000_diff1": 0.005103, + "nauc_map_at_1_max": 0.027726, + "nauc_map_at_1_std": -0.280202, + "nauc_map_at_1_diff1": 0.53996, + "nauc_map_at_3_max": 0.040165, + "nauc_map_at_3_std": -0.268485, + "nauc_map_at_3_diff1": 0.463524, + "nauc_map_at_5_max": 0.087174, + "nauc_map_at_5_std": -0.229069, + "nauc_map_at_5_diff1": 0.414971, + "nauc_map_at_10_max": 0.078557, + "nauc_map_at_10_std": -0.214444, + "nauc_map_at_10_diff1": 0.370581, + "nauc_map_at_20_max": 0.110153, + "nauc_map_at_20_std": -0.136878, + "nauc_map_at_20_diff1": 0.333297, + "nauc_map_at_100_max": 0.137899, + "nauc_map_at_100_std": -0.022404, + "nauc_map_at_100_diff1": 0.178577, + "nauc_map_at_1000_max": 0.121968, + "nauc_map_at_1000_std": -0.00471, + "nauc_map_at_1000_diff1": 0.137922, + "nauc_recall_at_1_max": 0.027726, + "nauc_recall_at_1_std": -0.280202, + "nauc_recall_at_1_diff1": 0.53996, + "nauc_recall_at_3_max": 0.051414, + "nauc_recall_at_3_std": -0.25093, + "nauc_recall_at_3_diff1": 0.48223, + "nauc_recall_at_5_max": 0.115335, + "nauc_recall_at_5_std": -0.201683, + "nauc_recall_at_5_diff1": 0.392333, + "nauc_recall_at_10_max": 0.001065, + "nauc_recall_at_10_std": -0.238091, + "nauc_recall_at_10_diff1": 0.303844, + "nauc_recall_at_20_max": 0.084466, + "nauc_recall_at_20_std": -0.11606, + "nauc_recall_at_20_diff1": 0.314337, + "nauc_recall_at_100_max": 0.314363, + "nauc_recall_at_100_std": 0.23885, + "nauc_recall_at_100_diff1": -0.051122, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.348887, + "nauc_precision_at_1_std": 0.297446, + "nauc_precision_at_1_diff1": 0.043053, + "nauc_precision_at_3_max": 0.167822, + "nauc_precision_at_3_std": 0.211842, + "nauc_precision_at_3_diff1": -0.155128, + "nauc_precision_at_5_max": 0.187603, + "nauc_precision_at_5_std": 0.212691, + "nauc_precision_at_5_diff1": -0.168278, + "nauc_precision_at_10_max": 0.097522, + "nauc_precision_at_10_std": 0.251754, + "nauc_precision_at_10_diff1": -0.255014, + "nauc_precision_at_20_max": 0.099614, + "nauc_precision_at_20_std": 0.342041, + "nauc_precision_at_20_diff1": -0.336113, + "nauc_precision_at_100_max": 0.047116, + "nauc_precision_at_100_std": 0.301534, + "nauc_precision_at_100_diff1": -0.395588, + "nauc_precision_at_1000_max": 0.00281, + "nauc_precision_at_1000_std": 0.250892, + "nauc_precision_at_1000_diff1": -0.358412, + "nauc_cv_recall_at_1_max": 0.348887, + "nauc_cv_recall_at_1_std": 0.297446, + "nauc_cv_recall_at_1_diff1": 0.043053, + "nauc_cv_recall_at_3_max": 0.337076, + "nauc_cv_recall_at_3_std": 0.402697, + "nauc_cv_recall_at_3_diff1": 0.118365, + "nauc_cv_recall_at_5_max": 0.217597, + "nauc_cv_recall_at_5_std": 0.181207, + "nauc_cv_recall_at_5_diff1": -0.07877, + "nauc_cv_recall_at_10_max": 0.409492, + "nauc_cv_recall_at_10_std": 0.454257, + "nauc_cv_recall_at_10_diff1": 0.057923, + "nauc_cv_recall_at_20_max": 0.359055, + "nauc_cv_recall_at_20_std": 0.554013, + "nauc_cv_recall_at_20_diff1": -0.574298, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.348887, + "nauc_mrr_at_1_std": 0.297446, + "nauc_mrr_at_1_diff1": 0.043053, + "nauc_mrr_at_3_max": 0.304399, + "nauc_mrr_at_3_std": 0.312655, + "nauc_mrr_at_3_diff1": 0.042167, + "nauc_mrr_at_5_max": 0.289229, + "nauc_mrr_at_5_std": 0.278778, + "nauc_mrr_at_5_diff1": 0.012719, + "nauc_mrr_at_10_max": 0.297087, + "nauc_mrr_at_10_std": 0.292599, + "nauc_mrr_at_10_diff1": 0.018074, + "nauc_mrr_at_20_max": 0.295599, + "nauc_mrr_at_20_std": 0.290075, + "nauc_mrr_at_20_diff1": 0.014708, + "nauc_mrr_at_100_max": 0.295403, + "nauc_mrr_at_100_std": 0.28926, + "nauc_mrr_at_100_diff1": 0.016528, + "nauc_mrr_at_1000_max": 0.295403, + "nauc_mrr_at_1000_std": 0.28926, + "nauc_mrr_at_1000_diff1": 0.016528, + "main_score": 0.461, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.56897, + "ndcg_at_3": 0.56317, + "ndcg_at_5": 0.53807, + "ndcg_at_10": 0.52371, + "ndcg_at_20": 0.55764, + "ndcg_at_100": 0.6692, + "ndcg_at_1000": 0.70447, + "map_at_1": 0.10122, + "map_at_3": 0.17335, + "map_at_5": 0.22216, + "map_at_10": 0.27985, + "map_at_20": 0.33766, + "map_at_100": 0.42552, + "map_at_1000": 0.442, + "recall_at_1": 0.10122, + "recall_at_3": 0.20133, + "recall_at_5": 0.27493, + "recall_at_10": 0.40089, + "recall_at_20": 0.57106, + "recall_at_100": 0.90804, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.56897, + "cv_recall_at_3": 0.81034, + "cv_recall_at_5": 0.86207, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.56897, + "precision_at_3": 0.52299, + "precision_at_5": 0.46897, + "precision_at_10": 0.37931, + "precision_at_20": 0.30086, + "precision_at_100": 0.12655, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.568966, + "mrr_at_3": 0.683908, + "mrr_at_5": 0.695977, + "mrr_at_10": 0.708429, + "mrr_at_20": 0.711444, + "mrr_at_100": 0.711444, + "mrr_at_1000": 0.711444, + "nauc_ndcg_at_1_max": 0.335469, + "nauc_ndcg_at_1_std": 0.23798, + "nauc_ndcg_at_1_diff1": 0.209649, + "nauc_ndcg_at_3_max": 0.123498, + "nauc_ndcg_at_3_std": 0.162669, + "nauc_ndcg_at_3_diff1": 0.021954, + "nauc_ndcg_at_5_max": 0.119396, + "nauc_ndcg_at_5_std": 0.139305, + "nauc_ndcg_at_5_diff1": 0.033321, + "nauc_ndcg_at_10_max": 0.009619, + "nauc_ndcg_at_10_std": 0.017739, + "nauc_ndcg_at_10_diff1": -0.047733, + "nauc_ndcg_at_20_max": 0.106074, + "nauc_ndcg_at_20_std": 0.070228, + "nauc_ndcg_at_20_diff1": 0.021204, + "nauc_ndcg_at_100_max": 0.130902, + "nauc_ndcg_at_100_std": 0.151294, + "nauc_ndcg_at_100_diff1": 0.090047, + "nauc_ndcg_at_1000_max": 0.111039, + "nauc_ndcg_at_1000_std": 0.170896, + "nauc_ndcg_at_1000_diff1": 0.038444, + "nauc_map_at_1_max": -0.078867, + "nauc_map_at_1_std": -0.339527, + "nauc_map_at_1_diff1": 0.345487, + "nauc_map_at_3_max": -0.069671, + "nauc_map_at_3_std": -0.266362, + "nauc_map_at_3_diff1": 0.151329, + "nauc_map_at_5_max": -0.033398, + "nauc_map_at_5_std": -0.210546, + "nauc_map_at_5_diff1": 0.100903, + "nauc_map_at_10_max": -0.036363, + "nauc_map_at_10_std": -0.209838, + "nauc_map_at_10_diff1": 0.060729, + "nauc_map_at_20_max": 0.017011, + "nauc_map_at_20_std": -0.145334, + "nauc_map_at_20_diff1": 0.073453, + "nauc_map_at_100_max": 0.017828, + "nauc_map_at_100_std": -0.02808, + "nauc_map_at_100_diff1": 0.065696, + "nauc_map_at_1000_max": 0.010474, + "nauc_map_at_1000_std": 0.001268, + "nauc_map_at_1000_diff1": 0.035087, + "nauc_recall_at_1_max": -0.078867, + "nauc_recall_at_1_std": -0.339527, + "nauc_recall_at_1_diff1": 0.345487, + "nauc_recall_at_3_max": -0.067295, + "nauc_recall_at_3_std": -0.245749, + "nauc_recall_at_3_diff1": 0.074254, + "nauc_recall_at_5_max": -0.006604, + "nauc_recall_at_5_std": -0.165938, + "nauc_recall_at_5_diff1": 0.025195, + "nauc_recall_at_10_max": -0.099022, + "nauc_recall_at_10_std": -0.247695, + "nauc_recall_at_10_diff1": -0.051642, + "nauc_recall_at_20_max": 0.059185, + "nauc_recall_at_20_std": -0.163954, + "nauc_recall_at_20_diff1": 0.085316, + "nauc_recall_at_100_max": 0.163292, + "nauc_recall_at_100_std": 0.004805, + "nauc_recall_at_100_diff1": 0.205659, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.335469, + "nauc_precision_at_1_std": 0.23798, + "nauc_precision_at_1_diff1": 0.209649, + "nauc_precision_at_3_max": 0.120008, + "nauc_precision_at_3_std": 0.270748, + "nauc_precision_at_3_diff1": -0.143747, + "nauc_precision_at_5_max": 0.150168, + "nauc_precision_at_5_std": 0.276894, + "nauc_precision_at_5_diff1": -0.102574, + "nauc_precision_at_10_max": 0.076511, + "nauc_precision_at_10_std": 0.243122, + "nauc_precision_at_10_diff1": -0.169011, + "nauc_precision_at_20_max": 0.108992, + "nauc_precision_at_20_std": 0.356643, + "nauc_precision_at_20_diff1": -0.125853, + "nauc_precision_at_100_max": 0.007058, + "nauc_precision_at_100_std": 0.324181, + "nauc_precision_at_100_diff1": -0.129439, + "nauc_precision_at_1000_max": -0.016758, + "nauc_precision_at_1000_std": 0.308355, + "nauc_precision_at_1000_diff1": -0.181933, + "nauc_cv_recall_at_1_max": 0.335469, + "nauc_cv_recall_at_1_std": 0.23798, + "nauc_cv_recall_at_1_diff1": 0.209649, + "nauc_cv_recall_at_3_max": 0.372948, + "nauc_cv_recall_at_3_std": 0.441008, + "nauc_cv_recall_at_3_diff1": 0.140625, + "nauc_cv_recall_at_5_max": 0.31553, + "nauc_cv_recall_at_5_std": 0.466671, + "nauc_cv_recall_at_5_diff1": -0.125128, + "nauc_cv_recall_at_10_max": -0.021143, + "nauc_cv_recall_at_10_std": 0.424041, + "nauc_cv_recall_at_10_diff1": -0.239075, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.335469, + "nauc_mrr_at_1_std": 0.23798, + "nauc_mrr_at_1_diff1": 0.209649, + "nauc_mrr_at_3_max": 0.345833, + "nauc_mrr_at_3_std": 0.2964, + "nauc_mrr_at_3_diff1": 0.197136, + "nauc_mrr_at_5_max": 0.338276, + "nauc_mrr_at_5_std": 0.295876, + "nauc_mrr_at_5_diff1": 0.166568, + "nauc_mrr_at_10_max": 0.327818, + "nauc_mrr_at_10_std": 0.283696, + "nauc_mrr_at_10_diff1": 0.183186, + "nauc_mrr_at_20_max": 0.331838, + "nauc_mrr_at_20_std": 0.281947, + "nauc_mrr_at_20_diff1": 0.18747, + "nauc_mrr_at_100_max": 0.331838, + "nauc_mrr_at_100_std": 0.281947, + "nauc_mrr_at_100_diff1": 0.18747, + "nauc_mrr_at_1000_max": 0.331838, + "nauc_mrr_at_1000_std": 0.281947, + "nauc_mrr_at_1000_diff1": 0.18747, + "main_score": 0.53807, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.5, + "ndcg_at_3": 0.48736, + "ndcg_at_5": 0.46239, + "ndcg_at_10": 0.45954, + "ndcg_at_20": 0.50071, + "ndcg_at_100": 0.63191, + "ndcg_at_1000": 0.67479, + "map_at_1": 0.1062, + "map_at_3": 0.16674, + "map_at_5": 0.20002, + "map_at_10": 0.25298, + "map_at_20": 0.29848, + "map_at_100": 0.38956, + "map_at_1000": 0.40769, + "recall_at_1": 0.1062, + "recall_at_3": 0.18047, + "recall_at_5": 0.2358, + "recall_at_10": 0.35335, + "recall_at_20": 0.51562, + "recall_at_100": 0.88122, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.5, + "cv_recall_at_3": 0.7069, + "cv_recall_at_5": 0.7931, + "cv_recall_at_10": 0.89655, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.5, + "precision_at_3": 0.44253, + "precision_at_5": 0.38966, + "precision_at_10": 0.32759, + "precision_at_20": 0.26379, + "precision_at_100": 0.12328, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.5, + "mrr_at_3": 0.597701, + "mrr_at_5": 0.617529, + "mrr_at_10": 0.630816, + "mrr_at_20": 0.63859, + "mrr_at_100": 0.63859, + "mrr_at_1000": 0.63859, + "nauc_ndcg_at_1_max": 0.127678, + "nauc_ndcg_at_1_std": 0.162249, + "nauc_ndcg_at_1_diff1": 0.022816, + "nauc_ndcg_at_3_max": 0.147008, + "nauc_ndcg_at_3_std": 0.161663, + "nauc_ndcg_at_3_diff1": 0.093525, + "nauc_ndcg_at_5_max": 0.159436, + "nauc_ndcg_at_5_std": 0.157312, + "nauc_ndcg_at_5_diff1": 0.067643, + "nauc_ndcg_at_10_max": 0.122771, + "nauc_ndcg_at_10_std": 0.114077, + "nauc_ndcg_at_10_diff1": 0.020776, + "nauc_ndcg_at_20_max": 0.139636, + "nauc_ndcg_at_20_std": 0.117454, + "nauc_ndcg_at_20_diff1": 0.05981, + "nauc_ndcg_at_100_max": 0.156716, + "nauc_ndcg_at_100_std": 0.154724, + "nauc_ndcg_at_100_diff1": 0.063668, + "nauc_ndcg_at_1000_max": 0.150747, + "nauc_ndcg_at_1000_std": 0.185122, + "nauc_ndcg_at_1000_diff1": 0.056843, + "nauc_map_at_1_max": -0.038187, + "nauc_map_at_1_std": -0.258363, + "nauc_map_at_1_diff1": 0.186575, + "nauc_map_at_3_max": -0.020783, + "nauc_map_at_3_std": -0.208115, + "nauc_map_at_3_diff1": 0.112749, + "nauc_map_at_5_max": -0.000273, + "nauc_map_at_5_std": -0.174837, + "nauc_map_at_5_diff1": 0.083089, + "nauc_map_at_10_max": 0.012401, + "nauc_map_at_10_std": -0.145972, + "nauc_map_at_10_diff1": 0.054661, + "nauc_map_at_20_max": 0.023236, + "nauc_map_at_20_std": -0.116443, + "nauc_map_at_20_diff1": 0.050159, + "nauc_map_at_100_max": 0.093042, + "nauc_map_at_100_std": 0.025629, + "nauc_map_at_100_diff1": 0.076777, + "nauc_map_at_1000_max": 0.104088, + "nauc_map_at_1000_std": 0.067538, + "nauc_map_at_1000_diff1": 0.076625, + "nauc_recall_at_1_max": -0.038187, + "nauc_recall_at_1_std": -0.258363, + "nauc_recall_at_1_diff1": 0.186575, + "nauc_recall_at_3_max": -0.011555, + "nauc_recall_at_3_std": -0.200659, + "nauc_recall_at_3_diff1": 0.109634, + "nauc_recall_at_5_max": 0.043399, + "nauc_recall_at_5_std": -0.133531, + "nauc_recall_at_5_diff1": 0.05132, + "nauc_recall_at_10_max": 0.033709, + "nauc_recall_at_10_std": -0.114321, + "nauc_recall_at_10_diff1": -0.010383, + "nauc_recall_at_20_max": -0.027582, + "nauc_recall_at_20_std": -0.163793, + "nauc_recall_at_20_diff1": 0.059281, + "nauc_recall_at_100_max": 0.176313, + "nauc_recall_at_100_std": 0.067047, + "nauc_recall_at_100_diff1": 0.118112, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.127678, + "nauc_precision_at_1_std": 0.162249, + "nauc_precision_at_1_diff1": 0.022816, + "nauc_precision_at_3_max": 0.147197, + "nauc_precision_at_3_std": 0.232176, + "nauc_precision_at_3_diff1": 0.024231, + "nauc_precision_at_5_max": 0.179044, + "nauc_precision_at_5_std": 0.278486, + "nauc_precision_at_5_diff1": -0.037558, + "nauc_precision_at_10_max": 0.146876, + "nauc_precision_at_10_std": 0.303225, + "nauc_precision_at_10_diff1": -0.07421, + "nauc_precision_at_20_max": 0.167999, + "nauc_precision_at_20_std": 0.375802, + "nauc_precision_at_20_diff1": -0.001646, + "nauc_precision_at_100_max": 0.197968, + "nauc_precision_at_100_std": 0.454412, + "nauc_precision_at_100_diff1": 0.063725, + "nauc_precision_at_1000_max": 0.167879, + "nauc_precision_at_1000_std": 0.429159, + "nauc_precision_at_1000_diff1": 0.040312, + "nauc_cv_recall_at_1_max": 0.127678, + "nauc_cv_recall_at_1_std": 0.162249, + "nauc_cv_recall_at_1_diff1": 0.022816, + "nauc_cv_recall_at_3_max": 0.180195, + "nauc_cv_recall_at_3_std": 0.286227, + "nauc_cv_recall_at_3_diff1": 0.074939, + "nauc_cv_recall_at_5_max": 0.303254, + "nauc_cv_recall_at_5_std": 0.383355, + "nauc_cv_recall_at_5_diff1": -0.120434, + "nauc_cv_recall_at_10_max": 0.135127, + "nauc_cv_recall_at_10_std": 0.238225, + "nauc_cv_recall_at_10_diff1": -0.392684, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.127678, + "nauc_mrr_at_1_std": 0.162249, + "nauc_mrr_at_1_diff1": 0.022816, + "nauc_mrr_at_3_max": 0.150028, + "nauc_mrr_at_3_std": 0.21408, + "nauc_mrr_at_3_diff1": 0.05367, + "nauc_mrr_at_5_max": 0.167759, + "nauc_mrr_at_5_std": 0.225004, + "nauc_mrr_at_5_diff1": 0.024517, + "nauc_mrr_at_10_max": 0.153792, + "nauc_mrr_at_10_std": 0.210845, + "nauc_mrr_at_10_diff1": 0.016311, + "nauc_mrr_at_20_max": 0.157503, + "nauc_mrr_at_20_std": 0.212811, + "nauc_mrr_at_20_diff1": 0.028026, + "nauc_mrr_at_100_max": 0.157503, + "nauc_mrr_at_100_std": 0.212811, + "nauc_mrr_at_100_diff1": 0.028026, + "nauc_mrr_at_1000_max": 0.157503, + "nauc_mrr_at_1000_std": 0.212811, + "nauc_mrr_at_1000_diff1": 0.028026, + "main_score": 0.46239, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 71.68568730354309, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreArxivQARetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..b63fe17f94 --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", + "task_name": "VidoreArxivQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.712, + "ndcg_at_3": 0.76638, + "ndcg_at_5": 0.77877, + "ndcg_at_10": 0.79655, + "ndcg_at_20": 0.80519, + "ndcg_at_100": 0.81575, + "ndcg_at_1000": 0.8187, + "map_at_1": 0.712, + "map_at_3": 0.75333, + "map_at_5": 0.76023, + "map_at_10": 0.76776, + "map_at_20": 0.77015, + "map_at_100": 0.77168, + "map_at_1000": 0.77181, + "recall_at_1": 0.712, + "recall_at_3": 0.804, + "recall_at_5": 0.834, + "recall_at_10": 0.888, + "recall_at_20": 0.922, + "recall_at_100": 0.978, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.712, + "cv_recall_at_3": 0.804, + "cv_recall_at_5": 0.834, + "cv_recall_at_10": 0.888, + "cv_recall_at_20": 0.922, + "cv_recall_at_100": 0.978, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.712, + "precision_at_3": 0.268, + "precision_at_5": 0.1668, + "precision_at_10": 0.0888, + "precision_at_20": 0.0461, + "precision_at_100": 0.00978, + "precision_at_1000": 0.001, + "mrr_at_1": 0.712, + "mrr_at_3": 0.753333, + "mrr_at_5": 0.760233, + "mrr_at_10": 0.76776, + "mrr_at_20": 0.770148, + "mrr_at_100": 0.771677, + "mrr_at_1000": 0.771811, + "nauc_ndcg_at_1_max": 0.687462, + "nauc_ndcg_at_1_std": -0.121103, + "nauc_ndcg_at_1_diff1": 0.890449, + "nauc_ndcg_at_3_max": 0.685299, + "nauc_ndcg_at_3_std": -0.134966, + "nauc_ndcg_at_3_diff1": 0.868707, + "nauc_ndcg_at_5_max": 0.682409, + "nauc_ndcg_at_5_std": -0.14884, + "nauc_ndcg_at_5_diff1": 0.863141, + "nauc_ndcg_at_10_max": 0.695363, + "nauc_ndcg_at_10_std": -0.130886, + "nauc_ndcg_at_10_diff1": 0.85922, + "nauc_ndcg_at_20_max": 0.694967, + "nauc_ndcg_at_20_std": -0.137091, + "nauc_ndcg_at_20_diff1": 0.864112, + "nauc_ndcg_at_100_max": 0.692761, + "nauc_ndcg_at_100_std": -0.125704, + "nauc_ndcg_at_100_diff1": 0.869324, + "nauc_ndcg_at_1000_max": 0.690523, + "nauc_ndcg_at_1000_std": -0.13204, + "nauc_ndcg_at_1000_diff1": 0.869389, + "nauc_map_at_1_max": 0.687462, + "nauc_map_at_1_std": -0.121103, + "nauc_map_at_1_diff1": 0.890449, + "nauc_map_at_3_max": 0.684728, + "nauc_map_at_3_std": -0.13142, + "nauc_map_at_3_diff1": 0.874817, + "nauc_map_at_5_max": 0.683099, + "nauc_map_at_5_std": -0.13881, + "nauc_map_at_5_diff1": 0.872011, + "nauc_map_at_10_max": 0.688705, + "nauc_map_at_10_std": -0.131594, + "nauc_map_at_10_diff1": 0.871121, + "nauc_map_at_20_max": 0.688527, + "nauc_map_at_20_std": -0.132804, + "nauc_map_at_20_diff1": 0.872501, + "nauc_map_at_100_max": 0.688185, + "nauc_map_at_100_std": -0.131945, + "nauc_map_at_100_diff1": 0.873109, + "nauc_map_at_1000_max": 0.688097, + "nauc_map_at_1000_std": -0.132132, + "nauc_map_at_1000_diff1": 0.873101, + "nauc_recall_at_1_max": 0.687462, + "nauc_recall_at_1_std": -0.121103, + "nauc_recall_at_1_diff1": 0.890449, + "nauc_recall_at_3_max": 0.687855, + "nauc_recall_at_3_std": -0.147138, + "nauc_recall_at_3_diff1": 0.847442, + "nauc_recall_at_5_max": 0.68001, + "nauc_recall_at_5_std": -0.190274, + "nauc_recall_at_5_diff1": 0.827264, + "nauc_recall_at_10_max": 0.737723, + "nauc_recall_at_10_std": -0.118168, + "nauc_recall_at_10_diff1": 0.789054, + "nauc_recall_at_20_max": 0.751993, + "nauc_recall_at_20_std": -0.173274, + "nauc_recall_at_20_diff1": 0.801827, + "nauc_recall_at_100_max": 0.816017, + "nauc_recall_at_100_std": 0.247899, + "nauc_recall_at_100_diff1": 0.859944, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.687462, + "nauc_precision_at_1_std": -0.121103, + "nauc_precision_at_1_diff1": 0.890449, + "nauc_precision_at_3_max": 0.687855, + "nauc_precision_at_3_std": -0.147138, + "nauc_precision_at_3_diff1": 0.847442, + "nauc_precision_at_5_max": 0.68001, + "nauc_precision_at_5_std": -0.190274, + "nauc_precision_at_5_diff1": 0.827264, + "nauc_precision_at_10_max": 0.737723, + "nauc_precision_at_10_std": -0.118168, + "nauc_precision_at_10_diff1": 0.789054, + "nauc_precision_at_20_max": 0.751993, + "nauc_precision_at_20_std": -0.173274, + "nauc_precision_at_20_diff1": 0.801827, + "nauc_precision_at_100_max": 0.816017, + "nauc_precision_at_100_std": 0.247899, + "nauc_precision_at_100_diff1": 0.859944, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_cv_recall_at_1_max": 0.687462, + "nauc_cv_recall_at_1_std": -0.121103, + "nauc_cv_recall_at_1_diff1": 0.890449, + "nauc_cv_recall_at_3_max": 0.687855, + "nauc_cv_recall_at_3_std": -0.147138, + "nauc_cv_recall_at_3_diff1": 0.847442, + "nauc_cv_recall_at_5_max": 0.68001, + "nauc_cv_recall_at_5_std": -0.190274, + "nauc_cv_recall_at_5_diff1": 0.827264, + "nauc_cv_recall_at_10_max": 0.737723, + "nauc_cv_recall_at_10_std": -0.118168, + "nauc_cv_recall_at_10_diff1": 0.789054, + "nauc_cv_recall_at_20_max": 0.751993, + "nauc_cv_recall_at_20_std": -0.173274, + "nauc_cv_recall_at_20_diff1": 0.801827, + "nauc_cv_recall_at_100_max": 0.816017, + "nauc_cv_recall_at_100_std": 0.247899, + "nauc_cv_recall_at_100_diff1": 0.859944, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.687462, + "nauc_mrr_at_1_std": -0.121103, + "nauc_mrr_at_1_diff1": 0.890449, + "nauc_mrr_at_3_max": 0.684728, + "nauc_mrr_at_3_std": -0.13142, + "nauc_mrr_at_3_diff1": 0.874817, + "nauc_mrr_at_5_max": 0.683099, + "nauc_mrr_at_5_std": -0.13881, + "nauc_mrr_at_5_diff1": 0.872011, + "nauc_mrr_at_10_max": 0.688705, + "nauc_mrr_at_10_std": -0.131594, + "nauc_mrr_at_10_diff1": 0.871121, + "nauc_mrr_at_20_max": 0.688527, + "nauc_mrr_at_20_std": -0.132804, + "nauc_mrr_at_20_diff1": 0.872501, + "nauc_mrr_at_100_max": 0.688185, + "nauc_mrr_at_100_std": -0.131945, + "nauc_mrr_at_100_diff1": 0.873109, + "nauc_mrr_at_1000_max": 0.688097, + "nauc_mrr_at_1000_std": -0.132132, + "nauc_mrr_at_1000_diff1": 0.873101, + "main_score": 0.77877, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 33.64434361457825, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreDocVQARetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..fdfbe3361c --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", + "task_name": "VidoreDocVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.48115, + "ndcg_at_3": 0.54299, + "ndcg_at_5": 0.56514, + "ndcg_at_10": 0.58656, + "ndcg_at_20": 0.60072, + "ndcg_at_100": 0.62668, + "ndcg_at_1000": 0.64143, + "map_at_1": 0.47746, + "map_at_3": 0.52506, + "map_at_5": 0.53779, + "map_at_10": 0.54722, + "map_at_20": 0.55143, + "map_at_100": 0.55511, + "map_at_1000": 0.5558, + "recall_at_1": 0.47746, + "recall_at_3": 0.5857, + "recall_at_5": 0.63954, + "recall_at_10": 0.70403, + "recall_at_20": 0.75887, + "recall_at_100": 0.89314, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.48115, + "cv_recall_at_3": 0.59645, + "cv_recall_at_5": 0.64967, + "cv_recall_at_10": 0.71397, + "cv_recall_at_20": 0.76497, + "cv_recall_at_100": 0.90022, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.48115, + "precision_at_3": 0.2003, + "precision_at_5": 0.13215, + "precision_at_10": 0.07295, + "precision_at_20": 0.03969, + "precision_at_100": 0.00965, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.481153, + "mrr_at_3": 0.531781, + "mrr_at_5": 0.543755, + "mrr_at_10": 0.552889, + "mrr_at_20": 0.556345, + "mrr_at_100": 0.559891, + "mrr_at_1000": 0.560427, + "nauc_ndcg_at_1_max": 0.295038, + "nauc_ndcg_at_1_std": 0.293511, + "nauc_ndcg_at_1_diff1": 0.841076, + "nauc_ndcg_at_3_max": 0.249753, + "nauc_ndcg_at_3_std": 0.311061, + "nauc_ndcg_at_3_diff1": 0.812961, + "nauc_ndcg_at_5_max": 0.226917, + "nauc_ndcg_at_5_std": 0.324406, + "nauc_ndcg_at_5_diff1": 0.81283, + "nauc_ndcg_at_10_max": 0.199757, + "nauc_ndcg_at_10_std": 0.342818, + "nauc_ndcg_at_10_diff1": 0.807858, + "nauc_ndcg_at_20_max": 0.190178, + "nauc_ndcg_at_20_std": 0.353141, + "nauc_ndcg_at_20_diff1": 0.804807, + "nauc_ndcg_at_100_max": 0.191221, + "nauc_ndcg_at_100_std": 0.352019, + "nauc_ndcg_at_100_diff1": 0.806626, + "nauc_ndcg_at_1000_max": 0.211043, + "nauc_ndcg_at_1000_std": 0.330243, + "nauc_ndcg_at_1000_diff1": 0.809992, + "nauc_map_at_1_max": 0.299106, + "nauc_map_at_1_std": 0.294459, + "nauc_map_at_1_diff1": 0.848514, + "nauc_map_at_3_max": 0.265628, + "nauc_map_at_3_std": 0.309732, + "nauc_map_at_3_diff1": 0.823804, + "nauc_map_at_5_max": 0.25296, + "nauc_map_at_5_std": 0.31568, + "nauc_map_at_5_diff1": 0.822827, + "nauc_map_at_10_max": 0.241506, + "nauc_map_at_10_std": 0.321565, + "nauc_map_at_10_diff1": 0.820625, + "nauc_map_at_20_max": 0.238391, + "nauc_map_at_20_std": 0.322756, + "nauc_map_at_20_diff1": 0.819332, + "nauc_map_at_100_max": 0.23892, + "nauc_map_at_100_std": 0.322231, + "nauc_map_at_100_diff1": 0.819521, + "nauc_map_at_1000_max": 0.239316, + "nauc_map_at_1000_std": 0.321296, + "nauc_map_at_1000_diff1": 0.819689, + "nauc_recall_at_1_max": 0.299106, + "nauc_recall_at_1_std": 0.294459, + "nauc_recall_at_1_diff1": 0.848514, + "nauc_recall_at_3_max": 0.22125, + "nauc_recall_at_3_std": 0.328602, + "nauc_recall_at_3_diff1": 0.790018, + "nauc_recall_at_5_max": 0.155551, + "nauc_recall_at_5_std": 0.366902, + "nauc_recall_at_5_diff1": 0.784459, + "nauc_recall_at_10_max": 0.047051, + "nauc_recall_at_10_std": 0.444229, + "nauc_recall_at_10_diff1": 0.762568, + "nauc_recall_at_20_max": -0.026091, + "nauc_recall_at_20_std": 0.529771, + "nauc_recall_at_20_diff1": 0.74674, + "nauc_recall_at_100_max": -0.23922, + "nauc_recall_at_100_std": 0.734675, + "nauc_recall_at_100_diff1": 0.734426, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.295038, + "nauc_precision_at_1_std": 0.293511, + "nauc_precision_at_1_diff1": 0.841076, + "nauc_precision_at_3_max": 0.182829, + "nauc_precision_at_3_std": 0.305535, + "nauc_precision_at_3_diff1": 0.750352, + "nauc_precision_at_5_max": 0.0948, + "nauc_precision_at_5_std": 0.319627, + "nauc_precision_at_5_diff1": 0.705953, + "nauc_precision_at_10_max": -0.029806, + "nauc_precision_at_10_std": 0.364317, + "nauc_precision_at_10_diff1": 0.650254, + "nauc_precision_at_20_max": -0.131517, + "nauc_precision_at_20_std": 0.369296, + "nauc_precision_at_20_diff1": 0.530128, + "nauc_precision_at_100_max": -0.323185, + "nauc_precision_at_100_std": 0.127195, + "nauc_precision_at_100_diff1": 0.132041, + "nauc_precision_at_1000_max": -0.348405, + "nauc_precision_at_1000_std": -0.454871, + "nauc_precision_at_1000_diff1": -0.411814, + "nauc_cv_recall_at_1_max": 0.295038, + "nauc_cv_recall_at_1_std": 0.293511, + "nauc_cv_recall_at_1_diff1": 0.841076, + "nauc_cv_recall_at_3_max": 0.198235, + "nauc_cv_recall_at_3_std": 0.314131, + "nauc_cv_recall_at_3_diff1": 0.777719, + "nauc_cv_recall_at_5_max": 0.129613, + "nauc_cv_recall_at_5_std": 0.354371, + "nauc_cv_recall_at_5_diff1": 0.779489, + "nauc_cv_recall_at_10_max": 0.016348, + "nauc_cv_recall_at_10_std": 0.434473, + "nauc_cv_recall_at_10_diff1": 0.758985, + "nauc_cv_recall_at_20_max": -0.04917, + "nauc_cv_recall_at_20_std": 0.523056, + "nauc_cv_recall_at_20_diff1": 0.743474, + "nauc_cv_recall_at_100_max": -0.25421, + "nauc_cv_recall_at_100_std": 0.709046, + "nauc_cv_recall_at_100_diff1": 0.722474, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.295038, + "nauc_mrr_at_1_std": 0.293511, + "nauc_mrr_at_1_diff1": 0.841076, + "nauc_mrr_at_3_max": 0.253714, + "nauc_mrr_at_3_std": 0.301751, + "nauc_mrr_at_3_diff1": 0.814645, + "nauc_mrr_at_5_max": 0.241682, + "nauc_mrr_at_5_std": 0.308246, + "nauc_mrr_at_5_diff1": 0.815839, + "nauc_mrr_at_10_max": 0.230809, + "nauc_mrr_at_10_std": 0.314688, + "nauc_mrr_at_10_diff1": 0.813628, + "nauc_mrr_at_20_max": 0.229484, + "nauc_mrr_at_20_std": 0.31679, + "nauc_mrr_at_20_diff1": 0.81303, + "nauc_mrr_at_100_max": 0.230728, + "nauc_mrr_at_100_std": 0.316224, + "nauc_mrr_at_100_diff1": 0.813188, + "nauc_mrr_at_1000_max": 0.23125, + "nauc_mrr_at_1000_std": 0.315612, + "nauc_mrr_at_1000_diff1": 0.813389, + "main_score": 0.56514, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 35.82968521118164, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreInfoVQARetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..ffa8da7a19 --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", + "task_name": "VidoreInfoVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.76113, + "ndcg_at_3": 0.81477, + "ndcg_at_5": 0.82358, + "ndcg_at_10": 0.83438, + "ndcg_at_20": 0.84178, + "ndcg_at_100": 0.8483, + "ndcg_at_1000": 0.8525, + "map_at_1": 0.76113, + "map_at_3": 0.80196, + "map_at_5": 0.80702, + "map_at_10": 0.81129, + "map_at_20": 0.81332, + "map_at_100": 0.81432, + "map_at_1000": 0.8145, + "recall_at_1": 0.76113, + "recall_at_3": 0.85121, + "recall_at_5": 0.87247, + "recall_at_10": 0.90688, + "recall_at_20": 0.93563, + "recall_at_100": 0.96903, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.76113, + "cv_recall_at_3": 0.85223, + "cv_recall_at_5": 0.87247, + "cv_recall_at_10": 0.90688, + "cv_recall_at_20": 0.93725, + "cv_recall_at_100": 0.97166, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.76113, + "precision_at_3": 0.28408, + "precision_at_5": 0.1749, + "precision_at_10": 0.09089, + "precision_at_20": 0.04696, + "precision_at_100": 0.00974, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.761134, + "mrr_at_3": 0.802294, + "mrr_at_5": 0.806849, + "mrr_at_10": 0.811119, + "mrr_at_20": 0.813301, + "mrr_at_100": 0.814316, + "mrr_at_1000": 0.814458, + "nauc_ndcg_at_1_max": 0.537472, + "nauc_ndcg_at_1_std": -0.022861, + "nauc_ndcg_at_1_diff1": 0.898498, + "nauc_ndcg_at_3_max": 0.570031, + "nauc_ndcg_at_3_std": 0.057037, + "nauc_ndcg_at_3_diff1": 0.858985, + "nauc_ndcg_at_5_max": 0.564459, + "nauc_ndcg_at_5_std": 0.036992, + "nauc_ndcg_at_5_diff1": 0.855596, + "nauc_ndcg_at_10_max": 0.558297, + "nauc_ndcg_at_10_std": 0.042421, + "nauc_ndcg_at_10_diff1": 0.859733, + "nauc_ndcg_at_20_max": 0.556266, + "nauc_ndcg_at_20_std": 0.059923, + "nauc_ndcg_at_20_diff1": 0.863516, + "nauc_ndcg_at_100_max": 0.564727, + "nauc_ndcg_at_100_std": 0.062839, + "nauc_ndcg_at_100_diff1": 0.866256, + "nauc_ndcg_at_1000_max": 0.557681, + "nauc_ndcg_at_1000_std": 0.046829, + "nauc_ndcg_at_1000_diff1": 0.866671, + "nauc_map_at_1_max": 0.537472, + "nauc_map_at_1_std": -0.022861, + "nauc_map_at_1_diff1": 0.898498, + "nauc_map_at_3_max": 0.558386, + "nauc_map_at_3_std": 0.033306, + "nauc_map_at_3_diff1": 0.869983, + "nauc_map_at_5_max": 0.555278, + "nauc_map_at_5_std": 0.021821, + "nauc_map_at_5_diff1": 0.868198, + "nauc_map_at_10_max": 0.552519, + "nauc_map_at_10_std": 0.022801, + "nauc_map_at_10_diff1": 0.869613, + "nauc_map_at_20_max": 0.552276, + "nauc_map_at_20_std": 0.026898, + "nauc_map_at_20_diff1": 0.870671, + "nauc_map_at_100_max": 0.553672, + "nauc_map_at_100_std": 0.027675, + "nauc_map_at_100_diff1": 0.871062, + "nauc_map_at_1000_max": 0.553367, + "nauc_map_at_1000_std": 0.027353, + "nauc_map_at_1000_diff1": 0.87105, + "nauc_recall_at_1_max": 0.537472, + "nauc_recall_at_1_std": -0.022861, + "nauc_recall_at_1_diff1": 0.898498, + "nauc_recall_at_3_max": 0.613875, + "nauc_recall_at_3_std": 0.147009, + "nauc_recall_at_3_diff1": 0.819308, + "nauc_recall_at_5_max": 0.602923, + "nauc_recall_at_5_std": 0.097252, + "nauc_recall_at_5_diff1": 0.803193, + "nauc_recall_at_10_max": 0.58677, + "nauc_recall_at_10_std": 0.151299, + "nauc_recall_at_10_diff1": 0.810931, + "nauc_recall_at_20_max": 0.578476, + "nauc_recall_at_20_std": 0.340832, + "nauc_recall_at_20_diff1": 0.823367, + "nauc_recall_at_100_max": 0.777773, + "nauc_recall_at_100_std": 0.67625, + "nauc_recall_at_100_diff1": 0.843209, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.537472, + "nauc_precision_at_1_std": -0.022861, + "nauc_precision_at_1_diff1": 0.898498, + "nauc_precision_at_3_max": 0.615133, + "nauc_precision_at_3_std": 0.143956, + "nauc_precision_at_3_diff1": 0.817249, + "nauc_precision_at_5_max": 0.580524, + "nauc_precision_at_5_std": 0.085746, + "nauc_precision_at_5_diff1": 0.765039, + "nauc_precision_at_10_max": 0.557843, + "nauc_precision_at_10_std": 0.134163, + "nauc_precision_at_10_diff1": 0.76037, + "nauc_precision_at_20_max": 0.526441, + "nauc_precision_at_20_std": 0.356683, + "nauc_precision_at_20_diff1": 0.744666, + "nauc_precision_at_100_max": 0.654695, + "nauc_precision_at_100_std": 0.678829, + "nauc_precision_at_100_diff1": 0.677506, + "nauc_precision_at_1000_max": -0.49966, + "nauc_precision_at_1000_std": 0.518098, + "nauc_precision_at_1000_diff1": -0.56103, + "nauc_cv_recall_at_1_max": 0.537472, + "nauc_cv_recall_at_1_std": -0.022861, + "nauc_cv_recall_at_1_diff1": 0.898498, + "nauc_cv_recall_at_3_max": 0.615133, + "nauc_cv_recall_at_3_std": 0.143956, + "nauc_cv_recall_at_3_diff1": 0.817249, + "nauc_cv_recall_at_5_max": 0.602923, + "nauc_cv_recall_at_5_std": 0.097252, + "nauc_cv_recall_at_5_diff1": 0.803193, + "nauc_cv_recall_at_10_max": 0.58677, + "nauc_cv_recall_at_10_std": 0.151299, + "nauc_cv_recall_at_10_diff1": 0.810931, + "nauc_cv_recall_at_20_max": 0.567598, + "nauc_cv_recall_at_20_std": 0.394625, + "nauc_cv_recall_at_20_diff1": 0.818809, + "nauc_cv_recall_at_100_max": 0.761802, + "nauc_cv_recall_at_100_std": 0.802967, + "nauc_cv_recall_at_100_diff1": 0.833314, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.537472, + "nauc_mrr_at_1_std": -0.022861, + "nauc_mrr_at_1_diff1": 0.898498, + "nauc_mrr_at_3_max": 0.558622, + "nauc_mrr_at_3_std": 0.032332, + "nauc_mrr_at_3_diff1": 0.869532, + "nauc_mrr_at_5_max": 0.55516, + "nauc_mrr_at_5_std": 0.022328, + "nauc_mrr_at_5_diff1": 0.86843, + "nauc_mrr_at_10_max": 0.552401, + "nauc_mrr_at_10_std": 0.023319, + "nauc_mrr_at_10_diff1": 0.869849, + "nauc_mrr_at_20_max": 0.551773, + "nauc_mrr_at_20_std": 0.028934, + "nauc_mrr_at_20_diff1": 0.870799, + "nauc_mrr_at_100_max": 0.553136, + "nauc_mrr_at_100_std": 0.029628, + "nauc_mrr_at_100_diff1": 0.871191, + "nauc_mrr_at_1000_max": 0.55293, + "nauc_mrr_at_1000_std": 0.028973, + "nauc_mrr_at_1000_diff1": 0.871204, + "main_score": 0.82358, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 34.392455101013184, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreShiftProjectRetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..e99e2be864 --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "84a382e05c4473fed9cff2bbae95fe2379416117", + "task_name": "VidoreShiftProjectRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.63, + "ndcg_at_3": 0.75833, + "ndcg_at_5": 0.78286, + "ndcg_at_10": 0.79957, + "ndcg_at_20": 0.80445, + "ndcg_at_100": 0.80631, + "ndcg_at_1000": 0.8077, + "map_at_1": 0.63, + "map_at_3": 0.72667, + "map_at_5": 0.74017, + "map_at_10": 0.74737, + "map_at_20": 0.74861, + "map_at_100": 0.74886, + "map_at_1000": 0.74893, + "recall_at_1": 0.63, + "recall_at_3": 0.85, + "recall_at_5": 0.91, + "recall_at_10": 0.96, + "recall_at_20": 0.98, + "recall_at_100": 0.99, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.63, + "cv_recall_at_3": 0.85, + "cv_recall_at_5": 0.91, + "cv_recall_at_10": 0.96, + "cv_recall_at_20": 0.98, + "cv_recall_at_100": 0.99, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.63, + "precision_at_3": 0.28333, + "precision_at_5": 0.182, + "precision_at_10": 0.096, + "precision_at_20": 0.049, + "precision_at_100": 0.0099, + "precision_at_1000": 0.001, + "mrr_at_1": 0.63, + "mrr_at_3": 0.726667, + "mrr_at_5": 0.740167, + "mrr_at_10": 0.747369, + "mrr_at_20": 0.74861, + "mrr_at_100": 0.74886, + "mrr_at_1000": 0.748929, + "nauc_ndcg_at_1_max": -0.06263, + "nauc_ndcg_at_1_std": -0.385951, + "nauc_ndcg_at_1_diff1": 0.724659, + "nauc_ndcg_at_3_max": -0.058792, + "nauc_ndcg_at_3_std": -0.424915, + "nauc_ndcg_at_3_diff1": 0.710137, + "nauc_ndcg_at_5_max": -0.119563, + "nauc_ndcg_at_5_std": -0.512383, + "nauc_ndcg_at_5_diff1": 0.701719, + "nauc_ndcg_at_10_max": -0.118386, + "nauc_ndcg_at_10_std": -0.498751, + "nauc_ndcg_at_10_diff1": 0.725264, + "nauc_ndcg_at_20_max": -0.119214, + "nauc_ndcg_at_20_std": -0.48414, + "nauc_ndcg_at_20_diff1": 0.723408, + "nauc_ndcg_at_100_max": -0.100737, + "nauc_ndcg_at_100_std": -0.469823, + "nauc_ndcg_at_100_diff1": 0.720255, + "nauc_ndcg_at_1000_max": -0.104668, + "nauc_ndcg_at_1000_std": -0.46902, + "nauc_ndcg_at_1000_diff1": 0.718978, + "nauc_map_at_1_max": -0.06263, + "nauc_map_at_1_std": -0.385951, + "nauc_map_at_1_diff1": 0.724659, + "nauc_map_at_3_max": -0.070118, + "nauc_map_at_3_std": -0.425685, + "nauc_map_at_3_diff1": 0.714427, + "nauc_map_at_5_max": -0.100303, + "nauc_map_at_5_std": -0.467583, + "nauc_map_at_5_diff1": 0.710402, + "nauc_map_at_10_max": -0.099618, + "nauc_map_at_10_std": -0.462385, + "nauc_map_at_10_diff1": 0.719013, + "nauc_map_at_20_max": -0.100273, + "nauc_map_at_20_std": -0.458788, + "nauc_map_at_20_diff1": 0.718473, + "nauc_map_at_100_max": -0.098329, + "nauc_map_at_100_std": -0.457269, + "nauc_map_at_100_diff1": 0.71814, + "nauc_map_at_1000_max": -0.098478, + "nauc_map_at_1000_std": -0.457234, + "nauc_map_at_1000_diff1": 0.71809, + "nauc_recall_at_1_max": -0.06263, + "nauc_recall_at_1_std": -0.385951, + "nauc_recall_at_1_diff1": 0.724659, + "nauc_recall_at_3_max": 0.000716, + "nauc_recall_at_3_std": -0.412756, + "nauc_recall_at_3_diff1": 0.690303, + "nauc_recall_at_5_max": -0.262527, + "nauc_recall_at_5_std": -0.842359, + "nauc_recall_at_5_diff1": 0.638811, + "nauc_recall_at_10_max": -0.416083, + "nauc_recall_at_10_std": -1.053688, + "nauc_recall_at_10_diff1": 0.855859, + "nauc_recall_at_20_max": -0.690943, + "nauc_recall_at_20_std": -1.151727, + "nauc_recall_at_20_diff1": 0.934641, + "nauc_recall_at_100_max": 0.358077, + "nauc_recall_at_100_std": -0.563492, + "nauc_recall_at_100_diff1": 0.869281, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.06263, + "nauc_precision_at_1_std": -0.385951, + "nauc_precision_at_1_diff1": 0.724659, + "nauc_precision_at_3_max": 0.000716, + "nauc_precision_at_3_std": -0.412756, + "nauc_precision_at_3_diff1": 0.690303, + "nauc_precision_at_5_max": -0.262527, + "nauc_precision_at_5_std": -0.842359, + "nauc_precision_at_5_diff1": 0.638811, + "nauc_precision_at_10_max": -0.416083, + "nauc_precision_at_10_std": -1.053688, + "nauc_precision_at_10_diff1": 0.855859, + "nauc_precision_at_20_max": -0.690943, + "nauc_precision_at_20_std": -1.151727, + "nauc_precision_at_20_diff1": 0.934641, + "nauc_precision_at_100_max": 0.358077, + "nauc_precision_at_100_std": -0.563492, + "nauc_precision_at_100_diff1": 0.869281, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": -0.06263, + "nauc_cv_recall_at_1_std": -0.385951, + "nauc_cv_recall_at_1_diff1": 0.724659, + "nauc_cv_recall_at_3_max": 0.000716, + "nauc_cv_recall_at_3_std": -0.412756, + "nauc_cv_recall_at_3_diff1": 0.690303, + "nauc_cv_recall_at_5_max": -0.262527, + "nauc_cv_recall_at_5_std": -0.842359, + "nauc_cv_recall_at_5_diff1": 0.638811, + "nauc_cv_recall_at_10_max": -0.416083, + "nauc_cv_recall_at_10_std": -1.053688, + "nauc_cv_recall_at_10_diff1": 0.855859, + "nauc_cv_recall_at_20_max": -0.690943, + "nauc_cv_recall_at_20_std": -1.151727, + "nauc_cv_recall_at_20_diff1": 0.934641, + "nauc_cv_recall_at_100_max": 0.358077, + "nauc_cv_recall_at_100_std": -0.563492, + "nauc_cv_recall_at_100_diff1": 0.869281, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.06263, + "nauc_mrr_at_1_std": -0.385951, + "nauc_mrr_at_1_diff1": 0.724659, + "nauc_mrr_at_3_max": -0.070118, + "nauc_mrr_at_3_std": -0.425685, + "nauc_mrr_at_3_diff1": 0.714427, + "nauc_mrr_at_5_max": -0.100303, + "nauc_mrr_at_5_std": -0.467583, + "nauc_mrr_at_5_diff1": 0.710402, + "nauc_mrr_at_10_max": -0.099618, + "nauc_mrr_at_10_std": -0.462385, + "nauc_mrr_at_10_diff1": 0.719013, + "nauc_mrr_at_20_max": -0.100273, + "nauc_mrr_at_20_std": -0.458788, + "nauc_mrr_at_20_diff1": 0.718473, + "nauc_mrr_at_100_max": -0.098329, + "nauc_mrr_at_100_std": -0.457269, + "nauc_mrr_at_100_diff1": 0.71814, + "nauc_mrr_at_1000_max": -0.098478, + "nauc_mrr_at_1000_std": -0.457234, + "nauc_mrr_at_1000_diff1": 0.71809, + "main_score": 0.78286, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 71.49851417541504, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreSyntheticDocQAAIRetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..9c4ede0b7c --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", + "task_name": "VidoreSyntheticDocQAAIRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.95, + "ndcg_at_3": 0.97524, + "ndcg_at_5": 0.97524, + "ndcg_at_10": 0.97524, + "ndcg_at_20": 0.97524, + "ndcg_at_100": 0.9769, + "ndcg_at_1000": 0.9769, + "map_at_1": 0.95, + "map_at_3": 0.97, + "map_at_5": 0.97, + "map_at_10": 0.97, + "map_at_20": 0.97, + "map_at_100": 0.97016, + "map_at_1000": 0.97016, + "recall_at_1": 0.95, + "recall_at_3": 0.99, + "recall_at_5": 0.99, + "recall_at_10": 0.99, + "recall_at_20": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.95, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 0.99, + "cv_recall_at_10": 0.99, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.95, + "precision_at_3": 0.33, + "precision_at_5": 0.198, + "precision_at_10": 0.099, + "precision_at_20": 0.0495, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.95, + "mrr_at_3": 0.97, + "mrr_at_5": 0.97, + "mrr_at_10": 0.97, + "mrr_at_20": 0.97, + "mrr_at_100": 0.970156, + "mrr_at_1000": 0.970156, + "nauc_ndcg_at_1_max": 0.709244, + "nauc_ndcg_at_1_std": -0.161251, + "nauc_ndcg_at_1_diff1": 0.944444, + "nauc_ndcg_at_3_max": 0.783325, + "nauc_ndcg_at_3_std": -0.413614, + "nauc_ndcg_at_3_diff1": 0.958599, + "nauc_ndcg_at_5_max": 0.783325, + "nauc_ndcg_at_5_std": -0.413614, + "nauc_ndcg_at_5_diff1": 0.958599, + "nauc_ndcg_at_10_max": 0.783325, + "nauc_ndcg_at_10_std": -0.413614, + "nauc_ndcg_at_10_diff1": 0.958599, + "nauc_ndcg_at_20_max": 0.783325, + "nauc_ndcg_at_20_std": -0.413614, + "nauc_ndcg_at_20_diff1": 0.958599, + "nauc_ndcg_at_100_max": 0.767752, + "nauc_ndcg_at_100_std": -0.360562, + "nauc_ndcg_at_100_diff1": 0.955624, + "nauc_ndcg_at_1000_max": 0.767752, + "nauc_ndcg_at_1000_std": -0.360562, + "nauc_ndcg_at_1000_diff1": 0.955624, + "nauc_map_at_1_max": 0.709244, + "nauc_map_at_1_std": -0.161251, + "nauc_map_at_1_diff1": 0.944444, + "nauc_map_at_3_max": 0.757703, + "nauc_map_at_3_std": -0.326331, + "nauc_map_at_3_diff1": 0.953704, + "nauc_map_at_5_max": 0.757703, + "nauc_map_at_5_std": -0.326331, + "nauc_map_at_5_diff1": 0.953704, + "nauc_map_at_10_max": 0.757703, + "nauc_map_at_10_std": -0.326331, + "nauc_map_at_10_diff1": 0.953704, + "nauc_map_at_20_max": 0.757703, + "nauc_map_at_20_std": -0.326331, + "nauc_map_at_20_diff1": 0.953704, + "nauc_map_at_100_max": 0.756435, + "nauc_map_at_100_std": -0.322009, + "nauc_map_at_100_diff1": 0.953461, + "nauc_map_at_1000_max": 0.756435, + "nauc_map_at_1000_std": -0.322009, + "nauc_map_at_1000_diff1": 0.953461, + "nauc_recall_at_1_max": 0.709244, + "nauc_recall_at_1_std": -0.161251, + "nauc_recall_at_1_diff1": 0.944444, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": -1.151727, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": -1.151727, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": -1.151727, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": -1.151727, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.709244, + "nauc_precision_at_1_std": -0.161251, + "nauc_precision_at_1_diff1": 0.944444, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": -1.151727, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": -1.151727, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": -1.151727, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": -1.151727, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.709244, + "nauc_cv_recall_at_1_std": -0.161251, + "nauc_cv_recall_at_1_diff1": 0.944444, + "nauc_cv_recall_at_3_max": 1.0, + "nauc_cv_recall_at_3_std": -1.151727, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 1.0, + "nauc_cv_recall_at_5_std": -1.151727, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": 1.0, + "nauc_cv_recall_at_10_std": -1.151727, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": 1.0, + "nauc_cv_recall_at_20_std": -1.151727, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.709244, + "nauc_mrr_at_1_std": -0.161251, + "nauc_mrr_at_1_diff1": 0.944444, + "nauc_mrr_at_3_max": 0.757703, + "nauc_mrr_at_3_std": -0.326331, + "nauc_mrr_at_3_diff1": 0.953704, + "nauc_mrr_at_5_max": 0.757703, + "nauc_mrr_at_5_std": -0.326331, + "nauc_mrr_at_5_diff1": 0.953704, + "nauc_mrr_at_10_max": 0.757703, + "nauc_mrr_at_10_std": -0.326331, + "nauc_mrr_at_10_diff1": 0.953704, + "nauc_mrr_at_20_max": 0.757703, + "nauc_mrr_at_20_std": -0.326331, + "nauc_mrr_at_20_diff1": 0.953704, + "nauc_mrr_at_100_max": 0.756435, + "nauc_mrr_at_100_std": -0.322009, + "nauc_mrr_at_100_diff1": 0.953461, + "nauc_mrr_at_1000_max": 0.756435, + "nauc_mrr_at_1000_std": -0.322009, + "nauc_mrr_at_1000_diff1": 0.953461, + "main_score": 0.97524, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 66.35040092468262, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreSyntheticDocQAEnergyRetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..a188a8baaf --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", + "task_name": "VidoreSyntheticDocQAEnergyRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.92, + "ndcg_at_3": 0.94393, + "ndcg_at_5": 0.94393, + "ndcg_at_10": 0.94983, + "ndcg_at_20": 0.94983, + "ndcg_at_100": 0.95161, + "ndcg_at_1000": 0.95307, + "map_at_1": 0.92, + "map_at_3": 0.93833, + "map_at_5": 0.93833, + "map_at_10": 0.94044, + "map_at_20": 0.94044, + "map_at_100": 0.94065, + "map_at_1000": 0.94074, + "recall_at_1": 0.92, + "recall_at_3": 0.96, + "recall_at_5": 0.96, + "recall_at_10": 0.98, + "recall_at_20": 0.98, + "recall_at_100": 0.99, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.92, + "cv_recall_at_3": 0.96, + "cv_recall_at_5": 0.96, + "cv_recall_at_10": 0.98, + "cv_recall_at_20": 0.98, + "cv_recall_at_100": 0.99, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.92, + "precision_at_3": 0.32, + "precision_at_5": 0.192, + "precision_at_10": 0.098, + "precision_at_20": 0.049, + "precision_at_100": 0.0099, + "precision_at_1000": 0.001, + "mrr_at_1": 0.92, + "mrr_at_3": 0.938333, + "mrr_at_5": 0.938333, + "mrr_at_10": 0.940444, + "mrr_at_20": 0.940444, + "mrr_at_100": 0.940653, + "mrr_at_1000": 0.94074, + "nauc_ndcg_at_1_max": 0.448646, + "nauc_ndcg_at_1_std": -0.638072, + "nauc_ndcg_at_1_diff1": 0.95098, + "nauc_ndcg_at_3_max": 0.431823, + "nauc_ndcg_at_3_std": -0.961179, + "nauc_ndcg_at_3_diff1": 0.944771, + "nauc_ndcg_at_5_max": 0.431823, + "nauc_ndcg_at_5_std": -0.961179, + "nauc_ndcg_at_5_diff1": 0.944771, + "nauc_ndcg_at_10_max": 0.495791, + "nauc_ndcg_at_10_std": -0.869581, + "nauc_ndcg_at_10_diff1": 0.946118, + "nauc_ndcg_at_20_max": 0.495791, + "nauc_ndcg_at_20_std": -0.869581, + "nauc_ndcg_at_20_diff1": 0.946118, + "nauc_ndcg_at_100_max": 0.482044, + "nauc_ndcg_at_100_std": -0.859196, + "nauc_ndcg_at_100_diff1": 0.944135, + "nauc_ndcg_at_1000_max": 0.465922, + "nauc_ndcg_at_1000_std": -0.868401, + "nauc_ndcg_at_1000_diff1": 0.946465, + "nauc_map_at_1_max": 0.448646, + "nauc_map_at_1_std": -0.638072, + "nauc_map_at_1_diff1": 0.95098, + "nauc_map_at_3_max": 0.435599, + "nauc_map_at_3_std": -0.882138, + "nauc_map_at_3_diff1": 0.947006, + "nauc_map_at_5_max": 0.435599, + "nauc_map_at_5_std": -0.882138, + "nauc_map_at_5_diff1": 0.947006, + "nauc_map_at_10_max": 0.455541, + "nauc_map_at_10_std": -0.85173, + "nauc_map_at_10_diff1": 0.947566, + "nauc_map_at_20_max": 0.455541, + "nauc_map_at_20_std": -0.85173, + "nauc_map_at_20_diff1": 0.947566, + "nauc_map_at_100_max": 0.454089, + "nauc_map_at_100_std": -0.850677, + "nauc_map_at_100_diff1": 0.947382, + "nauc_map_at_1000_max": 0.45328, + "nauc_map_at_1000_std": -0.851102, + "nauc_map_at_1000_diff1": 0.947498, + "nauc_recall_at_1_max": 0.448646, + "nauc_recall_at_1_std": -0.638072, + "nauc_recall_at_1_diff1": 0.95098, + "nauc_recall_at_3_max": 0.415966, + "nauc_recall_at_3_std": -1.298786, + "nauc_recall_at_3_diff1": 0.934641, + "nauc_recall_at_5_max": 0.415966, + "nauc_recall_at_5_std": -1.298786, + "nauc_recall_at_5_diff1": 0.934641, + "nauc_recall_at_10_max": 0.934641, + "nauc_recall_at_10_std": -0.85761, + "nauc_recall_at_10_diff1": 0.934641, + "nauc_recall_at_20_max": 0.934641, + "nauc_recall_at_20_std": -0.85761, + "nauc_recall_at_20_diff1": 0.934641, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": -0.563492, + "nauc_recall_at_100_diff1": 0.869281, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.448646, + "nauc_precision_at_1_std": -0.638072, + "nauc_precision_at_1_diff1": 0.95098, + "nauc_precision_at_3_max": 0.415966, + "nauc_precision_at_3_std": -1.298786, + "nauc_precision_at_3_diff1": 0.934641, + "nauc_precision_at_5_max": 0.415966, + "nauc_precision_at_5_std": -1.298786, + "nauc_precision_at_5_diff1": 0.934641, + "nauc_precision_at_10_max": 0.934641, + "nauc_precision_at_10_std": -0.85761, + "nauc_precision_at_10_diff1": 0.934641, + "nauc_precision_at_20_max": 0.934641, + "nauc_precision_at_20_std": -0.85761, + "nauc_precision_at_20_diff1": 0.934641, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": -0.563492, + "nauc_precision_at_100_diff1": 0.869281, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.448646, + "nauc_cv_recall_at_1_std": -0.638072, + "nauc_cv_recall_at_1_diff1": 0.95098, + "nauc_cv_recall_at_3_max": 0.415966, + "nauc_cv_recall_at_3_std": -1.298786, + "nauc_cv_recall_at_3_diff1": 0.934641, + "nauc_cv_recall_at_5_max": 0.415966, + "nauc_cv_recall_at_5_std": -1.298786, + "nauc_cv_recall_at_5_diff1": 0.934641, + "nauc_cv_recall_at_10_max": 0.934641, + "nauc_cv_recall_at_10_std": -0.85761, + "nauc_cv_recall_at_10_diff1": 0.934641, + "nauc_cv_recall_at_20_max": 0.934641, + "nauc_cv_recall_at_20_std": -0.85761, + "nauc_cv_recall_at_20_diff1": 0.934641, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": -0.563492, + "nauc_cv_recall_at_100_diff1": 0.869281, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.448646, + "nauc_mrr_at_1_std": -0.638072, + "nauc_mrr_at_1_diff1": 0.95098, + "nauc_mrr_at_3_max": 0.435599, + "nauc_mrr_at_3_std": -0.882138, + "nauc_mrr_at_3_diff1": 0.947006, + "nauc_mrr_at_5_max": 0.435599, + "nauc_mrr_at_5_std": -0.882138, + "nauc_mrr_at_5_diff1": 0.947006, + "nauc_mrr_at_10_max": 0.455541, + "nauc_mrr_at_10_std": -0.85173, + "nauc_mrr_at_10_diff1": 0.947566, + "nauc_mrr_at_20_max": 0.455541, + "nauc_mrr_at_20_std": -0.85173, + "nauc_mrr_at_20_diff1": 0.947566, + "nauc_mrr_at_100_max": 0.454089, + "nauc_mrr_at_100_std": -0.850677, + "nauc_mrr_at_100_diff1": 0.947382, + "nauc_mrr_at_1000_max": 0.45328, + "nauc_mrr_at_1000_std": -0.851102, + "nauc_mrr_at_1000_diff1": 0.947498, + "main_score": 0.94393, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 64.58769083023071, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..9d5588cb78 --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b4909afa930f81282fd20601e860668073ad02aa", + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.89, + "ndcg_at_3": 0.94917, + "ndcg_at_5": 0.94917, + "ndcg_at_10": 0.95273, + "ndcg_at_20": 0.95273, + "ndcg_at_100": 0.95273, + "ndcg_at_1000": 0.95273, + "map_at_1": 0.89, + "map_at_3": 0.935, + "map_at_5": 0.935, + "map_at_10": 0.93667, + "map_at_20": 0.93667, + "map_at_100": 0.93667, + "map_at_1000": 0.93667, + "recall_at_1": 0.89, + "recall_at_3": 0.99, + "recall_at_5": 0.99, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.89, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 0.99, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.89, + "precision_at_3": 0.33, + "precision_at_5": 0.198, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.89, + "mrr_at_3": 0.935, + "mrr_at_5": 0.935, + "mrr_at_10": 0.936667, + "mrr_at_20": 0.936667, + "mrr_at_100": 0.936667, + "mrr_at_1000": 0.936667, + "nauc_ndcg_at_1_max": 0.493042, + "nauc_ndcg_at_1_std": 0.197062, + "nauc_ndcg_at_1_diff1": 0.785242, + "nauc_ndcg_at_3_max": 0.540465, + "nauc_ndcg_at_3_std": 0.086062, + "nauc_ndcg_at_3_diff1": 0.822106, + "nauc_ndcg_at_5_max": 0.540465, + "nauc_ndcg_at_5_std": 0.086062, + "nauc_ndcg_at_5_diff1": 0.822106, + "nauc_ndcg_at_10_max": 0.505481, + "nauc_ndcg_at_10_std": 0.135512, + "nauc_ndcg_at_10_diff1": 0.808564, + "nauc_ndcg_at_20_max": 0.505481, + "nauc_ndcg_at_20_std": 0.135512, + "nauc_ndcg_at_20_diff1": 0.808564, + "nauc_ndcg_at_100_max": 0.505481, + "nauc_ndcg_at_100_std": 0.135512, + "nauc_ndcg_at_100_diff1": 0.808564, + "nauc_ndcg_at_1000_max": 0.505481, + "nauc_ndcg_at_1000_std": 0.135512, + "nauc_ndcg_at_1000_diff1": 0.808564, + "nauc_map_at_1_max": 0.493042, + "nauc_map_at_1_std": 0.197062, + "nauc_map_at_1_diff1": 0.785242, + "nauc_map_at_3_max": 0.516397, + "nauc_map_at_3_std": 0.122325, + "nauc_map_at_3_diff1": 0.811846, + "nauc_map_at_5_max": 0.516397, + "nauc_map_at_5_std": 0.122325, + "nauc_map_at_5_diff1": 0.811846, + "nauc_map_at_10_max": 0.503538, + "nauc_map_at_10_std": 0.140561, + "nauc_map_at_10_diff1": 0.806842, + "nauc_map_at_20_max": 0.503538, + "nauc_map_at_20_std": 0.140561, + "nauc_map_at_20_diff1": 0.806842, + "nauc_map_at_100_max": 0.503538, + "nauc_map_at_100_std": 0.140561, + "nauc_map_at_100_diff1": 0.806842, + "nauc_map_at_1000_max": 0.503538, + "nauc_map_at_1000_std": 0.140561, + "nauc_map_at_1000_diff1": 0.806842, + "nauc_recall_at_1_max": 0.493042, + "nauc_recall_at_1_std": 0.197062, + "nauc_recall_at_1_diff1": 0.785242, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": -0.563492, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": -0.563492, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.493042, + "nauc_precision_at_1_std": 0.197062, + "nauc_precision_at_1_diff1": 0.785242, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": -0.563492, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": -0.563492, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.493042, + "nauc_cv_recall_at_1_std": 0.197062, + "nauc_cv_recall_at_1_diff1": 0.785242, + "nauc_cv_recall_at_3_max": 1.0, + "nauc_cv_recall_at_3_std": -0.563492, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 1.0, + "nauc_cv_recall_at_5_std": -0.563492, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.493042, + "nauc_mrr_at_1_std": 0.197062, + "nauc_mrr_at_1_diff1": 0.785242, + "nauc_mrr_at_3_max": 0.516397, + "nauc_mrr_at_3_std": 0.122325, + "nauc_mrr_at_3_diff1": 0.811846, + "nauc_mrr_at_5_max": 0.516397, + "nauc_mrr_at_5_std": 0.122325, + "nauc_mrr_at_5_diff1": 0.811846, + "nauc_mrr_at_10_max": 0.503538, + "nauc_mrr_at_10_std": 0.140561, + "nauc_mrr_at_10_diff1": 0.806842, + "nauc_mrr_at_20_max": 0.503538, + "nauc_mrr_at_20_std": 0.140561, + "nauc_mrr_at_20_diff1": 0.806842, + "nauc_mrr_at_100_max": 0.503538, + "nauc_mrr_at_100_std": 0.140561, + "nauc_mrr_at_100_diff1": 0.806842, + "nauc_mrr_at_1000_max": 0.503538, + "nauc_mrr_at_1000_std": 0.140561, + "nauc_mrr_at_1000_diff1": 0.806842, + "main_score": 0.94917, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 65.28345394134521, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..11a0697ba3 --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.91, + "ndcg_at_3": 0.95393, + "ndcg_at_5": 0.95393, + "ndcg_at_10": 0.95393, + "ndcg_at_20": 0.95672, + "ndcg_at_100": 0.95672, + "ndcg_at_1000": 0.95672, + "map_at_1": 0.91, + "map_at_3": 0.94167, + "map_at_5": 0.94167, + "map_at_10": 0.94167, + "map_at_20": 0.94258, + "map_at_100": 0.94258, + "map_at_1000": 0.94258, + "recall_at_1": 0.91, + "recall_at_3": 0.99, + "recall_at_5": 0.99, + "recall_at_10": 0.99, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.91, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 0.99, + "cv_recall_at_10": 0.99, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.91, + "precision_at_3": 0.33, + "precision_at_5": 0.198, + "precision_at_10": 0.099, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.91, + "mrr_at_3": 0.941667, + "mrr_at_5": 0.941667, + "mrr_at_10": 0.941667, + "mrr_at_20": 0.942576, + "mrr_at_100": 0.942576, + "mrr_at_1000": 0.942576, + "nauc_ndcg_at_1_max": 0.423021, + "nauc_ndcg_at_1_std": -0.4446, + "nauc_ndcg_at_1_diff1": 0.940087, + "nauc_ndcg_at_3_max": 0.322611, + "nauc_ndcg_at_3_std": -0.605691, + "nauc_ndcg_at_3_diff1": 0.95309, + "nauc_ndcg_at_5_max": 0.322611, + "nauc_ndcg_at_5_std": -0.605691, + "nauc_ndcg_at_5_diff1": 0.95309, + "nauc_ndcg_at_10_max": 0.322611, + "nauc_ndcg_at_10_std": -0.605691, + "nauc_ndcg_at_10_diff1": 0.95309, + "nauc_ndcg_at_20_max": 0.379717, + "nauc_ndcg_at_20_std": -0.532591, + "nauc_ndcg_at_20_diff1": 0.950067, + "nauc_ndcg_at_100_max": 0.379717, + "nauc_ndcg_at_100_std": -0.532591, + "nauc_ndcg_at_100_diff1": 0.950067, + "nauc_ndcg_at_1000_max": 0.379717, + "nauc_ndcg_at_1000_std": -0.532591, + "nauc_ndcg_at_1000_diff1": 0.950067, + "nauc_map_at_1_max": 0.423021, + "nauc_map_at_1_std": -0.4446, + "nauc_map_at_1_diff1": 0.940087, + "nauc_map_at_3_max": 0.373336, + "nauc_map_at_3_std": -0.539242, + "nauc_map_at_3_diff1": 0.950047, + "nauc_map_at_5_max": 0.373336, + "nauc_map_at_5_std": -0.539242, + "nauc_map_at_5_diff1": 0.950047, + "nauc_map_at_10_max": 0.373336, + "nauc_map_at_10_std": -0.539242, + "nauc_map_at_10_diff1": 0.950047, + "nauc_map_at_20_max": 0.388167, + "nauc_map_at_20_std": -0.520234, + "nauc_map_at_20_diff1": 0.949256, + "nauc_map_at_100_max": 0.388167, + "nauc_map_at_100_std": -0.520234, + "nauc_map_at_100_diff1": 0.949256, + "nauc_map_at_1000_max": 0.388167, + "nauc_map_at_1000_std": -0.520234, + "nauc_map_at_1000_diff1": 0.949256, + "nauc_recall_at_1_max": 0.423021, + "nauc_recall_at_1_std": -0.4446, + "nauc_recall_at_1_diff1": 0.940087, + "nauc_recall_at_3_max": -0.563492, + "nauc_recall_at_3_std": -1.739963, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": -0.563492, + "nauc_recall_at_5_std": -1.739963, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": -0.563492, + "nauc_recall_at_10_std": -1.739963, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.423021, + "nauc_precision_at_1_std": -0.4446, + "nauc_precision_at_1_diff1": 0.940087, + "nauc_precision_at_3_max": -0.563492, + "nauc_precision_at_3_std": -1.739963, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": -0.563492, + "nauc_precision_at_5_std": -1.739963, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": -0.563492, + "nauc_precision_at_10_std": -1.739963, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.423021, + "nauc_cv_recall_at_1_std": -0.4446, + "nauc_cv_recall_at_1_diff1": 0.940087, + "nauc_cv_recall_at_3_max": -0.563492, + "nauc_cv_recall_at_3_std": -1.739963, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": -0.563492, + "nauc_cv_recall_at_5_std": -1.739963, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": -0.563492, + "nauc_cv_recall_at_10_std": -1.739963, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.423021, + "nauc_mrr_at_1_std": -0.4446, + "nauc_mrr_at_1_diff1": 0.940087, + "nauc_mrr_at_3_max": 0.373336, + "nauc_mrr_at_3_std": -0.539242, + "nauc_mrr_at_3_diff1": 0.950047, + "nauc_mrr_at_5_max": 0.373336, + "nauc_mrr_at_5_std": -0.539242, + "nauc_mrr_at_5_diff1": 0.950047, + "nauc_mrr_at_10_max": 0.373336, + "nauc_mrr_at_10_std": -0.539242, + "nauc_mrr_at_10_diff1": 0.950047, + "nauc_mrr_at_20_max": 0.388167, + "nauc_mrr_at_20_std": -0.520234, + "nauc_mrr_at_20_diff1": 0.949256, + "nauc_mrr_at_100_max": 0.388167, + "nauc_mrr_at_100_std": -0.520234, + "nauc_mrr_at_100_diff1": 0.949256, + "nauc_mrr_at_1000_max": 0.388167, + "nauc_mrr_at_1000_std": -0.520234, + "nauc_mrr_at_1000_diff1": 0.949256, + "main_score": 0.95393, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 62.64856839179993, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreTabfquadRetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..050e2137a3 --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", + "task_name": "VidoreTabfquadRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.82857, + "ndcg_at_3": 0.86905, + "ndcg_at_5": 0.88365, + "ndcg_at_10": 0.89193, + "ndcg_at_20": 0.89914, + "ndcg_at_100": 0.90218, + "ndcg_at_1000": 0.90218, + "map_at_1": 0.82857, + "map_at_3": 0.85952, + "map_at_5": 0.86756, + "map_at_10": 0.8711, + "map_at_20": 0.87306, + "map_at_100": 0.87364, + "map_at_1000": 0.87364, + "recall_at_1": 0.82857, + "recall_at_3": 0.89643, + "recall_at_5": 0.93214, + "recall_at_10": 0.95714, + "recall_at_20": 0.98571, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.82857, + "cv_recall_at_3": 0.89643, + "cv_recall_at_5": 0.93214, + "cv_recall_at_10": 0.95714, + "cv_recall_at_20": 0.98571, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.82857, + "precision_at_3": 0.29881, + "precision_at_5": 0.18643, + "precision_at_10": 0.09571, + "precision_at_20": 0.04929, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.828571, + "mrr_at_3": 0.859524, + "mrr_at_5": 0.86756, + "mrr_at_10": 0.871096, + "mrr_at_20": 0.873062, + "mrr_at_100": 0.873635, + "mrr_at_1000": 0.873635, + "nauc_ndcg_at_1_max": 0.663584, + "nauc_ndcg_at_1_std": 0.334935, + "nauc_ndcg_at_1_diff1": 0.852977, + "nauc_ndcg_at_3_max": 0.678198, + "nauc_ndcg_at_3_std": 0.378608, + "nauc_ndcg_at_3_diff1": 0.834712, + "nauc_ndcg_at_5_max": 0.684393, + "nauc_ndcg_at_5_std": 0.395781, + "nauc_ndcg_at_5_diff1": 0.833615, + "nauc_ndcg_at_10_max": 0.687327, + "nauc_ndcg_at_10_std": 0.418382, + "nauc_ndcg_at_10_diff1": 0.834049, + "nauc_ndcg_at_20_max": 0.682296, + "nauc_ndcg_at_20_std": 0.395518, + "nauc_ndcg_at_20_diff1": 0.833607, + "nauc_ndcg_at_100_max": 0.6792, + "nauc_ndcg_at_100_std": 0.383363, + "nauc_ndcg_at_100_diff1": 0.837811, + "nauc_ndcg_at_1000_max": 0.6792, + "nauc_ndcg_at_1000_std": 0.383363, + "nauc_ndcg_at_1000_diff1": 0.837811, + "nauc_map_at_1_max": 0.663584, + "nauc_map_at_1_std": 0.334935, + "nauc_map_at_1_diff1": 0.852977, + "nauc_map_at_3_max": 0.675809, + "nauc_map_at_3_std": 0.369577, + "nauc_map_at_3_diff1": 0.839403, + "nauc_map_at_5_max": 0.678236, + "nauc_map_at_5_std": 0.376389, + "nauc_map_at_5_diff1": 0.839203, + "nauc_map_at_10_max": 0.678825, + "nauc_map_at_10_std": 0.383279, + "nauc_map_at_10_diff1": 0.839188, + "nauc_map_at_20_max": 0.677655, + "nauc_map_at_20_std": 0.377796, + "nauc_map_at_20_diff1": 0.839098, + "nauc_map_at_100_max": 0.677208, + "nauc_map_at_100_std": 0.375999, + "nauc_map_at_100_diff1": 0.839775, + "nauc_map_at_1000_max": 0.677208, + "nauc_map_at_1000_std": 0.375999, + "nauc_map_at_1000_diff1": 0.839775, + "nauc_recall_at_1_max": 0.663584, + "nauc_recall_at_1_std": 0.334935, + "nauc_recall_at_1_diff1": 0.852977, + "nauc_recall_at_3_max": 0.686245, + "nauc_recall_at_3_std": 0.410832, + "nauc_recall_at_3_diff1": 0.817281, + "nauc_recall_at_5_max": 0.718954, + "nauc_recall_at_5_std": 0.504865, + "nauc_recall_at_5_diff1": 0.803455, + "nauc_recall_at_10_max": 0.765056, + "nauc_recall_at_10_std": 0.744398, + "nauc_recall_at_10_diff1": 0.79404, + "nauc_recall_at_20_max": 0.780696, + "nauc_recall_at_20_std": 0.777077, + "nauc_recall_at_20_diff1": 0.707166, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.663584, + "nauc_precision_at_1_std": 0.334935, + "nauc_precision_at_1_diff1": 0.852977, + "nauc_precision_at_3_max": 0.686245, + "nauc_precision_at_3_std": 0.410832, + "nauc_precision_at_3_diff1": 0.817281, + "nauc_precision_at_5_max": 0.718954, + "nauc_precision_at_5_std": 0.504865, + "nauc_precision_at_5_diff1": 0.803455, + "nauc_precision_at_10_max": 0.765056, + "nauc_precision_at_10_std": 0.744398, + "nauc_precision_at_10_diff1": 0.79404, + "nauc_precision_at_20_max": 0.780696, + "nauc_precision_at_20_std": 0.777077, + "nauc_precision_at_20_diff1": 0.707166, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.663584, + "nauc_cv_recall_at_1_std": 0.334935, + "nauc_cv_recall_at_1_diff1": 0.852977, + "nauc_cv_recall_at_3_max": 0.686245, + "nauc_cv_recall_at_3_std": 0.410832, + "nauc_cv_recall_at_3_diff1": 0.817281, + "nauc_cv_recall_at_5_max": 0.718954, + "nauc_cv_recall_at_5_std": 0.504865, + "nauc_cv_recall_at_5_diff1": 0.803455, + "nauc_cv_recall_at_10_max": 0.765056, + "nauc_cv_recall_at_10_std": 0.744398, + "nauc_cv_recall_at_10_diff1": 0.79404, + "nauc_cv_recall_at_20_max": 0.780696, + "nauc_cv_recall_at_20_std": 0.777077, + "nauc_cv_recall_at_20_diff1": 0.707166, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.663584, + "nauc_mrr_at_1_std": 0.334935, + "nauc_mrr_at_1_diff1": 0.852977, + "nauc_mrr_at_3_max": 0.675809, + "nauc_mrr_at_3_std": 0.369577, + "nauc_mrr_at_3_diff1": 0.839403, + "nauc_mrr_at_5_max": 0.678236, + "nauc_mrr_at_5_std": 0.376389, + "nauc_mrr_at_5_diff1": 0.839203, + "nauc_mrr_at_10_max": 0.678825, + "nauc_mrr_at_10_std": 0.383279, + "nauc_mrr_at_10_diff1": 0.839188, + "nauc_mrr_at_20_max": 0.677655, + "nauc_mrr_at_20_std": 0.377796, + "nauc_mrr_at_20_diff1": 0.839098, + "nauc_mrr_at_100_max": 0.677208, + "nauc_mrr_at_100_std": 0.375999, + "nauc_mrr_at_100_diff1": 0.839775, + "nauc_mrr_at_1000_max": 0.677208, + "nauc_mrr_at_1000_std": 0.375999, + "nauc_mrr_at_1000_diff1": 0.839775, + "main_score": 0.88365, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.9640450477600098, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreTatdqaRetrieval.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..0b9cb37cdf --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", + "task_name": "VidoreTatdqaRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.56015, + "ndcg_at_3": 0.6529, + "ndcg_at_5": 0.68148, + "ndcg_at_10": 0.70353, + "ndcg_at_20": 0.71885, + "ndcg_at_100": 0.73018, + "ndcg_at_1000": 0.73351, + "map_at_1": 0.55923, + "map_at_3": 0.63037, + "map_at_5": 0.64615, + "map_at_10": 0.65524, + "map_at_20": 0.65951, + "map_at_100": 0.66108, + "map_at_1000": 0.66125, + "recall_at_1": 0.55923, + "recall_at_3": 0.7175, + "recall_at_5": 0.78706, + "recall_at_10": 0.85541, + "recall_at_20": 0.91537, + "recall_at_100": 0.97606, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.56015, + "cv_recall_at_3": 0.7181, + "cv_recall_at_5": 0.78797, + "cv_recall_at_10": 0.85601, + "cv_recall_at_20": 0.91616, + "cv_recall_at_100": 0.97691, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.56015, + "precision_at_3": 0.23977, + "precision_at_5": 0.15796, + "precision_at_10": 0.08584, + "precision_at_20": 0.04602, + "precision_at_100": 0.00984, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.560146, + "mrr_at_3": 0.630822, + "mrr_at_5": 0.646618, + "mrr_at_10": 0.655631, + "mrr_at_20": 0.659849, + "mrr_at_100": 0.661418, + "mrr_at_1000": 0.661567, + "nauc_ndcg_at_1_max": 0.248167, + "nauc_ndcg_at_1_std": -0.129017, + "nauc_ndcg_at_1_diff1": 0.70486, + "nauc_ndcg_at_3_max": 0.263698, + "nauc_ndcg_at_3_std": -0.115138, + "nauc_ndcg_at_3_diff1": 0.651065, + "nauc_ndcg_at_5_max": 0.277947, + "nauc_ndcg_at_5_std": -0.108071, + "nauc_ndcg_at_5_diff1": 0.647766, + "nauc_ndcg_at_10_max": 0.278518, + "nauc_ndcg_at_10_std": -0.091666, + "nauc_ndcg_at_10_diff1": 0.646236, + "nauc_ndcg_at_20_max": 0.273691, + "nauc_ndcg_at_20_std": -0.093459, + "nauc_ndcg_at_20_diff1": 0.650085, + "nauc_ndcg_at_100_max": 0.271956, + "nauc_ndcg_at_100_std": -0.093807, + "nauc_ndcg_at_100_diff1": 0.657244, + "nauc_ndcg_at_1000_max": 0.268763, + "nauc_ndcg_at_1000_std": -0.102972, + "nauc_ndcg_at_1000_diff1": 0.658934, + "nauc_map_at_1_max": 0.248579, + "nauc_map_at_1_std": -0.128227, + "nauc_map_at_1_diff1": 0.706277, + "nauc_map_at_3_max": 0.258425, + "nauc_map_at_3_std": -0.119219, + "nauc_map_at_3_diff1": 0.665595, + "nauc_map_at_5_max": 0.265596, + "nauc_map_at_5_std": -0.115713, + "nauc_map_at_5_diff1": 0.664401, + "nauc_map_at_10_max": 0.265379, + "nauc_map_at_10_std": -0.110412, + "nauc_map_at_10_diff1": 0.664155, + "nauc_map_at_20_max": 0.264287, + "nauc_map_at_20_std": -0.110869, + "nauc_map_at_20_diff1": 0.66539, + "nauc_map_at_100_max": 0.26412, + "nauc_map_at_100_std": -0.110729, + "nauc_map_at_100_diff1": 0.666279, + "nauc_map_at_1000_max": 0.263954, + "nauc_map_at_1000_std": -0.111137, + "nauc_map_at_1000_diff1": 0.666324, + "nauc_recall_at_1_max": 0.248579, + "nauc_recall_at_1_std": -0.128227, + "nauc_recall_at_1_diff1": 0.706277, + "nauc_recall_at_3_max": 0.279909, + "nauc_recall_at_3_std": -0.102737, + "nauc_recall_at_3_diff1": 0.601764, + "nauc_recall_at_5_max": 0.331228, + "nauc_recall_at_5_std": -0.076608, + "nauc_recall_at_5_diff1": 0.578861, + "nauc_recall_at_10_max": 0.357739, + "nauc_recall_at_10_std": 0.03026, + "nauc_recall_at_10_diff1": 0.543445, + "nauc_recall_at_20_max": 0.353989, + "nauc_recall_at_20_std": 0.086979, + "nauc_recall_at_20_diff1": 0.518631, + "nauc_recall_at_100_max": 0.466052, + "nauc_recall_at_100_std": 0.488485, + "nauc_recall_at_100_diff1": 0.536024, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.248167, + "nauc_precision_at_1_std": -0.129017, + "nauc_precision_at_1_diff1": 0.70486, + "nauc_precision_at_3_max": 0.277715, + "nauc_precision_at_3_std": -0.10452, + "nauc_precision_at_3_diff1": 0.592719, + "nauc_precision_at_5_max": 0.327625, + "nauc_precision_at_5_std": -0.078113, + "nauc_precision_at_5_diff1": 0.564041, + "nauc_precision_at_10_max": 0.350468, + "nauc_precision_at_10_std": 0.028481, + "nauc_precision_at_10_diff1": 0.517016, + "nauc_precision_at_20_max": 0.312498, + "nauc_precision_at_20_std": 0.063285, + "nauc_precision_at_20_diff1": 0.444913, + "nauc_precision_at_100_max": 0.20796, + "nauc_precision_at_100_std": 0.219401, + "nauc_precision_at_100_diff1": 0.202839, + "nauc_precision_at_1000_max": -0.374894, + "nauc_precision_at_1000_std": -0.382682, + "nauc_precision_at_1000_diff1": -0.478052, + "nauc_cv_recall_at_1_max": 0.248167, + "nauc_cv_recall_at_1_std": -0.129017, + "nauc_cv_recall_at_1_diff1": 0.70486, + "nauc_cv_recall_at_3_max": 0.28325, + "nauc_cv_recall_at_3_std": -0.100803, + "nauc_cv_recall_at_3_diff1": 0.601322, + "nauc_cv_recall_at_5_max": 0.331496, + "nauc_cv_recall_at_5_std": -0.075324, + "nauc_cv_recall_at_5_diff1": 0.578711, + "nauc_cv_recall_at_10_max": 0.35667, + "nauc_cv_recall_at_10_std": 0.030027, + "nauc_cv_recall_at_10_diff1": 0.544464, + "nauc_cv_recall_at_20_max": 0.347903, + "nauc_cv_recall_at_20_std": 0.078378, + "nauc_cv_recall_at_20_diff1": 0.514096, + "nauc_cv_recall_at_100_max": 0.44638, + "nauc_cv_recall_at_100_std": 0.46964, + "nauc_cv_recall_at_100_diff1": 0.51893, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.248167, + "nauc_mrr_at_1_std": -0.129017, + "nauc_mrr_at_1_diff1": 0.70486, + "nauc_mrr_at_3_max": 0.26009, + "nauc_mrr_at_3_std": -0.117813, + "nauc_mrr_at_3_diff1": 0.664965, + "nauc_mrr_at_5_max": 0.266335, + "nauc_mrr_at_5_std": -0.114504, + "nauc_mrr_at_5_diff1": 0.66383, + "nauc_mrr_at_10_max": 0.265965, + "nauc_mrr_at_10_std": -0.109478, + "nauc_mrr_at_10_diff1": 0.663767, + "nauc_mrr_at_20_max": 0.264872, + "nauc_mrr_at_20_std": -0.110054, + "nauc_mrr_at_20_diff1": 0.664809, + "nauc_mrr_at_100_max": 0.264716, + "nauc_mrr_at_100_std": -0.109899, + "nauc_mrr_at_100_diff1": 0.665699, + "nauc_mrr_at_1000_max": 0.264615, + "nauc_mrr_at_1000_std": -0.110209, + "nauc_mrr_at_1000_diff1": 0.665775, + "main_score": 0.68148, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 23.929676055908203, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/model_meta.json b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/model_meta.json new file mode 100644 index 0000000000..3d6ca7f780 --- /dev/null +++ b/results/vidore__colpali-v1.2/6b89bc63c16809af4d111bfe412e2ac6bc3c9451/model_meta.json @@ -0,0 +1 @@ +{"name": "vidore/colpali-v1.2", "revision": "6b89bc63c16809af4d111bfe412e2ac6bc3c9451", "release_date": "2024-08-26", "languages": ["eng-Latn"], "n_parameters": 2920000000, "memory_usage_mb": 4700.0, "max_tokens": 16384.0, "embed_dim": 128, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/illuin-tech/colpali", "public_training_data": "https://huggingface.co/datasets/vidore/colpali_train_set", "framework": ["ColPali"], "reference": "https://huggingface.co/vidore/colpali-v1.2", "similarity_fn_name": "max_sim", "use_instructions": true, "training_datasets": {"DocVQA": ["train"], "InfoVQA": ["train"], "TATDQA": ["train"], "arXivQA": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "ColPaliWrapper"} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/Vidore2BioMedicalLecturesRetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..c252a6ed5c --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.475, + "ndcg_at_3": 0.51735, + "ndcg_at_5": 0.53183, + "ndcg_at_10": 0.57936, + "ndcg_at_20": 0.61023, + "ndcg_at_100": 0.64473, + "ndcg_at_1000": 0.6633, + "map_at_1": 0.28334, + "map_at_3": 0.40942, + "map_at_5": 0.43749, + "map_at_10": 0.47699, + "map_at_20": 0.49464, + "map_at_100": 0.50538, + "map_at_1000": 0.50727, + "recall_at_1": 0.28334, + "recall_at_3": 0.49977, + "recall_at_5": 0.57227, + "recall_at_10": 0.71001, + "recall_at_20": 0.79518, + "recall_at_100": 0.91142, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.475, + "cv_recall_at_3": 0.73125, + "cv_recall_at_5": 0.80625, + "cv_recall_at_10": 0.9125, + "cv_recall_at_20": 0.95, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.475, + "precision_at_3": 0.33333, + "precision_at_5": 0.24625, + "precision_at_10": 0.1675, + "precision_at_20": 0.10219, + "precision_at_100": 0.0265, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.475, + "mrr_at_3": 0.586458, + "mrr_at_5": 0.603646, + "mrr_at_10": 0.618581, + "mrr_at_20": 0.621505, + "mrr_at_100": 0.622361, + "mrr_at_1000": 0.622392, + "nauc_ndcg_at_1_max": 0.335353, + "nauc_ndcg_at_1_std": 0.032469, + "nauc_ndcg_at_1_diff1": 0.596399, + "nauc_ndcg_at_3_max": 0.377731, + "nauc_ndcg_at_3_std": 0.11403, + "nauc_ndcg_at_3_diff1": 0.505017, + "nauc_ndcg_at_5_max": 0.395318, + "nauc_ndcg_at_5_std": 0.134593, + "nauc_ndcg_at_5_diff1": 0.482427, + "nauc_ndcg_at_10_max": 0.412288, + "nauc_ndcg_at_10_std": 0.134733, + "nauc_ndcg_at_10_diff1": 0.480024, + "nauc_ndcg_at_20_max": 0.380106, + "nauc_ndcg_at_20_std": 0.113245, + "nauc_ndcg_at_20_diff1": 0.464642, + "nauc_ndcg_at_100_max": 0.372812, + "nauc_ndcg_at_100_std": 0.112353, + "nauc_ndcg_at_100_diff1": 0.452578, + "nauc_ndcg_at_1000_max": 0.355183, + "nauc_ndcg_at_1000_std": 0.085958, + "nauc_ndcg_at_1000_diff1": 0.459707, + "nauc_map_at_1_max": 0.384277, + "nauc_map_at_1_std": 0.121075, + "nauc_map_at_1_diff1": 0.694405, + "nauc_map_at_3_max": 0.401735, + "nauc_map_at_3_std": 0.114794, + "nauc_map_at_3_diff1": 0.596059, + "nauc_map_at_5_max": 0.398999, + "nauc_map_at_5_std": 0.113554, + "nauc_map_at_5_diff1": 0.558811, + "nauc_map_at_10_max": 0.400709, + "nauc_map_at_10_std": 0.10804, + "nauc_map_at_10_diff1": 0.53854, + "nauc_map_at_20_max": 0.3848, + "nauc_map_at_20_std": 0.099393, + "nauc_map_at_20_diff1": 0.527362, + "nauc_map_at_100_max": 0.386714, + "nauc_map_at_100_std": 0.102031, + "nauc_map_at_100_diff1": 0.523426, + "nauc_map_at_1000_max": 0.385072, + "nauc_map_at_1000_std": 0.099884, + "nauc_map_at_1000_diff1": 0.523079, + "nauc_recall_at_1_max": 0.384277, + "nauc_recall_at_1_std": 0.121075, + "nauc_recall_at_1_diff1": 0.694405, + "nauc_recall_at_3_max": 0.405597, + "nauc_recall_at_3_std": 0.165277, + "nauc_recall_at_3_diff1": 0.481208, + "nauc_recall_at_5_max": 0.392269, + "nauc_recall_at_5_std": 0.189225, + "nauc_recall_at_5_diff1": 0.363358, + "nauc_recall_at_10_max": 0.444665, + "nauc_recall_at_10_std": 0.248418, + "nauc_recall_at_10_diff1": 0.272905, + "nauc_recall_at_20_max": 0.364061, + "nauc_recall_at_20_std": 0.208919, + "nauc_recall_at_20_diff1": 0.198852, + "nauc_recall_at_100_max": 0.369967, + "nauc_recall_at_100_std": 0.346911, + "nauc_recall_at_100_diff1": 0.040276, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.335353, + "nauc_precision_at_1_std": 0.032469, + "nauc_precision_at_1_diff1": 0.596399, + "nauc_precision_at_3_max": 0.139871, + "nauc_precision_at_3_std": 0.009892, + "nauc_precision_at_3_diff1": 0.059017, + "nauc_precision_at_5_max": 0.058873, + "nauc_precision_at_5_std": -0.024562, + "nauc_precision_at_5_diff1": -0.125385, + "nauc_precision_at_10_max": -0.065176, + "nauc_precision_at_10_std": -0.092904, + "nauc_precision_at_10_diff1": -0.265477, + "nauc_precision_at_20_max": -0.16654, + "nauc_precision_at_20_std": -0.122316, + "nauc_precision_at_20_diff1": -0.326555, + "nauc_precision_at_100_max": -0.211366, + "nauc_precision_at_100_std": -0.129267, + "nauc_precision_at_100_diff1": -0.380666, + "nauc_precision_at_1000_max": -0.303626, + "nauc_precision_at_1000_std": -0.208554, + "nauc_precision_at_1000_diff1": -0.387762, + "nauc_cv_recall_at_1_max": 0.335353, + "nauc_cv_recall_at_1_std": 0.032469, + "nauc_cv_recall_at_1_diff1": 0.596399, + "nauc_cv_recall_at_3_max": 0.33009, + "nauc_cv_recall_at_3_std": 0.109976, + "nauc_cv_recall_at_3_diff1": 0.35874, + "nauc_cv_recall_at_5_max": 0.31954, + "nauc_cv_recall_at_5_std": 0.171474, + "nauc_cv_recall_at_5_diff1": 0.209816, + "nauc_cv_recall_at_10_max": 0.495231, + "nauc_cv_recall_at_10_std": 0.440276, + "nauc_cv_recall_at_10_diff1": -0.074863, + "nauc_cv_recall_at_20_max": 0.586893, + "nauc_cv_recall_at_20_std": 0.569153, + "nauc_cv_recall_at_20_diff1": -0.294935, + "nauc_cv_recall_at_100_max": 0.134143, + "nauc_cv_recall_at_100_std": 0.450047, + "nauc_cv_recall_at_100_diff1": -1.021008, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.335353, + "nauc_mrr_at_1_std": 0.032469, + "nauc_mrr_at_1_diff1": 0.596399, + "nauc_mrr_at_3_max": 0.335567, + "nauc_mrr_at_3_std": 0.056881, + "nauc_mrr_at_3_diff1": 0.508226, + "nauc_mrr_at_5_max": 0.333383, + "nauc_mrr_at_5_std": 0.063366, + "nauc_mrr_at_5_diff1": 0.493458, + "nauc_mrr_at_10_max": 0.34354, + "nauc_mrr_at_10_std": 0.072214, + "nauc_mrr_at_10_diff1": 0.49655, + "nauc_mrr_at_20_max": 0.343166, + "nauc_mrr_at_20_std": 0.070135, + "nauc_mrr_at_20_diff1": 0.499444, + "nauc_mrr_at_100_max": 0.34147, + "nauc_mrr_at_100_std": 0.068275, + "nauc_mrr_at_100_diff1": 0.500223, + "nauc_mrr_at_1000_max": 0.341479, + "nauc_mrr_at_1000_std": 0.068224, + "nauc_mrr_at_1000_diff1": 0.500383, + "main_score": 0.53183, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.54375, + "ndcg_at_3": 0.54826, + "ndcg_at_5": 0.57156, + "ndcg_at_10": 0.61338, + "ndcg_at_20": 0.63512, + "ndcg_at_100": 0.67197, + "ndcg_at_1000": 0.68996, + "map_at_1": 0.32423, + "map_at_3": 0.43923, + "map_at_5": 0.47695, + "map_at_10": 0.51434, + "map_at_20": 0.52708, + "map_at_100": 0.53835, + "map_at_1000": 0.5402, + "recall_at_1": 0.32423, + "recall_at_3": 0.50955, + "recall_at_5": 0.59856, + "recall_at_10": 0.72724, + "recall_at_20": 0.78814, + "recall_at_100": 0.91428, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.54375, + "cv_recall_at_3": 0.7625, + "cv_recall_at_5": 0.85, + "cv_recall_at_10": 0.93125, + "cv_recall_at_20": 0.94375, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.54375, + "precision_at_3": 0.3375, + "precision_at_5": 0.2625, + "precision_at_10": 0.17375, + "precision_at_20": 0.10094, + "precision_at_100": 0.02656, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.54375, + "mrr_at_3": 0.641667, + "mrr_at_5": 0.660417, + "mrr_at_10": 0.671347, + "mrr_at_20": 0.672436, + "mrr_at_100": 0.673265, + "mrr_at_1000": 0.673298, + "nauc_ndcg_at_1_max": 0.324956, + "nauc_ndcg_at_1_std": 0.003156, + "nauc_ndcg_at_1_diff1": 0.502001, + "nauc_ndcg_at_3_max": 0.329677, + "nauc_ndcg_at_3_std": 0.045092, + "nauc_ndcg_at_3_diff1": 0.413294, + "nauc_ndcg_at_5_max": 0.358566, + "nauc_ndcg_at_5_std": 0.074986, + "nauc_ndcg_at_5_diff1": 0.399908, + "nauc_ndcg_at_10_max": 0.376096, + "nauc_ndcg_at_10_std": 0.076335, + "nauc_ndcg_at_10_diff1": 0.386269, + "nauc_ndcg_at_20_max": 0.353295, + "nauc_ndcg_at_20_std": 0.061317, + "nauc_ndcg_at_20_diff1": 0.374513, + "nauc_ndcg_at_100_max": 0.339338, + "nauc_ndcg_at_100_std": 0.050413, + "nauc_ndcg_at_100_diff1": 0.367688, + "nauc_ndcg_at_1000_max": 0.327708, + "nauc_ndcg_at_1000_std": 0.03873, + "nauc_ndcg_at_1000_diff1": 0.368573, + "nauc_map_at_1_max": 0.398932, + "nauc_map_at_1_std": 0.133386, + "nauc_map_at_1_diff1": 0.541293, + "nauc_map_at_3_max": 0.372136, + "nauc_map_at_3_std": 0.10276, + "nauc_map_at_3_diff1": 0.469136, + "nauc_map_at_5_max": 0.364078, + "nauc_map_at_5_std": 0.084926, + "nauc_map_at_5_diff1": 0.442082, + "nauc_map_at_10_max": 0.365942, + "nauc_map_at_10_std": 0.069918, + "nauc_map_at_10_diff1": 0.421892, + "nauc_map_at_20_max": 0.355654, + "nauc_map_at_20_std": 0.061601, + "nauc_map_at_20_diff1": 0.418068, + "nauc_map_at_100_max": 0.353543, + "nauc_map_at_100_std": 0.061426, + "nauc_map_at_100_diff1": 0.413007, + "nauc_map_at_1000_max": 0.352408, + "nauc_map_at_1000_std": 0.060583, + "nauc_map_at_1000_diff1": 0.411705, + "nauc_recall_at_1_max": 0.398932, + "nauc_recall_at_1_std": 0.133386, + "nauc_recall_at_1_diff1": 0.541293, + "nauc_recall_at_3_max": 0.343719, + "nauc_recall_at_3_std": 0.099084, + "nauc_recall_at_3_diff1": 0.388904, + "nauc_recall_at_5_max": 0.378981, + "nauc_recall_at_5_std": 0.124814, + "nauc_recall_at_5_diff1": 0.316109, + "nauc_recall_at_10_max": 0.393171, + "nauc_recall_at_10_std": 0.156633, + "nauc_recall_at_10_diff1": 0.203692, + "nauc_recall_at_20_max": 0.332306, + "nauc_recall_at_20_std": 0.128358, + "nauc_recall_at_20_diff1": 0.139882, + "nauc_recall_at_100_max": 0.305658, + "nauc_recall_at_100_std": 0.130393, + "nauc_recall_at_100_diff1": 0.037992, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.324956, + "nauc_precision_at_1_std": 0.003156, + "nauc_precision_at_1_diff1": 0.502001, + "nauc_precision_at_3_max": 0.058261, + "nauc_precision_at_3_std": -0.087335, + "nauc_precision_at_3_diff1": 0.048377, + "nauc_precision_at_5_max": -0.034758, + "nauc_precision_at_5_std": -0.125666, + "nauc_precision_at_5_diff1": -0.122196, + "nauc_precision_at_10_max": -0.105951, + "nauc_precision_at_10_std": -0.163394, + "nauc_precision_at_10_diff1": -0.223701, + "nauc_precision_at_20_max": -0.178936, + "nauc_precision_at_20_std": -0.180597, + "nauc_precision_at_20_diff1": -0.256049, + "nauc_precision_at_100_max": -0.250061, + "nauc_precision_at_100_std": -0.184208, + "nauc_precision_at_100_diff1": -0.308433, + "nauc_precision_at_1000_max": -0.314711, + "nauc_precision_at_1000_std": -0.21942, + "nauc_precision_at_1000_diff1": -0.346832, + "nauc_cv_recall_at_1_max": 0.324956, + "nauc_cv_recall_at_1_std": 0.003156, + "nauc_cv_recall_at_1_diff1": 0.502001, + "nauc_cv_recall_at_3_max": 0.284133, + "nauc_cv_recall_at_3_std": 0.007935, + "nauc_cv_recall_at_3_diff1": 0.347122, + "nauc_cv_recall_at_5_max": 0.416124, + "nauc_cv_recall_at_5_std": 0.155365, + "nauc_cv_recall_at_5_diff1": 0.153352, + "nauc_cv_recall_at_10_max": 0.503947, + "nauc_cv_recall_at_10_std": 0.375817, + "nauc_cv_recall_at_10_diff1": -0.036839, + "nauc_cv_recall_at_20_max": 0.455441, + "nauc_cv_recall_at_20_std": 0.357973, + "nauc_cv_recall_at_20_diff1": -0.098454, + "nauc_cv_recall_at_100_max": -0.085901, + "nauc_cv_recall_at_100_std": 0.286648, + "nauc_cv_recall_at_100_diff1": -1.151727, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.324956, + "nauc_mrr_at_1_std": 0.003156, + "nauc_mrr_at_1_diff1": 0.502001, + "nauc_mrr_at_3_max": 0.310245, + "nauc_mrr_at_3_std": 0.007737, + "nauc_mrr_at_3_diff1": 0.455631, + "nauc_mrr_at_5_max": 0.3277, + "nauc_mrr_at_5_std": 0.02405, + "nauc_mrr_at_5_diff1": 0.438797, + "nauc_mrr_at_10_max": 0.326613, + "nauc_mrr_at_10_std": 0.025432, + "nauc_mrr_at_10_diff1": 0.442905, + "nauc_mrr_at_20_max": 0.324933, + "nauc_mrr_at_20_std": 0.023584, + "nauc_mrr_at_20_diff1": 0.443787, + "nauc_mrr_at_100_max": 0.323529, + "nauc_mrr_at_100_std": 0.022166, + "nauc_mrr_at_100_diff1": 0.443362, + "nauc_mrr_at_1000_max": 0.323568, + "nauc_mrr_at_1000_std": 0.022129, + "nauc_mrr_at_1000_diff1": 0.443556, + "main_score": 0.57156, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.5625, + "ndcg_at_3": 0.58607, + "ndcg_at_5": 0.60053, + "ndcg_at_10": 0.63867, + "ndcg_at_20": 0.66212, + "ndcg_at_100": 0.69824, + "ndcg_at_1000": 0.71449, + "map_at_1": 0.34522, + "map_at_3": 0.47085, + "map_at_5": 0.5054, + "map_at_10": 0.5429, + "map_at_20": 0.55653, + "map_at_100": 0.56899, + "map_at_1000": 0.57061, + "recall_at_1": 0.34522, + "recall_at_3": 0.5482, + "recall_at_5": 0.62566, + "recall_at_10": 0.73898, + "recall_at_20": 0.80658, + "recall_at_100": 0.92065, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.5625, + "cv_recall_at_3": 0.80625, + "cv_recall_at_5": 0.86875, + "cv_recall_at_10": 0.925, + "cv_recall_at_20": 0.95, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.5625, + "precision_at_3": 0.36875, + "precision_at_5": 0.275, + "precision_at_10": 0.17813, + "precision_at_20": 0.10281, + "precision_at_100": 0.02731, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.5625, + "mrr_at_3": 0.671875, + "mrr_at_5": 0.685312, + "mrr_at_10": 0.693068, + "mrr_at_20": 0.695119, + "mrr_at_100": 0.695862, + "mrr_at_1000": 0.695979, + "nauc_ndcg_at_1_max": 0.244311, + "nauc_ndcg_at_1_std": 0.01225, + "nauc_ndcg_at_1_diff1": 0.405685, + "nauc_ndcg_at_3_max": 0.343494, + "nauc_ndcg_at_3_std": 0.0794, + "nauc_ndcg_at_3_diff1": 0.364077, + "nauc_ndcg_at_5_max": 0.361719, + "nauc_ndcg_at_5_std": 0.064545, + "nauc_ndcg_at_5_diff1": 0.39062, + "nauc_ndcg_at_10_max": 0.315988, + "nauc_ndcg_at_10_std": 0.036356, + "nauc_ndcg_at_10_diff1": 0.393609, + "nauc_ndcg_at_20_max": 0.314389, + "nauc_ndcg_at_20_std": 0.038386, + "nauc_ndcg_at_20_diff1": 0.364327, + "nauc_ndcg_at_100_max": 0.301187, + "nauc_ndcg_at_100_std": 0.046126, + "nauc_ndcg_at_100_diff1": 0.340772, + "nauc_ndcg_at_1000_max": 0.28694, + "nauc_ndcg_at_1000_std": 0.033766, + "nauc_ndcg_at_1000_diff1": 0.339161, + "nauc_map_at_1_max": 0.335189, + "nauc_map_at_1_std": 0.110924, + "nauc_map_at_1_diff1": 0.536379, + "nauc_map_at_3_max": 0.362688, + "nauc_map_at_3_std": 0.101508, + "nauc_map_at_3_diff1": 0.472171, + "nauc_map_at_5_max": 0.357981, + "nauc_map_at_5_std": 0.063234, + "nauc_map_at_5_diff1": 0.462558, + "nauc_map_at_10_max": 0.334715, + "nauc_map_at_10_std": 0.050672, + "nauc_map_at_10_diff1": 0.432745, + "nauc_map_at_20_max": 0.332374, + "nauc_map_at_20_std": 0.048999, + "nauc_map_at_20_diff1": 0.41845, + "nauc_map_at_100_max": 0.32932, + "nauc_map_at_100_std": 0.050824, + "nauc_map_at_100_diff1": 0.407618, + "nauc_map_at_1000_max": 0.328196, + "nauc_map_at_1000_std": 0.050259, + "nauc_map_at_1000_diff1": 0.406553, + "nauc_recall_at_1_max": 0.335189, + "nauc_recall_at_1_std": 0.110924, + "nauc_recall_at_1_diff1": 0.536379, + "nauc_recall_at_3_max": 0.371548, + "nauc_recall_at_3_std": 0.098445, + "nauc_recall_at_3_diff1": 0.408501, + "nauc_recall_at_5_max": 0.392809, + "nauc_recall_at_5_std": 0.052218, + "nauc_recall_at_5_diff1": 0.391437, + "nauc_recall_at_10_max": 0.288744, + "nauc_recall_at_10_std": 0.005464, + "nauc_recall_at_10_diff1": 0.342588, + "nauc_recall_at_20_max": 0.283021, + "nauc_recall_at_20_std": 0.032608, + "nauc_recall_at_20_diff1": 0.231633, + "nauc_recall_at_100_max": 0.288196, + "nauc_recall_at_100_std": 0.150914, + "nauc_recall_at_100_diff1": 0.123547, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.244311, + "nauc_precision_at_1_std": 0.01225, + "nauc_precision_at_1_diff1": 0.405685, + "nauc_precision_at_3_max": 0.087896, + "nauc_precision_at_3_std": -0.033994, + "nauc_precision_at_3_diff1": -0.076149, + "nauc_precision_at_5_max": 0.00789, + "nauc_precision_at_5_std": -0.102593, + "nauc_precision_at_5_diff1": -0.180256, + "nauc_precision_at_10_max": -0.126207, + "nauc_precision_at_10_std": -0.111304, + "nauc_precision_at_10_diff1": -0.303242, + "nauc_precision_at_20_max": -0.154484, + "nauc_precision_at_20_std": -0.109298, + "nauc_precision_at_20_diff1": -0.361762, + "nauc_precision_at_100_max": -0.224203, + "nauc_precision_at_100_std": -0.094251, + "nauc_precision_at_100_diff1": -0.396856, + "nauc_precision_at_1000_max": -0.287754, + "nauc_precision_at_1000_std": -0.135299, + "nauc_precision_at_1000_diff1": -0.404773, + "nauc_cv_recall_at_1_max": 0.244311, + "nauc_cv_recall_at_1_std": 0.01225, + "nauc_cv_recall_at_1_diff1": 0.405685, + "nauc_cv_recall_at_3_max": 0.308157, + "nauc_cv_recall_at_3_std": 0.044806, + "nauc_cv_recall_at_3_diff1": 0.187148, + "nauc_cv_recall_at_5_max": 0.447272, + "nauc_cv_recall_at_5_std": 0.163508, + "nauc_cv_recall_at_5_diff1": 0.121426, + "nauc_cv_recall_at_10_max": 0.366363, + "nauc_cv_recall_at_10_std": 0.186547, + "nauc_cv_recall_at_10_diff1": 0.070962, + "nauc_cv_recall_at_20_max": 0.425303, + "nauc_cv_recall_at_20_std": 0.449872, + "nauc_cv_recall_at_20_diff1": -0.161006, + "nauc_cv_recall_at_100_max": 0.022176, + "nauc_cv_recall_at_100_std": 0.327381, + "nauc_cv_recall_at_100_diff1": -0.389122, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.244311, + "nauc_mrr_at_1_std": 0.01225, + "nauc_mrr_at_1_diff1": 0.405685, + "nauc_mrr_at_3_max": 0.264286, + "nauc_mrr_at_3_std": 0.024335, + "nauc_mrr_at_3_diff1": 0.336539, + "nauc_mrr_at_5_max": 0.275725, + "nauc_mrr_at_5_std": 0.034192, + "nauc_mrr_at_5_diff1": 0.336451, + "nauc_mrr_at_10_max": 0.267679, + "nauc_mrr_at_10_std": 0.032618, + "nauc_mrr_at_10_diff1": 0.34167, + "nauc_mrr_at_20_max": 0.268242, + "nauc_mrr_at_20_std": 0.036238, + "nauc_mrr_at_20_diff1": 0.3401, + "nauc_mrr_at_100_max": 0.266521, + "nauc_mrr_at_100_std": 0.035018, + "nauc_mrr_at_100_diff1": 0.340649, + "nauc_mrr_at_1000_max": 0.266343, + "nauc_mrr_at_1000_std": 0.034816, + "nauc_mrr_at_1000_diff1": 0.340782, + "main_score": 0.60053, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.49375, + "ndcg_at_3": 0.5113, + "ndcg_at_5": 0.5387, + "ndcg_at_10": 0.57397, + "ndcg_at_20": 0.59942, + "ndcg_at_100": 0.63523, + "ndcg_at_1000": 0.65855, + "map_at_1": 0.29642, + "map_at_3": 0.40658, + "map_at_5": 0.44544, + "map_at_10": 0.47805, + "map_at_20": 0.49193, + "map_at_100": 0.50252, + "map_at_1000": 0.50468, + "recall_at_1": 0.29642, + "recall_at_3": 0.48554, + "recall_at_5": 0.58296, + "recall_at_10": 0.69032, + "recall_at_20": 0.76293, + "recall_at_100": 0.88588, + "recall_at_1000": 0.99963, + "cv_recall_at_1": 0.49375, + "cv_recall_at_3": 0.70625, + "cv_recall_at_5": 0.80625, + "cv_recall_at_10": 0.8875, + "cv_recall_at_20": 0.91875, + "cv_recall_at_100": 0.9625, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.49375, + "precision_at_3": 0.32083, + "precision_at_5": 0.25, + "precision_at_10": 0.16313, + "precision_at_20": 0.0975, + "precision_at_100": 0.02531, + "precision_at_1000": 0.00321, + "mrr_at_1": 0.49375, + "mrr_at_3": 0.58125, + "mrr_at_5": 0.604688, + "mrr_at_10": 0.616803, + "mrr_at_20": 0.618884, + "mrr_at_100": 0.620239, + "mrr_at_1000": 0.620442, + "nauc_ndcg_at_1_max": 0.313512, + "nauc_ndcg_at_1_std": 0.164031, + "nauc_ndcg_at_1_diff1": 0.37153, + "nauc_ndcg_at_3_max": 0.338071, + "nauc_ndcg_at_3_std": 0.164985, + "nauc_ndcg_at_3_diff1": 0.356327, + "nauc_ndcg_at_5_max": 0.332775, + "nauc_ndcg_at_5_std": 0.141894, + "nauc_ndcg_at_5_diff1": 0.37733, + "nauc_ndcg_at_10_max": 0.345221, + "nauc_ndcg_at_10_std": 0.13061, + "nauc_ndcg_at_10_diff1": 0.354153, + "nauc_ndcg_at_20_max": 0.322545, + "nauc_ndcg_at_20_std": 0.118045, + "nauc_ndcg_at_20_diff1": 0.340573, + "nauc_ndcg_at_100_max": 0.312193, + "nauc_ndcg_at_100_std": 0.121741, + "nauc_ndcg_at_100_diff1": 0.317008, + "nauc_ndcg_at_1000_max": 0.307293, + "nauc_ndcg_at_1000_std": 0.128905, + "nauc_ndcg_at_1000_diff1": 0.332466, + "nauc_map_at_1_max": 0.368848, + "nauc_map_at_1_std": 0.255006, + "nauc_map_at_1_diff1": 0.507504, + "nauc_map_at_3_max": 0.360978, + "nauc_map_at_3_std": 0.202499, + "nauc_map_at_3_diff1": 0.425483, + "nauc_map_at_5_max": 0.346723, + "nauc_map_at_5_std": 0.167604, + "nauc_map_at_5_diff1": 0.406427, + "nauc_map_at_10_max": 0.338036, + "nauc_map_at_10_std": 0.14503, + "nauc_map_at_10_diff1": 0.377454, + "nauc_map_at_20_max": 0.326775, + "nauc_map_at_20_std": 0.137433, + "nauc_map_at_20_diff1": 0.372569, + "nauc_map_at_100_max": 0.326399, + "nauc_map_at_100_std": 0.138306, + "nauc_map_at_100_diff1": 0.365188, + "nauc_map_at_1000_max": 0.32606, + "nauc_map_at_1000_std": 0.139216, + "nauc_map_at_1000_diff1": 0.365491, + "nauc_recall_at_1_max": 0.368848, + "nauc_recall_at_1_std": 0.255006, + "nauc_recall_at_1_diff1": 0.507504, + "nauc_recall_at_3_max": 0.348283, + "nauc_recall_at_3_std": 0.171748, + "nauc_recall_at_3_diff1": 0.368674, + "nauc_recall_at_5_max": 0.30573, + "nauc_recall_at_5_std": 0.086948, + "nauc_recall_at_5_diff1": 0.338382, + "nauc_recall_at_10_max": 0.331597, + "nauc_recall_at_10_std": 0.059849, + "nauc_recall_at_10_diff1": 0.221537, + "nauc_recall_at_20_max": 0.288412, + "nauc_recall_at_20_std": 0.030797, + "nauc_recall_at_20_diff1": 0.172216, + "nauc_recall_at_100_max": 0.213075, + "nauc_recall_at_100_std": -0.024167, + "nauc_recall_at_100_diff1": -0.039337, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 0.722222, + "nauc_precision_at_1_max": 0.313512, + "nauc_precision_at_1_std": 0.164031, + "nauc_precision_at_1_diff1": 0.37153, + "nauc_precision_at_3_max": 0.138163, + "nauc_precision_at_3_std": -0.027625, + "nauc_precision_at_3_diff1": 0.030776, + "nauc_precision_at_5_max": 0.030696, + "nauc_precision_at_5_std": -0.124224, + "nauc_precision_at_5_diff1": -0.066226, + "nauc_precision_at_10_max": -0.049848, + "nauc_precision_at_10_std": -0.150374, + "nauc_precision_at_10_diff1": -0.196022, + "nauc_precision_at_20_max": -0.126853, + "nauc_precision_at_20_std": -0.17505, + "nauc_precision_at_20_diff1": -0.216096, + "nauc_precision_at_100_max": -0.154714, + "nauc_precision_at_100_std": -0.118722, + "nauc_precision_at_100_diff1": -0.275639, + "nauc_precision_at_1000_max": -0.239769, + "nauc_precision_at_1000_std": -0.151243, + "nauc_precision_at_1000_diff1": -0.291271, + "nauc_cv_recall_at_1_max": 0.313512, + "nauc_cv_recall_at_1_std": 0.164031, + "nauc_cv_recall_at_1_diff1": 0.37153, + "nauc_cv_recall_at_3_max": 0.377339, + "nauc_cv_recall_at_3_std": 0.154992, + "nauc_cv_recall_at_3_diff1": 0.35732, + "nauc_cv_recall_at_5_max": 0.30454, + "nauc_cv_recall_at_5_std": 0.017909, + "nauc_cv_recall_at_5_diff1": 0.40698, + "nauc_cv_recall_at_10_max": 0.445863, + "nauc_cv_recall_at_10_std": 0.14112, + "nauc_cv_recall_at_10_diff1": 0.230998, + "nauc_cv_recall_at_20_max": 0.439525, + "nauc_cv_recall_at_20_std": 0.20265, + "nauc_cv_recall_at_20_diff1": 0.170581, + "nauc_cv_recall_at_100_max": 0.155618, + "nauc_cv_recall_at_100_std": 0.137411, + "nauc_cv_recall_at_100_diff1": -0.427093, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.313512, + "nauc_mrr_at_1_std": 0.164031, + "nauc_mrr_at_1_diff1": 0.37153, + "nauc_mrr_at_3_max": 0.333998, + "nauc_mrr_at_3_std": 0.164765, + "nauc_mrr_at_3_diff1": 0.373675, + "nauc_mrr_at_5_max": 0.321233, + "nauc_mrr_at_5_std": 0.146522, + "nauc_mrr_at_5_diff1": 0.381877, + "nauc_mrr_at_10_max": 0.327015, + "nauc_mrr_at_10_std": 0.155494, + "nauc_mrr_at_10_diff1": 0.369744, + "nauc_mrr_at_20_max": 0.326265, + "nauc_mrr_at_20_std": 0.156873, + "nauc_mrr_at_20_diff1": 0.369803, + "nauc_mrr_at_100_max": 0.32458, + "nauc_mrr_at_100_std": 0.156465, + "nauc_mrr_at_100_diff1": 0.368431, + "nauc_mrr_at_1000_max": 0.32444, + "nauc_mrr_at_1000_std": 0.156418, + "nauc_mrr_at_1000_diff1": 0.368713, + "main_score": 0.5387, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 153.49859976768494, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/Vidore2ESGReportsHLRetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..e895584f31 --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.50641, + "ndcg_at_3": 0.57155, + "ndcg_at_5": 0.60446, + "ndcg_at_10": 0.62703, + "ndcg_at_20": 0.65675, + "ndcg_at_100": 0.67969, + "ndcg_at_1000": 0.6943, + "map_at_1": 0.33686, + "map_at_3": 0.48323, + "map_at_5": 0.52465, + "map_at_10": 0.5465, + "map_at_20": 0.56121, + "map_at_100": 0.56753, + "map_at_1000": 0.56887, + "recall_at_1": 0.33686, + "recall_at_3": 0.59295, + "recall_at_5": 0.69121, + "recall_at_10": 0.73815, + "recall_at_20": 0.8305, + "recall_at_100": 0.91948, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.51923, + "cv_recall_at_3": 0.76923, + "cv_recall_at_5": 0.84615, + "cv_recall_at_10": 0.86538, + "cv_recall_at_20": 0.92308, + "cv_recall_at_100": 0.98077, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.51923, + "precision_at_3": 0.32692, + "precision_at_5": 0.24615, + "precision_at_10": 0.14808, + "precision_at_20": 0.08942, + "precision_at_100": 0.02115, + "precision_at_1000": 0.00246, + "mrr_at_1": 0.519231, + "mrr_at_3": 0.637821, + "mrr_at_5": 0.655128, + "mrr_at_10": 0.657532, + "mrr_at_20": 0.661108, + "mrr_at_100": 0.66325, + "mrr_at_1000": 0.663317, + "nauc_ndcg_at_1_max": 0.17372, + "nauc_ndcg_at_1_std": 0.20764, + "nauc_ndcg_at_1_diff1": 0.551664, + "nauc_ndcg_at_3_max": 0.133448, + "nauc_ndcg_at_3_std": 0.196954, + "nauc_ndcg_at_3_diff1": 0.606798, + "nauc_ndcg_at_5_max": 0.226665, + "nauc_ndcg_at_5_std": 0.249826, + "nauc_ndcg_at_5_diff1": 0.632506, + "nauc_ndcg_at_10_max": 0.211194, + "nauc_ndcg_at_10_std": 0.235927, + "nauc_ndcg_at_10_diff1": 0.624351, + "nauc_ndcg_at_20_max": 0.192765, + "nauc_ndcg_at_20_std": 0.244563, + "nauc_ndcg_at_20_diff1": 0.620096, + "nauc_ndcg_at_100_max": 0.212663, + "nauc_ndcg_at_100_std": 0.254371, + "nauc_ndcg_at_100_diff1": 0.625697, + "nauc_ndcg_at_1000_max": 0.179079, + "nauc_ndcg_at_1000_std": 0.216451, + "nauc_ndcg_at_1000_diff1": 0.608083, + "nauc_map_at_1_max": 0.275354, + "nauc_map_at_1_std": 0.229158, + "nauc_map_at_1_diff1": 0.708342, + "nauc_map_at_3_max": 0.181403, + "nauc_map_at_3_std": 0.170427, + "nauc_map_at_3_diff1": 0.657973, + "nauc_map_at_5_max": 0.233171, + "nauc_map_at_5_std": 0.209065, + "nauc_map_at_5_diff1": 0.655261, + "nauc_map_at_10_max": 0.230068, + "nauc_map_at_10_std": 0.210988, + "nauc_map_at_10_diff1": 0.643373, + "nauc_map_at_20_max": 0.216178, + "nauc_map_at_20_std": 0.223874, + "nauc_map_at_20_diff1": 0.646782, + "nauc_map_at_100_max": 0.221249, + "nauc_map_at_100_std": 0.231277, + "nauc_map_at_100_diff1": 0.644268, + "nauc_map_at_1000_max": 0.218925, + "nauc_map_at_1000_std": 0.228542, + "nauc_map_at_1000_diff1": 0.643013, + "nauc_recall_at_1_max": 0.275354, + "nauc_recall_at_1_std": 0.229158, + "nauc_recall_at_1_diff1": 0.708342, + "nauc_recall_at_3_max": 0.096372, + "nauc_recall_at_3_std": 0.091876, + "nauc_recall_at_3_diff1": 0.54665, + "nauc_recall_at_5_max": 0.335235, + "nauc_recall_at_5_std": 0.28845, + "nauc_recall_at_5_diff1": 0.550651, + "nauc_recall_at_10_max": 0.315472, + "nauc_recall_at_10_std": 0.291701, + "nauc_recall_at_10_diff1": 0.570237, + "nauc_recall_at_20_max": 0.286079, + "nauc_recall_at_20_std": 0.417942, + "nauc_recall_at_20_diff1": 0.59223, + "nauc_recall_at_100_max": 0.64213, + "nauc_recall_at_100_std": 0.75358, + "nauc_recall_at_100_diff1": 0.75902, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.149048, + "nauc_precision_at_1_std": 0.189751, + "nauc_precision_at_1_diff1": 0.579569, + "nauc_precision_at_3_max": -0.081672, + "nauc_precision_at_3_std": 0.01809, + "nauc_precision_at_3_diff1": 0.192829, + "nauc_precision_at_5_max": 0.031765, + "nauc_precision_at_5_std": 0.054654, + "nauc_precision_at_5_diff1": 0.042902, + "nauc_precision_at_10_max": -0.06486, + "nauc_precision_at_10_std": 0.049072, + "nauc_precision_at_10_diff1": -0.049913, + "nauc_precision_at_20_max": -0.135187, + "nauc_precision_at_20_std": 0.066593, + "nauc_precision_at_20_diff1": -0.151797, + "nauc_precision_at_100_max": -0.143625, + "nauc_precision_at_100_std": 0.019572, + "nauc_precision_at_100_diff1": -0.254781, + "nauc_precision_at_1000_max": -0.219508, + "nauc_precision_at_1000_std": -0.123489, + "nauc_precision_at_1000_diff1": -0.345385, + "nauc_cv_recall_at_1_max": 0.149048, + "nauc_cv_recall_at_1_std": 0.189751, + "nauc_cv_recall_at_1_diff1": 0.579569, + "nauc_cv_recall_at_3_max": -0.157805, + "nauc_cv_recall_at_3_std": -0.026732, + "nauc_cv_recall_at_3_diff1": 0.567649, + "nauc_cv_recall_at_5_max": 0.281394, + "nauc_cv_recall_at_5_std": 0.21806, + "nauc_cv_recall_at_5_diff1": 0.528353, + "nauc_cv_recall_at_10_max": 0.235475, + "nauc_cv_recall_at_10_std": 0.138754, + "nauc_cv_recall_at_10_diff1": 0.489093, + "nauc_cv_recall_at_20_max": 0.099959, + "nauc_cv_recall_at_20_std": 0.196772, + "nauc_cv_recall_at_20_diff1": 0.39905, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 0.869837, + "nauc_cv_recall_at_100_diff1": 0.724179, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.149048, + "nauc_mrr_at_1_std": 0.189751, + "nauc_mrr_at_1_diff1": 0.579569, + "nauc_mrr_at_3_max": 0.051224, + "nauc_mrr_at_3_std": 0.127852, + "nauc_mrr_at_3_diff1": 0.571901, + "nauc_mrr_at_5_max": 0.119855, + "nauc_mrr_at_5_std": 0.168429, + "nauc_mrr_at_5_diff1": 0.564559, + "nauc_mrr_at_10_max": 0.115769, + "nauc_mrr_at_10_std": 0.163184, + "nauc_mrr_at_10_diff1": 0.562443, + "nauc_mrr_at_20_max": 0.110317, + "nauc_mrr_at_20_std": 0.162559, + "nauc_mrr_at_20_diff1": 0.560825, + "nauc_mrr_at_100_max": 0.112922, + "nauc_mrr_at_100_std": 0.164673, + "nauc_mrr_at_100_diff1": 0.561863, + "nauc_mrr_at_1000_max": 0.11269, + "nauc_mrr_at_1000_std": 0.164489, + "nauc_mrr_at_1000_diff1": 0.561821, + "main_score": 0.60446, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 147.7470908164978, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/Vidore2ESGReportsRetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..6c812da7e2 --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.61404, + "ndcg_at_3": 0.5454, + "ndcg_at_5": 0.55921, + "ndcg_at_10": 0.58837, + "ndcg_at_20": 0.62379, + "ndcg_at_100": 0.66881, + "ndcg_at_1000": 0.68318, + "map_at_1": 0.32533, + "map_at_3": 0.406, + "map_at_5": 0.44818, + "map_at_10": 0.48376, + "map_at_20": 0.50319, + "map_at_100": 0.52117, + "map_at_1000": 0.52279, + "recall_at_1": 0.32533, + "recall_at_3": 0.45656, + "recall_at_5": 0.54004, + "recall_at_10": 0.66381, + "recall_at_20": 0.77707, + "recall_at_100": 0.92118, + "recall_at_1000": 0.98977, + "cv_recall_at_1": 0.61404, + "cv_recall_at_3": 0.75439, + "cv_recall_at_5": 0.78947, + "cv_recall_at_10": 0.84211, + "cv_recall_at_20": 0.91228, + "cv_recall_at_100": 0.96491, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.61404, + "precision_at_3": 0.34503, + "precision_at_5": 0.26667, + "precision_at_10": 0.17368, + "precision_at_20": 0.11316, + "precision_at_100": 0.03316, + "precision_at_1000": 0.00386, + "mrr_at_1": 0.614035, + "mrr_at_3": 0.675439, + "mrr_at_5": 0.684211, + "mrr_at_10": 0.689912, + "mrr_at_20": 0.694356, + "mrr_at_100": 0.696213, + "mrr_at_1000": 0.696404, + "nauc_ndcg_at_1_max": -0.193435, + "nauc_ndcg_at_1_std": -0.024512, + "nauc_ndcg_at_1_diff1": 0.333707, + "nauc_ndcg_at_3_max": -0.106424, + "nauc_ndcg_at_3_std": -0.039297, + "nauc_ndcg_at_3_diff1": 0.154997, + "nauc_ndcg_at_5_max": -0.074913, + "nauc_ndcg_at_5_std": -0.006925, + "nauc_ndcg_at_5_diff1": 0.101165, + "nauc_ndcg_at_10_max": -0.010028, + "nauc_ndcg_at_10_std": 0.050504, + "nauc_ndcg_at_10_diff1": 0.138468, + "nauc_ndcg_at_20_max": -0.029896, + "nauc_ndcg_at_20_std": 0.065991, + "nauc_ndcg_at_20_diff1": 0.163513, + "nauc_ndcg_at_100_max": -0.077312, + "nauc_ndcg_at_100_std": 0.03835, + "nauc_ndcg_at_100_diff1": 0.14983, + "nauc_ndcg_at_1000_max": -0.115529, + "nauc_ndcg_at_1000_std": -0.00655, + "nauc_ndcg_at_1000_diff1": 0.164917, + "nauc_map_at_1_max": -0.000174, + "nauc_map_at_1_std": 0.020903, + "nauc_map_at_1_diff1": 0.246163, + "nauc_map_at_3_max": 0.017174, + "nauc_map_at_3_std": 0.038951, + "nauc_map_at_3_diff1": 0.177329, + "nauc_map_at_5_max": 0.014563, + "nauc_map_at_5_std": 0.045814, + "nauc_map_at_5_diff1": 0.133605, + "nauc_map_at_10_max": 0.029742, + "nauc_map_at_10_std": 0.05906, + "nauc_map_at_10_diff1": 0.168173, + "nauc_map_at_20_max": 0.004569, + "nauc_map_at_20_std": 0.052578, + "nauc_map_at_20_diff1": 0.162685, + "nauc_map_at_100_max": -0.018153, + "nauc_map_at_100_std": 0.037446, + "nauc_map_at_100_diff1": 0.150934, + "nauc_map_at_1000_max": -0.022514, + "nauc_map_at_1000_std": 0.033947, + "nauc_map_at_1000_diff1": 0.151596, + "nauc_recall_at_1_max": -0.000174, + "nauc_recall_at_1_std": 0.020903, + "nauc_recall_at_1_diff1": 0.246163, + "nauc_recall_at_3_max": 0.13451, + "nauc_recall_at_3_std": 0.132324, + "nauc_recall_at_3_diff1": 0.08804, + "nauc_recall_at_5_max": 0.147058, + "nauc_recall_at_5_std": 0.150067, + "nauc_recall_at_5_diff1": -0.015516, + "nauc_recall_at_10_max": 0.192352, + "nauc_recall_at_10_std": 0.208404, + "nauc_recall_at_10_diff1": -0.003044, + "nauc_recall_at_20_max": 0.223354, + "nauc_recall_at_20_std": 0.365581, + "nauc_recall_at_20_diff1": 0.10133, + "nauc_recall_at_100_max": 0.279571, + "nauc_recall_at_100_std": 0.643495, + "nauc_recall_at_100_diff1": 0.03675, + "nauc_recall_at_1000_max": 0.881171, + "nauc_recall_at_1000_std": 0.943648, + "nauc_recall_at_1000_diff1": 0.442381, + "nauc_precision_at_1_max": -0.193435, + "nauc_precision_at_1_std": -0.024512, + "nauc_precision_at_1_diff1": 0.333707, + "nauc_precision_at_3_max": -0.250102, + "nauc_precision_at_3_std": -0.111526, + "nauc_precision_at_3_diff1": 0.019213, + "nauc_precision_at_5_max": -0.239677, + "nauc_precision_at_5_std": -0.122729, + "nauc_precision_at_5_diff1": -0.072633, + "nauc_precision_at_10_max": -0.199279, + "nauc_precision_at_10_std": -0.124981, + "nauc_precision_at_10_diff1": -0.026108, + "nauc_precision_at_20_max": -0.327942, + "nauc_precision_at_20_std": -0.187234, + "nauc_precision_at_20_diff1": -0.081736, + "nauc_precision_at_100_max": -0.355689, + "nauc_precision_at_100_std": -0.197597, + "nauc_precision_at_100_diff1": -0.182991, + "nauc_precision_at_1000_max": -0.402984, + "nauc_precision_at_1000_std": -0.273332, + "nauc_precision_at_1000_diff1": -0.107338, + "nauc_cv_recall_at_1_max": -0.193435, + "nauc_cv_recall_at_1_std": -0.024512, + "nauc_cv_recall_at_1_diff1": 0.333707, + "nauc_cv_recall_at_3_max": -0.172238, + "nauc_cv_recall_at_3_std": 0.009623, + "nauc_cv_recall_at_3_diff1": 0.14768, + "nauc_cv_recall_at_5_max": -0.117077, + "nauc_cv_recall_at_5_std": -0.017054, + "nauc_cv_recall_at_5_diff1": 0.103388, + "nauc_cv_recall_at_10_max": -0.164752, + "nauc_cv_recall_at_10_std": -0.013221, + "nauc_cv_recall_at_10_diff1": -0.007342, + "nauc_cv_recall_at_20_max": -0.176566, + "nauc_cv_recall_at_20_std": 0.011259, + "nauc_cv_recall_at_20_diff1": 0.406936, + "nauc_cv_recall_at_100_max": -0.502563, + "nauc_cv_recall_at_100_std": 0.351784, + "nauc_cv_recall_at_100_diff1": 0.242235, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.193435, + "nauc_mrr_at_1_std": -0.024512, + "nauc_mrr_at_1_diff1": 0.333707, + "nauc_mrr_at_3_max": -0.210329, + "nauc_mrr_at_3_std": -0.047561, + "nauc_mrr_at_3_diff1": 0.263681, + "nauc_mrr_at_5_max": -0.201063, + "nauc_mrr_at_5_std": -0.053866, + "nauc_mrr_at_5_diff1": 0.2583, + "nauc_mrr_at_10_max": -0.205715, + "nauc_mrr_at_10_std": -0.054538, + "nauc_mrr_at_10_diff1": 0.254125, + "nauc_mrr_at_20_max": -0.203542, + "nauc_mrr_at_20_std": -0.051392, + "nauc_mrr_at_20_diff1": 0.264625, + "nauc_mrr_at_100_max": -0.206101, + "nauc_mrr_at_100_std": -0.051694, + "nauc_mrr_at_100_diff1": 0.262381, + "nauc_mrr_at_1000_max": -0.206447, + "nauc_mrr_at_1000_std": -0.052239, + "nauc_mrr_at_1000_diff1": 0.262342, + "main_score": 0.55921, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.52632, + "ndcg_at_3": 0.52988, + "ndcg_at_5": 0.55222, + "ndcg_at_10": 0.58584, + "ndcg_at_20": 0.61753, + "ndcg_at_100": 0.66404, + "ndcg_at_1000": 0.67388, + "map_at_1": 0.27343, + "map_at_3": 0.38746, + "map_at_5": 0.4317, + "map_at_10": 0.47231, + "map_at_20": 0.49094, + "map_at_100": 0.51025, + "map_at_1000": 0.51176, + "recall_at_1": 0.27343, + "recall_at_3": 0.45301, + "recall_at_5": 0.54662, + "recall_at_10": 0.68427, + "recall_at_20": 0.80336, + "recall_at_100": 0.94509, + "recall_at_1000": 0.98977, + "cv_recall_at_1": 0.52632, + "cv_recall_at_3": 0.75439, + "cv_recall_at_5": 0.80702, + "cv_recall_at_10": 0.85965, + "cv_recall_at_20": 0.92982, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.52632, + "precision_at_3": 0.35673, + "precision_at_5": 0.28421, + "precision_at_10": 0.18772, + "precision_at_20": 0.11491, + "precision_at_100": 0.03491, + "precision_at_1000": 0.00386, + "mrr_at_1": 0.526316, + "mrr_at_3": 0.634503, + "mrr_at_5": 0.647661, + "mrr_at_10": 0.654553, + "mrr_at_20": 0.659155, + "mrr_at_100": 0.661056, + "mrr_at_1000": 0.661095, + "nauc_ndcg_at_1_max": -0.030435, + "nauc_ndcg_at_1_std": 0.089728, + "nauc_ndcg_at_1_diff1": 0.154683, + "nauc_ndcg_at_3_max": -0.123816, + "nauc_ndcg_at_3_std": 0.002802, + "nauc_ndcg_at_3_diff1": -0.026521, + "nauc_ndcg_at_5_max": -0.074397, + "nauc_ndcg_at_5_std": 0.070869, + "nauc_ndcg_at_5_diff1": -0.008578, + "nauc_ndcg_at_10_max": 0.03248, + "nauc_ndcg_at_10_std": 0.099597, + "nauc_ndcg_at_10_diff1": 0.040426, + "nauc_ndcg_at_20_max": 0.049417, + "nauc_ndcg_at_20_std": 0.157319, + "nauc_ndcg_at_20_diff1": 0.01287, + "nauc_ndcg_at_100_max": -0.028363, + "nauc_ndcg_at_100_std": 0.119624, + "nauc_ndcg_at_100_diff1": -0.017853, + "nauc_ndcg_at_1000_max": -0.0603, + "nauc_ndcg_at_1000_std": 0.089981, + "nauc_ndcg_at_1000_diff1": 0.009537, + "nauc_map_at_1_max": 0.089738, + "nauc_map_at_1_std": 0.119303, + "nauc_map_at_1_diff1": 0.138385, + "nauc_map_at_3_max": 0.043204, + "nauc_map_at_3_std": 0.105668, + "nauc_map_at_3_diff1": 0.013022, + "nauc_map_at_5_max": 0.053757, + "nauc_map_at_5_std": 0.145387, + "nauc_map_at_5_diff1": 0.010286, + "nauc_map_at_10_max": 0.053282, + "nauc_map_at_10_std": 0.128527, + "nauc_map_at_10_diff1": 0.02649, + "nauc_map_at_20_max": 0.043013, + "nauc_map_at_20_std": 0.139885, + "nauc_map_at_20_diff1": 0.011581, + "nauc_map_at_100_max": 0.011337, + "nauc_map_at_100_std": 0.129078, + "nauc_map_at_100_diff1": -0.010944, + "nauc_map_at_1000_max": 0.007063, + "nauc_map_at_1000_std": 0.125942, + "nauc_map_at_1000_diff1": -0.008309, + "nauc_recall_at_1_max": 0.089738, + "nauc_recall_at_1_std": 0.119303, + "nauc_recall_at_1_diff1": 0.138385, + "nauc_recall_at_3_max": 0.035652, + "nauc_recall_at_3_std": 0.075368, + "nauc_recall_at_3_diff1": -0.04629, + "nauc_recall_at_5_max": 0.08914, + "nauc_recall_at_5_std": 0.156084, + "nauc_recall_at_5_diff1": 0.004377, + "nauc_recall_at_10_max": 0.279487, + "nauc_recall_at_10_std": 0.143437, + "nauc_recall_at_10_diff1": 0.091998, + "nauc_recall_at_20_max": 0.420757, + "nauc_recall_at_20_std": 0.417333, + "nauc_recall_at_20_diff1": -0.032394, + "nauc_recall_at_100_max": 0.553113, + "nauc_recall_at_100_std": 0.705716, + "nauc_recall_at_100_diff1": -0.333804, + "nauc_recall_at_1000_max": 0.924864, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 0.371455, + "nauc_precision_at_1_max": -0.030435, + "nauc_precision_at_1_std": 0.089728, + "nauc_precision_at_1_diff1": 0.154683, + "nauc_precision_at_3_max": -0.251609, + "nauc_precision_at_3_std": -0.049926, + "nauc_precision_at_3_diff1": -0.149806, + "nauc_precision_at_5_max": -0.241017, + "nauc_precision_at_5_std": -0.025293, + "nauc_precision_at_5_diff1": -0.113127, + "nauc_precision_at_10_max": -0.220329, + "nauc_precision_at_10_std": -0.074403, + "nauc_precision_at_10_diff1": -0.077859, + "nauc_precision_at_20_max": -0.262111, + "nauc_precision_at_20_std": -0.075332, + "nauc_precision_at_20_diff1": -0.124609, + "nauc_precision_at_100_max": -0.343422, + "nauc_precision_at_100_std": -0.167414, + "nauc_precision_at_100_diff1": -0.15979, + "nauc_precision_at_1000_max": -0.379701, + "nauc_precision_at_1000_std": -0.223846, + "nauc_precision_at_1000_diff1": -0.050452, + "nauc_cv_recall_at_1_max": -0.030435, + "nauc_cv_recall_at_1_std": 0.089728, + "nauc_cv_recall_at_1_diff1": 0.154683, + "nauc_cv_recall_at_3_max": -0.334307, + "nauc_cv_recall_at_3_std": -0.06442, + "nauc_cv_recall_at_3_diff1": -0.110664, + "nauc_cv_recall_at_5_max": -0.25598, + "nauc_cv_recall_at_5_std": -0.069374, + "nauc_cv_recall_at_5_diff1": 0.103575, + "nauc_cv_recall_at_10_max": 0.032311, + "nauc_cv_recall_at_10_std": -0.070858, + "nauc_cv_recall_at_10_diff1": 0.226934, + "nauc_cv_recall_at_20_max": -0.005196, + "nauc_cv_recall_at_20_std": 0.231265, + "nauc_cv_recall_at_20_diff1": -0.109279, + "nauc_cv_recall_at_100_max": -0.164944, + "nauc_cv_recall_at_100_std": 0.357853, + "nauc_cv_recall_at_100_diff1": -0.559408, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.030435, + "nauc_mrr_at_1_std": 0.089728, + "nauc_mrr_at_1_diff1": 0.154683, + "nauc_mrr_at_3_max": -0.144595, + "nauc_mrr_at_3_std": 0.018221, + "nauc_mrr_at_3_diff1": 0.058614, + "nauc_mrr_at_5_max": -0.125114, + "nauc_mrr_at_5_std": 0.020262, + "nauc_mrr_at_5_diff1": 0.099808, + "nauc_mrr_at_10_max": -0.103626, + "nauc_mrr_at_10_std": 0.022744, + "nauc_mrr_at_10_diff1": 0.107462, + "nauc_mrr_at_20_max": -0.104427, + "nauc_mrr_at_20_std": 0.03361, + "nauc_mrr_at_20_diff1": 0.10094, + "nauc_mrr_at_100_max": -0.105192, + "nauc_mrr_at_100_std": 0.032903, + "nauc_mrr_at_100_diff1": 0.101327, + "nauc_mrr_at_1000_max": -0.105183, + "nauc_mrr_at_1000_std": 0.032854, + "nauc_mrr_at_1000_diff1": 0.101426, + "main_score": 0.55222, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.54386, + "ndcg_at_3": 0.52544, + "ndcg_at_5": 0.55668, + "ndcg_at_10": 0.60259, + "ndcg_at_20": 0.62464, + "ndcg_at_100": 0.67517, + "ndcg_at_1000": 0.68293, + "map_at_1": 0.28523, + "map_at_3": 0.38855, + "map_at_5": 0.44339, + "map_at_10": 0.48887, + "map_at_20": 0.5079, + "map_at_100": 0.52736, + "map_at_1000": 0.52834, + "recall_at_1": 0.28523, + "recall_at_3": 0.44628, + "recall_at_5": 0.57089, + "recall_at_10": 0.72959, + "recall_at_20": 0.79799, + "recall_at_100": 0.95313, + "recall_at_1000": 0.99561, + "cv_recall_at_1": 0.54386, + "cv_recall_at_3": 0.73684, + "cv_recall_at_5": 0.80702, + "cv_recall_at_10": 0.89474, + "cv_recall_at_20": 0.91228, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.54386, + "precision_at_3": 0.35088, + "precision_at_5": 0.2807, + "precision_at_10": 0.19474, + "precision_at_20": 0.11842, + "precision_at_100": 0.03649, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.54386, + "mrr_at_3": 0.625731, + "mrr_at_5": 0.64152, + "mrr_at_10": 0.653599, + "mrr_at_20": 0.655194, + "mrr_at_100": 0.657947, + "mrr_at_1000": 0.657998, + "nauc_ndcg_at_1_max": -0.108049, + "nauc_ndcg_at_1_std": -0.184276, + "nauc_ndcg_at_1_diff1": 0.438554, + "nauc_ndcg_at_3_max": -0.040322, + "nauc_ndcg_at_3_std": -0.081235, + "nauc_ndcg_at_3_diff1": 0.27519, + "nauc_ndcg_at_5_max": -0.037282, + "nauc_ndcg_at_5_std": 0.032016, + "nauc_ndcg_at_5_diff1": 0.184235, + "nauc_ndcg_at_10_max": 0.029739, + "nauc_ndcg_at_10_std": 0.097903, + "nauc_ndcg_at_10_diff1": 0.219084, + "nauc_ndcg_at_20_max": 0.036861, + "nauc_ndcg_at_20_std": 0.078983, + "nauc_ndcg_at_20_diff1": 0.203469, + "nauc_ndcg_at_100_max": -0.035274, + "nauc_ndcg_at_100_std": 0.026445, + "nauc_ndcg_at_100_diff1": 0.200782, + "nauc_ndcg_at_1000_max": -0.047797, + "nauc_ndcg_at_1000_std": 0.009529, + "nauc_ndcg_at_1000_diff1": 0.21076, + "nauc_map_at_1_max": 0.110819, + "nauc_map_at_1_std": 0.007407, + "nauc_map_at_1_diff1": 0.374948, + "nauc_map_at_3_max": 0.107368, + "nauc_map_at_3_std": 0.04765, + "nauc_map_at_3_diff1": 0.346757, + "nauc_map_at_5_max": 0.09747, + "nauc_map_at_5_std": 0.093228, + "nauc_map_at_5_diff1": 0.288895, + "nauc_map_at_10_max": 0.091303, + "nauc_map_at_10_std": 0.101554, + "nauc_map_at_10_diff1": 0.287703, + "nauc_map_at_20_max": 0.074978, + "nauc_map_at_20_std": 0.087001, + "nauc_map_at_20_diff1": 0.271607, + "nauc_map_at_100_max": 0.048792, + "nauc_map_at_100_std": 0.07464, + "nauc_map_at_100_diff1": 0.254691, + "nauc_map_at_1000_max": 0.046715, + "nauc_map_at_1000_std": 0.07358, + "nauc_map_at_1000_diff1": 0.253236, + "nauc_recall_at_1_max": 0.110819, + "nauc_recall_at_1_std": 0.007407, + "nauc_recall_at_1_diff1": 0.374948, + "nauc_recall_at_3_max": 0.09948, + "nauc_recall_at_3_std": 0.027449, + "nauc_recall_at_3_diff1": 0.26822, + "nauc_recall_at_5_max": 0.070367, + "nauc_recall_at_5_std": 0.216325, + "nauc_recall_at_5_diff1": 0.069157, + "nauc_recall_at_10_max": 0.158164, + "nauc_recall_at_10_std": 0.336194, + "nauc_recall_at_10_diff1": 0.031528, + "nauc_recall_at_20_max": 0.235273, + "nauc_recall_at_20_std": 0.331222, + "nauc_recall_at_20_diff1": -0.109564, + "nauc_recall_at_100_max": 0.127854, + "nauc_recall_at_100_std": 0.477092, + "nauc_recall_at_100_diff1": -0.391428, + "nauc_recall_at_1000_max": 0.868512, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 0.126616, + "nauc_precision_at_1_max": -0.108049, + "nauc_precision_at_1_std": -0.184276, + "nauc_precision_at_1_diff1": 0.438554, + "nauc_precision_at_3_max": -0.180384, + "nauc_precision_at_3_std": -0.112419, + "nauc_precision_at_3_diff1": 0.042242, + "nauc_precision_at_5_max": -0.186076, + "nauc_precision_at_5_std": -0.029555, + "nauc_precision_at_5_diff1": -0.108753, + "nauc_precision_at_10_max": -0.216411, + "nauc_precision_at_10_std": -0.032875, + "nauc_precision_at_10_diff1": -0.158311, + "nauc_precision_at_20_max": -0.255455, + "nauc_precision_at_20_std": -0.073265, + "nauc_precision_at_20_diff1": -0.230566, + "nauc_precision_at_100_max": -0.32701, + "nauc_precision_at_100_std": -0.133966, + "nauc_precision_at_100_diff1": -0.357449, + "nauc_precision_at_1000_max": -0.34514, + "nauc_precision_at_1000_std": -0.156261, + "nauc_precision_at_1000_diff1": -0.356257, + "nauc_cv_recall_at_1_max": -0.108049, + "nauc_cv_recall_at_1_std": -0.184276, + "nauc_cv_recall_at_1_diff1": 0.438554, + "nauc_cv_recall_at_3_max": -0.281745, + "nauc_cv_recall_at_3_std": -0.224764, + "nauc_cv_recall_at_3_diff1": 0.140288, + "nauc_cv_recall_at_5_max": -0.297858, + "nauc_cv_recall_at_5_std": 0.088382, + "nauc_cv_recall_at_5_diff1": -0.124711, + "nauc_cv_recall_at_10_max": -0.251222, + "nauc_cv_recall_at_10_std": 0.202706, + "nauc_cv_recall_at_10_diff1": -0.139245, + "nauc_cv_recall_at_20_max": -0.06766, + "nauc_cv_recall_at_20_std": 0.13223, + "nauc_cv_recall_at_20_diff1": -0.340796, + "nauc_cv_recall_at_100_max": -1.169034, + "nauc_cv_recall_at_100_std": -0.164944, + "nauc_cv_recall_at_100_diff1": -1.169034, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.108049, + "nauc_mrr_at_1_std": -0.184276, + "nauc_mrr_at_1_diff1": 0.438554, + "nauc_mrr_at_3_max": -0.161585, + "nauc_mrr_at_3_std": -0.191461, + "nauc_mrr_at_3_diff1": 0.337688, + "nauc_mrr_at_5_max": -0.154666, + "nauc_mrr_at_5_std": -0.14357, + "nauc_mrr_at_5_diff1": 0.308143, + "nauc_mrr_at_10_max": -0.145163, + "nauc_mrr_at_10_std": -0.146228, + "nauc_mrr_at_10_diff1": 0.322319, + "nauc_mrr_at_20_max": -0.139095, + "nauc_mrr_at_20_std": -0.150385, + "nauc_mrr_at_20_diff1": 0.319082, + "nauc_mrr_at_100_max": -0.143497, + "nauc_mrr_at_100_std": -0.15562, + "nauc_mrr_at_100_diff1": 0.325107, + "nauc_mrr_at_1000_max": -0.143302, + "nauc_mrr_at_1000_std": -0.155619, + "nauc_mrr_at_1000_diff1": 0.325391, + "main_score": 0.55668, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.54386, + "ndcg_at_3": 0.51611, + "ndcg_at_5": 0.55746, + "ndcg_at_10": 0.58219, + "ndcg_at_20": 0.61652, + "ndcg_at_100": 0.66162, + "ndcg_at_1000": 0.67195, + "map_at_1": 0.27782, + "map_at_3": 0.38019, + "map_at_5": 0.43568, + "map_at_10": 0.46935, + "map_at_20": 0.49273, + "map_at_100": 0.51105, + "map_at_1000": 0.51249, + "recall_at_1": 0.27782, + "recall_at_3": 0.44472, + "recall_at_5": 0.56552, + "recall_at_10": 0.666, + "recall_at_20": 0.78668, + "recall_at_100": 0.9336, + "recall_at_1000": 0.98977, + "cv_recall_at_1": 0.54386, + "cv_recall_at_3": 0.73684, + "cv_recall_at_5": 0.78947, + "cv_recall_at_10": 0.85965, + "cv_recall_at_20": 0.92982, + "cv_recall_at_100": 0.96491, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.54386, + "precision_at_3": 0.33918, + "precision_at_5": 0.28772, + "precision_at_10": 0.18947, + "precision_at_20": 0.1193, + "precision_at_100": 0.03544, + "precision_at_1000": 0.00386, + "mrr_at_1": 0.54386, + "mrr_at_3": 0.631579, + "mrr_at_5": 0.644737, + "mrr_at_10": 0.654288, + "mrr_at_20": 0.658804, + "mrr_at_100": 0.659925, + "mrr_at_1000": 0.660058, + "nauc_ndcg_at_1_max": -0.000995, + "nauc_ndcg_at_1_std": 0.124077, + "nauc_ndcg_at_1_diff1": 0.20303, + "nauc_ndcg_at_3_max": -0.004653, + "nauc_ndcg_at_3_std": 0.035556, + "nauc_ndcg_at_3_diff1": 0.232382, + "nauc_ndcg_at_5_max": 0.013934, + "nauc_ndcg_at_5_std": 0.081926, + "nauc_ndcg_at_5_diff1": 0.152656, + "nauc_ndcg_at_10_max": 0.034647, + "nauc_ndcg_at_10_std": 0.086149, + "nauc_ndcg_at_10_diff1": 0.1508, + "nauc_ndcg_at_20_max": 0.066413, + "nauc_ndcg_at_20_std": 0.099289, + "nauc_ndcg_at_20_diff1": 0.17324, + "nauc_ndcg_at_100_max": 0.029618, + "nauc_ndcg_at_100_std": 0.107703, + "nauc_ndcg_at_100_diff1": 0.115142, + "nauc_ndcg_at_1000_max": 0.009323, + "nauc_ndcg_at_1000_std": 0.078498, + "nauc_ndcg_at_1000_diff1": 0.13055, + "nauc_map_at_1_max": -0.002336, + "nauc_map_at_1_std": 0.018019, + "nauc_map_at_1_diff1": 0.298254, + "nauc_map_at_3_max": 0.034991, + "nauc_map_at_3_std": 0.054531, + "nauc_map_at_3_diff1": 0.283209, + "nauc_map_at_5_max": 0.046048, + "nauc_map_at_5_std": 0.091252, + "nauc_map_at_5_diff1": 0.220468, + "nauc_map_at_10_max": 0.044266, + "nauc_map_at_10_std": 0.085005, + "nauc_map_at_10_diff1": 0.208768, + "nauc_map_at_20_max": 0.062652, + "nauc_map_at_20_std": 0.105342, + "nauc_map_at_20_diff1": 0.208613, + "nauc_map_at_100_max": 0.044468, + "nauc_map_at_100_std": 0.101251, + "nauc_map_at_100_diff1": 0.187682, + "nauc_map_at_1000_max": 0.041188, + "nauc_map_at_1000_std": 0.098766, + "nauc_map_at_1000_diff1": 0.18668, + "nauc_recall_at_1_max": -0.002336, + "nauc_recall_at_1_std": 0.018019, + "nauc_recall_at_1_diff1": 0.298254, + "nauc_recall_at_3_max": 0.05044, + "nauc_recall_at_3_std": 0.032399, + "nauc_recall_at_3_diff1": 0.289047, + "nauc_recall_at_5_max": 0.14188, + "nauc_recall_at_5_std": 0.146268, + "nauc_recall_at_5_diff1": 0.159075, + "nauc_recall_at_10_max": 0.118186, + "nauc_recall_at_10_std": 0.096489, + "nauc_recall_at_10_diff1": 0.111507, + "nauc_recall_at_20_max": 0.25537, + "nauc_recall_at_20_std": 0.143189, + "nauc_recall_at_20_diff1": 0.162231, + "nauc_recall_at_100_max": 0.383113, + "nauc_recall_at_100_std": 0.725319, + "nauc_recall_at_100_diff1": -0.40513, + "nauc_recall_at_1000_max": 0.806035, + "nauc_recall_at_1000_std": 0.943648, + "nauc_recall_at_1000_diff1": -0.009919, + "nauc_precision_at_1_max": -0.000995, + "nauc_precision_at_1_std": 0.124077, + "nauc_precision_at_1_diff1": 0.20303, + "nauc_precision_at_3_max": -0.033754, + "nauc_precision_at_3_std": 0.04407, + "nauc_precision_at_3_diff1": 0.016351, + "nauc_precision_at_5_max": -0.068191, + "nauc_precision_at_5_std": 0.042227, + "nauc_precision_at_5_diff1": -0.164485, + "nauc_precision_at_10_max": -0.027831, + "nauc_precision_at_10_std": 0.051906, + "nauc_precision_at_10_diff1": -0.199309, + "nauc_precision_at_20_max": -0.026248, + "nauc_precision_at_20_std": 0.056827, + "nauc_precision_at_20_diff1": -0.210588, + "nauc_precision_at_100_max": -0.159989, + "nauc_precision_at_100_std": -0.042608, + "nauc_precision_at_100_diff1": -0.265537, + "nauc_precision_at_1000_max": -0.199371, + "nauc_precision_at_1000_std": -0.095072, + "nauc_precision_at_1000_diff1": -0.252846, + "nauc_cv_recall_at_1_max": -0.000995, + "nauc_cv_recall_at_1_std": 0.124077, + "nauc_cv_recall_at_1_diff1": 0.20303, + "nauc_cv_recall_at_3_max": -0.13967, + "nauc_cv_recall_at_3_std": -0.078918, + "nauc_cv_recall_at_3_diff1": 0.101057, + "nauc_cv_recall_at_5_max": 0.145317, + "nauc_cv_recall_at_5_std": 0.112378, + "nauc_cv_recall_at_5_diff1": 0.107831, + "nauc_cv_recall_at_10_max": 0.175657, + "nauc_cv_recall_at_10_std": 0.095836, + "nauc_cv_recall_at_10_diff1": -0.057541, + "nauc_cv_recall_at_20_max": 0.100566, + "nauc_cv_recall_at_20_std": -0.384759, + "nauc_cv_recall_at_20_diff1": 0.104618, + "nauc_cv_recall_at_100_max": 0.220296, + "nauc_cv_recall_at_100_std": 0.678927, + "nauc_cv_recall_at_100_diff1": -0.586386, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.000995, + "nauc_mrr_at_1_std": 0.124077, + "nauc_mrr_at_1_diff1": 0.20303, + "nauc_mrr_at_3_max": -0.041778, + "nauc_mrr_at_3_std": 0.052623, + "nauc_mrr_at_3_diff1": 0.157583, + "nauc_mrr_at_5_max": 0.010977, + "nauc_mrr_at_5_std": 0.090272, + "nauc_mrr_at_5_diff1": 0.160679, + "nauc_mrr_at_10_max": 0.006667, + "nauc_mrr_at_10_std": 0.084489, + "nauc_mrr_at_10_diff1": 0.149594, + "nauc_mrr_at_20_max": 0.003669, + "nauc_mrr_at_20_std": 0.077504, + "nauc_mrr_at_20_diff1": 0.157397, + "nauc_mrr_at_100_max": 0.003986, + "nauc_mrr_at_100_std": 0.083494, + "nauc_mrr_at_100_diff1": 0.154825, + "nauc_mrr_at_1000_max": 0.003812, + "nauc_mrr_at_1000_std": 0.083169, + "nauc_mrr_at_1000_diff1": 0.155099, + "main_score": 0.55746, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 601.0417277812958, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/Vidore2EconomicsReportsRetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..907f856376 --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.55172, + "ndcg_at_3": 0.54082, + "ndcg_at_5": 0.50974, + "ndcg_at_10": 0.50915, + "ndcg_at_20": 0.51569, + "ndcg_at_100": 0.64368, + "ndcg_at_1000": 0.68434, + "map_at_1": 0.08858, + "map_at_3": 0.16644, + "map_at_5": 0.20521, + "map_at_10": 0.27386, + "map_at_20": 0.31437, + "map_at_100": 0.40432, + "map_at_1000": 0.42198, + "recall_at_1": 0.08858, + "recall_at_3": 0.21247, + "recall_at_5": 0.27584, + "recall_at_10": 0.40507, + "recall_at_20": 0.50964, + "recall_at_100": 0.88839, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.55172, + "cv_recall_at_3": 0.84483, + "cv_recall_at_5": 0.89655, + "cv_recall_at_10": 0.93103, + "cv_recall_at_20": 0.94828, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.55172, + "precision_at_3": 0.50575, + "precision_at_5": 0.44483, + "precision_at_10": 0.38103, + "precision_at_20": 0.27672, + "precision_at_100": 0.12414, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.551724, + "mrr_at_3": 0.675287, + "mrr_at_5": 0.686494, + "mrr_at_10": 0.689943, + "mrr_at_20": 0.69102, + "mrr_at_100": 0.692433, + "mrr_at_1000": 0.692433, + "nauc_ndcg_at_1_max": 0.335875, + "nauc_ndcg_at_1_std": 0.378385, + "nauc_ndcg_at_1_diff1": -0.031226, + "nauc_ndcg_at_3_max": 0.283928, + "nauc_ndcg_at_3_std": 0.282572, + "nauc_ndcg_at_3_diff1": 0.112509, + "nauc_ndcg_at_5_max": 0.21139, + "nauc_ndcg_at_5_std": 0.149676, + "nauc_ndcg_at_5_diff1": 0.121018, + "nauc_ndcg_at_10_max": 0.193292, + "nauc_ndcg_at_10_std": 0.113912, + "nauc_ndcg_at_10_diff1": 0.044308, + "nauc_ndcg_at_20_max": 0.221835, + "nauc_ndcg_at_20_std": 0.178449, + "nauc_ndcg_at_20_diff1": -0.041092, + "nauc_ndcg_at_100_max": 0.294946, + "nauc_ndcg_at_100_std": 0.303936, + "nauc_ndcg_at_100_diff1": -0.078798, + "nauc_ndcg_at_1000_max": 0.25471, + "nauc_ndcg_at_1000_std": 0.244248, + "nauc_ndcg_at_1000_diff1": 0.051335, + "nauc_map_at_1_max": -0.123914, + "nauc_map_at_1_std": -0.127597, + "nauc_map_at_1_diff1": -0.399901, + "nauc_map_at_3_max": 0.041633, + "nauc_map_at_3_std": -0.003677, + "nauc_map_at_3_diff1": -0.19942, + "nauc_map_at_5_max": 0.126787, + "nauc_map_at_5_std": 0.026502, + "nauc_map_at_5_diff1": -0.168073, + "nauc_map_at_10_max": 0.198415, + "nauc_map_at_10_std": 0.072541, + "nauc_map_at_10_diff1": -0.170349, + "nauc_map_at_20_max": 0.213947, + "nauc_map_at_20_std": 0.107072, + "nauc_map_at_20_diff1": -0.170884, + "nauc_map_at_100_max": 0.19364, + "nauc_map_at_100_std": 0.147494, + "nauc_map_at_100_diff1": -0.153479, + "nauc_map_at_1000_max": 0.156906, + "nauc_map_at_1000_std": 0.119112, + "nauc_map_at_1000_diff1": -0.110913, + "nauc_recall_at_1_max": -0.123914, + "nauc_recall_at_1_std": -0.127597, + "nauc_recall_at_1_diff1": -0.399901, + "nauc_recall_at_3_max": -0.007793, + "nauc_recall_at_3_std": -0.068932, + "nauc_recall_at_3_diff1": -0.086917, + "nauc_recall_at_5_max": 0.036946, + "nauc_recall_at_5_std": -0.07848, + "nauc_recall_at_5_diff1": -0.09747, + "nauc_recall_at_10_max": 0.123919, + "nauc_recall_at_10_std": -0.004497, + "nauc_recall_at_10_diff1": -0.144355, + "nauc_recall_at_20_max": 0.191298, + "nauc_recall_at_20_std": 0.098855, + "nauc_recall_at_20_diff1": -0.165383, + "nauc_recall_at_100_max": 0.258382, + "nauc_recall_at_100_std": 0.355174, + "nauc_recall_at_100_diff1": -0.471179, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.335875, + "nauc_precision_at_1_std": 0.378385, + "nauc_precision_at_1_diff1": -0.031226, + "nauc_precision_at_3_max": 0.364495, + "nauc_precision_at_3_std": 0.363797, + "nauc_precision_at_3_diff1": 0.241436, + "nauc_precision_at_5_max": 0.28683, + "nauc_precision_at_5_std": 0.209783, + "nauc_precision_at_5_diff1": 0.263509, + "nauc_precision_at_10_max": 0.228476, + "nauc_precision_at_10_std": 0.179727, + "nauc_precision_at_10_diff1": 0.211002, + "nauc_precision_at_20_max": 0.086829, + "nauc_precision_at_20_std": 0.119619, + "nauc_precision_at_20_diff1": 0.173921, + "nauc_precision_at_100_max": -0.127909, + "nauc_precision_at_100_std": -0.017523, + "nauc_precision_at_100_diff1": 0.173396, + "nauc_precision_at_1000_max": -0.168003, + "nauc_precision_at_1000_std": -0.074863, + "nauc_precision_at_1000_diff1": 0.226314, + "nauc_cv_recall_at_1_max": 0.335875, + "nauc_cv_recall_at_1_std": 0.378385, + "nauc_cv_recall_at_1_diff1": -0.031226, + "nauc_cv_recall_at_3_max": 0.471246, + "nauc_cv_recall_at_3_std": 0.477439, + "nauc_cv_recall_at_3_diff1": 0.478919, + "nauc_cv_recall_at_5_max": 0.324613, + "nauc_cv_recall_at_5_std": 0.253297, + "nauc_cv_recall_at_5_diff1": 0.650935, + "nauc_cv_recall_at_10_max": 0.450384, + "nauc_cv_recall_at_10_std": 0.34341, + "nauc_cv_recall_at_10_diff1": 0.695462, + "nauc_cv_recall_at_20_max": 0.559258, + "nauc_cv_recall_at_20_std": 0.515517, + "nauc_cv_recall_at_20_diff1": 0.593949, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.335875, + "nauc_mrr_at_1_std": 0.378385, + "nauc_mrr_at_1_diff1": -0.031226, + "nauc_mrr_at_3_max": 0.382484, + "nauc_mrr_at_3_std": 0.406267, + "nauc_mrr_at_3_diff1": 0.091823, + "nauc_mrr_at_5_max": 0.36626, + "nauc_mrr_at_5_std": 0.383707, + "nauc_mrr_at_5_diff1": 0.096397, + "nauc_mrr_at_10_max": 0.370395, + "nauc_mrr_at_10_std": 0.388088, + "nauc_mrr_at_10_diff1": 0.089832, + "nauc_mrr_at_20_max": 0.371487, + "nauc_mrr_at_20_std": 0.390571, + "nauc_mrr_at_20_diff1": 0.085803, + "nauc_mrr_at_100_max": 0.371033, + "nauc_mrr_at_100_std": 0.390396, + "nauc_mrr_at_100_diff1": 0.082455, + "nauc_mrr_at_1000_max": 0.371033, + "nauc_mrr_at_1000_std": 0.390396, + "nauc_mrr_at_1000_diff1": 0.082455, + "main_score": 0.50974, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.53448, + "ndcg_at_3": 0.49871, + "ndcg_at_5": 0.49608, + "ndcg_at_10": 0.4858, + "ndcg_at_20": 0.49841, + "ndcg_at_100": 0.63066, + "ndcg_at_1000": 0.66736, + "map_at_1": 0.05569, + "map_at_3": 0.13523, + "map_at_5": 0.1856, + "map_at_10": 0.2494, + "map_at_20": 0.29266, + "map_at_100": 0.38283, + "map_at_1000": 0.39921, + "recall_at_1": 0.05569, + "recall_at_3": 0.16645, + "recall_at_5": 0.27415, + "recall_at_10": 0.39964, + "recall_at_20": 0.51367, + "recall_at_100": 0.90267, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.53448, + "cv_recall_at_3": 0.72414, + "cv_recall_at_5": 0.82759, + "cv_recall_at_10": 0.91379, + "cv_recall_at_20": 0.96552, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.53448, + "precision_at_3": 0.47126, + "precision_at_5": 0.44483, + "precision_at_10": 0.37069, + "precision_at_20": 0.275, + "precision_at_100": 0.12638, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.534483, + "mrr_at_3": 0.623563, + "mrr_at_5": 0.646839, + "mrr_at_10": 0.657013, + "mrr_at_20": 0.660435, + "mrr_at_100": 0.661812, + "mrr_at_1000": 0.661812, + "nauc_ndcg_at_1_max": 0.107121, + "nauc_ndcg_at_1_std": 0.023052, + "nauc_ndcg_at_1_diff1": 0.055195, + "nauc_ndcg_at_3_max": 0.155586, + "nauc_ndcg_at_3_std": 0.040707, + "nauc_ndcg_at_3_diff1": 0.106434, + "nauc_ndcg_at_5_max": 0.110904, + "nauc_ndcg_at_5_std": -0.010394, + "nauc_ndcg_at_5_diff1": 0.164279, + "nauc_ndcg_at_10_max": 0.157575, + "nauc_ndcg_at_10_std": -0.021367, + "nauc_ndcg_at_10_diff1": 0.110662, + "nauc_ndcg_at_20_max": 0.214762, + "nauc_ndcg_at_20_std": 0.036425, + "nauc_ndcg_at_20_diff1": 0.122714, + "nauc_ndcg_at_100_max": 0.24658, + "nauc_ndcg_at_100_std": 0.129258, + "nauc_ndcg_at_100_diff1": 0.004781, + "nauc_ndcg_at_1000_max": 0.203114, + "nauc_ndcg_at_1000_std": 0.098152, + "nauc_ndcg_at_1000_diff1": 0.025268, + "nauc_map_at_1_max": -0.010371, + "nauc_map_at_1_std": -0.09382, + "nauc_map_at_1_diff1": 0.106084, + "nauc_map_at_3_max": 0.053952, + "nauc_map_at_3_std": -0.1332, + "nauc_map_at_3_diff1": 0.257599, + "nauc_map_at_5_max": 0.067165, + "nauc_map_at_5_std": -0.124089, + "nauc_map_at_5_diff1": 0.257684, + "nauc_map_at_10_max": 0.20329, + "nauc_map_at_10_std": -0.056424, + "nauc_map_at_10_diff1": 0.155759, + "nauc_map_at_20_max": 0.235795, + "nauc_map_at_20_std": -0.0043, + "nauc_map_at_20_diff1": 0.152001, + "nauc_map_at_100_max": 0.250371, + "nauc_map_at_100_std": 0.053378, + "nauc_map_at_100_diff1": 0.093918, + "nauc_map_at_1000_max": 0.216549, + "nauc_map_at_1000_std": 0.039505, + "nauc_map_at_1000_diff1": 0.102363, + "nauc_recall_at_1_max": -0.010371, + "nauc_recall_at_1_std": -0.09382, + "nauc_recall_at_1_diff1": 0.106084, + "nauc_recall_at_3_max": 0.012088, + "nauc_recall_at_3_std": -0.171721, + "nauc_recall_at_3_diff1": 0.352271, + "nauc_recall_at_5_max": -0.090889, + "nauc_recall_at_5_std": -0.238153, + "nauc_recall_at_5_diff1": 0.256745, + "nauc_recall_at_10_max": 0.040177, + "nauc_recall_at_10_std": -0.157779, + "nauc_recall_at_10_diff1": 0.09362, + "nauc_recall_at_20_max": 0.116883, + "nauc_recall_at_20_std": -0.054635, + "nauc_recall_at_20_diff1": 0.080912, + "nauc_recall_at_100_max": 0.223924, + "nauc_recall_at_100_std": 0.200702, + "nauc_recall_at_100_diff1": -0.081422, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.107121, + "nauc_precision_at_1_std": 0.023052, + "nauc_precision_at_1_diff1": 0.055195, + "nauc_precision_at_3_max": 0.207339, + "nauc_precision_at_3_std": 0.099521, + "nauc_precision_at_3_diff1": 0.027364, + "nauc_precision_at_5_max": 0.204795, + "nauc_precision_at_5_std": 0.108574, + "nauc_precision_at_5_diff1": 0.069762, + "nauc_precision_at_10_max": 0.218419, + "nauc_precision_at_10_std": 0.136004, + "nauc_precision_at_10_diff1": -0.061872, + "nauc_precision_at_20_max": 0.156412, + "nauc_precision_at_20_std": 0.135048, + "nauc_precision_at_20_diff1": -0.045679, + "nauc_precision_at_100_max": 0.005362, + "nauc_precision_at_100_std": 0.090402, + "nauc_precision_at_100_diff1": -0.057917, + "nauc_precision_at_1000_max": -0.040334, + "nauc_precision_at_1000_std": 0.036277, + "nauc_precision_at_1000_diff1": -0.007528, + "nauc_cv_recall_at_1_max": 0.107121, + "nauc_cv_recall_at_1_std": 0.023052, + "nauc_cv_recall_at_1_diff1": 0.055195, + "nauc_cv_recall_at_3_max": 0.185828, + "nauc_cv_recall_at_3_std": 0.143729, + "nauc_cv_recall_at_3_diff1": 0.054849, + "nauc_cv_recall_at_5_max": -0.089104, + "nauc_cv_recall_at_5_std": -0.14992, + "nauc_cv_recall_at_5_diff1": 0.241642, + "nauc_cv_recall_at_10_max": 0.017996, + "nauc_cv_recall_at_10_std": -0.020524, + "nauc_cv_recall_at_10_diff1": 0.072387, + "nauc_cv_recall_at_20_max": 0.123761, + "nauc_cv_recall_at_20_std": 0.093071, + "nauc_cv_recall_at_20_diff1": -0.294456, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.107121, + "nauc_mrr_at_1_std": 0.023052, + "nauc_mrr_at_1_diff1": 0.055195, + "nauc_mrr_at_3_max": 0.115975, + "nauc_mrr_at_3_std": 0.044744, + "nauc_mrr_at_3_diff1": 0.056184, + "nauc_mrr_at_5_max": 0.071649, + "nauc_mrr_at_5_std": -0.004367, + "nauc_mrr_at_5_diff1": 0.083505, + "nauc_mrr_at_10_max": 0.079817, + "nauc_mrr_at_10_std": 0.003972, + "nauc_mrr_at_10_diff1": 0.072155, + "nauc_mrr_at_20_max": 0.084186, + "nauc_mrr_at_20_std": 0.007894, + "nauc_mrr_at_20_diff1": 0.068924, + "nauc_mrr_at_100_max": 0.083978, + "nauc_mrr_at_100_std": 0.007178, + "nauc_mrr_at_100_diff1": 0.071689, + "nauc_mrr_at_1000_max": 0.083978, + "nauc_mrr_at_1000_std": 0.007178, + "nauc_mrr_at_1000_diff1": 0.071689, + "main_score": 0.49608, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.58621, + "ndcg_at_3": 0.55262, + "ndcg_at_5": 0.52229, + "ndcg_at_10": 0.51588, + "ndcg_at_20": 0.55242, + "ndcg_at_100": 0.65461, + "ndcg_at_1000": 0.69375, + "map_at_1": 0.06729, + "map_at_3": 0.1529, + "map_at_5": 0.19548, + "map_at_10": 0.26768, + "map_at_20": 0.32919, + "map_at_100": 0.41358, + "map_at_1000": 0.43139, + "recall_at_1": 0.06729, + "recall_at_3": 0.2023, + "recall_at_5": 0.26507, + "recall_at_10": 0.40359, + "recall_at_20": 0.57831, + "recall_at_100": 0.89106, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.58621, + "cv_recall_at_3": 0.81034, + "cv_recall_at_5": 0.86207, + "cv_recall_at_10": 0.91379, + "cv_recall_at_20": 0.96552, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.58621, + "precision_at_3": 0.52299, + "precision_at_5": 0.46897, + "precision_at_10": 0.39483, + "precision_at_20": 0.30948, + "precision_at_100": 0.12638, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.586207, + "mrr_at_3": 0.686782, + "mrr_at_5": 0.698851, + "mrr_at_10": 0.706513, + "mrr_at_20": 0.710221, + "mrr_at_100": 0.711731, + "mrr_at_1000": 0.711731, + "nauc_ndcg_at_1_max": 0.343639, + "nauc_ndcg_at_1_std": 0.212318, + "nauc_ndcg_at_1_diff1": 0.036489, + "nauc_ndcg_at_3_max": 0.157979, + "nauc_ndcg_at_3_std": 0.048049, + "nauc_ndcg_at_3_diff1": 0.043923, + "nauc_ndcg_at_5_max": 0.119111, + "nauc_ndcg_at_5_std": -0.010699, + "nauc_ndcg_at_5_diff1": 0.002972, + "nauc_ndcg_at_10_max": 0.119125, + "nauc_ndcg_at_10_std": 0.007179, + "nauc_ndcg_at_10_diff1": -0.057591, + "nauc_ndcg_at_20_max": 0.168117, + "nauc_ndcg_at_20_std": 0.071229, + "nauc_ndcg_at_20_diff1": -0.010798, + "nauc_ndcg_at_100_max": 0.262981, + "nauc_ndcg_at_100_std": 0.213177, + "nauc_ndcg_at_100_diff1": -0.109383, + "nauc_ndcg_at_1000_max": 0.228022, + "nauc_ndcg_at_1000_std": 0.168174, + "nauc_ndcg_at_1000_diff1": -0.13517, + "nauc_map_at_1_max": 0.119362, + "nauc_map_at_1_std": 0.039806, + "nauc_map_at_1_diff1": -0.046072, + "nauc_map_at_3_max": -0.041113, + "nauc_map_at_3_std": -0.143614, + "nauc_map_at_3_diff1": 0.096037, + "nauc_map_at_5_max": -0.020221, + "nauc_map_at_5_std": -0.161014, + "nauc_map_at_5_diff1": 0.06453, + "nauc_map_at_10_max": 0.054967, + "nauc_map_at_10_std": -0.086891, + "nauc_map_at_10_diff1": -0.010527, + "nauc_map_at_20_max": 0.112557, + "nauc_map_at_20_std": -0.016652, + "nauc_map_at_20_diff1": -0.010179, + "nauc_map_at_100_max": 0.138176, + "nauc_map_at_100_std": 0.065776, + "nauc_map_at_100_diff1": -0.085648, + "nauc_map_at_1000_max": 0.118411, + "nauc_map_at_1000_std": 0.057707, + "nauc_map_at_1000_diff1": -0.096079, + "nauc_recall_at_1_max": 0.119362, + "nauc_recall_at_1_std": 0.039806, + "nauc_recall_at_1_diff1": -0.046072, + "nauc_recall_at_3_max": -0.181177, + "nauc_recall_at_3_std": -0.276235, + "nauc_recall_at_3_diff1": 0.288692, + "nauc_recall_at_5_max": -0.151543, + "nauc_recall_at_5_std": -0.276609, + "nauc_recall_at_5_diff1": 0.239616, + "nauc_recall_at_10_max": -0.045266, + "nauc_recall_at_10_std": -0.14598, + "nauc_recall_at_10_diff1": 0.113955, + "nauc_recall_at_20_max": 0.021043, + "nauc_recall_at_20_std": -0.062098, + "nauc_recall_at_20_diff1": 0.166167, + "nauc_recall_at_100_max": 0.286235, + "nauc_recall_at_100_std": 0.293367, + "nauc_recall_at_100_diff1": 0.067976, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.343639, + "nauc_precision_at_1_std": 0.212318, + "nauc_precision_at_1_diff1": 0.036489, + "nauc_precision_at_3_max": 0.137922, + "nauc_precision_at_3_std": 0.057958, + "nauc_precision_at_3_diff1": -0.02922, + "nauc_precision_at_5_max": 0.122421, + "nauc_precision_at_5_std": 0.014417, + "nauc_precision_at_5_diff1": -0.093063, + "nauc_precision_at_10_max": 0.178685, + "nauc_precision_at_10_std": 0.129516, + "nauc_precision_at_10_diff1": -0.157006, + "nauc_precision_at_20_max": 0.141919, + "nauc_precision_at_20_std": 0.15797, + "nauc_precision_at_20_diff1": -0.130312, + "nauc_precision_at_100_max": -0.000129, + "nauc_precision_at_100_std": 0.092998, + "nauc_precision_at_100_diff1": -0.154686, + "nauc_precision_at_1000_max": -0.027252, + "nauc_precision_at_1000_std": 0.053167, + "nauc_precision_at_1000_diff1": -0.152254, + "nauc_cv_recall_at_1_max": 0.343639, + "nauc_cv_recall_at_1_std": 0.212318, + "nauc_cv_recall_at_1_diff1": 0.036489, + "nauc_cv_recall_at_3_max": 0.477198, + "nauc_cv_recall_at_3_std": 0.259075, + "nauc_cv_recall_at_3_diff1": 0.211426, + "nauc_cv_recall_at_5_max": 0.418765, + "nauc_cv_recall_at_5_std": 0.252534, + "nauc_cv_recall_at_5_diff1": 0.159756, + "nauc_cv_recall_at_10_max": 0.394451, + "nauc_cv_recall_at_10_std": 0.415265, + "nauc_cv_recall_at_10_diff1": -0.263578, + "nauc_cv_recall_at_20_max": 0.338887, + "nauc_cv_recall_at_20_std": 0.456534, + "nauc_cv_recall_at_20_diff1": -0.225269, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.343639, + "nauc_mrr_at_1_std": 0.212318, + "nauc_mrr_at_1_diff1": 0.036489, + "nauc_mrr_at_3_max": 0.388091, + "nauc_mrr_at_3_std": 0.242072, + "nauc_mrr_at_3_diff1": 0.057422, + "nauc_mrr_at_5_max": 0.378457, + "nauc_mrr_at_5_std": 0.243172, + "nauc_mrr_at_5_diff1": 0.043691, + "nauc_mrr_at_10_max": 0.377044, + "nauc_mrr_at_10_std": 0.255107, + "nauc_mrr_at_10_diff1": 0.018354, + "nauc_mrr_at_20_max": 0.376114, + "nauc_mrr_at_20_std": 0.253, + "nauc_mrr_at_20_diff1": 0.022519, + "nauc_mrr_at_100_max": 0.376474, + "nauc_mrr_at_100_std": 0.251744, + "nauc_mrr_at_100_diff1": 0.023919, + "nauc_mrr_at_1000_max": 0.376474, + "nauc_mrr_at_1000_std": 0.251744, + "nauc_mrr_at_1000_diff1": 0.023919, + "main_score": 0.52229, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.48276, + "ndcg_at_3": 0.4784, + "ndcg_at_5": 0.47715, + "ndcg_at_10": 0.47335, + "ndcg_at_20": 0.49916, + "ndcg_at_100": 0.62214, + "ndcg_at_1000": 0.66272, + "map_at_1": 0.04966, + "map_at_3": 0.1202, + "map_at_5": 0.16903, + "map_at_10": 0.23813, + "map_at_20": 0.28502, + "map_at_100": 0.37122, + "map_at_1000": 0.38916, + "recall_at_1": 0.04966, + "recall_at_3": 0.15964, + "recall_at_5": 0.24254, + "recall_at_10": 0.39146, + "recall_at_20": 0.53849, + "recall_at_100": 0.88627, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.48276, + "cv_recall_at_3": 0.77586, + "cv_recall_at_5": 0.87931, + "cv_recall_at_10": 0.96552, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.48276, + "precision_at_3": 0.45402, + "precision_at_5": 0.44483, + "precision_at_10": 0.37069, + "precision_at_20": 0.27845, + "precision_at_100": 0.125, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.482759, + "mrr_at_3": 0.614943, + "mrr_at_5": 0.639943, + "mrr_at_10": 0.651054, + "mrr_at_20": 0.654058, + "mrr_at_100": 0.654058, + "mrr_at_1000": 0.654058, + "nauc_ndcg_at_1_max": 0.172456, + "nauc_ndcg_at_1_std": 0.158663, + "nauc_ndcg_at_1_diff1": 0.077735, + "nauc_ndcg_at_3_max": 0.127243, + "nauc_ndcg_at_3_std": 0.117721, + "nauc_ndcg_at_3_diff1": 0.095495, + "nauc_ndcg_at_5_max": 0.130637, + "nauc_ndcg_at_5_std": 0.093806, + "nauc_ndcg_at_5_diff1": 0.11147, + "nauc_ndcg_at_10_max": 0.075359, + "nauc_ndcg_at_10_std": 0.008974, + "nauc_ndcg_at_10_diff1": 0.14035, + "nauc_ndcg_at_20_max": 0.116427, + "nauc_ndcg_at_20_std": 0.070224, + "nauc_ndcg_at_20_diff1": 0.108078, + "nauc_ndcg_at_100_max": 0.121987, + "nauc_ndcg_at_100_std": 0.151014, + "nauc_ndcg_at_100_diff1": 0.006687, + "nauc_ndcg_at_1000_max": 0.129136, + "nauc_ndcg_at_1000_std": 0.169711, + "nauc_ndcg_at_1000_diff1": 0.052221, + "nauc_map_at_1_max": 0.113497, + "nauc_map_at_1_std": 0.051322, + "nauc_map_at_1_diff1": 0.03929, + "nauc_map_at_3_max": 0.074917, + "nauc_map_at_3_std": -0.043743, + "nauc_map_at_3_diff1": 0.143433, + "nauc_map_at_5_max": 0.116397, + "nauc_map_at_5_std": -0.032448, + "nauc_map_at_5_diff1": 0.090864, + "nauc_map_at_10_max": 0.092893, + "nauc_map_at_10_std": -0.045911, + "nauc_map_at_10_diff1": 0.042304, + "nauc_map_at_20_max": 0.076214, + "nauc_map_at_20_std": -0.020589, + "nauc_map_at_20_diff1": 0.005855, + "nauc_map_at_100_max": 0.053651, + "nauc_map_at_100_std": 0.038806, + "nauc_map_at_100_diff1": 0.01852, + "nauc_map_at_1000_max": 0.053094, + "nauc_map_at_1000_std": 0.048924, + "nauc_map_at_1000_diff1": 0.05228, + "nauc_recall_at_1_max": 0.113497, + "nauc_recall_at_1_std": 0.051322, + "nauc_recall_at_1_diff1": 0.03929, + "nauc_recall_at_3_max": 0.083436, + "nauc_recall_at_3_std": -0.114854, + "nauc_recall_at_3_diff1": 0.332327, + "nauc_recall_at_5_max": 0.149064, + "nauc_recall_at_5_std": -0.070115, + "nauc_recall_at_5_diff1": 0.231459, + "nauc_recall_at_10_max": 0.017366, + "nauc_recall_at_10_std": -0.170092, + "nauc_recall_at_10_diff1": 0.175981, + "nauc_recall_at_20_max": -0.001672, + "nauc_recall_at_20_std": -0.151724, + "nauc_recall_at_20_diff1": 0.048594, + "nauc_recall_at_100_max": 0.080094, + "nauc_recall_at_100_std": 0.0223, + "nauc_recall_at_100_diff1": -0.108807, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.172456, + "nauc_precision_at_1_std": 0.158663, + "nauc_precision_at_1_diff1": 0.077735, + "nauc_precision_at_3_max": 0.114381, + "nauc_precision_at_3_std": 0.129629, + "nauc_precision_at_3_diff1": 0.029854, + "nauc_precision_at_5_max": 0.115294, + "nauc_precision_at_5_std": 0.11134, + "nauc_precision_at_5_diff1": 0.031626, + "nauc_precision_at_10_max": 0.051737, + "nauc_precision_at_10_std": 0.090501, + "nauc_precision_at_10_diff1": 0.038702, + "nauc_precision_at_20_max": -0.008043, + "nauc_precision_at_20_std": 0.124857, + "nauc_precision_at_20_diff1": 0.009376, + "nauc_precision_at_100_max": -0.037272, + "nauc_precision_at_100_std": 0.122189, + "nauc_precision_at_100_diff1": 0.088347, + "nauc_precision_at_1000_max": -0.025903, + "nauc_precision_at_1000_std": 0.109912, + "nauc_precision_at_1000_diff1": 0.148686, + "nauc_cv_recall_at_1_max": 0.172456, + "nauc_cv_recall_at_1_std": 0.158663, + "nauc_cv_recall_at_1_diff1": 0.077735, + "nauc_cv_recall_at_3_max": 0.418315, + "nauc_cv_recall_at_3_std": 0.377445, + "nauc_cv_recall_at_3_diff1": 0.300946, + "nauc_cv_recall_at_5_max": 0.651893, + "nauc_cv_recall_at_5_std": 0.636742, + "nauc_cv_recall_at_5_diff1": 0.199945, + "nauc_cv_recall_at_10_max": 0.413543, + "nauc_cv_recall_at_10_std": 0.413543, + "nauc_cv_recall_at_10_diff1": 0.422101, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.172456, + "nauc_mrr_at_1_std": 0.158663, + "nauc_mrr_at_1_diff1": 0.077735, + "nauc_mrr_at_3_max": 0.234485, + "nauc_mrr_at_3_std": 0.22339, + "nauc_mrr_at_3_diff1": 0.126001, + "nauc_mrr_at_5_max": 0.246994, + "nauc_mrr_at_5_std": 0.24168, + "nauc_mrr_at_5_diff1": 0.100608, + "nauc_mrr_at_10_max": 0.225705, + "nauc_mrr_at_10_std": 0.22028, + "nauc_mrr_at_10_diff1": 0.101779, + "nauc_mrr_at_20_max": 0.223181, + "nauc_mrr_at_20_std": 0.217692, + "nauc_mrr_at_20_diff1": 0.097836, + "nauc_mrr_at_100_max": 0.223181, + "nauc_mrr_at_100_std": 0.217692, + "nauc_mrr_at_100_diff1": 0.097836, + "nauc_mrr_at_1000_max": 0.223181, + "nauc_mrr_at_1000_std": 0.217692, + "nauc_mrr_at_1000_diff1": 0.097836, + "main_score": 0.47715, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 71.68689393997192, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreArxivQARetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..69ebe4afb4 --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", + "task_name": "VidoreArxivQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.786, + "ndcg_at_3": 0.82071, + "ndcg_at_5": 0.83655, + "ndcg_at_10": 0.84639, + "ndcg_at_20": 0.8565, + "ndcg_at_100": 0.8641, + "ndcg_at_1000": 0.86491, + "map_at_1": 0.786, + "map_at_3": 0.812, + "map_at_5": 0.8209, + "map_at_10": 0.82504, + "map_at_20": 0.82781, + "map_at_100": 0.82892, + "map_at_1000": 0.82896, + "recall_at_1": 0.786, + "recall_at_3": 0.846, + "recall_at_5": 0.884, + "recall_at_10": 0.914, + "recall_at_20": 0.954, + "recall_at_100": 0.994, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.786, + "cv_recall_at_3": 0.846, + "cv_recall_at_5": 0.884, + "cv_recall_at_10": 0.914, + "cv_recall_at_20": 0.954, + "cv_recall_at_100": 0.994, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.786, + "precision_at_3": 0.282, + "precision_at_5": 0.1768, + "precision_at_10": 0.0914, + "precision_at_20": 0.0477, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "mrr_at_1": 0.786, + "mrr_at_3": 0.812, + "mrr_at_5": 0.8209, + "mrr_at_10": 0.82504, + "mrr_at_20": 0.827807, + "mrr_at_100": 0.828924, + "mrr_at_1000": 0.828961, + "nauc_ndcg_at_1_max": 0.651839, + "nauc_ndcg_at_1_std": 0.002801, + "nauc_ndcg_at_1_diff1": 0.914017, + "nauc_ndcg_at_3_max": 0.67999, + "nauc_ndcg_at_3_std": 0.035753, + "nauc_ndcg_at_3_diff1": 0.91406, + "nauc_ndcg_at_5_max": 0.677788, + "nauc_ndcg_at_5_std": 0.05173, + "nauc_ndcg_at_5_diff1": 0.91281, + "nauc_ndcg_at_10_max": 0.680205, + "nauc_ndcg_at_10_std": 0.060098, + "nauc_ndcg_at_10_diff1": 0.910376, + "nauc_ndcg_at_20_max": 0.674304, + "nauc_ndcg_at_20_std": 0.058032, + "nauc_ndcg_at_20_diff1": 0.912703, + "nauc_ndcg_at_100_max": 0.673099, + "nauc_ndcg_at_100_std": 0.051814, + "nauc_ndcg_at_100_diff1": 0.912161, + "nauc_ndcg_at_1000_max": 0.67229, + "nauc_ndcg_at_1000_std": 0.046561, + "nauc_ndcg_at_1000_diff1": 0.912141, + "nauc_map_at_1_max": 0.651839, + "nauc_map_at_1_std": 0.002801, + "nauc_map_at_1_diff1": 0.914017, + "nauc_map_at_3_max": 0.671615, + "nauc_map_at_3_std": 0.026721, + "nauc_map_at_3_diff1": 0.913671, + "nauc_map_at_5_max": 0.670071, + "nauc_map_at_5_std": 0.034843, + "nauc_map_at_5_diff1": 0.913108, + "nauc_map_at_10_max": 0.670547, + "nauc_map_at_10_std": 0.036827, + "nauc_map_at_10_diff1": 0.912111, + "nauc_map_at_20_max": 0.668932, + "nauc_map_at_20_std": 0.035987, + "nauc_map_at_20_diff1": 0.912644, + "nauc_map_at_100_max": 0.669038, + "nauc_map_at_100_std": 0.036019, + "nauc_map_at_100_diff1": 0.912511, + "nauc_map_at_1000_max": 0.668996, + "nauc_map_at_1000_std": 0.03582, + "nauc_map_at_1000_diff1": 0.912508, + "nauc_recall_at_1_max": 0.651839, + "nauc_recall_at_1_std": 0.002801, + "nauc_recall_at_1_diff1": 0.914017, + "nauc_recall_at_3_max": 0.709221, + "nauc_recall_at_3_std": 0.06693, + "nauc_recall_at_3_diff1": 0.915583, + "nauc_recall_at_5_max": 0.710654, + "nauc_recall_at_5_std": 0.126242, + "nauc_recall_at_5_diff1": 0.91135, + "nauc_recall_at_10_max": 0.736315, + "nauc_recall_at_10_std": 0.201008, + "nauc_recall_at_10_diff1": 0.899301, + "nauc_recall_at_20_max": 0.718812, + "nauc_recall_at_20_std": 0.298786, + "nauc_recall_at_20_diff1": 0.91759, + "nauc_recall_at_100_max": 0.786026, + "nauc_recall_at_100_std": 0.863834, + "nauc_recall_at_100_diff1": 0.912854, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.651839, + "nauc_precision_at_1_std": 0.002801, + "nauc_precision_at_1_diff1": 0.914017, + "nauc_precision_at_3_max": 0.709221, + "nauc_precision_at_3_std": 0.06693, + "nauc_precision_at_3_diff1": 0.915583, + "nauc_precision_at_5_max": 0.710654, + "nauc_precision_at_5_std": 0.126242, + "nauc_precision_at_5_diff1": 0.91135, + "nauc_precision_at_10_max": 0.736315, + "nauc_precision_at_10_std": 0.201008, + "nauc_precision_at_10_diff1": 0.899301, + "nauc_precision_at_20_max": 0.718812, + "nauc_precision_at_20_std": 0.298786, + "nauc_precision_at_20_diff1": 0.91759, + "nauc_precision_at_100_max": 0.786026, + "nauc_precision_at_100_std": 0.863834, + "nauc_precision_at_100_diff1": 0.912854, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_cv_recall_at_1_max": 0.651839, + "nauc_cv_recall_at_1_std": 0.002801, + "nauc_cv_recall_at_1_diff1": 0.914017, + "nauc_cv_recall_at_3_max": 0.709221, + "nauc_cv_recall_at_3_std": 0.06693, + "nauc_cv_recall_at_3_diff1": 0.915583, + "nauc_cv_recall_at_5_max": 0.710654, + "nauc_cv_recall_at_5_std": 0.126242, + "nauc_cv_recall_at_5_diff1": 0.91135, + "nauc_cv_recall_at_10_max": 0.736315, + "nauc_cv_recall_at_10_std": 0.201008, + "nauc_cv_recall_at_10_diff1": 0.899301, + "nauc_cv_recall_at_20_max": 0.718812, + "nauc_cv_recall_at_20_std": 0.298786, + "nauc_cv_recall_at_20_diff1": 0.91759, + "nauc_cv_recall_at_100_max": 0.786026, + "nauc_cv_recall_at_100_std": 0.863834, + "nauc_cv_recall_at_100_diff1": 0.912854, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.651839, + "nauc_mrr_at_1_std": 0.002801, + "nauc_mrr_at_1_diff1": 0.914017, + "nauc_mrr_at_3_max": 0.671615, + "nauc_mrr_at_3_std": 0.026721, + "nauc_mrr_at_3_diff1": 0.913671, + "nauc_mrr_at_5_max": 0.670071, + "nauc_mrr_at_5_std": 0.034843, + "nauc_mrr_at_5_diff1": 0.913108, + "nauc_mrr_at_10_max": 0.670547, + "nauc_mrr_at_10_std": 0.036827, + "nauc_mrr_at_10_diff1": 0.912111, + "nauc_mrr_at_20_max": 0.668932, + "nauc_mrr_at_20_std": 0.035987, + "nauc_mrr_at_20_diff1": 0.912644, + "nauc_mrr_at_100_max": 0.669038, + "nauc_mrr_at_100_std": 0.036019, + "nauc_mrr_at_100_diff1": 0.912511, + "nauc_mrr_at_1000_max": 0.668996, + "nauc_mrr_at_1000_std": 0.03582, + "nauc_mrr_at_1000_diff1": 0.912508, + "main_score": 0.83655, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 31.698575973510742, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreDocVQARetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..1617a64ca7 --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", + "task_name": "VidoreDocVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.50554, + "ndcg_at_3": 0.56773, + "ndcg_at_5": 0.58735, + "ndcg_at_10": 0.60377, + "ndcg_at_20": 0.61448, + "ndcg_at_100": 0.6364, + "ndcg_at_1000": 0.6553, + "map_at_1": 0.49952, + "map_at_3": 0.54978, + "map_at_5": 0.56131, + "map_at_10": 0.5683, + "map_at_20": 0.57145, + "map_at_100": 0.57447, + "map_at_1000": 0.57539, + "recall_at_1": 0.49952, + "recall_at_3": 0.6126, + "recall_at_5": 0.66138, + "recall_at_10": 0.71024, + "recall_at_20": 0.75048, + "recall_at_100": 0.86545, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.50554, + "cv_recall_at_3": 0.61863, + "cv_recall_at_5": 0.66519, + "cv_recall_at_10": 0.7184, + "cv_recall_at_20": 0.76053, + "cv_recall_at_100": 0.87583, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.50554, + "precision_at_3": 0.20695, + "precision_at_5": 0.13437, + "precision_at_10": 0.07339, + "precision_at_20": 0.03902, + "precision_at_100": 0.00922, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.505543, + "mrr_at_3": 0.555063, + "mrr_at_5": 0.565595, + "mrr_at_10": 0.572709, + "mrr_at_20": 0.575866, + "mrr_at_100": 0.578735, + "mrr_at_1000": 0.579445, + "nauc_ndcg_at_1_max": -0.251974, + "nauc_ndcg_at_1_std": 0.719505, + "nauc_ndcg_at_1_diff1": 0.86157, + "nauc_ndcg_at_3_max": -0.342895, + "nauc_ndcg_at_3_std": 0.779356, + "nauc_ndcg_at_3_diff1": 0.855412, + "nauc_ndcg_at_5_max": -0.360988, + "nauc_ndcg_at_5_std": 0.787099, + "nauc_ndcg_at_5_diff1": 0.861074, + "nauc_ndcg_at_10_max": -0.390103, + "nauc_ndcg_at_10_std": 0.794203, + "nauc_ndcg_at_10_diff1": 0.857906, + "nauc_ndcg_at_20_max": -0.395524, + "nauc_ndcg_at_20_std": 0.793977, + "nauc_ndcg_at_20_diff1": 0.861338, + "nauc_ndcg_at_100_max": -0.389104, + "nauc_ndcg_at_100_std": 0.78474, + "nauc_ndcg_at_100_diff1": 0.858501, + "nauc_ndcg_at_1000_max": -0.365726, + "nauc_ndcg_at_1000_std": 0.777999, + "nauc_ndcg_at_1000_diff1": 0.858376, + "nauc_map_at_1_max": -0.249821, + "nauc_map_at_1_std": 0.729239, + "nauc_map_at_1_diff1": 0.870873, + "nauc_map_at_3_max": -0.316462, + "nauc_map_at_3_std": 0.767745, + "nauc_map_at_3_diff1": 0.860553, + "nauc_map_at_5_max": -0.327377, + "nauc_map_at_5_std": 0.772392, + "nauc_map_at_5_diff1": 0.863479, + "nauc_map_at_10_max": -0.338659, + "nauc_map_at_10_std": 0.77455, + "nauc_map_at_10_diff1": 0.861838, + "nauc_map_at_20_max": -0.33985, + "nauc_map_at_20_std": 0.774473, + "nauc_map_at_20_diff1": 0.862849, + "nauc_map_at_100_max": -0.339198, + "nauc_map_at_100_std": 0.773047, + "nauc_map_at_100_diff1": 0.862495, + "nauc_map_at_1000_max": -0.338179, + "nauc_map_at_1000_std": 0.772638, + "nauc_map_at_1000_diff1": 0.862422, + "nauc_recall_at_1_max": -0.249821, + "nauc_recall_at_1_std": 0.729239, + "nauc_recall_at_1_diff1": 0.870873, + "nauc_recall_at_3_max": -0.417332, + "nauc_recall_at_3_std": 0.820956, + "nauc_recall_at_3_diff1": 0.846459, + "nauc_recall_at_5_max": -0.471337, + "nauc_recall_at_5_std": 0.841674, + "nauc_recall_at_5_diff1": 0.85894, + "nauc_recall_at_10_max": -0.586163, + "nauc_recall_at_10_std": 0.874607, + "nauc_recall_at_10_diff1": 0.845696, + "nauc_recall_at_20_max": -0.640171, + "nauc_recall_at_20_std": 0.884802, + "nauc_recall_at_20_diff1": 0.86281, + "nauc_recall_at_100_max": -0.729332, + "nauc_recall_at_100_std": 0.85452, + "nauc_recall_at_100_diff1": 0.835606, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.251974, + "nauc_precision_at_1_std": 0.719505, + "nauc_precision_at_1_diff1": 0.86157, + "nauc_precision_at_3_max": -0.418077, + "nauc_precision_at_3_std": 0.807729, + "nauc_precision_at_3_diff1": 0.825551, + "nauc_precision_at_5_max": -0.462338, + "nauc_precision_at_5_std": 0.806855, + "nauc_precision_at_5_diff1": 0.807553, + "nauc_precision_at_10_max": -0.554235, + "nauc_precision_at_10_std": 0.760217, + "nauc_precision_at_10_diff1": 0.719351, + "nauc_precision_at_20_max": -0.581229, + "nauc_precision_at_20_std": 0.715137, + "nauc_precision_at_20_diff1": 0.681991, + "nauc_precision_at_100_max": -0.444429, + "nauc_precision_at_100_std": 0.337071, + "nauc_precision_at_100_diff1": 0.316479, + "nauc_precision_at_1000_max": 0.132169, + "nauc_precision_at_1000_std": -0.487261, + "nauc_precision_at_1000_diff1": -0.491032, + "nauc_cv_recall_at_1_max": -0.251974, + "nauc_cv_recall_at_1_std": 0.719505, + "nauc_cv_recall_at_1_diff1": 0.86157, + "nauc_cv_recall_at_3_max": -0.427514, + "nauc_cv_recall_at_3_std": 0.823268, + "nauc_cv_recall_at_3_diff1": 0.842389, + "nauc_cv_recall_at_5_max": -0.475942, + "nauc_cv_recall_at_5_std": 0.835211, + "nauc_cv_recall_at_5_diff1": 0.853545, + "nauc_cv_recall_at_10_max": -0.602207, + "nauc_cv_recall_at_10_std": 0.868009, + "nauc_cv_recall_at_10_diff1": 0.842094, + "nauc_cv_recall_at_20_max": -0.652863, + "nauc_cv_recall_at_20_std": 0.870822, + "nauc_cv_recall_at_20_diff1": 0.853006, + "nauc_cv_recall_at_100_max": -0.745192, + "nauc_cv_recall_at_100_std": 0.837586, + "nauc_cv_recall_at_100_diff1": 0.821353, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.251974, + "nauc_mrr_at_1_std": 0.719505, + "nauc_mrr_at_1_diff1": 0.86157, + "nauc_mrr_at_3_max": -0.322062, + "nauc_mrr_at_3_std": 0.762614, + "nauc_mrr_at_3_diff1": 0.854059, + "nauc_mrr_at_5_max": -0.330774, + "nauc_mrr_at_5_std": 0.764394, + "nauc_mrr_at_5_diff1": 0.856858, + "nauc_mrr_at_10_max": -0.342287, + "nauc_mrr_at_10_std": 0.766701, + "nauc_mrr_at_10_diff1": 0.855634, + "nauc_mrr_at_20_max": -0.342994, + "nauc_mrr_at_20_std": 0.766315, + "nauc_mrr_at_20_diff1": 0.856371, + "nauc_mrr_at_100_max": -0.341983, + "nauc_mrr_at_100_std": 0.765095, + "nauc_mrr_at_100_diff1": 0.8561, + "nauc_mrr_at_1000_max": -0.341111, + "nauc_mrr_at_1000_std": 0.764897, + "nauc_mrr_at_1000_diff1": 0.856177, + "main_score": 0.58735, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 39.14582061767578, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreInfoVQARetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..eb1dd005bc --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", + "task_name": "VidoreInfoVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.78947, + "ndcg_at_3": 0.83954, + "ndcg_at_5": 0.85675, + "ndcg_at_10": 0.86406, + "ndcg_at_20": 0.86719, + "ndcg_at_100": 0.87322, + "ndcg_at_1000": 0.87584, + "map_at_1": 0.78947, + "map_at_3": 0.8276, + "map_at_5": 0.83711, + "map_at_10": 0.84031, + "map_at_20": 0.8412, + "map_at_100": 0.84205, + "map_at_1000": 0.84216, + "recall_at_1": 0.78947, + "recall_at_3": 0.87348, + "recall_at_5": 0.91498, + "recall_at_10": 0.93664, + "recall_at_20": 0.94879, + "recall_at_100": 0.98117, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.78947, + "cv_recall_at_3": 0.87449, + "cv_recall_at_5": 0.917, + "cv_recall_at_10": 0.93927, + "cv_recall_at_20": 0.95142, + "cv_recall_at_100": 0.98381, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.78947, + "precision_at_3": 0.2915, + "precision_at_5": 0.1834, + "precision_at_10": 0.09413, + "precision_at_20": 0.04767, + "precision_at_100": 0.00986, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.789474, + "mrr_at_3": 0.827935, + "mrr_at_5": 0.837652, + "mrr_at_10": 0.840696, + "mrr_at_20": 0.84158, + "mrr_at_100": 0.842429, + "mrr_at_1000": 0.842519, + "nauc_ndcg_at_1_max": 0.533127, + "nauc_ndcg_at_1_std": 0.032682, + "nauc_ndcg_at_1_diff1": 0.905323, + "nauc_ndcg_at_3_max": 0.527035, + "nauc_ndcg_at_3_std": 0.064903, + "nauc_ndcg_at_3_diff1": 0.881765, + "nauc_ndcg_at_5_max": 0.562256, + "nauc_ndcg_at_5_std": 0.096282, + "nauc_ndcg_at_5_diff1": 0.886653, + "nauc_ndcg_at_10_max": 0.583106, + "nauc_ndcg_at_10_std": 0.127423, + "nauc_ndcg_at_10_diff1": 0.891117, + "nauc_ndcg_at_20_max": 0.576269, + "nauc_ndcg_at_20_std": 0.128941, + "nauc_ndcg_at_20_diff1": 0.889586, + "nauc_ndcg_at_100_max": 0.561968, + "nauc_ndcg_at_100_std": 0.117238, + "nauc_ndcg_at_100_diff1": 0.889647, + "nauc_ndcg_at_1000_max": 0.559755, + "nauc_ndcg_at_1000_std": 0.099333, + "nauc_ndcg_at_1000_diff1": 0.890718, + "nauc_map_at_1_max": 0.533127, + "nauc_map_at_1_std": 0.032682, + "nauc_map_at_1_diff1": 0.905323, + "nauc_map_at_3_max": 0.529466, + "nauc_map_at_3_std": 0.055356, + "nauc_map_at_3_diff1": 0.888737, + "nauc_map_at_5_max": 0.547631, + "nauc_map_at_5_std": 0.071683, + "nauc_map_at_5_diff1": 0.891579, + "nauc_map_at_10_max": 0.555936, + "nauc_map_at_10_std": 0.083232, + "nauc_map_at_10_diff1": 0.893066, + "nauc_map_at_20_max": 0.554138, + "nauc_map_at_20_std": 0.083225, + "nauc_map_at_20_diff1": 0.892723, + "nauc_map_at_100_max": 0.552349, + "nauc_map_at_100_std": 0.081583, + "nauc_map_at_100_diff1": 0.892641, + "nauc_map_at_1000_max": 0.552232, + "nauc_map_at_1000_std": 0.080942, + "nauc_map_at_1000_diff1": 0.89268, + "nauc_recall_at_1_max": 0.533127, + "nauc_recall_at_1_std": 0.032682, + "nauc_recall_at_1_diff1": 0.905323, + "nauc_recall_at_3_max": 0.514964, + "nauc_recall_at_3_std": 0.100457, + "nauc_recall_at_3_diff1": 0.856068, + "nauc_recall_at_5_max": 0.642002, + "nauc_recall_at_5_std": 0.232357, + "nauc_recall_at_5_diff1": 0.861809, + "nauc_recall_at_10_max": 0.79266, + "nauc_recall_at_10_std": 0.470067, + "nauc_recall_at_10_diff1": 0.88302, + "nauc_recall_at_20_max": 0.777245, + "nauc_recall_at_20_std": 0.568014, + "nauc_recall_at_20_diff1": 0.866255, + "nauc_recall_at_100_max": 0.639139, + "nauc_recall_at_100_std": 0.922211, + "nauc_recall_at_100_diff1": 0.839872, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.533127, + "nauc_precision_at_1_std": 0.032682, + "nauc_precision_at_1_diff1": 0.905323, + "nauc_precision_at_3_max": 0.520093, + "nauc_precision_at_3_std": 0.102598, + "nauc_precision_at_3_diff1": 0.853947, + "nauc_precision_at_5_max": 0.650937, + "nauc_precision_at_5_std": 0.227913, + "nauc_precision_at_5_diff1": 0.861826, + "nauc_precision_at_10_max": 0.753678, + "nauc_precision_at_10_std": 0.427927, + "nauc_precision_at_10_diff1": 0.80589, + "nauc_precision_at_20_max": 0.729443, + "nauc_precision_at_20_std": 0.514365, + "nauc_precision_at_20_diff1": 0.771438, + "nauc_precision_at_100_max": 0.519129, + "nauc_precision_at_100_std": 0.764337, + "nauc_precision_at_100_diff1": 0.590555, + "nauc_precision_at_1000_max": -0.385114, + "nauc_precision_at_1000_std": -0.461596, + "nauc_precision_at_1000_diff1": -0.497032, + "nauc_cv_recall_at_1_max": 0.533127, + "nauc_cv_recall_at_1_std": 0.032682, + "nauc_cv_recall_at_1_diff1": 0.905323, + "nauc_cv_recall_at_3_max": 0.520093, + "nauc_cv_recall_at_3_std": 0.102598, + "nauc_cv_recall_at_3_diff1": 0.853947, + "nauc_cv_recall_at_5_max": 0.650937, + "nauc_cv_recall_at_5_std": 0.227913, + "nauc_cv_recall_at_5_diff1": 0.861826, + "nauc_cv_recall_at_10_max": 0.791787, + "nauc_cv_recall_at_10_std": 0.447103, + "nauc_cv_recall_at_10_diff1": 0.886062, + "nauc_cv_recall_at_20_max": 0.775318, + "nauc_cv_recall_at_20_std": 0.544615, + "nauc_cv_recall_at_20_diff1": 0.86915, + "nauc_cv_recall_at_100_max": 0.610918, + "nauc_cv_recall_at_100_std": 0.90957, + "nauc_cv_recall_at_100_diff1": 0.844269, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.533127, + "nauc_mrr_at_1_std": 0.032682, + "nauc_mrr_at_1_diff1": 0.905323, + "nauc_mrr_at_3_max": 0.530782, + "nauc_mrr_at_3_std": 0.055813, + "nauc_mrr_at_3_diff1": 0.888263, + "nauc_mrr_at_5_max": 0.548859, + "nauc_mrr_at_5_std": 0.070973, + "nauc_mrr_at_5_diff1": 0.891358, + "nauc_mrr_at_10_max": 0.555335, + "nauc_mrr_at_10_std": 0.080894, + "nauc_mrr_at_10_diff1": 0.893325, + "nauc_mrr_at_20_max": 0.553529, + "nauc_mrr_at_20_std": 0.080872, + "nauc_mrr_at_20_diff1": 0.892983, + "nauc_mrr_at_100_max": 0.551732, + "nauc_mrr_at_100_std": 0.079213, + "nauc_mrr_at_100_diff1": 0.892902, + "nauc_mrr_at_1000_max": 0.551666, + "nauc_mrr_at_1000_std": 0.078736, + "nauc_mrr_at_1000_diff1": 0.892931, + "main_score": 0.85675, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 37.24749732017517, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreShiftProjectRetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..d69e41718b --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "84a382e05c4473fed9cff2bbae95fe2379416117", + "task_name": "VidoreShiftProjectRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.61, + "ndcg_at_3": 0.73071, + "ndcg_at_5": 0.76473, + "ndcg_at_10": 0.78132, + "ndcg_at_20": 0.78627, + "ndcg_at_100": 0.79143, + "ndcg_at_1000": 0.79143, + "map_at_1": 0.61, + "map_at_3": 0.7, + "map_at_5": 0.7195, + "map_at_10": 0.72658, + "map_at_20": 0.72789, + "map_at_100": 0.72844, + "map_at_1000": 0.72844, + "recall_at_1": 0.61, + "recall_at_3": 0.82, + "recall_at_5": 0.9, + "recall_at_10": 0.95, + "recall_at_20": 0.97, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.61, + "cv_recall_at_3": 0.82, + "cv_recall_at_5": 0.9, + "cv_recall_at_10": 0.95, + "cv_recall_at_20": 0.97, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.61, + "precision_at_3": 0.27333, + "precision_at_5": 0.18, + "precision_at_10": 0.095, + "precision_at_20": 0.0485, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.61, + "mrr_at_3": 0.7, + "mrr_at_5": 0.7195, + "mrr_at_10": 0.726583, + "mrr_at_20": 0.727886, + "mrr_at_100": 0.728443, + "mrr_at_1000": 0.728443, + "nauc_ndcg_at_1_max": -0.056365, + "nauc_ndcg_at_1_std": -0.234768, + "nauc_ndcg_at_1_diff1": 0.700406, + "nauc_ndcg_at_3_max": -0.185589, + "nauc_ndcg_at_3_std": -0.416493, + "nauc_ndcg_at_3_diff1": 0.617428, + "nauc_ndcg_at_5_max": -0.106431, + "nauc_ndcg_at_5_std": -0.32448, + "nauc_ndcg_at_5_diff1": 0.623438, + "nauc_ndcg_at_10_max": -0.115383, + "nauc_ndcg_at_10_std": -0.315257, + "nauc_ndcg_at_10_diff1": 0.634082, + "nauc_ndcg_at_20_max": -0.115508, + "nauc_ndcg_at_20_std": -0.333188, + "nauc_ndcg_at_20_diff1": 0.636851, + "nauc_ndcg_at_100_max": -0.110691, + "nauc_ndcg_at_100_std": -0.319816, + "nauc_ndcg_at_100_diff1": 0.640984, + "nauc_ndcg_at_1000_max": -0.110691, + "nauc_ndcg_at_1000_std": -0.319816, + "nauc_ndcg_at_1000_diff1": 0.640984, + "nauc_map_at_1_max": -0.056365, + "nauc_map_at_1_std": -0.234768, + "nauc_map_at_1_diff1": 0.700406, + "nauc_map_at_3_max": -0.147724, + "nauc_map_at_3_std": -0.363987, + "nauc_map_at_3_diff1": 0.639195, + "nauc_map_at_5_max": -0.106619, + "nauc_map_at_5_std": -0.315829, + "nauc_map_at_5_diff1": 0.643022, + "nauc_map_at_10_max": -0.109356, + "nauc_map_at_10_std": -0.311393, + "nauc_map_at_10_diff1": 0.647242, + "nauc_map_at_20_max": -0.108947, + "nauc_map_at_20_std": -0.31503, + "nauc_map_at_20_diff1": 0.647963, + "nauc_map_at_100_max": -0.108083, + "nauc_map_at_100_std": -0.313456, + "nauc_map_at_100_diff1": 0.648071, + "nauc_map_at_1000_max": -0.108083, + "nauc_map_at_1000_std": -0.313456, + "nauc_map_at_1000_diff1": 0.648071, + "nauc_recall_at_1_max": -0.056365, + "nauc_recall_at_1_std": -0.234768, + "nauc_recall_at_1_diff1": 0.700406, + "nauc_recall_at_3_max": -0.346762, + "nauc_recall_at_3_std": -0.639525, + "nauc_recall_at_3_diff1": 0.526352, + "nauc_recall_at_5_max": -0.080532, + "nauc_recall_at_5_std": -0.352568, + "nauc_recall_at_5_diff1": 0.498179, + "nauc_recall_at_10_max": -0.162278, + "nauc_recall_at_10_std": -0.300747, + "nauc_recall_at_10_diff1": 0.492624, + "nauc_recall_at_20_max": -0.219888, + "nauc_recall_at_20_std": -0.726891, + "nauc_recall_at_20_diff1": 0.46094, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.056365, + "nauc_precision_at_1_std": -0.234768, + "nauc_precision_at_1_diff1": 0.700406, + "nauc_precision_at_3_max": -0.346762, + "nauc_precision_at_3_std": -0.639525, + "nauc_precision_at_3_diff1": 0.526352, + "nauc_precision_at_5_max": -0.080532, + "nauc_precision_at_5_std": -0.352568, + "nauc_precision_at_5_diff1": 0.498179, + "nauc_precision_at_10_max": -0.162278, + "nauc_precision_at_10_std": -0.300747, + "nauc_precision_at_10_diff1": 0.492624, + "nauc_precision_at_20_max": -0.219888, + "nauc_precision_at_20_std": -0.726891, + "nauc_precision_at_20_diff1": 0.46094, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": -0.056365, + "nauc_cv_recall_at_1_std": -0.234768, + "nauc_cv_recall_at_1_diff1": 0.700406, + "nauc_cv_recall_at_3_max": -0.346762, + "nauc_cv_recall_at_3_std": -0.639525, + "nauc_cv_recall_at_3_diff1": 0.526352, + "nauc_cv_recall_at_5_max": -0.080532, + "nauc_cv_recall_at_5_std": -0.352568, + "nauc_cv_recall_at_5_diff1": 0.498179, + "nauc_cv_recall_at_10_max": -0.162278, + "nauc_cv_recall_at_10_std": -0.300747, + "nauc_cv_recall_at_10_diff1": 0.492624, + "nauc_cv_recall_at_20_max": -0.219888, + "nauc_cv_recall_at_20_std": -0.726891, + "nauc_cv_recall_at_20_diff1": 0.46094, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.056365, + "nauc_mrr_at_1_std": -0.234768, + "nauc_mrr_at_1_diff1": 0.700406, + "nauc_mrr_at_3_max": -0.147724, + "nauc_mrr_at_3_std": -0.363987, + "nauc_mrr_at_3_diff1": 0.639195, + "nauc_mrr_at_5_max": -0.106619, + "nauc_mrr_at_5_std": -0.315829, + "nauc_mrr_at_5_diff1": 0.643022, + "nauc_mrr_at_10_max": -0.109356, + "nauc_mrr_at_10_std": -0.311393, + "nauc_mrr_at_10_diff1": 0.647242, + "nauc_mrr_at_20_max": -0.108947, + "nauc_mrr_at_20_std": -0.31503, + "nauc_mrr_at_20_diff1": 0.647963, + "nauc_mrr_at_100_max": -0.108083, + "nauc_mrr_at_100_std": -0.313456, + "nauc_mrr_at_100_diff1": 0.648071, + "nauc_mrr_at_1000_max": -0.108083, + "nauc_mrr_at_1000_std": -0.313456, + "nauc_mrr_at_1000_diff1": 0.648071, + "main_score": 0.76473, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 73.16034507751465, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreSyntheticDocQAAIRetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..29ab758c72 --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", + "task_name": "VidoreSyntheticDocQAAIRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.95, + "ndcg_at_3": 0.96262, + "ndcg_at_5": 0.96649, + "ndcg_at_10": 0.97253, + "ndcg_at_20": 0.97253, + "ndcg_at_100": 0.97253, + "ndcg_at_1000": 0.97253, + "map_at_1": 0.95, + "map_at_3": 0.96, + "map_at_5": 0.962, + "map_at_10": 0.96425, + "map_at_20": 0.96425, + "map_at_100": 0.96425, + "map_at_1000": 0.96425, + "recall_at_1": 0.95, + "recall_at_3": 0.97, + "recall_at_5": 0.98, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.95, + "cv_recall_at_3": 0.97, + "cv_recall_at_5": 0.98, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.95, + "precision_at_3": 0.32333, + "precision_at_5": 0.196, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.95, + "mrr_at_3": 0.96, + "mrr_at_5": 0.962, + "mrr_at_10": 0.96425, + "mrr_at_20": 0.96425, + "mrr_at_100": 0.96425, + "mrr_at_1000": 0.96425, + "nauc_ndcg_at_1_max": 0.713445, + "nauc_ndcg_at_1_std": -0.35733, + "nauc_ndcg_at_1_diff1": 0.947712, + "nauc_ndcg_at_3_max": 0.638778, + "nauc_ndcg_at_3_std": -0.570931, + "nauc_ndcg_at_3_diff1": 0.930062, + "nauc_ndcg_at_5_max": 0.732293, + "nauc_ndcg_at_5_std": -0.503887, + "nauc_ndcg_at_5_diff1": 0.937078, + "nauc_ndcg_at_10_max": 0.68713, + "nauc_ndcg_at_10_std": -0.49532, + "nauc_ndcg_at_10_diff1": 0.936987, + "nauc_ndcg_at_20_max": 0.68713, + "nauc_ndcg_at_20_std": -0.49532, + "nauc_ndcg_at_20_diff1": 0.936987, + "nauc_ndcg_at_100_max": 0.68713, + "nauc_ndcg_at_100_std": -0.49532, + "nauc_ndcg_at_100_diff1": 0.936987, + "nauc_ndcg_at_1000_max": 0.68713, + "nauc_ndcg_at_1000_std": -0.49532, + "nauc_ndcg_at_1000_diff1": 0.936987, + "nauc_map_at_1_max": 0.713445, + "nauc_map_at_1_std": -0.35733, + "nauc_map_at_1_diff1": 0.947712, + "nauc_map_at_3_max": 0.658147, + "nauc_map_at_3_std": -0.515523, + "nauc_map_at_3_diff1": 0.934641, + "nauc_map_at_5_max": 0.701804, + "nauc_map_at_5_std": -0.482038, + "nauc_map_at_5_diff1": 0.93808, + "nauc_map_at_10_max": 0.686692, + "nauc_map_at_10_std": -0.483082, + "nauc_map_at_10_diff1": 0.93784, + "nauc_map_at_20_max": 0.686692, + "nauc_map_at_20_std": -0.483082, + "nauc_map_at_20_diff1": 0.93784, + "nauc_map_at_100_max": 0.686692, + "nauc_map_at_100_std": -0.483082, + "nauc_map_at_100_diff1": 0.93784, + "nauc_map_at_1000_max": 0.686692, + "nauc_map_at_1000_std": -0.483082, + "nauc_map_at_1000_diff1": 0.93784, + "nauc_recall_at_1_max": 0.713445, + "nauc_recall_at_1_std": -0.35733, + "nauc_recall_at_1_diff1": 0.947712, + "nauc_recall_at_3_max": 0.565982, + "nauc_recall_at_3_std": -0.779178, + "nauc_recall_at_3_diff1": 0.912854, + "nauc_recall_at_5_max": 0.934641, + "nauc_recall_at_5_std": -0.592904, + "nauc_recall_at_5_diff1": 0.934641, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.713445, + "nauc_precision_at_1_std": -0.35733, + "nauc_precision_at_1_diff1": 0.947712, + "nauc_precision_at_3_max": 0.565982, + "nauc_precision_at_3_std": -0.779178, + "nauc_precision_at_3_diff1": 0.912854, + "nauc_precision_at_5_max": 0.934641, + "nauc_precision_at_5_std": -0.592904, + "nauc_precision_at_5_diff1": 0.934641, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.713445, + "nauc_cv_recall_at_1_std": -0.35733, + "nauc_cv_recall_at_1_diff1": 0.947712, + "nauc_cv_recall_at_3_max": 0.565982, + "nauc_cv_recall_at_3_std": -0.779178, + "nauc_cv_recall_at_3_diff1": 0.912854, + "nauc_cv_recall_at_5_max": 0.934641, + "nauc_cv_recall_at_5_std": -0.592904, + "nauc_cv_recall_at_5_diff1": 0.934641, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.713445, + "nauc_mrr_at_1_std": -0.35733, + "nauc_mrr_at_1_diff1": 0.947712, + "nauc_mrr_at_3_max": 0.658147, + "nauc_mrr_at_3_std": -0.515523, + "nauc_mrr_at_3_diff1": 0.934641, + "nauc_mrr_at_5_max": 0.701804, + "nauc_mrr_at_5_std": -0.482038, + "nauc_mrr_at_5_diff1": 0.93808, + "nauc_mrr_at_10_max": 0.686692, + "nauc_mrr_at_10_std": -0.483082, + "nauc_mrr_at_10_diff1": 0.93784, + "nauc_mrr_at_20_max": 0.686692, + "nauc_mrr_at_20_std": -0.483082, + "nauc_mrr_at_20_diff1": 0.93784, + "nauc_mrr_at_100_max": 0.686692, + "nauc_mrr_at_100_std": -0.483082, + "nauc_mrr_at_100_diff1": 0.93784, + "nauc_mrr_at_1000_max": 0.686692, + "nauc_mrr_at_1000_std": -0.483082, + "nauc_mrr_at_1000_diff1": 0.93784, + "main_score": 0.96649, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 70.06839275360107, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreSyntheticDocQAEnergyRetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..49bb25026a --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", + "task_name": "VidoreSyntheticDocQAEnergyRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.93, + "ndcg_at_3": 0.94262, + "ndcg_at_5": 0.94649, + "ndcg_at_10": 0.94964, + "ndcg_at_20": 0.95458, + "ndcg_at_100": 0.95635, + "ndcg_at_1000": 0.95635, + "map_at_1": 0.93, + "map_at_3": 0.94, + "map_at_5": 0.942, + "map_at_10": 0.94325, + "map_at_20": 0.94455, + "map_at_100": 0.94475, + "map_at_1000": 0.94475, + "recall_at_1": 0.93, + "recall_at_3": 0.95, + "recall_at_5": 0.96, + "recall_at_10": 0.97, + "recall_at_20": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.93, + "cv_recall_at_3": 0.95, + "cv_recall_at_5": 0.96, + "cv_recall_at_10": 0.97, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.93, + "precision_at_3": 0.31667, + "precision_at_5": 0.192, + "precision_at_10": 0.097, + "precision_at_20": 0.0495, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.93, + "mrr_at_3": 0.94, + "mrr_at_5": 0.942, + "mrr_at_10": 0.94325, + "mrr_at_20": 0.944546, + "mrr_at_100": 0.94475, + "mrr_at_1000": 0.94475, + "nauc_ndcg_at_1_max": 0.553288, + "nauc_ndcg_at_1_std": -0.434641, + "nauc_ndcg_at_1_diff1": 0.922969, + "nauc_ndcg_at_3_max": 0.485595, + "nauc_ndcg_at_3_std": -0.630524, + "nauc_ndcg_at_3_diff1": 0.920403, + "nauc_ndcg_at_5_max": 0.468489, + "nauc_ndcg_at_5_std": -0.716167, + "nauc_ndcg_at_5_diff1": 0.914648, + "nauc_ndcg_at_10_max": 0.490146, + "nauc_ndcg_at_10_std": -0.652032, + "nauc_ndcg_at_10_diff1": 0.909302, + "nauc_ndcg_at_20_max": 0.531762, + "nauc_ndcg_at_20_std": -0.563661, + "nauc_ndcg_at_20_diff1": 0.922159, + "nauc_ndcg_at_100_max": 0.512754, + "nauc_ndcg_at_100_std": -0.591527, + "nauc_ndcg_at_100_diff1": 0.918999, + "nauc_ndcg_at_1000_max": 0.512754, + "nauc_ndcg_at_1000_std": -0.591527, + "nauc_ndcg_at_1000_diff1": 0.918999, + "nauc_map_at_1_max": 0.553288, + "nauc_map_at_1_std": -0.434641, + "nauc_map_at_1_diff1": 0.922969, + "nauc_map_at_3_max": 0.501984, + "nauc_map_at_3_std": -0.5831, + "nauc_map_at_3_diff1": 0.921024, + "nauc_map_at_5_max": 0.49439, + "nauc_map_at_5_std": -0.622316, + "nauc_map_at_5_diff1": 0.918301, + "nauc_map_at_10_max": 0.502575, + "nauc_map_at_10_std": -0.597698, + "nauc_map_at_10_diff1": 0.916501, + "nauc_map_at_20_max": 0.513882, + "nauc_map_at_20_std": -0.576595, + "nauc_map_at_20_diff1": 0.919644, + "nauc_map_at_100_max": 0.512087, + "nauc_map_at_100_std": -0.579178, + "nauc_map_at_100_diff1": 0.919347, + "nauc_map_at_1000_max": 0.512087, + "nauc_map_at_1000_std": -0.579178, + "nauc_map_at_1000_diff1": 0.919347, + "nauc_recall_at_1_max": 0.553288, + "nauc_recall_at_1_std": -0.434641, + "nauc_recall_at_1_diff1": 0.922969, + "nauc_recall_at_3_max": 0.430159, + "nauc_recall_at_3_std": -0.790943, + "nauc_recall_at_3_diff1": 0.918301, + "nauc_recall_at_5_max": 0.357143, + "nauc_recall_at_5_std": -1.127218, + "nauc_recall_at_5_diff1": 0.897876, + "nauc_recall_at_10_max": 0.435263, + "nauc_recall_at_10_std": -0.922969, + "nauc_recall_at_10_diff1": 0.863834, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 0.122782, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.553288, + "nauc_precision_at_1_std": -0.434641, + "nauc_precision_at_1_diff1": 0.922969, + "nauc_precision_at_3_max": 0.430159, + "nauc_precision_at_3_std": -0.790943, + "nauc_precision_at_3_diff1": 0.918301, + "nauc_precision_at_5_max": 0.357143, + "nauc_precision_at_5_std": -1.127218, + "nauc_precision_at_5_diff1": 0.897876, + "nauc_precision_at_10_max": 0.435263, + "nauc_precision_at_10_std": -0.922969, + "nauc_precision_at_10_diff1": 0.863834, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 0.122782, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.553288, + "nauc_cv_recall_at_1_std": -0.434641, + "nauc_cv_recall_at_1_diff1": 0.922969, + "nauc_cv_recall_at_3_max": 0.430159, + "nauc_cv_recall_at_3_std": -0.790943, + "nauc_cv_recall_at_3_diff1": 0.918301, + "nauc_cv_recall_at_5_max": 0.357143, + "nauc_cv_recall_at_5_std": -1.127218, + "nauc_cv_recall_at_5_diff1": 0.897876, + "nauc_cv_recall_at_10_max": 0.435263, + "nauc_cv_recall_at_10_std": -0.922969, + "nauc_cv_recall_at_10_diff1": 0.863834, + "nauc_cv_recall_at_20_max": 1.0, + "nauc_cv_recall_at_20_std": 0.122782, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.553288, + "nauc_mrr_at_1_std": -0.434641, + "nauc_mrr_at_1_diff1": 0.922969, + "nauc_mrr_at_3_max": 0.501984, + "nauc_mrr_at_3_std": -0.5831, + "nauc_mrr_at_3_diff1": 0.921024, + "nauc_mrr_at_5_max": 0.49439, + "nauc_mrr_at_5_std": -0.622316, + "nauc_mrr_at_5_diff1": 0.918301, + "nauc_mrr_at_10_max": 0.502575, + "nauc_mrr_at_10_std": -0.597698, + "nauc_mrr_at_10_diff1": 0.916501, + "nauc_mrr_at_20_max": 0.513882, + "nauc_mrr_at_20_std": -0.576595, + "nauc_mrr_at_20_diff1": 0.919644, + "nauc_mrr_at_100_max": 0.512087, + "nauc_mrr_at_100_std": -0.579178, + "nauc_mrr_at_100_diff1": 0.919347, + "nauc_mrr_at_1000_max": 0.512087, + "nauc_mrr_at_1000_std": -0.579178, + "nauc_mrr_at_1000_diff1": 0.919347, + "main_score": 0.94649, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 67.0736289024353, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..7627ba90f2 --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b4909afa930f81282fd20601e860668073ad02aa", + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.9, + "ndcg_at_3": 0.95917, + "ndcg_at_5": 0.95917, + "ndcg_at_10": 0.95917, + "ndcg_at_20": 0.95917, + "ndcg_at_100": 0.95917, + "ndcg_at_1000": 0.95917, + "map_at_1": 0.9, + "map_at_3": 0.945, + "map_at_5": 0.945, + "map_at_10": 0.945, + "map_at_20": 0.945, + "map_at_100": 0.945, + "map_at_1000": 0.945, + "recall_at_1": 0.9, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.9, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.9, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.9, + "mrr_at_3": 0.945, + "mrr_at_5": 0.945, + "mrr_at_10": 0.945, + "mrr_at_20": 0.945, + "mrr_at_100": 0.945, + "mrr_at_1000": 0.945, + "nauc_ndcg_at_1_max": 0.298506, + "nauc_ndcg_at_1_std": 0.412698, + "nauc_ndcg_at_1_diff1": 0.827965, + "nauc_ndcg_at_3_max": 0.328426, + "nauc_ndcg_at_3_std": 0.432682, + "nauc_ndcg_at_3_diff1": 0.831415, + "nauc_ndcg_at_5_max": 0.328426, + "nauc_ndcg_at_5_std": 0.432682, + "nauc_ndcg_at_5_diff1": 0.831415, + "nauc_ndcg_at_10_max": 0.328426, + "nauc_ndcg_at_10_std": 0.432682, + "nauc_ndcg_at_10_diff1": 0.831415, + "nauc_ndcg_at_20_max": 0.328426, + "nauc_ndcg_at_20_std": 0.432682, + "nauc_ndcg_at_20_diff1": 0.831415, + "nauc_ndcg_at_100_max": 0.328426, + "nauc_ndcg_at_100_std": 0.432682, + "nauc_ndcg_at_100_diff1": 0.831415, + "nauc_ndcg_at_1000_max": 0.328426, + "nauc_ndcg_at_1000_std": 0.432682, + "nauc_ndcg_at_1000_diff1": 0.831415, + "nauc_map_at_1_max": 0.298506, + "nauc_map_at_1_std": 0.412698, + "nauc_map_at_1_diff1": 0.827965, + "nauc_map_at_3_max": 0.326783, + "nauc_map_at_3_std": 0.431585, + "nauc_map_at_3_diff1": 0.831225, + "nauc_map_at_5_max": 0.326783, + "nauc_map_at_5_std": 0.431585, + "nauc_map_at_5_diff1": 0.831225, + "nauc_map_at_10_max": 0.326783, + "nauc_map_at_10_std": 0.431585, + "nauc_map_at_10_diff1": 0.831225, + "nauc_map_at_20_max": 0.326783, + "nauc_map_at_20_std": 0.431585, + "nauc_map_at_20_diff1": 0.831225, + "nauc_map_at_100_max": 0.326783, + "nauc_map_at_100_std": 0.431585, + "nauc_map_at_100_diff1": 0.831225, + "nauc_map_at_1000_max": 0.326783, + "nauc_map_at_1000_std": 0.431585, + "nauc_map_at_1000_diff1": 0.831225, + "nauc_recall_at_1_max": 0.298506, + "nauc_recall_at_1_std": 0.412698, + "nauc_recall_at_1_diff1": 0.827965, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.298506, + "nauc_precision_at_1_std": 0.412698, + "nauc_precision_at_1_diff1": 0.827965, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.298506, + "nauc_cv_recall_at_1_std": 0.412698, + "nauc_cv_recall_at_1_diff1": 0.827965, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.298506, + "nauc_mrr_at_1_std": 0.412698, + "nauc_mrr_at_1_diff1": 0.827965, + "nauc_mrr_at_3_max": 0.326783, + "nauc_mrr_at_3_std": 0.431585, + "nauc_mrr_at_3_diff1": 0.831225, + "nauc_mrr_at_5_max": 0.326783, + "nauc_mrr_at_5_std": 0.431585, + "nauc_mrr_at_5_diff1": 0.831225, + "nauc_mrr_at_10_max": 0.326783, + "nauc_mrr_at_10_std": 0.431585, + "nauc_mrr_at_10_diff1": 0.831225, + "nauc_mrr_at_20_max": 0.326783, + "nauc_mrr_at_20_std": 0.431585, + "nauc_mrr_at_20_diff1": 0.831225, + "nauc_mrr_at_100_max": 0.326783, + "nauc_mrr_at_100_std": 0.431585, + "nauc_mrr_at_100_diff1": 0.831225, + "nauc_mrr_at_1000_max": 0.326783, + "nauc_mrr_at_1000_std": 0.431585, + "nauc_mrr_at_1000_diff1": 0.831225, + "main_score": 0.95917, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 69.92950463294983, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..0f16d465ec --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.95, + "ndcg_at_3": 0.97393, + "ndcg_at_5": 0.97393, + "ndcg_at_10": 0.97749, + "ndcg_at_20": 0.97749, + "ndcg_at_100": 0.97749, + "ndcg_at_1000": 0.97749, + "map_at_1": 0.95, + "map_at_3": 0.96833, + "map_at_5": 0.96833, + "map_at_10": 0.97, + "map_at_20": 0.97, + "map_at_100": 0.97, + "map_at_1000": 0.97, + "recall_at_1": 0.95, + "recall_at_3": 0.99, + "recall_at_5": 0.99, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.95, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 0.99, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.95, + "precision_at_3": 0.33, + "precision_at_5": 0.198, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.95, + "mrr_at_3": 0.968333, + "mrr_at_5": 0.968333, + "mrr_at_10": 0.97, + "mrr_at_20": 0.97, + "mrr_at_100": 0.97, + "mrr_at_1000": 0.97, + "nauc_ndcg_at_1_max": 0.358637, + "nauc_ndcg_at_1_std": -0.916433, + "nauc_ndcg_at_1_diff1": 0.947712, + "nauc_ndcg_at_3_max": 0.135459, + "nauc_ndcg_at_3_std": -1.097998, + "nauc_ndcg_at_3_diff1": 0.931358, + "nauc_ndcg_at_5_max": 0.135459, + "nauc_ndcg_at_5_std": -1.097998, + "nauc_ndcg_at_5_diff1": 0.931358, + "nauc_ndcg_at_10_max": 0.246063, + "nauc_ndcg_at_10_std": -0.996412, + "nauc_ndcg_at_10_diff1": 0.941182, + "nauc_ndcg_at_20_max": 0.246063, + "nauc_ndcg_at_20_std": -0.996412, + "nauc_ndcg_at_20_diff1": 0.941182, + "nauc_ndcg_at_100_max": 0.246063, + "nauc_ndcg_at_100_std": -0.996412, + "nauc_ndcg_at_100_diff1": 0.941182, + "nauc_ndcg_at_1000_max": 0.246063, + "nauc_ndcg_at_1000_std": -0.996412, + "nauc_ndcg_at_1000_diff1": 0.941182, + "nauc_map_at_1_max": 0.358637, + "nauc_map_at_1_std": -0.916433, + "nauc_map_at_1_diff1": 0.947712, + "nauc_map_at_3_max": 0.213008, + "nauc_map_at_3_std": -1.027888, + "nauc_map_at_3_diff1": 0.93808, + "nauc_map_at_5_max": 0.213008, + "nauc_map_at_5_std": -1.027888, + "nauc_map_at_5_diff1": 0.93808, + "nauc_map_at_10_max": 0.256147, + "nauc_map_at_10_std": -0.988329, + "nauc_map_at_10_diff1": 0.941903, + "nauc_map_at_20_max": 0.256147, + "nauc_map_at_20_std": -0.988329, + "nauc_map_at_20_diff1": 0.941903, + "nauc_map_at_100_max": 0.256147, + "nauc_map_at_100_std": -0.988329, + "nauc_map_at_100_diff1": 0.941903, + "nauc_map_at_1000_max": 0.256147, + "nauc_map_at_1000_std": -0.988329, + "nauc_map_at_1000_diff1": 0.941903, + "nauc_recall_at_1_max": 0.358637, + "nauc_recall_at_1_std": -0.916433, + "nauc_recall_at_1_diff1": 0.947712, + "nauc_recall_at_3_max": -0.563492, + "nauc_recall_at_3_std": -1.739963, + "nauc_recall_at_3_diff1": 0.869281, + "nauc_recall_at_5_max": -0.563492, + "nauc_recall_at_5_std": -1.739963, + "nauc_recall_at_5_diff1": 0.869281, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.358637, + "nauc_precision_at_1_std": -0.916433, + "nauc_precision_at_1_diff1": 0.947712, + "nauc_precision_at_3_max": -0.563492, + "nauc_precision_at_3_std": -1.739963, + "nauc_precision_at_3_diff1": 0.869281, + "nauc_precision_at_5_max": -0.563492, + "nauc_precision_at_5_std": -1.739963, + "nauc_precision_at_5_diff1": 0.869281, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.358637, + "nauc_cv_recall_at_1_std": -0.916433, + "nauc_cv_recall_at_1_diff1": 0.947712, + "nauc_cv_recall_at_3_max": -0.563492, + "nauc_cv_recall_at_3_std": -1.739963, + "nauc_cv_recall_at_3_diff1": 0.869281, + "nauc_cv_recall_at_5_max": -0.563492, + "nauc_cv_recall_at_5_std": -1.739963, + "nauc_cv_recall_at_5_diff1": 0.869281, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.358637, + "nauc_mrr_at_1_std": -0.916433, + "nauc_mrr_at_1_diff1": 0.947712, + "nauc_mrr_at_3_max": 0.213008, + "nauc_mrr_at_3_std": -1.027888, + "nauc_mrr_at_3_diff1": 0.93808, + "nauc_mrr_at_5_max": 0.213008, + "nauc_mrr_at_5_std": -1.027888, + "nauc_mrr_at_5_diff1": 0.93808, + "nauc_mrr_at_10_max": 0.256147, + "nauc_mrr_at_10_std": -0.988329, + "nauc_mrr_at_10_diff1": 0.941903, + "nauc_mrr_at_20_max": 0.256147, + "nauc_mrr_at_20_std": -0.988329, + "nauc_mrr_at_20_diff1": 0.941903, + "nauc_mrr_at_100_max": 0.256147, + "nauc_mrr_at_100_std": -0.988329, + "nauc_mrr_at_100_diff1": 0.941903, + "nauc_mrr_at_1000_max": 0.256147, + "nauc_mrr_at_1000_std": -0.988329, + "nauc_mrr_at_1000_diff1": 0.941903, + "main_score": 0.97393, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 66.94136929512024, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreTabfquadRetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..66421abed4 --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", + "task_name": "VidoreTabfquadRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.80357, + "ndcg_at_3": 0.85842, + "ndcg_at_5": 0.86749, + "ndcg_at_10": 0.88031, + "ndcg_at_20": 0.88584, + "ndcg_at_100": 0.88996, + "ndcg_at_1000": 0.88996, + "map_at_1": 0.80357, + "map_at_3": 0.84524, + "map_at_5": 0.85042, + "map_at_10": 0.85577, + "map_at_20": 0.85735, + "map_at_100": 0.85797, + "map_at_1000": 0.85797, + "recall_at_1": 0.80357, + "recall_at_3": 0.89643, + "recall_at_5": 0.91786, + "recall_at_10": 0.95714, + "recall_at_20": 0.97857, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.80357, + "cv_recall_at_3": 0.89643, + "cv_recall_at_5": 0.91786, + "cv_recall_at_10": 0.95714, + "cv_recall_at_20": 0.97857, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.80357, + "precision_at_3": 0.29881, + "precision_at_5": 0.18357, + "precision_at_10": 0.09571, + "precision_at_20": 0.04893, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.803571, + "mrr_at_3": 0.845238, + "mrr_at_5": 0.850417, + "mrr_at_10": 0.855774, + "mrr_at_20": 0.857353, + "mrr_at_100": 0.857965, + "mrr_at_1000": 0.857965, + "nauc_ndcg_at_1_max": 0.474349, + "nauc_ndcg_at_1_std": 0.104608, + "nauc_ndcg_at_1_diff1": 0.879283, + "nauc_ndcg_at_3_max": 0.497364, + "nauc_ndcg_at_3_std": 0.130038, + "nauc_ndcg_at_3_diff1": 0.860098, + "nauc_ndcg_at_5_max": 0.487101, + "nauc_ndcg_at_5_std": 0.123286, + "nauc_ndcg_at_5_diff1": 0.86051, + "nauc_ndcg_at_10_max": 0.488621, + "nauc_ndcg_at_10_std": 0.120285, + "nauc_ndcg_at_10_diff1": 0.86294, + "nauc_ndcg_at_20_max": 0.508946, + "nauc_ndcg_at_20_std": 0.146242, + "nauc_ndcg_at_20_diff1": 0.864676, + "nauc_ndcg_at_100_max": 0.495969, + "nauc_ndcg_at_100_std": 0.128272, + "nauc_ndcg_at_100_diff1": 0.867376, + "nauc_ndcg_at_1000_max": 0.495969, + "nauc_ndcg_at_1000_std": 0.128272, + "nauc_ndcg_at_1000_diff1": 0.867376, + "nauc_map_at_1_max": 0.474349, + "nauc_map_at_1_std": 0.104608, + "nauc_map_at_1_diff1": 0.879283, + "nauc_map_at_3_max": 0.495653, + "nauc_map_at_3_std": 0.126755, + "nauc_map_at_3_diff1": 0.86653, + "nauc_map_at_5_max": 0.490502, + "nauc_map_at_5_std": 0.123134, + "nauc_map_at_5_diff1": 0.866978, + "nauc_map_at_10_max": 0.490445, + "nauc_map_at_10_std": 0.121194, + "nauc_map_at_10_diff1": 0.867744, + "nauc_map_at_20_max": 0.495003, + "nauc_map_at_20_std": 0.126594, + "nauc_map_at_20_diff1": 0.868357, + "nauc_map_at_100_max": 0.493573, + "nauc_map_at_100_std": 0.124716, + "nauc_map_at_100_diff1": 0.868852, + "nauc_map_at_1000_max": 0.493573, + "nauc_map_at_1000_std": 0.124716, + "nauc_map_at_1000_diff1": 0.868852, + "nauc_recall_at_1_max": 0.474349, + "nauc_recall_at_1_std": 0.104608, + "nauc_recall_at_1_diff1": 0.879283, + "nauc_recall_at_3_max": 0.501358, + "nauc_recall_at_3_std": 0.14103, + "nauc_recall_at_3_diff1": 0.833128, + "nauc_recall_at_5_max": 0.463809, + "nauc_recall_at_5_std": 0.119575, + "nauc_recall_at_5_diff1": 0.82769, + "nauc_recall_at_10_max": 0.466348, + "nauc_recall_at_10_std": 0.106209, + "nauc_recall_at_10_diff1": 0.823024, + "nauc_recall_at_20_max": 0.851385, + "nauc_recall_at_20_std": 0.629707, + "nauc_recall_at_20_diff1": 0.810224, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.474349, + "nauc_precision_at_1_std": 0.104608, + "nauc_precision_at_1_diff1": 0.879283, + "nauc_precision_at_3_max": 0.501358, + "nauc_precision_at_3_std": 0.14103, + "nauc_precision_at_3_diff1": 0.833128, + "nauc_precision_at_5_max": 0.463809, + "nauc_precision_at_5_std": 0.119575, + "nauc_precision_at_5_diff1": 0.82769, + "nauc_precision_at_10_max": 0.466348, + "nauc_precision_at_10_std": 0.106209, + "nauc_precision_at_10_diff1": 0.823024, + "nauc_precision_at_20_max": 0.851385, + "nauc_precision_at_20_std": 0.629707, + "nauc_precision_at_20_diff1": 0.810224, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.474349, + "nauc_cv_recall_at_1_std": 0.104608, + "nauc_cv_recall_at_1_diff1": 0.879283, + "nauc_cv_recall_at_3_max": 0.501358, + "nauc_cv_recall_at_3_std": 0.14103, + "nauc_cv_recall_at_3_diff1": 0.833128, + "nauc_cv_recall_at_5_max": 0.463809, + "nauc_cv_recall_at_5_std": 0.119575, + "nauc_cv_recall_at_5_diff1": 0.82769, + "nauc_cv_recall_at_10_max": 0.466348, + "nauc_cv_recall_at_10_std": 0.106209, + "nauc_cv_recall_at_10_diff1": 0.823024, + "nauc_cv_recall_at_20_max": 0.851385, + "nauc_cv_recall_at_20_std": 0.629707, + "nauc_cv_recall_at_20_diff1": 0.810224, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.474349, + "nauc_mrr_at_1_std": 0.104608, + "nauc_mrr_at_1_diff1": 0.879283, + "nauc_mrr_at_3_max": 0.495653, + "nauc_mrr_at_3_std": 0.126755, + "nauc_mrr_at_3_diff1": 0.86653, + "nauc_mrr_at_5_max": 0.490502, + "nauc_mrr_at_5_std": 0.123134, + "nauc_mrr_at_5_diff1": 0.866978, + "nauc_mrr_at_10_max": 0.490445, + "nauc_mrr_at_10_std": 0.121194, + "nauc_mrr_at_10_diff1": 0.867744, + "nauc_mrr_at_20_max": 0.495003, + "nauc_mrr_at_20_std": 0.126594, + "nauc_mrr_at_20_diff1": 0.868357, + "nauc_mrr_at_100_max": 0.493573, + "nauc_mrr_at_100_std": 0.124716, + "nauc_mrr_at_100_diff1": 0.868852, + "nauc_mrr_at_1000_max": 0.493573, + "nauc_mrr_at_1000_std": 0.124716, + "nauc_mrr_at_1000_diff1": 0.868852, + "main_score": 0.86749, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 4.569951295852661, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreTatdqaRetrieval.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..edea141e2c --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", + "task_name": "VidoreTatdqaRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.57047, + "ndcg_at_3": 0.67906, + "ndcg_at_5": 0.70652, + "ndcg_at_10": 0.72771, + "ndcg_at_20": 0.73709, + "ndcg_at_100": 0.74755, + "ndcg_at_1000": 0.75045, + "map_at_1": 0.56956, + "map_at_3": 0.65224, + "map_at_5": 0.66749, + "map_at_10": 0.67647, + "map_at_20": 0.67905, + "map_at_100": 0.68055, + "map_at_1000": 0.6807, + "recall_at_1": 0.56956, + "recall_at_3": 0.75638, + "recall_at_5": 0.8229, + "recall_at_10": 0.8873, + "recall_at_20": 0.92436, + "recall_at_100": 0.97977, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.57047, + "cv_recall_at_3": 0.75699, + "cv_recall_at_5": 0.82382, + "cv_recall_at_10": 0.88761, + "cv_recall_at_20": 0.92467, + "cv_recall_at_100": 0.98056, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.57047, + "precision_at_3": 0.25294, + "precision_at_5": 0.16513, + "precision_at_10": 0.08906, + "precision_at_20": 0.04639, + "precision_at_100": 0.00985, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.570474, + "mrr_at_3": 0.652491, + "mrr_at_5": 0.667801, + "mrr_at_10": 0.676633, + "mrr_at_20": 0.67922, + "mrr_at_100": 0.680699, + "mrr_at_1000": 0.680831, + "nauc_ndcg_at_1_max": 0.209396, + "nauc_ndcg_at_1_std": 0.278706, + "nauc_ndcg_at_1_diff1": 0.751376, + "nauc_ndcg_at_3_max": 0.18984, + "nauc_ndcg_at_3_std": 0.320788, + "nauc_ndcg_at_3_diff1": 0.699106, + "nauc_ndcg_at_5_max": 0.185327, + "nauc_ndcg_at_5_std": 0.332562, + "nauc_ndcg_at_5_diff1": 0.693195, + "nauc_ndcg_at_10_max": 0.204088, + "nauc_ndcg_at_10_std": 0.343184, + "nauc_ndcg_at_10_diff1": 0.700487, + "nauc_ndcg_at_20_max": 0.204426, + "nauc_ndcg_at_20_std": 0.339044, + "nauc_ndcg_at_20_diff1": 0.703991, + "nauc_ndcg_at_100_max": 0.202572, + "nauc_ndcg_at_100_std": 0.328496, + "nauc_ndcg_at_100_diff1": 0.709886, + "nauc_ndcg_at_1000_max": 0.201078, + "nauc_ndcg_at_1000_std": 0.323277, + "nauc_ndcg_at_1000_diff1": 0.710535, + "nauc_map_at_1_max": 0.209459, + "nauc_map_at_1_std": 0.279386, + "nauc_map_at_1_diff1": 0.752225, + "nauc_map_at_3_max": 0.194753, + "nauc_map_at_3_std": 0.307882, + "nauc_map_at_3_diff1": 0.713264, + "nauc_map_at_5_max": 0.193102, + "nauc_map_at_5_std": 0.313095, + "nauc_map_at_5_diff1": 0.7108, + "nauc_map_at_10_max": 0.200542, + "nauc_map_at_10_std": 0.316718, + "nauc_map_at_10_diff1": 0.714167, + "nauc_map_at_20_max": 0.200696, + "nauc_map_at_20_std": 0.315522, + "nauc_map_at_20_diff1": 0.715123, + "nauc_map_at_100_max": 0.200482, + "nauc_map_at_100_std": 0.314085, + "nauc_map_at_100_diff1": 0.715961, + "nauc_map_at_1000_max": 0.200415, + "nauc_map_at_1000_std": 0.313847, + "nauc_map_at_1000_diff1": 0.715972, + "nauc_recall_at_1_max": 0.209459, + "nauc_recall_at_1_std": 0.279386, + "nauc_recall_at_1_diff1": 0.752225, + "nauc_recall_at_3_max": 0.17071, + "nauc_recall_at_3_std": 0.367717, + "nauc_recall_at_3_diff1": 0.646709, + "nauc_recall_at_5_max": 0.14644, + "nauc_recall_at_5_std": 0.424954, + "nauc_recall_at_5_diff1": 0.610147, + "nauc_recall_at_10_max": 0.239187, + "nauc_recall_at_10_std": 0.53438, + "nauc_recall_at_10_diff1": 0.611418, + "nauc_recall_at_20_max": 0.256498, + "nauc_recall_at_20_std": 0.576971, + "nauc_recall_at_20_diff1": 0.608224, + "nauc_recall_at_100_max": 0.301577, + "nauc_recall_at_100_std": 0.683331, + "nauc_recall_at_100_diff1": 0.649805, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.209396, + "nauc_precision_at_1_std": 0.278706, + "nauc_precision_at_1_diff1": 0.751376, + "nauc_precision_at_3_max": 0.169426, + "nauc_precision_at_3_std": 0.36175, + "nauc_precision_at_3_diff1": 0.63335, + "nauc_precision_at_5_max": 0.144468, + "nauc_precision_at_5_std": 0.415569, + "nauc_precision_at_5_diff1": 0.593576, + "nauc_precision_at_10_max": 0.232068, + "nauc_precision_at_10_std": 0.508561, + "nauc_precision_at_10_diff1": 0.570911, + "nauc_precision_at_20_max": 0.245296, + "nauc_precision_at_20_std": 0.537426, + "nauc_precision_at_20_diff1": 0.550393, + "nauc_precision_at_100_max": 0.206156, + "nauc_precision_at_100_std": 0.443138, + "nauc_precision_at_100_diff1": 0.375514, + "nauc_precision_at_1000_max": -0.200435, + "nauc_precision_at_1000_std": -0.401901, + "nauc_precision_at_1000_diff1": -0.465184, + "nauc_cv_recall_at_1_max": 0.209396, + "nauc_cv_recall_at_1_std": 0.278706, + "nauc_cv_recall_at_1_diff1": 0.751376, + "nauc_cv_recall_at_3_max": 0.172704, + "nauc_cv_recall_at_3_std": 0.369874, + "nauc_cv_recall_at_3_diff1": 0.64734, + "nauc_cv_recall_at_5_max": 0.14829, + "nauc_cv_recall_at_5_std": 0.427248, + "nauc_cv_recall_at_5_diff1": 0.611163, + "nauc_cv_recall_at_10_max": 0.238238, + "nauc_cv_recall_at_10_std": 0.533673, + "nauc_cv_recall_at_10_diff1": 0.612538, + "nauc_cv_recall_at_20_max": 0.255298, + "nauc_cv_recall_at_20_std": 0.576386, + "nauc_cv_recall_at_20_diff1": 0.610182, + "nauc_cv_recall_at_100_max": 0.291318, + "nauc_cv_recall_at_100_std": 0.670466, + "nauc_cv_recall_at_100_diff1": 0.635579, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.209396, + "nauc_mrr_at_1_std": 0.278706, + "nauc_mrr_at_1_diff1": 0.751376, + "nauc_mrr_at_3_max": 0.195613, + "nauc_mrr_at_3_std": 0.308749, + "nauc_mrr_at_3_diff1": 0.713448, + "nauc_mrr_at_5_max": 0.193913, + "nauc_mrr_at_5_std": 0.313871, + "nauc_mrr_at_5_diff1": 0.711013, + "nauc_mrr_at_10_max": 0.200848, + "nauc_mrr_at_10_std": 0.317009, + "nauc_mrr_at_10_diff1": 0.714337, + "nauc_mrr_at_20_max": 0.201005, + "nauc_mrr_at_20_std": 0.315815, + "nauc_mrr_at_20_diff1": 0.715295, + "nauc_mrr_at_100_max": 0.200814, + "nauc_mrr_at_100_std": 0.314389, + "nauc_mrr_at_100_diff1": 0.716068, + "nauc_mrr_at_1000_max": 0.200781, + "nauc_mrr_at_1000_std": 0.314214, + "nauc_mrr_at_1000_diff1": 0.716105, + "main_score": 0.70652, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 26.27332615852356, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/model_meta.json b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/model_meta.json new file mode 100644 index 0000000000..5ad454f49f --- /dev/null +++ b/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0/model_meta.json @@ -0,0 +1 @@ +{"name": "vidore/colpali-v1.3", "revision": "1b5c8929330df1a66de441a9b5409a878f0de5b0", "release_date": "2024-11-01", "languages": ["eng-Latn"], "n_parameters": 2920000000, "memory_usage_mb": 4700.0, "max_tokens": 16384.0, "embed_dim": 128, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/illuin-tech/colpali", "public_training_data": "https://huggingface.co/datasets/vidore/colpali_train_set", "framework": ["ColPali"], "reference": "https://huggingface.co/vidore/colpali-v1.3", "similarity_fn_name": "max_sim", "use_instructions": true, "training_datasets": {"DocVQA": ["train"], "InfoVQA": ["train"], "TATDQA": ["train"], "arXivQA": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "ColPaliWrapper"} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..7bb2b9b5e2 --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.55625, + "ndcg_at_3": 0.5471, + "ndcg_at_5": 0.56311, + "ndcg_at_10": 0.59423, + "ndcg_at_20": 0.62644, + "ndcg_at_100": 0.66428, + "ndcg_at_1000": 0.6842, + "map_at_1": 0.33095, + "map_at_3": 0.43403, + "map_at_5": 0.46951, + "map_at_10": 0.50426, + "map_at_20": 0.51998, + "map_at_100": 0.53209, + "map_at_1000": 0.53402, + "recall_at_1": 0.33095, + "recall_at_3": 0.50627, + "recall_at_5": 0.58467, + "recall_at_10": 0.68165, + "recall_at_20": 0.78238, + "recall_at_100": 0.90335, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.55625, + "cv_recall_at_3": 0.75625, + "cv_recall_at_5": 0.825, + "cv_recall_at_10": 0.875, + "cv_recall_at_20": 0.94375, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.55625, + "precision_at_3": 0.33958, + "precision_at_5": 0.25875, + "precision_at_10": 0.165, + "precision_at_20": 0.09813, + "precision_at_100": 0.02637, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.55625, + "mrr_at_3": 0.6375, + "mrr_at_5": 0.654062, + "mrr_at_10": 0.661081, + "mrr_at_20": 0.666029, + "mrr_at_100": 0.667226, + "mrr_at_1000": 0.667306, + "nauc_ndcg_at_1_max": 0.160174, + "nauc_ndcg_at_1_std": -0.097582, + "nauc_ndcg_at_1_diff1": 0.526494, + "nauc_ndcg_at_3_max": 0.203005, + "nauc_ndcg_at_3_std": -0.101958, + "nauc_ndcg_at_3_diff1": 0.491344, + "nauc_ndcg_at_5_max": 0.211829, + "nauc_ndcg_at_5_std": -0.080904, + "nauc_ndcg_at_5_diff1": 0.47296, + "nauc_ndcg_at_10_max": 0.209027, + "nauc_ndcg_at_10_std": -0.085354, + "nauc_ndcg_at_10_diff1": 0.468238, + "nauc_ndcg_at_20_max": 0.23936, + "nauc_ndcg_at_20_std": -0.063215, + "nauc_ndcg_at_20_diff1": 0.467768, + "nauc_ndcg_at_100_max": 0.214409, + "nauc_ndcg_at_100_std": -0.067615, + "nauc_ndcg_at_100_diff1": 0.440359, + "nauc_ndcg_at_1000_max": 0.187074, + "nauc_ndcg_at_1000_std": -0.095333, + "nauc_ndcg_at_1000_diff1": 0.454443, + "nauc_map_at_1_max": 0.270599, + "nauc_map_at_1_std": -0.009681, + "nauc_map_at_1_diff1": 0.568607, + "nauc_map_at_3_max": 0.249616, + "nauc_map_at_3_std": -0.072127, + "nauc_map_at_3_diff1": 0.538588, + "nauc_map_at_5_max": 0.237692, + "nauc_map_at_5_std": -0.07232, + "nauc_map_at_5_diff1": 0.507443, + "nauc_map_at_10_max": 0.218224, + "nauc_map_at_10_std": -0.087388, + "nauc_map_at_10_diff1": 0.491534, + "nauc_map_at_20_max": 0.226352, + "nauc_map_at_20_std": -0.081818, + "nauc_map_at_20_diff1": 0.494233, + "nauc_map_at_100_max": 0.22236, + "nauc_map_at_100_std": -0.080454, + "nauc_map_at_100_diff1": 0.4884, + "nauc_map_at_1000_max": 0.220025, + "nauc_map_at_1000_std": -0.082228, + "nauc_map_at_1000_diff1": 0.488472, + "nauc_recall_at_1_max": 0.270599, + "nauc_recall_at_1_std": -0.009681, + "nauc_recall_at_1_diff1": 0.568607, + "nauc_recall_at_3_max": 0.24452, + "nauc_recall_at_3_std": -0.068494, + "nauc_recall_at_3_diff1": 0.45921, + "nauc_recall_at_5_max": 0.236292, + "nauc_recall_at_5_std": -0.04922, + "nauc_recall_at_5_diff1": 0.374854, + "nauc_recall_at_10_max": 0.234342, + "nauc_recall_at_10_std": -0.034423, + "nauc_recall_at_10_diff1": 0.345784, + "nauc_recall_at_20_max": 0.370557, + "nauc_recall_at_20_std": 0.104232, + "nauc_recall_at_20_diff1": 0.287777, + "nauc_recall_at_100_max": 0.365465, + "nauc_recall_at_100_std": 0.209744, + "nauc_recall_at_100_diff1": -0.001085, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.160174, + "nauc_precision_at_1_std": -0.097582, + "nauc_precision_at_1_diff1": 0.526494, + "nauc_precision_at_3_max": -0.02151, + "nauc_precision_at_3_std": -0.143231, + "nauc_precision_at_3_diff1": 0.107716, + "nauc_precision_at_5_max": -0.104471, + "nauc_precision_at_5_std": -0.134639, + "nauc_precision_at_5_diff1": -0.043968, + "nauc_precision_at_10_max": -0.1348, + "nauc_precision_at_10_std": -0.111834, + "nauc_precision_at_10_diff1": -0.115657, + "nauc_precision_at_20_max": -0.12209, + "nauc_precision_at_20_std": -0.061828, + "nauc_precision_at_20_diff1": -0.15637, + "nauc_precision_at_100_max": -0.181042, + "nauc_precision_at_100_std": -0.035876, + "nauc_precision_at_100_diff1": -0.244848, + "nauc_precision_at_1000_max": -0.281326, + "nauc_precision_at_1000_std": -0.111397, + "nauc_precision_at_1000_diff1": -0.28522, + "nauc_cv_recall_at_1_max": 0.160174, + "nauc_cv_recall_at_1_std": -0.097582, + "nauc_cv_recall_at_1_diff1": 0.526494, + "nauc_cv_recall_at_3_max": 0.089327, + "nauc_cv_recall_at_3_std": -0.212009, + "nauc_cv_recall_at_3_diff1": 0.400518, + "nauc_cv_recall_at_5_max": 0.070192, + "nauc_cv_recall_at_5_std": -0.212961, + "nauc_cv_recall_at_5_diff1": 0.291826, + "nauc_cv_recall_at_10_max": 0.169655, + "nauc_cv_recall_at_10_std": -0.069631, + "nauc_cv_recall_at_10_diff1": 0.317402, + "nauc_cv_recall_at_20_max": 0.616817, + "nauc_cv_recall_at_20_std": 0.467631, + "nauc_cv_recall_at_20_diff1": 0.189128, + "nauc_cv_recall_at_100_max": 0.18643, + "nauc_cv_recall_at_100_std": 0.264862, + "nauc_cv_recall_at_100_diff1": -0.955649, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.160174, + "nauc_mrr_at_1_std": -0.097582, + "nauc_mrr_at_1_diff1": 0.526494, + "nauc_mrr_at_3_max": 0.136296, + "nauc_mrr_at_3_std": -0.134499, + "nauc_mrr_at_3_diff1": 0.486913, + "nauc_mrr_at_5_max": 0.136467, + "nauc_mrr_at_5_std": -0.130048, + "nauc_mrr_at_5_diff1": 0.475522, + "nauc_mrr_at_10_max": 0.142717, + "nauc_mrr_at_10_std": -0.120022, + "nauc_mrr_at_10_diff1": 0.481236, + "nauc_mrr_at_20_max": 0.150568, + "nauc_mrr_at_20_std": -0.111692, + "nauc_mrr_at_20_diff1": 0.482232, + "nauc_mrr_at_100_max": 0.147795, + "nauc_mrr_at_100_std": -0.114231, + "nauc_mrr_at_100_diff1": 0.480981, + "nauc_mrr_at_1000_max": 0.147734, + "nauc_mrr_at_1000_std": -0.114338, + "nauc_mrr_at_1000_diff1": 0.481404, + "main_score": 0.56311, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.53125, + "ndcg_at_3": 0.5301, + "ndcg_at_5": 0.54601, + "ndcg_at_10": 0.59041, + "ndcg_at_20": 0.61843, + "ndcg_at_100": 0.65715, + "ndcg_at_1000": 0.6764, + "map_at_1": 0.31882, + "map_at_3": 0.42413, + "map_at_5": 0.46028, + "map_at_10": 0.49846, + "map_at_20": 0.51239, + "map_at_100": 0.52432, + "map_at_1000": 0.52617, + "recall_at_1": 0.31882, + "recall_at_3": 0.49092, + "recall_at_5": 0.56786, + "recall_at_10": 0.69759, + "recall_at_20": 0.77984, + "recall_at_100": 0.91025, + "recall_at_1000": 0.99978, + "cv_recall_at_1": 0.53125, + "cv_recall_at_3": 0.7375, + "cv_recall_at_5": 0.8, + "cv_recall_at_10": 0.88125, + "cv_recall_at_20": 0.94375, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.53125, + "precision_at_3": 0.325, + "precision_at_5": 0.24875, + "precision_at_10": 0.16687, + "precision_at_20": 0.09906, + "precision_at_100": 0.02612, + "precision_at_1000": 0.00321, + "mrr_at_1": 0.53125, + "mrr_at_3": 0.620833, + "mrr_at_5": 0.635521, + "mrr_at_10": 0.646349, + "mrr_at_20": 0.650803, + "mrr_at_100": 0.652094, + "mrr_at_1000": 0.65221, + "nauc_ndcg_at_1_max": 0.190468, + "nauc_ndcg_at_1_std": -0.075324, + "nauc_ndcg_at_1_diff1": 0.426393, + "nauc_ndcg_at_3_max": 0.20239, + "nauc_ndcg_at_3_std": -0.088144, + "nauc_ndcg_at_3_diff1": 0.405523, + "nauc_ndcg_at_5_max": 0.186164, + "nauc_ndcg_at_5_std": -0.085737, + "nauc_ndcg_at_5_diff1": 0.391756, + "nauc_ndcg_at_10_max": 0.219784, + "nauc_ndcg_at_10_std": -0.051498, + "nauc_ndcg_at_10_diff1": 0.408305, + "nauc_ndcg_at_20_max": 0.205854, + "nauc_ndcg_at_20_std": -0.062107, + "nauc_ndcg_at_20_diff1": 0.388601, + "nauc_ndcg_at_100_max": 0.208398, + "nauc_ndcg_at_100_std": -0.034483, + "nauc_ndcg_at_100_diff1": 0.383661, + "nauc_ndcg_at_1000_max": 0.180437, + "nauc_ndcg_at_1000_std": -0.068664, + "nauc_ndcg_at_1000_diff1": 0.380447, + "nauc_map_at_1_max": 0.242425, + "nauc_map_at_1_std": 0.00125, + "nauc_map_at_1_diff1": 0.491636, + "nauc_map_at_3_max": 0.230784, + "nauc_map_at_3_std": -0.057787, + "nauc_map_at_3_diff1": 0.45794, + "nauc_map_at_5_max": 0.198372, + "nauc_map_at_5_std": -0.07934, + "nauc_map_at_5_diff1": 0.424771, + "nauc_map_at_10_max": 0.194613, + "nauc_map_at_10_std": -0.07903, + "nauc_map_at_10_diff1": 0.415554, + "nauc_map_at_20_max": 0.190741, + "nauc_map_at_20_std": -0.078187, + "nauc_map_at_20_diff1": 0.409271, + "nauc_map_at_100_max": 0.19241, + "nauc_map_at_100_std": -0.06863, + "nauc_map_at_100_diff1": 0.40687, + "nauc_map_at_1000_max": 0.189987, + "nauc_map_at_1000_std": -0.070701, + "nauc_map_at_1000_diff1": 0.40596, + "nauc_recall_at_1_max": 0.242425, + "nauc_recall_at_1_std": 0.00125, + "nauc_recall_at_1_diff1": 0.491636, + "nauc_recall_at_3_max": 0.230714, + "nauc_recall_at_3_std": -0.072088, + "nauc_recall_at_3_diff1": 0.412528, + "nauc_recall_at_5_max": 0.164067, + "nauc_recall_at_5_std": -0.095916, + "nauc_recall_at_5_diff1": 0.339246, + "nauc_recall_at_10_max": 0.259813, + "nauc_recall_at_10_std": 0.0258, + "nauc_recall_at_10_diff1": 0.330859, + "nauc_recall_at_20_max": 0.222229, + "nauc_recall_at_20_std": 0.018872, + "nauc_recall_at_20_diff1": 0.236191, + "nauc_recall_at_100_max": 0.368668, + "nauc_recall_at_100_std": 0.339796, + "nauc_recall_at_100_diff1": 0.185592, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.869281, + "nauc_recall_at_1000_diff1": 0.554155, + "nauc_precision_at_1_max": 0.190468, + "nauc_precision_at_1_std": -0.075324, + "nauc_precision_at_1_diff1": 0.426393, + "nauc_precision_at_3_max": 0.040878, + "nauc_precision_at_3_std": -0.131069, + "nauc_precision_at_3_diff1": 0.097642, + "nauc_precision_at_5_max": -0.078112, + "nauc_precision_at_5_std": -0.138719, + "nauc_precision_at_5_diff1": -0.061916, + "nauc_precision_at_10_max": -0.110636, + "nauc_precision_at_10_std": -0.076601, + "nauc_precision_at_10_diff1": -0.123986, + "nauc_precision_at_20_max": -0.156648, + "nauc_precision_at_20_std": -0.062219, + "nauc_precision_at_20_diff1": -0.190162, + "nauc_precision_at_100_max": -0.166471, + "nauc_precision_at_100_std": 0.025621, + "nauc_precision_at_100_diff1": -0.227357, + "nauc_precision_at_1000_max": -0.284051, + "nauc_precision_at_1000_std": -0.09199, + "nauc_precision_at_1000_diff1": -0.285208, + "nauc_cv_recall_at_1_max": 0.190468, + "nauc_cv_recall_at_1_std": -0.075324, + "nauc_cv_recall_at_1_diff1": 0.426393, + "nauc_cv_recall_at_3_max": 0.209269, + "nauc_cv_recall_at_3_std": -0.132949, + "nauc_cv_recall_at_3_diff1": 0.381849, + "nauc_cv_recall_at_5_max": 0.176339, + "nauc_cv_recall_at_5_std": -0.106503, + "nauc_cv_recall_at_5_diff1": 0.322365, + "nauc_cv_recall_at_10_max": 0.487556, + "nauc_cv_recall_at_10_std": 0.262808, + "nauc_cv_recall_at_10_diff1": 0.436776, + "nauc_cv_recall_at_20_max": 0.373535, + "nauc_cv_recall_at_20_std": 0.298786, + "nauc_cv_recall_at_20_diff1": 0.124857, + "nauc_cv_recall_at_100_max": 0.435263, + "nauc_cv_recall_at_100_std": 0.764239, + "nauc_cv_recall_at_100_diff1": -0.5831, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.190468, + "nauc_mrr_at_1_std": -0.075324, + "nauc_mrr_at_1_diff1": 0.426393, + "nauc_mrr_at_3_max": 0.207894, + "nauc_mrr_at_3_std": -0.091335, + "nauc_mrr_at_3_diff1": 0.409237, + "nauc_mrr_at_5_max": 0.204209, + "nauc_mrr_at_5_std": -0.083939, + "nauc_mrr_at_5_diff1": 0.40068, + "nauc_mrr_at_10_max": 0.22337, + "nauc_mrr_at_10_std": -0.061775, + "nauc_mrr_at_10_diff1": 0.41063, + "nauc_mrr_at_20_max": 0.217141, + "nauc_mrr_at_20_std": -0.066689, + "nauc_mrr_at_20_diff1": 0.405809, + "nauc_mrr_at_100_max": 0.216135, + "nauc_mrr_at_100_std": -0.067807, + "nauc_mrr_at_100_diff1": 0.405162, + "nauc_mrr_at_1000_max": 0.215902, + "nauc_mrr_at_1000_std": -0.068195, + "nauc_mrr_at_1000_diff1": 0.405643, + "main_score": 0.54601, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.58125, + "ndcg_at_3": 0.59979, + "ndcg_at_5": 0.6161, + "ndcg_at_10": 0.64742, + "ndcg_at_20": 0.67787, + "ndcg_at_100": 0.71013, + "ndcg_at_1000": 0.7248, + "map_at_1": 0.35974, + "map_at_3": 0.48525, + "map_at_5": 0.51994, + "map_at_10": 0.55488, + "map_at_20": 0.57131, + "map_at_100": 0.58256, + "map_at_1000": 0.58398, + "recall_at_1": 0.35974, + "recall_at_3": 0.5592, + "recall_at_5": 0.64328, + "recall_at_10": 0.74377, + "recall_at_20": 0.83049, + "recall_at_100": 0.93064, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.58125, + "cv_recall_at_3": 0.8, + "cv_recall_at_5": 0.88125, + "cv_recall_at_10": 0.93125, + "cv_recall_at_20": 0.96875, + "cv_recall_at_100": 0.9875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.58125, + "precision_at_3": 0.37292, + "precision_at_5": 0.2775, + "precision_at_10": 0.17813, + "precision_at_20": 0.10594, + "precision_at_100": 0.02756, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.58125, + "mrr_at_3": 0.680208, + "mrr_at_5": 0.698646, + "mrr_at_10": 0.705206, + "mrr_at_20": 0.707561, + "mrr_at_100": 0.708049, + "mrr_at_1000": 0.708095, + "nauc_ndcg_at_1_max": 0.269969, + "nauc_ndcg_at_1_std": -0.050268, + "nauc_ndcg_at_1_diff1": 0.422409, + "nauc_ndcg_at_3_max": 0.265812, + "nauc_ndcg_at_3_std": -0.059497, + "nauc_ndcg_at_3_diff1": 0.342608, + "nauc_ndcg_at_5_max": 0.291628, + "nauc_ndcg_at_5_std": -0.020001, + "nauc_ndcg_at_5_diff1": 0.359446, + "nauc_ndcg_at_10_max": 0.294874, + "nauc_ndcg_at_10_std": -0.042392, + "nauc_ndcg_at_10_diff1": 0.355569, + "nauc_ndcg_at_20_max": 0.262718, + "nauc_ndcg_at_20_std": -0.071281, + "nauc_ndcg_at_20_diff1": 0.332792, + "nauc_ndcg_at_100_max": 0.264731, + "nauc_ndcg_at_100_std": -0.03742, + "nauc_ndcg_at_100_diff1": 0.33645, + "nauc_ndcg_at_1000_max": 0.247006, + "nauc_ndcg_at_1000_std": -0.055823, + "nauc_ndcg_at_1000_diff1": 0.336264, + "nauc_map_at_1_max": 0.338682, + "nauc_map_at_1_std": 0.031612, + "nauc_map_at_1_diff1": 0.503828, + "nauc_map_at_3_max": 0.302775, + "nauc_map_at_3_std": -0.021987, + "nauc_map_at_3_diff1": 0.425529, + "nauc_map_at_5_max": 0.297028, + "nauc_map_at_5_std": -0.020399, + "nauc_map_at_5_diff1": 0.404236, + "nauc_map_at_10_max": 0.28496, + "nauc_map_at_10_std": -0.03808, + "nauc_map_at_10_diff1": 0.373312, + "nauc_map_at_20_max": 0.270928, + "nauc_map_at_20_std": -0.049883, + "nauc_map_at_20_diff1": 0.365532, + "nauc_map_at_100_max": 0.270316, + "nauc_map_at_100_std": -0.044932, + "nauc_map_at_100_diff1": 0.364821, + "nauc_map_at_1000_max": 0.268833, + "nauc_map_at_1000_std": -0.04558, + "nauc_map_at_1000_diff1": 0.364275, + "nauc_recall_at_1_max": 0.338682, + "nauc_recall_at_1_std": 0.031612, + "nauc_recall_at_1_diff1": 0.503828, + "nauc_recall_at_3_max": 0.292009, + "nauc_recall_at_3_std": -0.047887, + "nauc_recall_at_3_diff1": 0.356626, + "nauc_recall_at_5_max": 0.304443, + "nauc_recall_at_5_std": 0.013298, + "nauc_recall_at_5_diff1": 0.316488, + "nauc_recall_at_10_max": 0.324771, + "nauc_recall_at_10_std": -0.026605, + "nauc_recall_at_10_diff1": 0.263777, + "nauc_recall_at_20_max": 0.202943, + "nauc_recall_at_20_std": -0.135714, + "nauc_recall_at_20_diff1": 0.136166, + "nauc_recall_at_100_max": 0.332108, + "nauc_recall_at_100_std": 0.224819, + "nauc_recall_at_100_diff1": 0.105241, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.269969, + "nauc_precision_at_1_std": -0.050268, + "nauc_precision_at_1_diff1": 0.422409, + "nauc_precision_at_3_max": -0.008618, + "nauc_precision_at_3_std": -0.093847, + "nauc_precision_at_3_diff1": -0.078755, + "nauc_precision_at_5_max": -0.096935, + "nauc_precision_at_5_std": -0.08291, + "nauc_precision_at_5_diff1": -0.190218, + "nauc_precision_at_10_max": -0.161502, + "nauc_precision_at_10_std": -0.110493, + "nauc_precision_at_10_diff1": -0.270686, + "nauc_precision_at_20_max": -0.237733, + "nauc_precision_at_20_std": -0.134458, + "nauc_precision_at_20_diff1": -0.325056, + "nauc_precision_at_100_max": -0.237177, + "nauc_precision_at_100_std": -0.043845, + "nauc_precision_at_100_diff1": -0.29487, + "nauc_precision_at_1000_max": -0.308697, + "nauc_precision_at_1000_std": -0.083206, + "nauc_precision_at_1000_diff1": -0.325735, + "nauc_cv_recall_at_1_max": 0.269969, + "nauc_cv_recall_at_1_std": -0.050268, + "nauc_cv_recall_at_1_diff1": 0.422409, + "nauc_cv_recall_at_3_max": 0.265219, + "nauc_cv_recall_at_3_std": -0.143029, + "nauc_cv_recall_at_3_diff1": 0.301948, + "nauc_cv_recall_at_5_max": 0.354358, + "nauc_cv_recall_at_5_std": 0.01548, + "nauc_cv_recall_at_5_diff1": 0.237593, + "nauc_cv_recall_at_10_max": 0.491639, + "nauc_cv_recall_at_10_std": 0.152576, + "nauc_cv_recall_at_10_diff1": 0.237501, + "nauc_cv_recall_at_20_max": 0.180859, + "nauc_cv_recall_at_20_std": -0.006349, + "nauc_cv_recall_at_20_diff1": -0.239683, + "nauc_cv_recall_at_100_max": -0.141223, + "nauc_cv_recall_at_100_std": 0.496032, + "nauc_cv_recall_at_100_diff1": -1.151727, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.269969, + "nauc_mrr_at_1_std": -0.050268, + "nauc_mrr_at_1_diff1": 0.422409, + "nauc_mrr_at_3_max": 0.257542, + "nauc_mrr_at_3_std": -0.083444, + "nauc_mrr_at_3_diff1": 0.380636, + "nauc_mrr_at_5_max": 0.267479, + "nauc_mrr_at_5_std": -0.060308, + "nauc_mrr_at_5_diff1": 0.376582, + "nauc_mrr_at_10_max": 0.270137, + "nauc_mrr_at_10_std": -0.056006, + "nauc_mrr_at_10_diff1": 0.380679, + "nauc_mrr_at_20_max": 0.26561, + "nauc_mrr_at_20_std": -0.058656, + "nauc_mrr_at_20_diff1": 0.378023, + "nauc_mrr_at_100_max": 0.265558, + "nauc_mrr_at_100_std": -0.057977, + "nauc_mrr_at_100_diff1": 0.378278, + "nauc_mrr_at_1000_max": 0.265526, + "nauc_mrr_at_1000_std": -0.058126, + "nauc_mrr_at_1000_diff1": 0.37864, + "main_score": 0.6161, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.5, + "ndcg_at_3": 0.50888, + "ndcg_at_5": 0.53216, + "ndcg_at_10": 0.57035, + "ndcg_at_20": 0.59757, + "ndcg_at_100": 0.64112, + "ndcg_at_1000": 0.66164, + "map_at_1": 0.30261, + "map_at_3": 0.41151, + "map_at_5": 0.44815, + "map_at_10": 0.48048, + "map_at_20": 0.49409, + "map_at_100": 0.50644, + "map_at_1000": 0.50845, + "recall_at_1": 0.30261, + "recall_at_3": 0.48059, + "recall_at_5": 0.56254, + "recall_at_10": 0.67844, + "recall_at_20": 0.75199, + "recall_at_100": 0.90487, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.5, + "cv_recall_at_3": 0.69375, + "cv_recall_at_5": 0.7625, + "cv_recall_at_10": 0.8625, + "cv_recall_at_20": 0.91875, + "cv_recall_at_100": 0.9875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.5, + "precision_at_3": 0.30625, + "precision_at_5": 0.23875, + "precision_at_10": 0.15625, + "precision_at_20": 0.09469, + "precision_at_100": 0.026, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.5, + "mrr_at_3": 0.5875, + "mrr_at_5": 0.60375, + "mrr_at_10": 0.617294, + "mrr_at_20": 0.621284, + "mrr_at_100": 0.62337, + "mrr_at_1000": 0.623431, + "nauc_ndcg_at_1_max": 0.263483, + "nauc_ndcg_at_1_std": 0.079047, + "nauc_ndcg_at_1_diff1": 0.494612, + "nauc_ndcg_at_3_max": 0.292756, + "nauc_ndcg_at_3_std": 0.043255, + "nauc_ndcg_at_3_diff1": 0.488893, + "nauc_ndcg_at_5_max": 0.285651, + "nauc_ndcg_at_5_std": 0.047828, + "nauc_ndcg_at_5_diff1": 0.473015, + "nauc_ndcg_at_10_max": 0.275561, + "nauc_ndcg_at_10_std": 0.037918, + "nauc_ndcg_at_10_diff1": 0.444724, + "nauc_ndcg_at_20_max": 0.276331, + "nauc_ndcg_at_20_std": 0.038269, + "nauc_ndcg_at_20_diff1": 0.437864, + "nauc_ndcg_at_100_max": 0.277149, + "nauc_ndcg_at_100_std": 0.05713, + "nauc_ndcg_at_100_diff1": 0.43976, + "nauc_ndcg_at_1000_max": 0.251901, + "nauc_ndcg_at_1000_std": 0.027807, + "nauc_ndcg_at_1000_diff1": 0.441701, + "nauc_map_at_1_max": 0.278501, + "nauc_map_at_1_std": 0.056522, + "nauc_map_at_1_diff1": 0.59891, + "nauc_map_at_3_max": 0.305461, + "nauc_map_at_3_std": 0.034224, + "nauc_map_at_3_diff1": 0.554621, + "nauc_map_at_5_max": 0.286099, + "nauc_map_at_5_std": 0.029155, + "nauc_map_at_5_diff1": 0.509078, + "nauc_map_at_10_max": 0.26362, + "nauc_map_at_10_std": 0.018139, + "nauc_map_at_10_diff1": 0.470964, + "nauc_map_at_20_max": 0.2586, + "nauc_map_at_20_std": 0.011969, + "nauc_map_at_20_diff1": 0.465436, + "nauc_map_at_100_max": 0.258074, + "nauc_map_at_100_std": 0.015607, + "nauc_map_at_100_diff1": 0.46412, + "nauc_map_at_1000_max": 0.255687, + "nauc_map_at_1000_std": 0.013262, + "nauc_map_at_1000_diff1": 0.46436, + "nauc_recall_at_1_max": 0.278501, + "nauc_recall_at_1_std": 0.056522, + "nauc_recall_at_1_diff1": 0.59891, + "nauc_recall_at_3_max": 0.325618, + "nauc_recall_at_3_std": 0.038963, + "nauc_recall_at_3_diff1": 0.527514, + "nauc_recall_at_5_max": 0.290025, + "nauc_recall_at_5_std": 0.035401, + "nauc_recall_at_5_diff1": 0.434415, + "nauc_recall_at_10_max": 0.23882, + "nauc_recall_at_10_std": 0.019697, + "nauc_recall_at_10_diff1": 0.308879, + "nauc_recall_at_20_max": 0.276556, + "nauc_recall_at_20_std": 0.04534, + "nauc_recall_at_20_diff1": 0.291377, + "nauc_recall_at_100_max": 0.384007, + "nauc_recall_at_100_std": 0.294699, + "nauc_recall_at_100_diff1": 0.247821, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.263483, + "nauc_precision_at_1_std": 0.079047, + "nauc_precision_at_1_diff1": 0.494612, + "nauc_precision_at_3_max": 0.144503, + "nauc_precision_at_3_std": 0.032135, + "nauc_precision_at_3_diff1": 0.097117, + "nauc_precision_at_5_max": 0.027204, + "nauc_precision_at_5_std": 0.01071, + "nauc_precision_at_5_diff1": -0.082594, + "nauc_precision_at_10_max": -0.089836, + "nauc_precision_at_10_std": -0.038502, + "nauc_precision_at_10_diff1": -0.25743, + "nauc_precision_at_20_max": -0.133355, + "nauc_precision_at_20_std": -0.051054, + "nauc_precision_at_20_diff1": -0.303702, + "nauc_precision_at_100_max": -0.164448, + "nauc_precision_at_100_std": -0.008156, + "nauc_precision_at_100_diff1": -0.326722, + "nauc_precision_at_1000_max": -0.275985, + "nauc_precision_at_1000_std": -0.137141, + "nauc_precision_at_1000_diff1": -0.350708, + "nauc_cv_recall_at_1_max": 0.263483, + "nauc_cv_recall_at_1_std": 0.079047, + "nauc_cv_recall_at_1_diff1": 0.494612, + "nauc_cv_recall_at_3_max": 0.379721, + "nauc_cv_recall_at_3_std": 0.13154, + "nauc_cv_recall_at_3_diff1": 0.531838, + "nauc_cv_recall_at_5_max": 0.384738, + "nauc_cv_recall_at_5_std": 0.174229, + "nauc_cv_recall_at_5_diff1": 0.448068, + "nauc_cv_recall_at_10_max": 0.405906, + "nauc_cv_recall_at_10_std": 0.180442, + "nauc_cv_recall_at_10_diff1": 0.388852, + "nauc_cv_recall_at_20_max": 0.538282, + "nauc_cv_recall_at_20_std": 0.388638, + "nauc_cv_recall_at_20_diff1": 0.333118, + "nauc_cv_recall_at_100_max": -0.214753, + "nauc_cv_recall_at_100_std": 0.152894, + "nauc_cv_recall_at_100_diff1": -0.367414, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.263483, + "nauc_mrr_at_1_std": 0.079047, + "nauc_mrr_at_1_diff1": 0.494612, + "nauc_mrr_at_3_max": 0.308697, + "nauc_mrr_at_3_std": 0.098538, + "nauc_mrr_at_3_diff1": 0.505784, + "nauc_mrr_at_5_max": 0.305599, + "nauc_mrr_at_5_std": 0.103889, + "nauc_mrr_at_5_diff1": 0.489606, + "nauc_mrr_at_10_max": 0.305171, + "nauc_mrr_at_10_std": 0.101628, + "nauc_mrr_at_10_diff1": 0.48762, + "nauc_mrr_at_20_max": 0.306019, + "nauc_mrr_at_20_std": 0.104078, + "nauc_mrr_at_20_diff1": 0.487717, + "nauc_mrr_at_100_max": 0.303688, + "nauc_mrr_at_100_std": 0.101792, + "nauc_mrr_at_100_diff1": 0.488429, + "nauc_mrr_at_1000_max": 0.303692, + "nauc_mrr_at_1000_std": 0.101699, + "nauc_mrr_at_1000_diff1": 0.488637, + "main_score": 0.53216, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 305.6200659275055, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..f9b87aa3e6 --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.5641, + "ndcg_at_3": 0.5664, + "ndcg_at_5": 0.60427, + "ndcg_at_10": 0.63597, + "ndcg_at_20": 0.66064, + "ndcg_at_100": 0.69653, + "ndcg_at_1000": 0.70409, + "map_at_1": 0.40577, + "map_at_3": 0.48199, + "map_at_5": 0.52783, + "map_at_10": 0.55346, + "map_at_20": 0.5649, + "map_at_100": 0.57567, + "map_at_1000": 0.57635, + "recall_at_1": 0.40577, + "recall_at_3": 0.53252, + "recall_at_5": 0.65303, + "recall_at_10": 0.74781, + "recall_at_20": 0.81948, + "recall_at_100": 0.96614, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.57692, + "cv_recall_at_3": 0.73077, + "cv_recall_at_5": 0.82692, + "cv_recall_at_10": 0.88462, + "cv_recall_at_20": 0.94231, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.57692, + "precision_at_3": 0.3141, + "precision_at_5": 0.24615, + "precision_at_10": 0.14808, + "precision_at_20": 0.08654, + "precision_at_100": 0.02212, + "precision_at_1000": 0.00246, + "mrr_at_1": 0.576923, + "mrr_at_3": 0.637821, + "mrr_at_5": 0.660897, + "mrr_at_10": 0.669712, + "mrr_at_20": 0.674194, + "mrr_at_100": 0.675773, + "mrr_at_1000": 0.675773, + "nauc_ndcg_at_1_max": -0.028907, + "nauc_ndcg_at_1_std": -0.243908, + "nauc_ndcg_at_1_diff1": 0.560018, + "nauc_ndcg_at_3_max": 0.139863, + "nauc_ndcg_at_3_std": -0.062285, + "nauc_ndcg_at_3_diff1": 0.479333, + "nauc_ndcg_at_5_max": 0.18588, + "nauc_ndcg_at_5_std": -0.007817, + "nauc_ndcg_at_5_diff1": 0.499535, + "nauc_ndcg_at_10_max": 0.198356, + "nauc_ndcg_at_10_std": -0.009071, + "nauc_ndcg_at_10_diff1": 0.468269, + "nauc_ndcg_at_20_max": 0.201849, + "nauc_ndcg_at_20_std": -0.029741, + "nauc_ndcg_at_20_diff1": 0.497507, + "nauc_ndcg_at_100_max": 0.179817, + "nauc_ndcg_at_100_std": -0.036407, + "nauc_ndcg_at_100_diff1": 0.494205, + "nauc_ndcg_at_1000_max": 0.162616, + "nauc_ndcg_at_1000_std": -0.057532, + "nauc_ndcg_at_1000_diff1": 0.494094, + "nauc_map_at_1_max": 0.193838, + "nauc_map_at_1_std": -0.026476, + "nauc_map_at_1_diff1": 0.529453, + "nauc_map_at_3_max": 0.201316, + "nauc_map_at_3_std": -0.043317, + "nauc_map_at_3_diff1": 0.536105, + "nauc_map_at_5_max": 0.215114, + "nauc_map_at_5_std": -0.010918, + "nauc_map_at_5_diff1": 0.550183, + "nauc_map_at_10_max": 0.212674, + "nauc_map_at_10_std": -0.016517, + "nauc_map_at_10_diff1": 0.520682, + "nauc_map_at_20_max": 0.216454, + "nauc_map_at_20_std": -0.019984, + "nauc_map_at_20_diff1": 0.528211, + "nauc_map_at_100_max": 0.213616, + "nauc_map_at_100_std": -0.013108, + "nauc_map_at_100_diff1": 0.522431, + "nauc_map_at_1000_max": 0.212415, + "nauc_map_at_1000_std": -0.014541, + "nauc_map_at_1000_diff1": 0.522903, + "nauc_recall_at_1_max": 0.193838, + "nauc_recall_at_1_std": -0.026476, + "nauc_recall_at_1_diff1": 0.529453, + "nauc_recall_at_3_max": 0.240657, + "nauc_recall_at_3_std": -0.012452, + "nauc_recall_at_3_diff1": 0.435621, + "nauc_recall_at_5_max": 0.325724, + "nauc_recall_at_5_std": 0.11795, + "nauc_recall_at_5_diff1": 0.482455, + "nauc_recall_at_10_max": 0.339218, + "nauc_recall_at_10_std": 0.154998, + "nauc_recall_at_10_diff1": 0.310784, + "nauc_recall_at_20_max": 0.392815, + "nauc_recall_at_20_std": 0.123982, + "nauc_recall_at_20_diff1": 0.401476, + "nauc_recall_at_100_max": 0.624479, + "nauc_recall_at_100_std": 0.585563, + "nauc_recall_at_100_diff1": 0.306433, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.05128, + "nauc_precision_at_1_std": -0.225479, + "nauc_precision_at_1_diff1": 0.553794, + "nauc_precision_at_3_max": -0.086774, + "nauc_precision_at_3_std": -0.118059, + "nauc_precision_at_3_diff1": 0.14661, + "nauc_precision_at_5_max": -0.103805, + "nauc_precision_at_5_std": -0.048848, + "nauc_precision_at_5_diff1": 0.016372, + "nauc_precision_at_10_max": -0.024072, + "nauc_precision_at_10_std": 0.044267, + "nauc_precision_at_10_diff1": -0.18152, + "nauc_precision_at_20_max": -0.05402, + "nauc_precision_at_20_std": 0.008532, + "nauc_precision_at_20_diff1": -0.211683, + "nauc_precision_at_100_max": -0.118572, + "nauc_precision_at_100_std": 0.028382, + "nauc_precision_at_100_diff1": -0.308696, + "nauc_precision_at_1000_max": -0.169504, + "nauc_precision_at_1000_std": -0.048042, + "nauc_precision_at_1000_diff1": -0.326593, + "nauc_cv_recall_at_1_max": -0.05128, + "nauc_cv_recall_at_1_std": -0.225479, + "nauc_cv_recall_at_1_diff1": 0.553794, + "nauc_cv_recall_at_3_max": 0.055611, + "nauc_cv_recall_at_3_std": -0.164475, + "nauc_cv_recall_at_3_diff1": 0.199553, + "nauc_cv_recall_at_5_max": 0.170414, + "nauc_cv_recall_at_5_std": -0.136308, + "nauc_cv_recall_at_5_diff1": 0.383075, + "nauc_cv_recall_at_10_max": 0.339677, + "nauc_cv_recall_at_10_std": 0.051452, + "nauc_cv_recall_at_10_diff1": 0.165217, + "nauc_cv_recall_at_20_max": 0.317299, + "nauc_cv_recall_at_20_std": -0.203119, + "nauc_cv_recall_at_20_diff1": 0.523322, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.05128, + "nauc_mrr_at_1_std": -0.225479, + "nauc_mrr_at_1_diff1": 0.553794, + "nauc_mrr_at_3_max": -0.015885, + "nauc_mrr_at_3_std": -0.184347, + "nauc_mrr_at_3_diff1": 0.434941, + "nauc_mrr_at_5_max": -0.003998, + "nauc_mrr_at_5_std": -0.178326, + "nauc_mrr_at_5_diff1": 0.474799, + "nauc_mrr_at_10_max": 0.003617, + "nauc_mrr_at_10_std": -0.167504, + "nauc_mrr_at_10_diff1": 0.463148, + "nauc_mrr_at_20_max": -0.000855, + "nauc_mrr_at_20_std": -0.175112, + "nauc_mrr_at_20_diff1": 0.474943, + "nauc_mrr_at_100_max": -0.001996, + "nauc_mrr_at_100_std": -0.173925, + "nauc_mrr_at_100_diff1": 0.473799, + "nauc_mrr_at_1000_max": -0.001996, + "nauc_mrr_at_1000_std": -0.173925, + "nauc_mrr_at_1000_diff1": 0.473799, + "main_score": 0.60427, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 210.90927386283875, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..2ee6e577ca --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.49123, + "ndcg_at_3": 0.48311, + "ndcg_at_5": 0.52498, + "ndcg_at_10": 0.5702, + "ndcg_at_20": 0.60806, + "ndcg_at_100": 0.65065, + "ndcg_at_1000": 0.65442, + "map_at_1": 0.21715, + "map_at_3": 0.33833, + "map_at_5": 0.39955, + "map_at_10": 0.44351, + "map_at_20": 0.46818, + "map_at_100": 0.49165, + "map_at_1000": 0.49186, + "recall_at_1": 0.21715, + "recall_at_3": 0.41898, + "recall_at_5": 0.57074, + "recall_at_10": 0.73237, + "recall_at_20": 0.84779, + "recall_at_100": 0.97368, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.49123, + "cv_recall_at_3": 0.70175, + "cv_recall_at_5": 0.80702, + "cv_recall_at_10": 0.91228, + "cv_recall_at_20": 0.96491, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.49123, + "precision_at_3": 0.36257, + "precision_at_5": 0.29825, + "precision_at_10": 0.2, + "precision_at_20": 0.12982, + "precision_at_100": 0.0386, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.491228, + "mrr_at_3": 0.573099, + "mrr_at_5": 0.594152, + "mrr_at_10": 0.608981, + "mrr_at_20": 0.613012, + "mrr_at_100": 0.613193, + "mrr_at_1000": 0.61327, + "nauc_ndcg_at_1_max": -0.036103, + "nauc_ndcg_at_1_std": -0.02146, + "nauc_ndcg_at_1_diff1": 0.086994, + "nauc_ndcg_at_3_max": 0.006854, + "nauc_ndcg_at_3_std": -0.072156, + "nauc_ndcg_at_3_diff1": 0.005066, + "nauc_ndcg_at_5_max": -0.006455, + "nauc_ndcg_at_5_std": -0.08267, + "nauc_ndcg_at_5_diff1": -0.065629, + "nauc_ndcg_at_10_max": 0.011306, + "nauc_ndcg_at_10_std": -0.062061, + "nauc_ndcg_at_10_diff1": -0.036953, + "nauc_ndcg_at_20_max": 0.033313, + "nauc_ndcg_at_20_std": -0.065892, + "nauc_ndcg_at_20_diff1": -0.048354, + "nauc_ndcg_at_100_max": -0.045521, + "nauc_ndcg_at_100_std": -0.043446, + "nauc_ndcg_at_100_diff1": -0.023067, + "nauc_ndcg_at_1000_max": -0.05477, + "nauc_ndcg_at_1000_std": -0.057345, + "nauc_ndcg_at_1000_diff1": -0.020009, + "nauc_map_at_1_max": 0.092321, + "nauc_map_at_1_std": -0.04973, + "nauc_map_at_1_diff1": 0.05035, + "nauc_map_at_3_max": 0.096196, + "nauc_map_at_3_std": -0.071161, + "nauc_map_at_3_diff1": -0.010778, + "nauc_map_at_5_max": 0.079734, + "nauc_map_at_5_std": -0.077037, + "nauc_map_at_5_diff1": -0.054089, + "nauc_map_at_10_max": 0.060328, + "nauc_map_at_10_std": -0.093499, + "nauc_map_at_10_diff1": -0.047887, + "nauc_map_at_20_max": 0.051934, + "nauc_map_at_20_std": -0.087699, + "nauc_map_at_20_diff1": -0.059106, + "nauc_map_at_100_max": 0.01058, + "nauc_map_at_100_std": -0.075888, + "nauc_map_at_100_diff1": -0.052562, + "nauc_map_at_1000_max": 0.010037, + "nauc_map_at_1000_std": -0.076452, + "nauc_map_at_1000_diff1": -0.0526, + "nauc_recall_at_1_max": 0.092321, + "nauc_recall_at_1_std": -0.04973, + "nauc_recall_at_1_diff1": 0.05035, + "nauc_recall_at_3_max": 0.132913, + "nauc_recall_at_3_std": -0.08923, + "nauc_recall_at_3_diff1": 0.052342, + "nauc_recall_at_5_max": 0.160393, + "nauc_recall_at_5_std": -0.036118, + "nauc_recall_at_5_diff1": -0.092803, + "nauc_recall_at_10_max": 0.144936, + "nauc_recall_at_10_std": 0.000177, + "nauc_recall_at_10_diff1": -0.084699, + "nauc_recall_at_20_max": 0.397716, + "nauc_recall_at_20_std": 0.029681, + "nauc_recall_at_20_diff1": -0.174609, + "nauc_recall_at_100_max": 0.417744, + "nauc_recall_at_100_std": 0.764036, + "nauc_recall_at_100_diff1": -0.253654, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.036103, + "nauc_precision_at_1_std": -0.02146, + "nauc_precision_at_1_diff1": 0.086994, + "nauc_precision_at_3_max": -0.097968, + "nauc_precision_at_3_std": -0.078782, + "nauc_precision_at_3_diff1": -0.055589, + "nauc_precision_at_5_max": -0.15036, + "nauc_precision_at_5_std": -0.079816, + "nauc_precision_at_5_diff1": -0.13261, + "nauc_precision_at_10_max": -0.197612, + "nauc_precision_at_10_std": -0.047321, + "nauc_precision_at_10_diff1": -0.096276, + "nauc_precision_at_20_max": -0.234336, + "nauc_precision_at_20_std": -0.001538, + "nauc_precision_at_20_diff1": -0.067805, + "nauc_precision_at_100_max": -0.30288, + "nauc_precision_at_100_std": 0.044094, + "nauc_precision_at_100_diff1": 0.043665, + "nauc_precision_at_1000_max": -0.310556, + "nauc_precision_at_1000_std": 0.037583, + "nauc_precision_at_1000_diff1": 0.04496, + "nauc_cv_recall_at_1_max": -0.036103, + "nauc_cv_recall_at_1_std": -0.02146, + "nauc_cv_recall_at_1_diff1": 0.086994, + "nauc_cv_recall_at_3_max": -0.180917, + "nauc_cv_recall_at_3_std": -0.159049, + "nauc_cv_recall_at_3_diff1": 0.190612, + "nauc_cv_recall_at_5_max": -0.132974, + "nauc_cv_recall_at_5_std": 0.027994, + "nauc_cv_recall_at_5_diff1": -0.046185, + "nauc_cv_recall_at_10_max": -0.121113, + "nauc_cv_recall_at_10_std": 0.510583, + "nauc_cv_recall_at_10_diff1": -0.060428, + "nauc_cv_recall_at_20_max": 0.563308, + "nauc_cv_recall_at_20_std": 0.934256, + "nauc_cv_recall_at_20_diff1": -0.362176, + "nauc_cv_recall_at_100_max": 0.126616, + "nauc_cv_recall_at_100_std": 0.868512, + "nauc_cv_recall_at_100_diff1": -0.559408, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.036103, + "nauc_mrr_at_1_std": -0.02146, + "nauc_mrr_at_1_diff1": 0.086994, + "nauc_mrr_at_3_max": -0.098549, + "nauc_mrr_at_3_std": -0.063194, + "nauc_mrr_at_3_diff1": 0.090987, + "nauc_mrr_at_5_max": -0.088577, + "nauc_mrr_at_5_std": -0.035721, + "nauc_mrr_at_5_diff1": 0.057308, + "nauc_mrr_at_10_max": -0.086415, + "nauc_mrr_at_10_std": -0.016873, + "nauc_mrr_at_10_diff1": 0.060377, + "nauc_mrr_at_20_max": -0.077478, + "nauc_mrr_at_20_std": -0.019464, + "nauc_mrr_at_20_diff1": 0.059892, + "nauc_mrr_at_100_max": -0.078167, + "nauc_mrr_at_100_std": -0.020116, + "nauc_mrr_at_100_diff1": 0.060036, + "nauc_mrr_at_1000_max": -0.078223, + "nauc_mrr_at_1000_std": -0.020359, + "nauc_mrr_at_1000_diff1": 0.060205, + "main_score": 0.52498, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.50877, + "ndcg_at_3": 0.5278, + "ndcg_at_5": 0.56029, + "ndcg_at_10": 0.5927, + "ndcg_at_20": 0.63273, + "ndcg_at_100": 0.67659, + "ndcg_at_1000": 0.68043, + "map_at_1": 0.2325, + "map_at_3": 0.38598, + "map_at_5": 0.43764, + "map_at_10": 0.47346, + "map_at_20": 0.5015, + "map_at_100": 0.5257, + "map_at_1000": 0.52594, + "recall_at_1": 0.2325, + "recall_at_3": 0.48722, + "recall_at_5": 0.60309, + "recall_at_10": 0.72101, + "recall_at_20": 0.84403, + "recall_at_100": 0.97368, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.50877, + "cv_recall_at_3": 0.77193, + "cv_recall_at_5": 0.84211, + "cv_recall_at_10": 0.91228, + "cv_recall_at_20": 0.96491, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.50877, + "precision_at_3": 0.38012, + "precision_at_5": 0.30175, + "precision_at_10": 0.19825, + "precision_at_20": 0.13158, + "precision_at_100": 0.0386, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.508772, + "mrr_at_3": 0.614035, + "mrr_at_5": 0.630702, + "mrr_at_10": 0.6401, + "mrr_at_20": 0.643923, + "mrr_at_100": 0.644573, + "mrr_at_1000": 0.644654, + "nauc_ndcg_at_1_max": -0.072665, + "nauc_ndcg_at_1_std": -0.031056, + "nauc_ndcg_at_1_diff1": 0.314358, + "nauc_ndcg_at_3_max": 0.093905, + "nauc_ndcg_at_3_std": 0.056453, + "nauc_ndcg_at_3_diff1": 0.176744, + "nauc_ndcg_at_5_max": 0.053355, + "nauc_ndcg_at_5_std": -0.035761, + "nauc_ndcg_at_5_diff1": 0.17707, + "nauc_ndcg_at_10_max": 0.029526, + "nauc_ndcg_at_10_std": -0.090822, + "nauc_ndcg_at_10_diff1": 0.120186, + "nauc_ndcg_at_20_max": 0.03808, + "nauc_ndcg_at_20_std": -0.092475, + "nauc_ndcg_at_20_diff1": 0.145686, + "nauc_ndcg_at_100_max": 0.005915, + "nauc_ndcg_at_100_std": -0.020194, + "nauc_ndcg_at_100_diff1": 0.165304, + "nauc_ndcg_at_1000_max": -0.009992, + "nauc_ndcg_at_1000_std": -0.036103, + "nauc_ndcg_at_1000_diff1": 0.172985, + "nauc_map_at_1_max": 0.029991, + "nauc_map_at_1_std": -0.139202, + "nauc_map_at_1_diff1": 0.308426, + "nauc_map_at_3_max": 0.144814, + "nauc_map_at_3_std": -0.045291, + "nauc_map_at_3_diff1": 0.175752, + "nauc_map_at_5_max": 0.114938, + "nauc_map_at_5_std": -0.086519, + "nauc_map_at_5_diff1": 0.176468, + "nauc_map_at_10_max": 0.069217, + "nauc_map_at_10_std": -0.118562, + "nauc_map_at_10_diff1": 0.135782, + "nauc_map_at_20_max": 0.06278, + "nauc_map_at_20_std": -0.104272, + "nauc_map_at_20_diff1": 0.159333, + "nauc_map_at_100_max": 0.035104, + "nauc_map_at_100_std": -0.078089, + "nauc_map_at_100_diff1": 0.169514, + "nauc_map_at_1000_max": 0.034298, + "nauc_map_at_1000_std": -0.07876, + "nauc_map_at_1000_diff1": 0.169345, + "nauc_recall_at_1_max": 0.029991, + "nauc_recall_at_1_std": -0.139202, + "nauc_recall_at_1_diff1": 0.308426, + "nauc_recall_at_3_max": 0.232643, + "nauc_recall_at_3_std": 0.034406, + "nauc_recall_at_3_diff1": 0.058253, + "nauc_recall_at_5_max": 0.201717, + "nauc_recall_at_5_std": -0.0899, + "nauc_recall_at_5_diff1": 0.063155, + "nauc_recall_at_10_max": 0.109719, + "nauc_recall_at_10_std": -0.198426, + "nauc_recall_at_10_diff1": -0.08611, + "nauc_recall_at_20_max": 0.225064, + "nauc_recall_at_20_std": -0.241035, + "nauc_recall_at_20_diff1": -0.079675, + "nauc_recall_at_100_max": 0.815155, + "nauc_recall_at_100_std": 0.851695, + "nauc_recall_at_100_diff1": -0.446022, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.072665, + "nauc_precision_at_1_std": -0.031056, + "nauc_precision_at_1_diff1": 0.314358, + "nauc_precision_at_3_max": 0.028185, + "nauc_precision_at_3_std": 0.114364, + "nauc_precision_at_3_diff1": 0.030192, + "nauc_precision_at_5_max": -0.092966, + "nauc_precision_at_5_std": 0.018258, + "nauc_precision_at_5_diff1": 0.049492, + "nauc_precision_at_10_max": -0.222648, + "nauc_precision_at_10_std": -0.033405, + "nauc_precision_at_10_diff1": -0.071109, + "nauc_precision_at_20_max": -0.227598, + "nauc_precision_at_20_std": 0.050951, + "nauc_precision_at_20_diff1": -0.011063, + "nauc_precision_at_100_max": -0.23045, + "nauc_precision_at_100_std": 0.166629, + "nauc_precision_at_100_diff1": 0.037895, + "nauc_precision_at_1000_max": -0.24045, + "nauc_precision_at_1000_std": 0.160402, + "nauc_precision_at_1000_diff1": 0.03899, + "nauc_cv_recall_at_1_max": -0.072665, + "nauc_cv_recall_at_1_std": -0.031056, + "nauc_cv_recall_at_1_diff1": 0.314358, + "nauc_cv_recall_at_3_max": -0.107553, + "nauc_cv_recall_at_3_std": 0.117526, + "nauc_cv_recall_at_3_diff1": -0.05514, + "nauc_cv_recall_at_5_max": -0.155201, + "nauc_cv_recall_at_5_std": -0.078556, + "nauc_cv_recall_at_5_diff1": -0.017555, + "nauc_cv_recall_at_10_max": 0.023324, + "nauc_cv_recall_at_10_std": 0.244221, + "nauc_cv_recall_at_10_diff1": 0.251453, + "nauc_cv_recall_at_20_max": 0.424674, + "nauc_cv_recall_at_20_std": 0.417528, + "nauc_cv_recall_at_20_diff1": -0.666989, + "nauc_cv_recall_at_100_max": 0.722732, + "nauc_cv_recall_at_100_std": 1.0, + "nauc_cv_recall_at_100_diff1": -1.169034, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.072665, + "nauc_mrr_at_1_std": -0.031056, + "nauc_mrr_at_1_diff1": 0.314358, + "nauc_mrr_at_3_max": -0.073612, + "nauc_mrr_at_3_std": 0.032505, + "nauc_mrr_at_3_diff1": 0.18923, + "nauc_mrr_at_5_max": -0.076399, + "nauc_mrr_at_5_std": 0.004923, + "nauc_mrr_at_5_diff1": 0.202281, + "nauc_mrr_at_10_max": -0.066226, + "nauc_mrr_at_10_std": 0.022552, + "nauc_mrr_at_10_diff1": 0.219964, + "nauc_mrr_at_20_max": -0.061413, + "nauc_mrr_at_20_std": 0.023136, + "nauc_mrr_at_20_diff1": 0.210923, + "nauc_mrr_at_100_max": -0.06187, + "nauc_mrr_at_100_std": 0.023593, + "nauc_mrr_at_100_diff1": 0.211836, + "nauc_mrr_at_1000_max": -0.062108, + "nauc_mrr_at_1000_std": 0.023296, + "nauc_mrr_at_1000_diff1": 0.212255, + "main_score": 0.56029, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.50877, + "ndcg_at_3": 0.52325, + "ndcg_at_5": 0.55228, + "ndcg_at_10": 0.59766, + "ndcg_at_20": 0.64272, + "ndcg_at_100": 0.67832, + "ndcg_at_1000": 0.68179, + "map_at_1": 0.23761, + "map_at_3": 0.37586, + "map_at_5": 0.42382, + "map_at_10": 0.47805, + "map_at_20": 0.50374, + "map_at_100": 0.52331, + "map_at_1000": 0.52394, + "recall_at_1": 0.23761, + "recall_at_3": 0.45407, + "recall_at_5": 0.56322, + "recall_at_10": 0.72302, + "recall_at_20": 0.87941, + "recall_at_100": 0.97953, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.50877, + "cv_recall_at_3": 0.73684, + "cv_recall_at_5": 0.82456, + "cv_recall_at_10": 0.87719, + "cv_recall_at_20": 0.98246, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.50877, + "precision_at_3": 0.37427, + "precision_at_5": 0.30175, + "precision_at_10": 0.20702, + "precision_at_20": 0.13246, + "precision_at_100": 0.03807, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.508772, + "mrr_at_3": 0.608187, + "mrr_at_5": 0.628363, + "mrr_at_10": 0.63616, + "mrr_at_20": 0.644501, + "mrr_at_100": 0.644501, + "mrr_at_1000": 0.644568, + "nauc_ndcg_at_1_max": 0.035273, + "nauc_ndcg_at_1_std": 0.110492, + "nauc_ndcg_at_1_diff1": 0.341229, + "nauc_ndcg_at_3_max": 0.03775, + "nauc_ndcg_at_3_std": 0.093358, + "nauc_ndcg_at_3_diff1": 0.152267, + "nauc_ndcg_at_5_max": 0.077876, + "nauc_ndcg_at_5_std": 0.138539, + "nauc_ndcg_at_5_diff1": 0.189211, + "nauc_ndcg_at_10_max": 0.064464, + "nauc_ndcg_at_10_std": 0.088623, + "nauc_ndcg_at_10_diff1": 0.218504, + "nauc_ndcg_at_20_max": 0.066083, + "nauc_ndcg_at_20_std": 0.046889, + "nauc_ndcg_at_20_diff1": 0.156793, + "nauc_ndcg_at_100_max": -0.003658, + "nauc_ndcg_at_100_std": 0.089972, + "nauc_ndcg_at_100_diff1": 0.197941, + "nauc_ndcg_at_1000_max": -0.004058, + "nauc_ndcg_at_1000_std": 0.081713, + "nauc_ndcg_at_1000_diff1": 0.214567, + "nauc_map_at_1_max": 0.224244, + "nauc_map_at_1_std": 0.078265, + "nauc_map_at_1_diff1": 0.394613, + "nauc_map_at_3_max": 0.177048, + "nauc_map_at_3_std": 0.082998, + "nauc_map_at_3_diff1": 0.202696, + "nauc_map_at_5_max": 0.19084, + "nauc_map_at_5_std": 0.122254, + "nauc_map_at_5_diff1": 0.205713, + "nauc_map_at_10_max": 0.129039, + "nauc_map_at_10_std": 0.055678, + "nauc_map_at_10_diff1": 0.207682, + "nauc_map_at_20_max": 0.11082, + "nauc_map_at_20_std": 0.043269, + "nauc_map_at_20_diff1": 0.17929, + "nauc_map_at_100_max": 0.078365, + "nauc_map_at_100_std": 0.068481, + "nauc_map_at_100_diff1": 0.189063, + "nauc_map_at_1000_max": 0.076462, + "nauc_map_at_1000_std": 0.06661, + "nauc_map_at_1000_diff1": 0.192119, + "nauc_recall_at_1_max": 0.224244, + "nauc_recall_at_1_std": 0.078265, + "nauc_recall_at_1_diff1": 0.394613, + "nauc_recall_at_3_max": 0.180347, + "nauc_recall_at_3_std": 0.090736, + "nauc_recall_at_3_diff1": 0.103266, + "nauc_recall_at_5_max": 0.274681, + "nauc_recall_at_5_std": 0.206538, + "nauc_recall_at_5_diff1": 0.133485, + "nauc_recall_at_10_max": 0.11483, + "nauc_recall_at_10_std": -0.006862, + "nauc_recall_at_10_diff1": 0.17681, + "nauc_recall_at_20_max": 0.257879, + "nauc_recall_at_20_std": -0.163983, + "nauc_recall_at_20_diff1": -0.256807, + "nauc_recall_at_100_max": -0.336635, + "nauc_recall_at_100_std": 0.430804, + "nauc_recall_at_100_diff1": -0.646497, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.035273, + "nauc_precision_at_1_std": 0.110492, + "nauc_precision_at_1_diff1": 0.341229, + "nauc_precision_at_3_max": -0.166471, + "nauc_precision_at_3_std": 0.045652, + "nauc_precision_at_3_diff1": -0.047578, + "nauc_precision_at_5_max": -0.190081, + "nauc_precision_at_5_std": 0.042078, + "nauc_precision_at_5_diff1": -0.054596, + "nauc_precision_at_10_max": -0.271337, + "nauc_precision_at_10_std": -0.043179, + "nauc_precision_at_10_diff1": -0.099342, + "nauc_precision_at_20_max": -0.333379, + "nauc_precision_at_20_std": -0.039613, + "nauc_precision_at_20_diff1": -0.150918, + "nauc_precision_at_100_max": -0.35953, + "nauc_precision_at_100_std": 0.055008, + "nauc_precision_at_100_diff1": -0.029626, + "nauc_precision_at_1000_max": -0.36585, + "nauc_precision_at_1000_std": 0.035033, + "nauc_precision_at_1000_diff1": -0.003627, + "nauc_cv_recall_at_1_max": 0.035273, + "nauc_cv_recall_at_1_std": 0.110492, + "nauc_cv_recall_at_1_diff1": 0.341229, + "nauc_cv_recall_at_3_max": -0.064278, + "nauc_cv_recall_at_3_std": 0.201978, + "nauc_cv_recall_at_3_diff1": 0.218777, + "nauc_cv_recall_at_5_max": 0.036172, + "nauc_cv_recall_at_5_std": 0.34747, + "nauc_cv_recall_at_5_diff1": 0.251036, + "nauc_cv_recall_at_10_max": -0.159845, + "nauc_cv_recall_at_10_std": 0.210031, + "nauc_cv_recall_at_10_diff1": 0.230018, + "nauc_cv_recall_at_20_max": -0.559408, + "nauc_cv_recall_at_20_std": 0.357853, + "nauc_cv_recall_at_20_diff1": -0.559408, + "nauc_cv_recall_at_100_max": -0.559408, + "nauc_cv_recall_at_100_std": 0.357853, + "nauc_cv_recall_at_100_diff1": -0.559408, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.035273, + "nauc_mrr_at_1_std": 0.110492, + "nauc_mrr_at_1_diff1": 0.341229, + "nauc_mrr_at_3_max": 0.005977, + "nauc_mrr_at_3_std": 0.152246, + "nauc_mrr_at_3_diff1": 0.276857, + "nauc_mrr_at_5_max": 0.022084, + "nauc_mrr_at_5_std": 0.166805, + "nauc_mrr_at_5_diff1": 0.286417, + "nauc_mrr_at_10_max": 0.010935, + "nauc_mrr_at_10_std": 0.153904, + "nauc_mrr_at_10_diff1": 0.28625, + "nauc_mrr_at_20_max": 0.010959, + "nauc_mrr_at_20_std": 0.151023, + "nauc_mrr_at_20_diff1": 0.284465, + "nauc_mrr_at_100_max": 0.010959, + "nauc_mrr_at_100_std": 0.151023, + "nauc_mrr_at_100_diff1": 0.284465, + "nauc_mrr_at_1000_max": 0.0111, + "nauc_mrr_at_1000_std": 0.150972, + "nauc_mrr_at_1000_diff1": 0.284675, + "main_score": 0.55228, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.52632, + "ndcg_at_3": 0.4735, + "ndcg_at_5": 0.52524, + "ndcg_at_10": 0.58033, + "ndcg_at_20": 0.61737, + "ndcg_at_100": 0.66758, + "ndcg_at_1000": 0.66909, + "map_at_1": 0.25297, + "map_at_3": 0.35064, + "map_at_5": 0.40302, + "map_at_10": 0.45662, + "map_at_20": 0.48091, + "map_at_100": 0.50582, + "map_at_1000": 0.50595, + "recall_at_1": 0.25297, + "recall_at_3": 0.40437, + "recall_at_5": 0.54666, + "recall_at_10": 0.71165, + "recall_at_20": 0.83711, + "recall_at_100": 0.99123, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.52632, + "cv_recall_at_3": 0.66667, + "cv_recall_at_5": 0.80702, + "cv_recall_at_10": 0.91228, + "cv_recall_at_20": 0.94737, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.52632, + "precision_at_3": 0.31579, + "precision_at_5": 0.27368, + "precision_at_10": 0.2, + "precision_at_20": 0.12719, + "precision_at_100": 0.03877, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.526316, + "mrr_at_3": 0.584795, + "mrr_at_5": 0.616374, + "mrr_at_10": 0.632769, + "mrr_at_20": 0.635328, + "mrr_at_100": 0.637028, + "mrr_at_1000": 0.637028, + "nauc_ndcg_at_1_max": -0.070994, + "nauc_ndcg_at_1_std": 0.028168, + "nauc_ndcg_at_1_diff1": 0.311639, + "nauc_ndcg_at_3_max": -0.046121, + "nauc_ndcg_at_3_std": -0.023975, + "nauc_ndcg_at_3_diff1": 0.204835, + "nauc_ndcg_at_5_max": -0.018841, + "nauc_ndcg_at_5_std": -0.027051, + "nauc_ndcg_at_5_diff1": 0.160402, + "nauc_ndcg_at_10_max": 0.026743, + "nauc_ndcg_at_10_std": -0.03494, + "nauc_ndcg_at_10_diff1": 0.166743, + "nauc_ndcg_at_20_max": 0.010269, + "nauc_ndcg_at_20_std": -0.075288, + "nauc_ndcg_at_20_diff1": 0.170295, + "nauc_ndcg_at_100_max": -0.019803, + "nauc_ndcg_at_100_std": -0.00323, + "nauc_ndcg_at_100_diff1": 0.16948, + "nauc_ndcg_at_1000_max": -0.026206, + "nauc_ndcg_at_1000_std": -0.007295, + "nauc_ndcg_at_1000_diff1": 0.169048, + "nauc_map_at_1_max": 0.036707, + "nauc_map_at_1_std": -0.030576, + "nauc_map_at_1_diff1": 0.288644, + "nauc_map_at_3_max": -0.005741, + "nauc_map_at_3_std": -0.079857, + "nauc_map_at_3_diff1": 0.221617, + "nauc_map_at_5_max": 0.008202, + "nauc_map_at_5_std": -0.089029, + "nauc_map_at_5_diff1": 0.184068, + "nauc_map_at_10_max": 0.027682, + "nauc_map_at_10_std": -0.08789, + "nauc_map_at_10_diff1": 0.178152, + "nauc_map_at_20_max": 0.014818, + "nauc_map_at_20_std": -0.093519, + "nauc_map_at_20_diff1": 0.176327, + "nauc_map_at_100_max": -0.004152, + "nauc_map_at_100_std": -0.04533, + "nauc_map_at_100_diff1": 0.166708, + "nauc_map_at_1000_max": -0.004553, + "nauc_map_at_1000_std": -0.04559, + "nauc_map_at_1000_diff1": 0.166659, + "nauc_recall_at_1_max": 0.036707, + "nauc_recall_at_1_std": -0.030576, + "nauc_recall_at_1_diff1": 0.288644, + "nauc_recall_at_3_max": -0.033126, + "nauc_recall_at_3_std": -0.11992, + "nauc_recall_at_3_diff1": 0.169308, + "nauc_recall_at_5_max": 0.066586, + "nauc_recall_at_5_std": -0.089469, + "nauc_recall_at_5_diff1": 0.105454, + "nauc_recall_at_10_max": 0.151212, + "nauc_recall_at_10_std": -0.112097, + "nauc_recall_at_10_diff1": 0.086438, + "nauc_recall_at_20_max": 0.154233, + "nauc_recall_at_20_std": -0.217384, + "nauc_recall_at_20_diff1": 0.070524, + "nauc_recall_at_100_max": 0.868512, + "nauc_recall_at_100_std": 0.555085, + "nauc_recall_at_100_diff1": 0.126616, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.070994, + "nauc_precision_at_1_std": 0.028168, + "nauc_precision_at_1_diff1": 0.311639, + "nauc_precision_at_3_max": -0.062535, + "nauc_precision_at_3_std": 0.039526, + "nauc_precision_at_3_diff1": 0.072921, + "nauc_precision_at_5_max": -0.015719, + "nauc_precision_at_5_std": 0.080031, + "nauc_precision_at_5_diff1": -0.061821, + "nauc_precision_at_10_max": -0.00377, + "nauc_precision_at_10_std": 0.099726, + "nauc_precision_at_10_diff1": -0.105757, + "nauc_precision_at_20_max": -0.066372, + "nauc_precision_at_20_std": 0.142395, + "nauc_precision_at_20_diff1": -0.127541, + "nauc_precision_at_100_max": -0.138315, + "nauc_precision_at_100_std": 0.273157, + "nauc_precision_at_100_diff1": -0.135377, + "nauc_precision_at_1000_max": -0.142858, + "nauc_precision_at_1000_std": 0.271219, + "nauc_precision_at_1000_diff1": -0.136347, + "nauc_cv_recall_at_1_max": -0.070994, + "nauc_cv_recall_at_1_std": 0.028168, + "nauc_cv_recall_at_1_diff1": 0.311639, + "nauc_cv_recall_at_3_max": -0.114132, + "nauc_cv_recall_at_3_std": -0.036108, + "nauc_cv_recall_at_3_diff1": 0.139252, + "nauc_cv_recall_at_5_max": 0.126326, + "nauc_cv_recall_at_5_std": 0.175198, + "nauc_cv_recall_at_5_diff1": 0.19956, + "nauc_cv_recall_at_10_max": 0.313551, + "nauc_cv_recall_at_10_std": 0.379295, + "nauc_cv_recall_at_10_diff1": -0.031272, + "nauc_cv_recall_at_20_max": 0.23947, + "nauc_cv_recall_at_20_std": 0.353807, + "nauc_cv_recall_at_20_diff1": -0.106562, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.070994, + "nauc_mrr_at_1_std": 0.028168, + "nauc_mrr_at_1_diff1": 0.311639, + "nauc_mrr_at_3_max": -0.10152, + "nauc_mrr_at_3_std": -0.018206, + "nauc_mrr_at_3_diff1": 0.249473, + "nauc_mrr_at_5_max": -0.06781, + "nauc_mrr_at_5_std": 0.010679, + "nauc_mrr_at_5_diff1": 0.260065, + "nauc_mrr_at_10_max": -0.068568, + "nauc_mrr_at_10_std": 0.010988, + "nauc_mrr_at_10_diff1": 0.251227, + "nauc_mrr_at_20_max": -0.073516, + "nauc_mrr_at_20_std": 0.006465, + "nauc_mrr_at_20_diff1": 0.251947, + "nauc_mrr_at_100_max": -0.074435, + "nauc_mrr_at_100_std": 0.005361, + "nauc_mrr_at_100_diff1": 0.255862, + "nauc_mrr_at_1000_max": -0.074435, + "nauc_mrr_at_1000_std": 0.005361, + "nauc_mrr_at_1000_diff1": 0.255862, + "main_score": 0.52524, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 825.627824306488, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..6b97a909b6 --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.51724, + "ndcg_at_3": 0.51442, + "ndcg_at_5": 0.50609, + "ndcg_at_10": 0.49123, + "ndcg_at_20": 0.52211, + "ndcg_at_100": 0.64312, + "ndcg_at_1000": 0.68743, + "map_at_1": 0.10295, + "map_at_3": 0.17131, + "map_at_5": 0.21494, + "map_at_10": 0.26768, + "map_at_20": 0.32481, + "map_at_100": 0.40826, + "map_at_1000": 0.42557, + "recall_at_1": 0.10295, + "recall_at_3": 0.19939, + "recall_at_5": 0.27131, + "recall_at_10": 0.38047, + "recall_at_20": 0.5126, + "recall_at_100": 0.87783, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.51724, + "cv_recall_at_3": 0.81034, + "cv_recall_at_5": 0.89655, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 0.94828, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.51724, + "precision_at_3": 0.46552, + "precision_at_5": 0.43793, + "precision_at_10": 0.3569, + "precision_at_20": 0.28362, + "precision_at_100": 0.1231, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.517241, + "mrr_at_3": 0.649425, + "mrr_at_5": 0.668391, + "mrr_at_10": 0.675335, + "mrr_at_20": 0.675335, + "mrr_at_100": 0.677062, + "mrr_at_1000": 0.677062, + "nauc_ndcg_at_1_max": 0.045036, + "nauc_ndcg_at_1_std": -0.0316, + "nauc_ndcg_at_1_diff1": 0.329091, + "nauc_ndcg_at_3_max": 0.110988, + "nauc_ndcg_at_3_std": 0.102454, + "nauc_ndcg_at_3_diff1": 0.136849, + "nauc_ndcg_at_5_max": 0.116639, + "nauc_ndcg_at_5_std": 0.142757, + "nauc_ndcg_at_5_diff1": 0.136024, + "nauc_ndcg_at_10_max": 0.089022, + "nauc_ndcg_at_10_std": 0.05747, + "nauc_ndcg_at_10_diff1": 0.047778, + "nauc_ndcg_at_20_max": 0.159788, + "nauc_ndcg_at_20_std": 0.115885, + "nauc_ndcg_at_20_diff1": 0.059593, + "nauc_ndcg_at_100_max": 0.175232, + "nauc_ndcg_at_100_std": 0.134844, + "nauc_ndcg_at_100_diff1": 0.018842, + "nauc_ndcg_at_1000_max": 0.107457, + "nauc_ndcg_at_1000_std": 0.102947, + "nauc_ndcg_at_1000_diff1": 0.070656, + "nauc_map_at_1_max": -0.155378, + "nauc_map_at_1_std": -0.277948, + "nauc_map_at_1_diff1": 0.293512, + "nauc_map_at_3_max": 0.033269, + "nauc_map_at_3_std": -0.092259, + "nauc_map_at_3_diff1": 0.201489, + "nauc_map_at_5_max": 0.081079, + "nauc_map_at_5_std": -0.02264, + "nauc_map_at_5_diff1": 0.225075, + "nauc_map_at_10_max": 0.137961, + "nauc_map_at_10_std": 0.005785, + "nauc_map_at_10_diff1": 0.13394, + "nauc_map_at_20_max": 0.17342, + "nauc_map_at_20_std": 0.062783, + "nauc_map_at_20_diff1": 0.108408, + "nauc_map_at_100_max": 0.147121, + "nauc_map_at_100_std": 0.085056, + "nauc_map_at_100_diff1": 0.035576, + "nauc_map_at_1000_max": 0.113576, + "nauc_map_at_1000_std": 0.073047, + "nauc_map_at_1000_diff1": 0.035845, + "nauc_recall_at_1_max": -0.155378, + "nauc_recall_at_1_std": -0.277948, + "nauc_recall_at_1_diff1": 0.293512, + "nauc_recall_at_3_max": 0.02859, + "nauc_recall_at_3_std": -0.072996, + "nauc_recall_at_3_diff1": 0.143624, + "nauc_recall_at_5_max": 0.065247, + "nauc_recall_at_5_std": -0.021863, + "nauc_recall_at_5_diff1": 0.165366, + "nauc_recall_at_10_max": 0.16249, + "nauc_recall_at_10_std": 0.016684, + "nauc_recall_at_10_diff1": 0.031729, + "nauc_recall_at_20_max": 0.255199, + "nauc_recall_at_20_std": 0.113898, + "nauc_recall_at_20_diff1": 0.020062, + "nauc_recall_at_100_max": 0.319628, + "nauc_recall_at_100_std": 0.17394, + "nauc_recall_at_100_diff1": -0.299727, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.045036, + "nauc_precision_at_1_std": -0.0316, + "nauc_precision_at_1_diff1": 0.329091, + "nauc_precision_at_3_max": 0.214909, + "nauc_precision_at_3_std": 0.256255, + "nauc_precision_at_3_diff1": 0.036113, + "nauc_precision_at_5_max": 0.242756, + "nauc_precision_at_5_std": 0.342064, + "nauc_precision_at_5_diff1": 0.015401, + "nauc_precision_at_10_max": 0.151721, + "nauc_precision_at_10_std": 0.229032, + "nauc_precision_at_10_diff1": -0.1283, + "nauc_precision_at_20_max": 0.041273, + "nauc_precision_at_20_std": 0.156167, + "nauc_precision_at_20_diff1": -0.16745, + "nauc_precision_at_100_max": -0.170454, + "nauc_precision_at_100_std": 0.006769, + "nauc_precision_at_100_diff1": -0.182705, + "nauc_precision_at_1000_max": -0.217319, + "nauc_precision_at_1000_std": -0.030019, + "nauc_precision_at_1000_diff1": -0.135806, + "nauc_cv_recall_at_1_max": 0.045036, + "nauc_cv_recall_at_1_std": -0.0316, + "nauc_cv_recall_at_1_diff1": 0.329091, + "nauc_cv_recall_at_3_max": -0.017733, + "nauc_cv_recall_at_3_std": 0.213482, + "nauc_cv_recall_at_3_diff1": 0.194602, + "nauc_cv_recall_at_5_max": -0.063076, + "nauc_cv_recall_at_5_std": 0.131086, + "nauc_cv_recall_at_5_diff1": 0.108687, + "nauc_cv_recall_at_10_max": -0.049532, + "nauc_cv_recall_at_10_std": -0.049532, + "nauc_cv_recall_at_10_diff1": -0.10528, + "nauc_cv_recall_at_20_max": -0.049532, + "nauc_cv_recall_at_20_std": -0.049532, + "nauc_cv_recall_at_20_diff1": -0.10528, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.045036, + "nauc_mrr_at_1_std": -0.0316, + "nauc_mrr_at_1_diff1": 0.329091, + "nauc_mrr_at_3_max": 0.022221, + "nauc_mrr_at_3_std": 0.035474, + "nauc_mrr_at_3_diff1": 0.247845, + "nauc_mrr_at_5_max": 0.027384, + "nauc_mrr_at_5_std": 0.022003, + "nauc_mrr_at_5_diff1": 0.244174, + "nauc_mrr_at_10_max": 0.033984, + "nauc_mrr_at_10_std": 0.015539, + "nauc_mrr_at_10_diff1": 0.242938, + "nauc_mrr_at_20_max": 0.033984, + "nauc_mrr_at_20_std": 0.015539, + "nauc_mrr_at_20_diff1": 0.242938, + "nauc_mrr_at_100_max": 0.035796, + "nauc_mrr_at_100_std": 0.017222, + "nauc_mrr_at_100_diff1": 0.244, + "nauc_mrr_at_1000_max": 0.035796, + "nauc_mrr_at_1000_std": 0.017222, + "nauc_mrr_at_1000_diff1": 0.244, + "main_score": 0.50609, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.56897, + "ndcg_at_3": 0.53704, + "ndcg_at_5": 0.52306, + "ndcg_at_10": 0.50358, + "ndcg_at_20": 0.5393, + "ndcg_at_100": 0.65527, + "ndcg_at_1000": 0.69692, + "map_at_1": 0.07947, + "map_at_3": 0.17402, + "map_at_5": 0.22228, + "map_at_10": 0.27698, + "map_at_20": 0.33097, + "map_at_100": 0.41735, + "map_at_1000": 0.43442, + "recall_at_1": 0.07947, + "recall_at_3": 0.21501, + "recall_at_5": 0.28758, + "recall_at_10": 0.39533, + "recall_at_20": 0.54763, + "recall_at_100": 0.88725, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.56897, + "cv_recall_at_3": 0.7931, + "cv_recall_at_5": 0.87931, + "cv_recall_at_10": 0.91379, + "cv_recall_at_20": 0.96552, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.56897, + "precision_at_3": 0.48851, + "precision_at_5": 0.45517, + "precision_at_10": 0.36034, + "precision_at_20": 0.28448, + "precision_at_100": 0.12345, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.568966, + "mrr_at_3": 0.675287, + "mrr_at_5": 0.695977, + "mrr_at_10": 0.700048, + "mrr_at_20": 0.70413, + "mrr_at_100": 0.705489, + "mrr_at_1000": 0.705489, + "nauc_ndcg_at_1_max": -0.09858, + "nauc_ndcg_at_1_std": -0.028781, + "nauc_ndcg_at_1_diff1": -0.161158, + "nauc_ndcg_at_3_max": 0.052402, + "nauc_ndcg_at_3_std": 0.066053, + "nauc_ndcg_at_3_diff1": -0.113489, + "nauc_ndcg_at_5_max": 0.015773, + "nauc_ndcg_at_5_std": 0.02751, + "nauc_ndcg_at_5_diff1": -0.148076, + "nauc_ndcg_at_10_max": -0.000217, + "nauc_ndcg_at_10_std": -0.037132, + "nauc_ndcg_at_10_diff1": -0.192376, + "nauc_ndcg_at_20_max": 0.076381, + "nauc_ndcg_at_20_std": 0.04151, + "nauc_ndcg_at_20_diff1": -0.178792, + "nauc_ndcg_at_100_max": 0.081505, + "nauc_ndcg_at_100_std": 0.083654, + "nauc_ndcg_at_100_diff1": -0.183744, + "nauc_ndcg_at_1000_max": 0.028202, + "nauc_ndcg_at_1000_std": 0.051258, + "nauc_ndcg_at_1000_diff1": -0.184949, + "nauc_map_at_1_max": -0.136884, + "nauc_map_at_1_std": -0.182469, + "nauc_map_at_1_diff1": -0.279515, + "nauc_map_at_3_max": -0.028634, + "nauc_map_at_3_std": -0.153025, + "nauc_map_at_3_diff1": -0.204523, + "nauc_map_at_5_max": 0.044732, + "nauc_map_at_5_std": -0.094176, + "nauc_map_at_5_diff1": -0.149182, + "nauc_map_at_10_max": 0.097067, + "nauc_map_at_10_std": -0.047523, + "nauc_map_at_10_diff1": -0.141951, + "nauc_map_at_20_max": 0.113639, + "nauc_map_at_20_std": -0.002769, + "nauc_map_at_20_diff1": -0.149704, + "nauc_map_at_100_max": 0.074348, + "nauc_map_at_100_std": 0.015441, + "nauc_map_at_100_diff1": -0.208119, + "nauc_map_at_1000_max": 0.049478, + "nauc_map_at_1000_std": 0.007282, + "nauc_map_at_1000_diff1": -0.216468, + "nauc_recall_at_1_max": -0.136884, + "nauc_recall_at_1_std": -0.182469, + "nauc_recall_at_1_diff1": -0.279515, + "nauc_recall_at_3_max": -0.059469, + "nauc_recall_at_3_std": -0.181738, + "nauc_recall_at_3_diff1": -0.180499, + "nauc_recall_at_5_max": 0.027237, + "nauc_recall_at_5_std": -0.111341, + "nauc_recall_at_5_diff1": -0.131006, + "nauc_recall_at_10_max": 0.064598, + "nauc_recall_at_10_std": -0.096058, + "nauc_recall_at_10_diff1": -0.130015, + "nauc_recall_at_20_max": 0.14448, + "nauc_recall_at_20_std": 0.001685, + "nauc_recall_at_20_diff1": -0.120785, + "nauc_recall_at_100_max": 0.273393, + "nauc_recall_at_100_std": 0.201324, + "nauc_recall_at_100_diff1": -0.152279, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.09858, + "nauc_precision_at_1_std": -0.028781, + "nauc_precision_at_1_diff1": -0.161158, + "nauc_precision_at_3_max": 0.145753, + "nauc_precision_at_3_std": 0.174518, + "nauc_precision_at_3_diff1": -0.003951, + "nauc_precision_at_5_max": 0.120968, + "nauc_precision_at_5_std": 0.16369, + "nauc_precision_at_5_diff1": -0.014865, + "nauc_precision_at_10_max": 0.045519, + "nauc_precision_at_10_std": 0.116812, + "nauc_precision_at_10_diff1": -0.03799, + "nauc_precision_at_20_max": -0.001405, + "nauc_precision_at_20_std": 0.126731, + "nauc_precision_at_20_diff1": -0.056317, + "nauc_precision_at_100_max": -0.118122, + "nauc_precision_at_100_std": 0.024481, + "nauc_precision_at_100_diff1": -0.100393, + "nauc_precision_at_1000_max": -0.14565, + "nauc_precision_at_1000_std": -0.008504, + "nauc_precision_at_1000_diff1": -0.103194, + "nauc_cv_recall_at_1_max": -0.09858, + "nauc_cv_recall_at_1_std": -0.028781, + "nauc_cv_recall_at_1_diff1": -0.161158, + "nauc_cv_recall_at_3_max": 0.04934, + "nauc_cv_recall_at_3_std": 0.224733, + "nauc_cv_recall_at_3_diff1": 0.008339, + "nauc_cv_recall_at_5_max": -0.054935, + "nauc_cv_recall_at_5_std": 0.071266, + "nauc_cv_recall_at_5_diff1": -0.253375, + "nauc_cv_recall_at_10_max": -0.160607, + "nauc_cv_recall_at_10_std": -0.147665, + "nauc_cv_recall_at_10_diff1": -0.293245, + "nauc_cv_recall_at_20_max": -0.107622, + "nauc_cv_recall_at_20_std": 0.073071, + "nauc_cv_recall_at_20_diff1": -0.509582, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.09858, + "nauc_mrr_at_1_std": -0.028781, + "nauc_mrr_at_1_diff1": -0.161158, + "nauc_mrr_at_3_max": -0.030478, + "nauc_mrr_at_3_std": 0.075547, + "nauc_mrr_at_3_diff1": -0.104567, + "nauc_mrr_at_5_max": -0.04711, + "nauc_mrr_at_5_std": 0.047948, + "nauc_mrr_at_5_diff1": -0.142493, + "nauc_mrr_at_10_max": -0.050655, + "nauc_mrr_at_10_std": 0.038608, + "nauc_mrr_at_10_diff1": -0.143415, + "nauc_mrr_at_20_max": -0.046842, + "nauc_mrr_at_20_std": 0.045302, + "nauc_mrr_at_20_diff1": -0.142808, + "nauc_mrr_at_100_max": -0.045934, + "nauc_mrr_at_100_std": 0.045916, + "nauc_mrr_at_100_diff1": -0.139939, + "nauc_mrr_at_1000_max": -0.045934, + "nauc_mrr_at_1000_std": 0.045916, + "nauc_mrr_at_1000_diff1": -0.139939, + "main_score": 0.52306, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.63793, + "ndcg_at_3": 0.63732, + "ndcg_at_5": 0.62326, + "ndcg_at_10": 0.58602, + "ndcg_at_20": 0.60401, + "ndcg_at_100": 0.70546, + "ndcg_at_1000": 0.73908, + "map_at_1": 0.07035, + "map_at_3": 0.19081, + "map_at_5": 0.26474, + "map_at_10": 0.33409, + "map_at_20": 0.39551, + "map_at_100": 0.47961, + "map_at_1000": 0.49543, + "recall_at_1": 0.07035, + "recall_at_3": 0.23734, + "recall_at_5": 0.34883, + "recall_at_10": 0.465, + "recall_at_20": 0.59997, + "recall_at_100": 0.9109, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.63793, + "cv_recall_at_3": 0.86207, + "cv_recall_at_5": 0.91379, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 0.96552, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.63793, + "precision_at_3": 0.6092, + "precision_at_5": 0.56897, + "precision_at_10": 0.43621, + "precision_at_20": 0.31897, + "precision_at_100": 0.12793, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.637931, + "mrr_at_3": 0.744253, + "mrr_at_5": 0.75546, + "mrr_at_10": 0.761207, + "mrr_at_20": 0.762069, + "mrr_at_100": 0.763356, + "mrr_at_1000": 0.763356, + "nauc_ndcg_at_1_max": 0.376684, + "nauc_ndcg_at_1_std": 0.448927, + "nauc_ndcg_at_1_diff1": 0.406021, + "nauc_ndcg_at_3_max": 0.292509, + "nauc_ndcg_at_3_std": 0.260579, + "nauc_ndcg_at_3_diff1": 0.325617, + "nauc_ndcg_at_5_max": 0.262735, + "nauc_ndcg_at_5_std": 0.231126, + "nauc_ndcg_at_5_diff1": 0.306615, + "nauc_ndcg_at_10_max": 0.265119, + "nauc_ndcg_at_10_std": 0.198498, + "nauc_ndcg_at_10_diff1": 0.251287, + "nauc_ndcg_at_20_max": 0.353965, + "nauc_ndcg_at_20_std": 0.250635, + "nauc_ndcg_at_20_diff1": 0.270088, + "nauc_ndcg_at_100_max": 0.335426, + "nauc_ndcg_at_100_std": 0.342741, + "nauc_ndcg_at_100_diff1": 0.164958, + "nauc_ndcg_at_1000_max": 0.307344, + "nauc_ndcg_at_1000_std": 0.316876, + "nauc_ndcg_at_1000_diff1": 0.283872, + "nauc_map_at_1_max": 0.187705, + "nauc_map_at_1_std": 0.211042, + "nauc_map_at_1_diff1": -0.003085, + "nauc_map_at_3_max": 0.128987, + "nauc_map_at_3_std": -0.008312, + "nauc_map_at_3_diff1": 0.006972, + "nauc_map_at_5_max": 0.174901, + "nauc_map_at_5_std": 0.057318, + "nauc_map_at_5_diff1": 0.034885, + "nauc_map_at_10_max": 0.25826, + "nauc_map_at_10_std": 0.118472, + "nauc_map_at_10_diff1": 0.082346, + "nauc_map_at_20_max": 0.323443, + "nauc_map_at_20_std": 0.185643, + "nauc_map_at_20_diff1": 0.12669, + "nauc_map_at_100_max": 0.311633, + "nauc_map_at_100_std": 0.256663, + "nauc_map_at_100_diff1": 0.160553, + "nauc_map_at_1000_max": 0.284921, + "nauc_map_at_1000_std": 0.24291, + "nauc_map_at_1000_diff1": 0.20564, + "nauc_recall_at_1_max": 0.187705, + "nauc_recall_at_1_std": 0.211042, + "nauc_recall_at_1_diff1": -0.003085, + "nauc_recall_at_3_max": 0.053798, + "nauc_recall_at_3_std": -0.142749, + "nauc_recall_at_3_diff1": -0.039802, + "nauc_recall_at_5_max": 0.078402, + "nauc_recall_at_5_std": -0.098009, + "nauc_recall_at_5_diff1": -0.041438, + "nauc_recall_at_10_max": 0.133232, + "nauc_recall_at_10_std": -0.079796, + "nauc_recall_at_10_diff1": -0.023655, + "nauc_recall_at_20_max": 0.251184, + "nauc_recall_at_20_std": 0.036754, + "nauc_recall_at_20_diff1": 0.024257, + "nauc_recall_at_100_max": 0.246073, + "nauc_recall_at_100_std": 0.237027, + "nauc_recall_at_100_diff1": -0.492332, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.376684, + "nauc_precision_at_1_std": 0.448927, + "nauc_precision_at_1_diff1": 0.406021, + "nauc_precision_at_3_max": 0.273996, + "nauc_precision_at_3_std": 0.266782, + "nauc_precision_at_3_diff1": 0.293151, + "nauc_precision_at_5_max": 0.249942, + "nauc_precision_at_5_std": 0.276125, + "nauc_precision_at_5_diff1": 0.29989, + "nauc_precision_at_10_max": 0.218522, + "nauc_precision_at_10_std": 0.243517, + "nauc_precision_at_10_diff1": 0.293041, + "nauc_precision_at_20_max": 0.176636, + "nauc_precision_at_20_std": 0.229914, + "nauc_precision_at_20_diff1": 0.257942, + "nauc_precision_at_100_max": -0.064532, + "nauc_precision_at_100_std": 0.078632, + "nauc_precision_at_100_diff1": 0.210866, + "nauc_precision_at_1000_max": -0.099913, + "nauc_precision_at_1000_std": 0.032966, + "nauc_precision_at_1000_diff1": 0.280801, + "nauc_cv_recall_at_1_max": 0.376684, + "nauc_cv_recall_at_1_std": 0.448927, + "nauc_cv_recall_at_1_diff1": 0.406021, + "nauc_cv_recall_at_3_max": 0.51161, + "nauc_cv_recall_at_3_std": 0.567953, + "nauc_cv_recall_at_3_diff1": 0.443232, + "nauc_cv_recall_at_5_max": 0.426306, + "nauc_cv_recall_at_5_std": 0.36107, + "nauc_cv_recall_at_5_diff1": 0.555072, + "nauc_cv_recall_at_10_max": 0.137029, + "nauc_cv_recall_at_10_std": 0.028303, + "nauc_cv_recall_at_10_diff1": 0.472102, + "nauc_cv_recall_at_20_max": -0.071463, + "nauc_cv_recall_at_20_std": -0.137074, + "nauc_cv_recall_at_20_diff1": 0.794609, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.376684, + "nauc_mrr_at_1_std": 0.448927, + "nauc_mrr_at_1_diff1": 0.406021, + "nauc_mrr_at_3_max": 0.391478, + "nauc_mrr_at_3_std": 0.472172, + "nauc_mrr_at_3_diff1": 0.39702, + "nauc_mrr_at_5_max": 0.376244, + "nauc_mrr_at_5_std": 0.448812, + "nauc_mrr_at_5_diff1": 0.406325, + "nauc_mrr_at_10_max": 0.362513, + "nauc_mrr_at_10_std": 0.437141, + "nauc_mrr_at_10_diff1": 0.398574, + "nauc_mrr_at_20_max": 0.361695, + "nauc_mrr_at_20_std": 0.437474, + "nauc_mrr_at_20_diff1": 0.401016, + "nauc_mrr_at_100_max": 0.366374, + "nauc_mrr_at_100_std": 0.442945, + "nauc_mrr_at_100_diff1": 0.39862, + "nauc_mrr_at_1000_max": 0.366374, + "nauc_mrr_at_1000_std": 0.442945, + "nauc_mrr_at_1000_diff1": 0.39862, + "main_score": 0.62326, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.48276, + "ndcg_at_3": 0.48973, + "ndcg_at_5": 0.47612, + "ndcg_at_10": 0.47768, + "ndcg_at_20": 0.51796, + "ndcg_at_100": 0.62295, + "ndcg_at_1000": 0.66875, + "map_at_1": 0.04772, + "map_at_3": 0.13987, + "map_at_5": 0.18236, + "map_at_10": 0.24431, + "map_at_20": 0.30179, + "map_at_100": 0.38164, + "map_at_1000": 0.39963, + "recall_at_1": 0.04772, + "recall_at_3": 0.19021, + "recall_at_5": 0.26507, + "recall_at_10": 0.40354, + "recall_at_20": 0.5604, + "recall_at_100": 0.87171, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.48276, + "cv_recall_at_3": 0.77586, + "cv_recall_at_5": 0.87931, + "cv_recall_at_10": 0.93103, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.48276, + "precision_at_3": 0.47126, + "precision_at_5": 0.43793, + "precision_at_10": 0.36897, + "precision_at_20": 0.29138, + "precision_at_100": 0.12259, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.482759, + "mrr_at_3": 0.612069, + "mrr_at_5": 0.635345, + "mrr_at_10": 0.642816, + "mrr_at_20": 0.647207, + "mrr_at_100": 0.647207, + "mrr_at_1000": 0.647207, + "nauc_ndcg_at_1_max": 0.185993, + "nauc_ndcg_at_1_std": 0.304849, + "nauc_ndcg_at_1_diff1": -0.070336, + "nauc_ndcg_at_3_max": 0.233534, + "nauc_ndcg_at_3_std": 0.26287, + "nauc_ndcg_at_3_diff1": -0.005377, + "nauc_ndcg_at_5_max": 0.188991, + "nauc_ndcg_at_5_std": 0.238624, + "nauc_ndcg_at_5_diff1": 0.04707, + "nauc_ndcg_at_10_max": 0.167138, + "nauc_ndcg_at_10_std": 0.206104, + "nauc_ndcg_at_10_diff1": 0.037465, + "nauc_ndcg_at_20_max": 0.190408, + "nauc_ndcg_at_20_std": 0.249297, + "nauc_ndcg_at_20_diff1": -0.000375, + "nauc_ndcg_at_100_max": 0.171079, + "nauc_ndcg_at_100_std": 0.263448, + "nauc_ndcg_at_100_diff1": -0.085506, + "nauc_ndcg_at_1000_max": 0.160103, + "nauc_ndcg_at_1000_std": 0.280587, + "nauc_ndcg_at_1000_diff1": -0.040813, + "nauc_map_at_1_max": 0.124559, + "nauc_map_at_1_std": 0.106496, + "nauc_map_at_1_diff1": -0.105234, + "nauc_map_at_3_max": 0.213632, + "nauc_map_at_3_std": 0.001729, + "nauc_map_at_3_diff1": 0.115656, + "nauc_map_at_5_max": 0.180831, + "nauc_map_at_5_std": 0.005408, + "nauc_map_at_5_diff1": 0.102667, + "nauc_map_at_10_max": 0.187102, + "nauc_map_at_10_std": 0.017037, + "nauc_map_at_10_diff1": 0.087568, + "nauc_map_at_20_max": 0.188846, + "nauc_map_at_20_std": 0.081331, + "nauc_map_at_20_diff1": 0.061803, + "nauc_map_at_100_max": 0.175062, + "nauc_map_at_100_std": 0.195612, + "nauc_map_at_100_diff1": -0.009692, + "nauc_map_at_1000_max": 0.173723, + "nauc_map_at_1000_std": 0.230043, + "nauc_map_at_1000_diff1": -0.01546, + "nauc_recall_at_1_max": 0.124559, + "nauc_recall_at_1_std": 0.106496, + "nauc_recall_at_1_diff1": -0.105234, + "nauc_recall_at_3_max": 0.124733, + "nauc_recall_at_3_std": -0.137781, + "nauc_recall_at_3_diff1": 0.197668, + "nauc_recall_at_5_max": 0.069886, + "nauc_recall_at_5_std": -0.156702, + "nauc_recall_at_5_diff1": 0.168916, + "nauc_recall_at_10_max": 0.046931, + "nauc_recall_at_10_std": -0.175455, + "nauc_recall_at_10_diff1": 0.102688, + "nauc_recall_at_20_max": 0.078128, + "nauc_recall_at_20_std": -0.105023, + "nauc_recall_at_20_diff1": 0.076236, + "nauc_recall_at_100_max": 0.169368, + "nauc_recall_at_100_std": 0.11647, + "nauc_recall_at_100_diff1": -0.196245, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.185993, + "nauc_precision_at_1_std": 0.304849, + "nauc_precision_at_1_diff1": -0.070336, + "nauc_precision_at_3_max": 0.210922, + "nauc_precision_at_3_std": 0.284307, + "nauc_precision_at_3_diff1": -0.067955, + "nauc_precision_at_5_max": 0.146708, + "nauc_precision_at_5_std": 0.283366, + "nauc_precision_at_5_diff1": -0.012617, + "nauc_precision_at_10_max": 0.094497, + "nauc_precision_at_10_std": 0.293309, + "nauc_precision_at_10_diff1": -0.019965, + "nauc_precision_at_20_max": 0.064105, + "nauc_precision_at_20_std": 0.337607, + "nauc_precision_at_20_diff1": -0.069452, + "nauc_precision_at_100_max": 0.009483, + "nauc_precision_at_100_std": 0.353091, + "nauc_precision_at_100_diff1": -0.13765, + "nauc_precision_at_1000_max": 0.012158, + "nauc_precision_at_1000_std": 0.348051, + "nauc_precision_at_1000_diff1": -0.108076, + "nauc_cv_recall_at_1_max": 0.185993, + "nauc_cv_recall_at_1_std": 0.304849, + "nauc_cv_recall_at_1_diff1": -0.070336, + "nauc_cv_recall_at_3_max": 0.063634, + "nauc_cv_recall_at_3_std": 0.169252, + "nauc_cv_recall_at_3_diff1": -0.136664, + "nauc_cv_recall_at_5_max": -0.150155, + "nauc_cv_recall_at_5_std": 0.029181, + "nauc_cv_recall_at_5_diff1": -0.084474, + "nauc_cv_recall_at_10_max": -0.681811, + "nauc_cv_recall_at_10_std": -0.373856, + "nauc_cv_recall_at_10_diff1": -0.441594, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.185993, + "nauc_mrr_at_1_std": 0.304849, + "nauc_mrr_at_1_diff1": -0.070336, + "nauc_mrr_at_3_max": 0.177926, + "nauc_mrr_at_3_std": 0.27475, + "nauc_mrr_at_3_diff1": -0.0465, + "nauc_mrr_at_5_max": 0.159744, + "nauc_mrr_at_5_std": 0.263738, + "nauc_mrr_at_5_diff1": -0.039461, + "nauc_mrr_at_10_max": 0.149348, + "nauc_mrr_at_10_std": 0.256139, + "nauc_mrr_at_10_diff1": -0.051884, + "nauc_mrr_at_20_max": 0.16161, + "nauc_mrr_at_20_std": 0.264826, + "nauc_mrr_at_20_diff1": -0.043952, + "nauc_mrr_at_100_max": 0.16161, + "nauc_mrr_at_100_std": 0.264826, + "nauc_mrr_at_100_diff1": -0.043952, + "nauc_mrr_at_1000_max": 0.16161, + "nauc_mrr_at_1000_std": 0.264826, + "nauc_mrr_at_1000_diff1": -0.043952, + "main_score": 0.47612, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 138.6242938041687, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..629aec419a --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", + "task_name": "VidoreArxivQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.826, + "ndcg_at_3": 0.87002, + "ndcg_at_5": 0.87983, + "ndcg_at_10": 0.88594, + "ndcg_at_20": 0.89273, + "ndcg_at_100": 0.89781, + "ndcg_at_1000": 0.89836, + "map_at_1": 0.826, + "map_at_3": 0.859, + "map_at_5": 0.8644, + "map_at_10": 0.86709, + "map_at_20": 0.86907, + "map_at_100": 0.86986, + "map_at_1000": 0.86989, + "recall_at_1": 0.826, + "recall_at_3": 0.902, + "recall_at_5": 0.926, + "recall_at_10": 0.944, + "recall_at_20": 0.97, + "recall_at_100": 0.996, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.826, + "cv_recall_at_3": 0.902, + "cv_recall_at_5": 0.926, + "cv_recall_at_10": 0.944, + "cv_recall_at_20": 0.97, + "cv_recall_at_100": 0.996, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.826, + "precision_at_3": 0.30067, + "precision_at_5": 0.1852, + "precision_at_10": 0.0944, + "precision_at_20": 0.0485, + "precision_at_100": 0.00996, + "precision_at_1000": 0.001, + "mrr_at_1": 0.826, + "mrr_at_3": 0.859, + "mrr_at_5": 0.8644, + "mrr_at_10": 0.86709, + "mrr_at_20": 0.869067, + "mrr_at_100": 0.869859, + "mrr_at_1000": 0.869886, + "nauc_ndcg_at_1_max": 0.775328, + "nauc_ndcg_at_1_std": -0.084099, + "nauc_ndcg_at_1_diff1": 0.922838, + "nauc_ndcg_at_3_max": 0.801018, + "nauc_ndcg_at_3_std": -0.066963, + "nauc_ndcg_at_3_diff1": 0.906373, + "nauc_ndcg_at_5_max": 0.800403, + "nauc_ndcg_at_5_std": -0.037711, + "nauc_ndcg_at_5_diff1": 0.910332, + "nauc_ndcg_at_10_max": 0.804924, + "nauc_ndcg_at_10_std": -0.019782, + "nauc_ndcg_at_10_diff1": 0.912082, + "nauc_ndcg_at_20_max": 0.801094, + "nauc_ndcg_at_20_std": -0.028095, + "nauc_ndcg_at_20_diff1": 0.912149, + "nauc_ndcg_at_100_max": 0.796494, + "nauc_ndcg_at_100_std": -0.041877, + "nauc_ndcg_at_100_diff1": 0.914204, + "nauc_ndcg_at_1000_max": 0.795351, + "nauc_ndcg_at_1000_std": -0.045095, + "nauc_ndcg_at_1000_diff1": 0.913722, + "nauc_map_at_1_max": 0.775328, + "nauc_map_at_1_std": -0.084099, + "nauc_map_at_1_diff1": 0.922838, + "nauc_map_at_3_max": 0.792454, + "nauc_map_at_3_std": -0.071767, + "nauc_map_at_3_diff1": 0.911826, + "nauc_map_at_5_max": 0.79181, + "nauc_map_at_5_std": -0.058104, + "nauc_map_at_5_diff1": 0.914112, + "nauc_map_at_10_max": 0.793372, + "nauc_map_at_10_std": -0.051643, + "nauc_map_at_10_diff1": 0.914775, + "nauc_map_at_20_max": 0.792111, + "nauc_map_at_20_std": -0.054027, + "nauc_map_at_20_diff1": 0.91461, + "nauc_map_at_100_max": 0.791549, + "nauc_map_at_100_std": -0.055548, + "nauc_map_at_100_diff1": 0.914898, + "nauc_map_at_1000_max": 0.791505, + "nauc_map_at_1000_std": -0.055648, + "nauc_map_at_1000_diff1": 0.914879, + "nauc_recall_at_1_max": 0.775328, + "nauc_recall_at_1_std": -0.084099, + "nauc_recall_at_1_diff1": 0.922838, + "nauc_recall_at_3_max": 0.835801, + "nauc_recall_at_3_std": -0.047905, + "nauc_recall_at_3_diff1": 0.884239, + "nauc_recall_at_5_max": 0.844626, + "nauc_recall_at_5_std": 0.07423, + "nauc_recall_at_5_diff1": 0.891425, + "nauc_recall_at_10_max": 0.884704, + "nauc_recall_at_10_std": 0.212985, + "nauc_recall_at_10_diff1": 0.896375, + "nauc_recall_at_20_max": 0.908185, + "nauc_recall_at_20_std": 0.30112, + "nauc_recall_at_20_diff1": 0.890756, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.561391, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.775328, + "nauc_precision_at_1_std": -0.084099, + "nauc_precision_at_1_diff1": 0.922838, + "nauc_precision_at_3_max": 0.835801, + "nauc_precision_at_3_std": -0.047905, + "nauc_precision_at_3_diff1": 0.884239, + "nauc_precision_at_5_max": 0.844626, + "nauc_precision_at_5_std": 0.07423, + "nauc_precision_at_5_diff1": 0.891425, + "nauc_precision_at_10_max": 0.884704, + "nauc_precision_at_10_std": 0.212985, + "nauc_precision_at_10_diff1": 0.896375, + "nauc_precision_at_20_max": 0.908185, + "nauc_precision_at_20_std": 0.30112, + "nauc_precision_at_20_diff1": 0.890756, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.561391, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_cv_recall_at_1_max": 0.775328, + "nauc_cv_recall_at_1_std": -0.084099, + "nauc_cv_recall_at_1_diff1": 0.922838, + "nauc_cv_recall_at_3_max": 0.835801, + "nauc_cv_recall_at_3_std": -0.047905, + "nauc_cv_recall_at_3_diff1": 0.884239, + "nauc_cv_recall_at_5_max": 0.844626, + "nauc_cv_recall_at_5_std": 0.07423, + "nauc_cv_recall_at_5_diff1": 0.891425, + "nauc_cv_recall_at_10_max": 0.884704, + "nauc_cv_recall_at_10_std": 0.212985, + "nauc_cv_recall_at_10_diff1": 0.896375, + "nauc_cv_recall_at_20_max": 0.908185, + "nauc_cv_recall_at_20_std": 0.30112, + "nauc_cv_recall_at_20_diff1": 0.890756, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 0.561391, + "nauc_cv_recall_at_100_diff1": 1.0, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.775328, + "nauc_mrr_at_1_std": -0.084099, + "nauc_mrr_at_1_diff1": 0.922838, + "nauc_mrr_at_3_max": 0.792454, + "nauc_mrr_at_3_std": -0.071767, + "nauc_mrr_at_3_diff1": 0.911826, + "nauc_mrr_at_5_max": 0.79181, + "nauc_mrr_at_5_std": -0.058104, + "nauc_mrr_at_5_diff1": 0.914112, + "nauc_mrr_at_10_max": 0.793372, + "nauc_mrr_at_10_std": -0.051643, + "nauc_mrr_at_10_diff1": 0.914775, + "nauc_mrr_at_20_max": 0.792111, + "nauc_mrr_at_20_std": -0.054027, + "nauc_mrr_at_20_diff1": 0.91461, + "nauc_mrr_at_100_max": 0.791549, + "nauc_mrr_at_100_std": -0.055548, + "nauc_mrr_at_100_diff1": 0.914898, + "nauc_mrr_at_1000_max": 0.791505, + "nauc_mrr_at_1000_std": -0.055648, + "nauc_mrr_at_1000_diff1": 0.914879, + "main_score": 0.87983, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 51.192665815353394, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..edaa8d1e0f --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", + "task_name": "VidoreDocVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.53659, + "ndcg_at_3": 0.59536, + "ndcg_at_5": 0.61483, + "ndcg_at_10": 0.6333, + "ndcg_at_20": 0.6464, + "ndcg_at_100": 0.66648, + "ndcg_at_1000": 0.68187, + "map_at_1": 0.53437, + "map_at_3": 0.58004, + "map_at_5": 0.59115, + "map_at_10": 0.59888, + "map_at_20": 0.60286, + "map_at_100": 0.60578, + "map_at_1000": 0.60653, + "recall_at_1": 0.53437, + "recall_at_3": 0.63533, + "recall_at_5": 0.68104, + "recall_at_10": 0.73718, + "recall_at_20": 0.78714, + "recall_at_100": 0.89191, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.53659, + "cv_recall_at_3": 0.6408, + "cv_recall_at_5": 0.68736, + "cv_recall_at_10": 0.74501, + "cv_recall_at_20": 0.79157, + "cv_recall_at_100": 0.898, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.53659, + "precision_at_3": 0.2136, + "precision_at_5": 0.13969, + "precision_at_10": 0.0765, + "precision_at_20": 0.04135, + "precision_at_100": 0.00951, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.536585, + "mrr_at_3": 0.583888, + "mrr_at_5": 0.594309, + "mrr_at_10": 0.601796, + "mrr_at_20": 0.604925, + "mrr_at_100": 0.607641, + "mrr_at_1000": 0.608237, + "nauc_ndcg_at_1_max": 0.143621, + "nauc_ndcg_at_1_std": 0.593158, + "nauc_ndcg_at_1_diff1": 0.900072, + "nauc_ndcg_at_3_max": 0.085632, + "nauc_ndcg_at_3_std": 0.651504, + "nauc_ndcg_at_3_diff1": 0.880907, + "nauc_ndcg_at_5_max": 0.063684, + "nauc_ndcg_at_5_std": 0.664786, + "nauc_ndcg_at_5_diff1": 0.874329, + "nauc_ndcg_at_10_max": 0.032378, + "nauc_ndcg_at_10_std": 0.688659, + "nauc_ndcg_at_10_diff1": 0.870744, + "nauc_ndcg_at_20_max": 0.021759, + "nauc_ndcg_at_20_std": 0.685605, + "nauc_ndcg_at_20_diff1": 0.870983, + "nauc_ndcg_at_100_max": 0.00882, + "nauc_ndcg_at_100_std": 0.681015, + "nauc_ndcg_at_100_diff1": 0.870229, + "nauc_ndcg_at_1000_max": 0.044122, + "nauc_ndcg_at_1000_std": 0.666653, + "nauc_ndcg_at_1000_diff1": 0.874428, + "nauc_map_at_1_max": 0.146968, + "nauc_map_at_1_std": 0.592278, + "nauc_map_at_1_diff1": 0.901593, + "nauc_map_at_3_max": 0.1015, + "nauc_map_at_3_std": 0.637282, + "nauc_map_at_3_diff1": 0.88678, + "nauc_map_at_5_max": 0.089609, + "nauc_map_at_5_std": 0.644204, + "nauc_map_at_5_diff1": 0.882952, + "nauc_map_at_10_max": 0.07774, + "nauc_map_at_10_std": 0.653383, + "nauc_map_at_10_diff1": 0.881467, + "nauc_map_at_20_max": 0.074311, + "nauc_map_at_20_std": 0.652731, + "nauc_map_at_20_diff1": 0.881195, + "nauc_map_at_100_max": 0.072499, + "nauc_map_at_100_std": 0.652202, + "nauc_map_at_100_diff1": 0.881269, + "nauc_map_at_1000_max": 0.073955, + "nauc_map_at_1000_std": 0.651542, + "nauc_map_at_1000_diff1": 0.881372, + "nauc_recall_at_1_max": 0.146968, + "nauc_recall_at_1_std": 0.592278, + "nauc_recall_at_1_diff1": 0.901593, + "nauc_recall_at_3_max": 0.042647, + "nauc_recall_at_3_std": 0.698, + "nauc_recall_at_3_diff1": 0.865603, + "nauc_recall_at_5_max": -0.019473, + "nauc_recall_at_5_std": 0.735312, + "nauc_recall_at_5_diff1": 0.849251, + "nauc_recall_at_10_max": -0.152357, + "nauc_recall_at_10_std": 0.832521, + "nauc_recall_at_10_diff1": 0.832244, + "nauc_recall_at_20_max": -0.234563, + "nauc_recall_at_20_std": 0.841053, + "nauc_recall_at_20_diff1": 0.830578, + "nauc_recall_at_100_max": -0.591791, + "nauc_recall_at_100_std": 0.90481, + "nauc_recall_at_100_diff1": 0.786248, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.143621, + "nauc_precision_at_1_std": 0.593158, + "nauc_precision_at_1_diff1": 0.900072, + "nauc_precision_at_3_max": 0.034831, + "nauc_precision_at_3_std": 0.694305, + "nauc_precision_at_3_diff1": 0.861442, + "nauc_precision_at_5_max": -0.045133, + "nauc_precision_at_5_std": 0.686157, + "nauc_precision_at_5_diff1": 0.759549, + "nauc_precision_at_10_max": -0.181792, + "nauc_precision_at_10_std": 0.716323, + "nauc_precision_at_10_diff1": 0.660922, + "nauc_precision_at_20_max": -0.243045, + "nauc_precision_at_20_std": 0.598924, + "nauc_precision_at_20_diff1": 0.518328, + "nauc_precision_at_100_max": -0.421156, + "nauc_precision_at_100_std": 0.345789, + "nauc_precision_at_100_diff1": 0.18648, + "nauc_precision_at_1000_max": -0.094542, + "nauc_precision_at_1000_std": -0.398002, + "nauc_precision_at_1000_diff1": -0.46031, + "nauc_cv_recall_at_1_max": 0.143621, + "nauc_cv_recall_at_1_std": 0.593158, + "nauc_cv_recall_at_1_diff1": 0.900072, + "nauc_cv_recall_at_3_max": 0.034831, + "nauc_cv_recall_at_3_std": 0.694305, + "nauc_cv_recall_at_3_diff1": 0.861442, + "nauc_cv_recall_at_5_max": -0.030417, + "nauc_cv_recall_at_5_std": 0.738396, + "nauc_cv_recall_at_5_diff1": 0.843303, + "nauc_cv_recall_at_10_max": -0.16803, + "nauc_cv_recall_at_10_std": 0.843355, + "nauc_cv_recall_at_10_diff1": 0.822678, + "nauc_cv_recall_at_20_max": -0.248731, + "nauc_cv_recall_at_20_std": 0.83588, + "nauc_cv_recall_at_20_diff1": 0.82654, + "nauc_cv_recall_at_100_max": -0.607791, + "nauc_cv_recall_at_100_std": 0.894863, + "nauc_cv_recall_at_100_diff1": 0.773208, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.143621, + "nauc_mrr_at_1_std": 0.593158, + "nauc_mrr_at_1_diff1": 0.900072, + "nauc_mrr_at_3_max": 0.09628, + "nauc_mrr_at_3_std": 0.635379, + "nauc_mrr_at_3_diff1": 0.883532, + "nauc_mrr_at_5_max": 0.085419, + "nauc_mrr_at_5_std": 0.642722, + "nauc_mrr_at_5_diff1": 0.880318, + "nauc_mrr_at_10_max": 0.074327, + "nauc_mrr_at_10_std": 0.652364, + "nauc_mrr_at_10_diff1": 0.878761, + "nauc_mrr_at_20_max": 0.071684, + "nauc_mrr_at_20_std": 0.650568, + "nauc_mrr_at_20_diff1": 0.87948, + "nauc_mrr_at_100_max": 0.070296, + "nauc_mrr_at_100_std": 0.650257, + "nauc_mrr_at_100_diff1": 0.879621, + "nauc_mrr_at_1000_max": 0.071658, + "nauc_mrr_at_1000_std": 0.649786, + "nauc_mrr_at_1000_diff1": 0.879802, + "main_score": 0.61483, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 56.31658387184143, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..b7c1016458 --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", + "task_name": "VidoreInfoVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.88866, + "ndcg_at_3": 0.91452, + "ndcg_at_5": 0.92507, + "ndcg_at_10": 0.92835, + "ndcg_at_20": 0.93058, + "ndcg_at_100": 0.93436, + "ndcg_at_1000": 0.93524, + "map_at_1": 0.88866, + "map_at_3": 0.90823, + "map_at_5": 0.9143, + "map_at_10": 0.91578, + "map_at_20": 0.91637, + "map_at_100": 0.91699, + "map_at_1000": 0.91703, + "recall_at_1": 0.88866, + "recall_at_3": 0.93219, + "recall_at_5": 0.95749, + "recall_at_10": 0.9666, + "recall_at_20": 0.9751, + "recall_at_100": 0.99352, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.88866, + "cv_recall_at_3": 0.9332, + "cv_recall_at_5": 0.95749, + "cv_recall_at_10": 0.96761, + "cv_recall_at_20": 0.97773, + "cv_recall_at_100": 0.99393, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.88866, + "precision_at_3": 0.31107, + "precision_at_5": 0.1919, + "precision_at_10": 0.09696, + "precision_at_20": 0.04899, + "precision_at_100": 0.01004, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.888664, + "mrr_at_3": 0.90857, + "mrr_at_5": 0.914136, + "mrr_at_10": 0.915775, + "mrr_at_20": 0.916501, + "mrr_at_100": 0.917012, + "mrr_at_1000": 0.91704, + "nauc_ndcg_at_1_max": 0.655827, + "nauc_ndcg_at_1_std": -0.044736, + "nauc_ndcg_at_1_diff1": 0.939973, + "nauc_ndcg_at_3_max": 0.672013, + "nauc_ndcg_at_3_std": -0.056866, + "nauc_ndcg_at_3_diff1": 0.939303, + "nauc_ndcg_at_5_max": 0.707708, + "nauc_ndcg_at_5_std": 0.008742, + "nauc_ndcg_at_5_diff1": 0.939565, + "nauc_ndcg_at_10_max": 0.710709, + "nauc_ndcg_at_10_std": 0.026842, + "nauc_ndcg_at_10_diff1": 0.940737, + "nauc_ndcg_at_20_max": 0.705235, + "nauc_ndcg_at_20_std": 0.044167, + "nauc_ndcg_at_20_diff1": 0.939857, + "nauc_ndcg_at_100_max": 0.695001, + "nauc_ndcg_at_100_std": 0.007567, + "nauc_ndcg_at_100_diff1": 0.937943, + "nauc_ndcg_at_1000_max": 0.691971, + "nauc_ndcg_at_1000_std": -0.000676, + "nauc_ndcg_at_1000_diff1": 0.938758, + "nauc_map_at_1_max": 0.655827, + "nauc_map_at_1_std": -0.044736, + "nauc_map_at_1_diff1": 0.939973, + "nauc_map_at_3_max": 0.668457, + "nauc_map_at_3_std": -0.052672, + "nauc_map_at_3_diff1": 0.939146, + "nauc_map_at_5_max": 0.687212, + "nauc_map_at_5_std": -0.017133, + "nauc_map_at_5_diff1": 0.939153, + "nauc_map_at_10_max": 0.688186, + "nauc_map_at_10_std": -0.009886, + "nauc_map_at_10_diff1": 0.939572, + "nauc_map_at_20_max": 0.686742, + "nauc_map_at_20_std": -0.00675, + "nauc_map_at_20_diff1": 0.939406, + "nauc_map_at_100_max": 0.6854, + "nauc_map_at_100_std": -0.011715, + "nauc_map_at_100_diff1": 0.939144, + "nauc_map_at_1000_max": 0.685277, + "nauc_map_at_1000_std": -0.012064, + "nauc_map_at_1000_diff1": 0.939161, + "nauc_recall_at_1_max": 0.655827, + "nauc_recall_at_1_std": -0.044736, + "nauc_recall_at_1_diff1": 0.939973, + "nauc_recall_at_3_max": 0.681245, + "nauc_recall_at_3_std": -0.081228, + "nauc_recall_at_3_diff1": 0.940535, + "nauc_recall_at_5_max": 0.832057, + "nauc_recall_at_5_std": 0.170039, + "nauc_recall_at_5_diff1": 0.942453, + "nauc_recall_at_10_max": 0.885843, + "nauc_recall_at_10_std": 0.328498, + "nauc_recall_at_10_diff1": 0.951507, + "nauc_recall_at_20_max": 0.892023, + "nauc_recall_at_20_std": 0.65289, + "nauc_recall_at_20_diff1": 0.945566, + "nauc_recall_at_100_max": 0.913208, + "nauc_recall_at_100_std": 0.593291, + "nauc_recall_at_100_diff1": 0.872395, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.655827, + "nauc_precision_at_1_std": -0.044736, + "nauc_precision_at_1_diff1": 0.939973, + "nauc_precision_at_3_max": 0.689709, + "nauc_precision_at_3_std": -0.065025, + "nauc_precision_at_3_diff1": 0.939634, + "nauc_precision_at_5_max": 0.762874, + "nauc_precision_at_5_std": 0.207593, + "nauc_precision_at_5_diff1": 0.826232, + "nauc_precision_at_10_max": 0.788663, + "nauc_precision_at_10_std": 0.342141, + "nauc_precision_at_10_diff1": 0.800279, + "nauc_precision_at_20_max": 0.749333, + "nauc_precision_at_20_std": 0.618437, + "nauc_precision_at_20_diff1": 0.732166, + "nauc_precision_at_100_max": -0.119506, + "nauc_precision_at_100_std": -0.098212, + "nauc_precision_at_100_diff1": -0.205521, + "nauc_precision_at_1000_max": -0.489653, + "nauc_precision_at_1000_std": -0.367947, + "nauc_precision_at_1000_diff1": -0.573514, + "nauc_cv_recall_at_1_max": 0.655827, + "nauc_cv_recall_at_1_std": -0.044736, + "nauc_cv_recall_at_1_diff1": 0.939973, + "nauc_cv_recall_at_3_max": 0.689709, + "nauc_cv_recall_at_3_std": -0.065025, + "nauc_cv_recall_at_3_diff1": 0.939634, + "nauc_cv_recall_at_5_max": 0.832057, + "nauc_cv_recall_at_5_std": 0.170039, + "nauc_cv_recall_at_5_diff1": 0.942453, + "nauc_cv_recall_at_10_max": 0.882276, + "nauc_cv_recall_at_10_std": 0.307514, + "nauc_cv_recall_at_10_diff1": 0.949991, + "nauc_cv_recall_at_20_max": 0.879262, + "nauc_cv_recall_at_20_std": 0.611867, + "nauc_cv_recall_at_20_diff1": 0.939133, + "nauc_cv_recall_at_100_max": 0.907422, + "nauc_cv_recall_at_100_std": 0.566178, + "nauc_cv_recall_at_100_diff1": 0.863888, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.655827, + "nauc_mrr_at_1_std": -0.044736, + "nauc_mrr_at_1_diff1": 0.939973, + "nauc_mrr_at_3_max": 0.670489, + "nauc_mrr_at_3_std": -0.048586, + "nauc_mrr_at_3_diff1": 0.93892, + "nauc_mrr_at_5_max": 0.686093, + "nauc_mrr_at_5_std": -0.019381, + "nauc_mrr_at_5_diff1": 0.939274, + "nauc_mrr_at_10_max": 0.686412, + "nauc_mrr_at_10_std": -0.014234, + "nauc_mrr_at_10_diff1": 0.939572, + "nauc_mrr_at_20_max": 0.684441, + "nauc_mrr_at_20_std": -0.012779, + "nauc_mrr_at_20_diff1": 0.939307, + "nauc_mrr_at_100_max": 0.683503, + "nauc_mrr_at_100_std": -0.016437, + "nauc_mrr_at_100_diff1": 0.939126, + "nauc_mrr_at_1000_max": 0.683427, + "nauc_mrr_at_1000_std": -0.016635, + "nauc_mrr_at_1000_diff1": 0.939152, + "main_score": 0.92507, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 56.54674196243286, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..7fe22370b6 --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "84a382e05c4473fed9cff2bbae95fe2379416117", + "task_name": "VidoreShiftProjectRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.79, + "ndcg_at_3": 0.88702, + "ndcg_at_5": 0.89906, + "ndcg_at_10": 0.90196, + "ndcg_at_20": 0.90196, + "ndcg_at_100": 0.90389, + "ndcg_at_1000": 0.90389, + "map_at_1": 0.79, + "map_at_3": 0.865, + "map_at_5": 0.8715, + "map_at_10": 0.8725, + "map_at_20": 0.8725, + "map_at_100": 0.87279, + "map_at_1000": 0.87279, + "recall_at_1": 0.79, + "recall_at_3": 0.95, + "recall_at_5": 0.98, + "recall_at_10": 0.99, + "recall_at_20": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.79, + "cv_recall_at_3": 0.95, + "cv_recall_at_5": 0.98, + "cv_recall_at_10": 0.99, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.79, + "precision_at_3": 0.31667, + "precision_at_5": 0.196, + "precision_at_10": 0.099, + "precision_at_20": 0.0495, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.79, + "mrr_at_3": 0.865, + "mrr_at_5": 0.8715, + "mrr_at_10": 0.8725, + "mrr_at_20": 0.8725, + "mrr_at_100": 0.872786, + "mrr_at_1000": 0.872786, + "nauc_ndcg_at_1_max": 0.006637, + "nauc_ndcg_at_1_std": -0.400558, + "nauc_ndcg_at_1_diff1": 0.73422, + "nauc_ndcg_at_3_max": 0.073019, + "nauc_ndcg_at_3_std": -0.341669, + "nauc_ndcg_at_3_diff1": 0.717213, + "nauc_ndcg_at_5_max": 0.097847, + "nauc_ndcg_at_5_std": -0.299964, + "nauc_ndcg_at_5_diff1": 0.704852, + "nauc_ndcg_at_10_max": 0.078271, + "nauc_ndcg_at_10_std": -0.313218, + "nauc_ndcg_at_10_diff1": 0.715724, + "nauc_ndcg_at_20_max": 0.078271, + "nauc_ndcg_at_20_std": -0.313218, + "nauc_ndcg_at_20_diff1": 0.715724, + "nauc_ndcg_at_100_max": 0.064471, + "nauc_ndcg_at_100_std": -0.331806, + "nauc_ndcg_at_100_diff1": 0.712434, + "nauc_ndcg_at_1000_max": 0.064471, + "nauc_ndcg_at_1000_std": -0.331806, + "nauc_ndcg_at_1000_diff1": 0.712434, + "nauc_map_at_1_max": 0.006637, + "nauc_map_at_1_std": -0.400558, + "nauc_map_at_1_diff1": 0.73422, + "nauc_map_at_3_max": 0.048642, + "nauc_map_at_3_std": -0.35616, + "nauc_map_at_3_diff1": 0.716453, + "nauc_map_at_5_max": 0.061561, + "nauc_map_at_5_std": -0.336767, + "nauc_map_at_5_diff1": 0.710731, + "nauc_map_at_10_max": 0.056036, + "nauc_map_at_10_std": -0.34061, + "nauc_map_at_10_diff1": 0.713681, + "nauc_map_at_20_max": 0.056036, + "nauc_map_at_20_std": -0.34061, + "nauc_map_at_20_diff1": 0.713681, + "nauc_map_at_100_max": 0.05444, + "nauc_map_at_100_std": -0.342753, + "nauc_map_at_100_diff1": 0.713308, + "nauc_map_at_1000_max": 0.05444, + "nauc_map_at_1000_std": -0.342753, + "nauc_map_at_1000_diff1": 0.713308, + "nauc_recall_at_1_max": 0.006637, + "nauc_recall_at_1_std": -0.400558, + "nauc_recall_at_1_diff1": 0.73422, + "nauc_recall_at_3_max": 0.251727, + "nauc_recall_at_3_std": -0.242951, + "nauc_recall_at_3_diff1": 0.730159, + "nauc_recall_at_5_max": 0.722222, + "nauc_recall_at_5_std": 0.338469, + "nauc_recall_at_5_diff1": 0.613679, + "nauc_recall_at_10_max": 0.722222, + "nauc_recall_at_10_std": 0.554155, + "nauc_recall_at_10_diff1": 0.869281, + "nauc_recall_at_20_max": 0.722222, + "nauc_recall_at_20_std": 0.554155, + "nauc_recall_at_20_diff1": 0.869281, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.006637, + "nauc_precision_at_1_std": -0.400558, + "nauc_precision_at_1_diff1": 0.73422, + "nauc_precision_at_3_max": 0.251727, + "nauc_precision_at_3_std": -0.242951, + "nauc_precision_at_3_diff1": 0.730159, + "nauc_precision_at_5_max": 0.722222, + "nauc_precision_at_5_std": 0.338469, + "nauc_precision_at_5_diff1": 0.613679, + "nauc_precision_at_10_max": 0.722222, + "nauc_precision_at_10_std": 0.554155, + "nauc_precision_at_10_diff1": 0.869281, + "nauc_precision_at_20_max": 0.722222, + "nauc_precision_at_20_std": 0.554155, + "nauc_precision_at_20_diff1": 0.869281, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.006637, + "nauc_cv_recall_at_1_std": -0.400558, + "nauc_cv_recall_at_1_diff1": 0.73422, + "nauc_cv_recall_at_3_max": 0.251727, + "nauc_cv_recall_at_3_std": -0.242951, + "nauc_cv_recall_at_3_diff1": 0.730159, + "nauc_cv_recall_at_5_max": 0.722222, + "nauc_cv_recall_at_5_std": 0.338469, + "nauc_cv_recall_at_5_diff1": 0.613679, + "nauc_cv_recall_at_10_max": 0.722222, + "nauc_cv_recall_at_10_std": 0.554155, + "nauc_cv_recall_at_10_diff1": 0.869281, + "nauc_cv_recall_at_20_max": 0.722222, + "nauc_cv_recall_at_20_std": 0.554155, + "nauc_cv_recall_at_20_diff1": 0.869281, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.006637, + "nauc_mrr_at_1_std": -0.400558, + "nauc_mrr_at_1_diff1": 0.73422, + "nauc_mrr_at_3_max": 0.048642, + "nauc_mrr_at_3_std": -0.35616, + "nauc_mrr_at_3_diff1": 0.716453, + "nauc_mrr_at_5_max": 0.061561, + "nauc_mrr_at_5_std": -0.336767, + "nauc_mrr_at_5_diff1": 0.710731, + "nauc_mrr_at_10_max": 0.056036, + "nauc_mrr_at_10_std": -0.34061, + "nauc_mrr_at_10_diff1": 0.713681, + "nauc_mrr_at_20_max": 0.056036, + "nauc_mrr_at_20_std": -0.34061, + "nauc_mrr_at_20_diff1": 0.713681, + "nauc_mrr_at_100_max": 0.05444, + "nauc_mrr_at_100_std": -0.342753, + "nauc_mrr_at_100_diff1": 0.713308, + "nauc_mrr_at_1000_max": 0.05444, + "nauc_mrr_at_1000_std": -0.342753, + "nauc_mrr_at_1000_diff1": 0.713308, + "main_score": 0.89906, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 105.73924612998962, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..a8fab1bf05 --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", + "task_name": "VidoreSyntheticDocQAAIRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.98, + "ndcg_at_3": 0.98631, + "ndcg_at_5": 0.99018, + "ndcg_at_10": 0.99018, + "ndcg_at_20": 0.99018, + "ndcg_at_100": 0.99018, + "ndcg_at_1000": 0.99018, + "map_at_1": 0.98, + "map_at_3": 0.985, + "map_at_5": 0.987, + "map_at_10": 0.987, + "map_at_20": 0.987, + "map_at_100": 0.987, + "map_at_1000": 0.987, + "recall_at_1": 0.98, + "recall_at_3": 0.99, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.98, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.98, + "precision_at_3": 0.33, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.98, + "mrr_at_3": 0.985, + "mrr_at_5": 0.987, + "mrr_at_10": 0.987, + "mrr_at_20": 0.987, + "mrr_at_100": 0.987, + "mrr_at_1000": 0.987, + "nauc_ndcg_at_1_max": 0.795752, + "nauc_ndcg_at_1_std": -1.445845, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.829637, + "nauc_ndcg_at_3_std": -1.310302, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.814023, + "nauc_ndcg_at_5_std": -1.372758, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.814023, + "nauc_ndcg_at_10_std": -1.372758, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.814023, + "nauc_ndcg_at_20_std": -1.372758, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.814023, + "nauc_ndcg_at_100_std": -1.372758, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.814023, + "nauc_ndcg_at_1000_std": -1.372758, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.795752, + "nauc_map_at_1_std": -1.445845, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.820261, + "nauc_map_at_3_std": -1.347806, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.81272, + "nauc_map_at_5_std": -1.377972, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.81272, + "nauc_map_at_10_std": -1.377972, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.81272, + "nauc_map_at_20_std": -1.377972, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.81272, + "nauc_map_at_100_std": -1.377972, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.81272, + "nauc_map_at_1000_std": -1.377972, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.795752, + "nauc_recall_at_1_std": -1.445845, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 0.869281, + "nauc_recall_at_3_std": -1.151727, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.795752, + "nauc_precision_at_1_std": -1.445845, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 0.869281, + "nauc_precision_at_3_std": -1.151727, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.795752, + "nauc_cv_recall_at_1_std": -1.445845, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": 0.869281, + "nauc_cv_recall_at_3_std": -1.151727, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.795752, + "nauc_mrr_at_1_std": -1.445845, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.820261, + "nauc_mrr_at_3_std": -1.347806, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.81272, + "nauc_mrr_at_5_std": -1.377972, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.81272, + "nauc_mrr_at_10_std": -1.377972, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.81272, + "nauc_mrr_at_20_std": -1.377972, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.81272, + "nauc_mrr_at_100_std": -1.377972, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.81272, + "nauc_mrr_at_1000_std": -1.377972, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.99018, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 108.3666422367096, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..aaee2576b2 --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", + "task_name": "VidoreSyntheticDocQAEnergyRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.94, + "ndcg_at_3": 0.95893, + "ndcg_at_5": 0.95893, + "ndcg_at_10": 0.96495, + "ndcg_at_20": 0.96739, + "ndcg_at_100": 0.96739, + "ndcg_at_1000": 0.96739, + "map_at_1": 0.94, + "map_at_3": 0.955, + "map_at_5": 0.955, + "map_at_10": 0.95722, + "map_at_20": 0.95785, + "map_at_100": 0.95785, + "map_at_1000": 0.95785, + "recall_at_1": 0.94, + "recall_at_3": 0.97, + "recall_at_5": 0.97, + "recall_at_10": 0.99, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.94, + "cv_recall_at_3": 0.97, + "cv_recall_at_5": 0.97, + "cv_recall_at_10": 0.99, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.94, + "precision_at_3": 0.32333, + "precision_at_5": 0.194, + "precision_at_10": 0.099, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.94, + "mrr_at_3": 0.955, + "mrr_at_5": 0.955, + "mrr_at_10": 0.957222, + "mrr_at_20": 0.957847, + "mrr_at_100": 0.957847, + "mrr_at_1000": 0.957847, + "nauc_ndcg_at_1_max": 0.068472, + "nauc_ndcg_at_1_std": -0.965453, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.320338, + "nauc_ndcg_at_3_std": -1.141658, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.320338, + "nauc_ndcg_at_5_std": -1.141658, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.278933, + "nauc_ndcg_at_10_std": -1.139929, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.245671, + "nauc_ndcg_at_20_std": -1.094906, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.245671, + "nauc_ndcg_at_100_std": -1.094906, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.245671, + "nauc_ndcg_at_1000_std": -1.094906, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.068472, + "nauc_map_at_1_std": -0.965453, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.250648, + "nauc_map_at_3_std": -1.092904, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.250648, + "nauc_map_at_5_std": -1.092904, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.234506, + "nauc_map_at_10_std": -1.089848, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.227275, + "nauc_map_at_20_std": -1.080209, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.227275, + "nauc_map_at_100_std": -1.080209, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.227275, + "nauc_map_at_1000_std": -1.080209, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.068472, + "nauc_recall_at_1_std": -0.965453, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 0.615002, + "nauc_recall_at_3_std": -1.347806, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 0.615002, + "nauc_recall_at_5_std": -1.347806, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 0.722222, + "nauc_recall_at_10_std": -1.739963, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.068472, + "nauc_precision_at_1_std": -0.965453, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 0.615002, + "nauc_precision_at_3_std": -1.347806, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 0.615002, + "nauc_precision_at_5_std": -1.347806, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 0.722222, + "nauc_precision_at_10_std": -1.739963, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.068472, + "nauc_cv_recall_at_1_std": -0.965453, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": 0.615002, + "nauc_cv_recall_at_3_std": -1.347806, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 0.615002, + "nauc_cv_recall_at_5_std": -1.347806, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": 0.722222, + "nauc_cv_recall_at_10_std": -1.739963, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.068472, + "nauc_mrr_at_1_std": -0.965453, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.250648, + "nauc_mrr_at_3_std": -1.092904, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.250648, + "nauc_mrr_at_5_std": -1.092904, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.234506, + "nauc_mrr_at_10_std": -1.089848, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.227275, + "nauc_mrr_at_20_std": -1.080209, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.227275, + "nauc_mrr_at_100_std": -1.080209, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.227275, + "nauc_mrr_at_1000_std": -1.080209, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.95893, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 106.55211758613586, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..3a33424b4b --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b4909afa930f81282fd20601e860668073ad02aa", + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.9, + "ndcg_at_3": 0.95047, + "ndcg_at_5": 0.95478, + "ndcg_at_10": 0.95834, + "ndcg_at_20": 0.95834, + "ndcg_at_100": 0.95834, + "ndcg_at_1000": 0.95834, + "map_at_1": 0.9, + "map_at_3": 0.94, + "map_at_5": 0.9425, + "map_at_10": 0.94417, + "map_at_20": 0.94417, + "map_at_100": 0.94417, + "map_at_1000": 0.94417, + "recall_at_1": 0.9, + "recall_at_3": 0.98, + "recall_at_5": 0.99, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.9, + "cv_recall_at_3": 0.98, + "cv_recall_at_5": 0.99, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.9, + "precision_at_3": 0.32667, + "precision_at_5": 0.198, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.9, + "mrr_at_3": 0.94, + "mrr_at_5": 0.9425, + "mrr_at_10": 0.944167, + "mrr_at_20": 0.944167, + "mrr_at_100": 0.944167, + "mrr_at_1000": 0.944167, + "nauc_ndcg_at_1_max": 0.683567, + "nauc_ndcg_at_1_std": 0.277451, + "nauc_ndcg_at_1_diff1": 0.960784, + "nauc_ndcg_at_3_max": 0.707392, + "nauc_ndcg_at_3_std": 0.344384, + "nauc_ndcg_at_3_diff1": 0.970776, + "nauc_ndcg_at_5_max": 0.721987, + "nauc_ndcg_at_5_std": 0.308397, + "nauc_ndcg_at_5_diff1": 0.967993, + "nauc_ndcg_at_10_max": 0.698214, + "nauc_ndcg_at_10_std": 0.304149, + "nauc_ndcg_at_10_diff1": 0.965256, + "nauc_ndcg_at_20_max": 0.698214, + "nauc_ndcg_at_20_std": 0.304149, + "nauc_ndcg_at_20_diff1": 0.965256, + "nauc_ndcg_at_100_max": 0.698214, + "nauc_ndcg_at_100_std": 0.304149, + "nauc_ndcg_at_100_diff1": 0.965256, + "nauc_ndcg_at_1000_max": 0.698214, + "nauc_ndcg_at_1000_std": 0.304149, + "nauc_ndcg_at_1000_diff1": 0.965256, + "nauc_map_at_1_max": 0.683567, + "nauc_map_at_1_std": 0.277451, + "nauc_map_at_1_diff1": 0.960784, + "nauc_map_at_3_max": 0.699152, + "nauc_map_at_3_std": 0.321234, + "nauc_map_at_3_diff1": 0.96732, + "nauc_map_at_5_max": 0.705456, + "nauc_map_at_5_std": 0.3038, + "nauc_map_at_5_diff1": 0.965899, + "nauc_map_at_10_max": 0.696664, + "nauc_map_at_10_std": 0.30218, + "nauc_map_at_10_diff1": 0.964881, + "nauc_map_at_20_max": 0.696664, + "nauc_map_at_20_std": 0.30218, + "nauc_map_at_20_diff1": 0.964881, + "nauc_map_at_100_max": 0.696664, + "nauc_map_at_100_std": 0.30218, + "nauc_map_at_100_diff1": 0.964881, + "nauc_map_at_1000_max": 0.696664, + "nauc_map_at_1000_std": 0.30218, + "nauc_map_at_1000_diff1": 0.964881, + "nauc_recall_at_1_max": 0.683567, + "nauc_recall_at_1_std": 0.277451, + "nauc_recall_at_1_diff1": 0.960784, + "nauc_recall_at_3_max": 0.777077, + "nauc_recall_at_3_std": 0.540149, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": 0.358077, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.683567, + "nauc_precision_at_1_std": 0.277451, + "nauc_precision_at_1_diff1": 0.960784, + "nauc_precision_at_3_max": 0.777077, + "nauc_precision_at_3_std": 0.540149, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 0.358077, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.683567, + "nauc_cv_recall_at_1_std": 0.277451, + "nauc_cv_recall_at_1_diff1": 0.960784, + "nauc_cv_recall_at_3_max": 0.777077, + "nauc_cv_recall_at_3_std": 0.540149, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 1.0, + "nauc_cv_recall_at_5_std": 0.358077, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.683567, + "nauc_mrr_at_1_std": 0.277451, + "nauc_mrr_at_1_diff1": 0.960784, + "nauc_mrr_at_3_max": 0.699152, + "nauc_mrr_at_3_std": 0.321234, + "nauc_mrr_at_3_diff1": 0.96732, + "nauc_mrr_at_5_max": 0.705456, + "nauc_mrr_at_5_std": 0.3038, + "nauc_mrr_at_5_diff1": 0.965899, + "nauc_mrr_at_10_max": 0.696664, + "nauc_mrr_at_10_std": 0.30218, + "nauc_mrr_at_10_diff1": 0.964881, + "nauc_mrr_at_20_max": 0.696664, + "nauc_mrr_at_20_std": 0.30218, + "nauc_mrr_at_20_diff1": 0.964881, + "nauc_mrr_at_100_max": 0.696664, + "nauc_mrr_at_100_std": 0.30218, + "nauc_mrr_at_100_diff1": 0.964881, + "nauc_mrr_at_1000_max": 0.696664, + "nauc_mrr_at_1000_std": 0.30218, + "nauc_mrr_at_1000_diff1": 0.964881, + "main_score": 0.95478, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 102.92445397377014, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..fc2e46ff63 --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.97, + "ndcg_at_3": 0.98762, + "ndcg_at_5": 0.98762, + "ndcg_at_10": 0.98762, + "ndcg_at_20": 0.98762, + "ndcg_at_100": 0.98762, + "ndcg_at_1000": 0.98762, + "map_at_1": 0.97, + "map_at_3": 0.98333, + "map_at_5": 0.98333, + "map_at_10": 0.98333, + "map_at_20": 0.98333, + "map_at_100": 0.98333, + "map_at_1000": 0.98333, + "recall_at_1": 0.97, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.97, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.97, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.97, + "mrr_at_3": 0.983333, + "mrr_at_5": 0.983333, + "mrr_at_10": 0.983333, + "mrr_at_20": 0.983333, + "mrr_at_100": 0.983333, + "mrr_at_1000": 0.983333, + "nauc_ndcg_at_1_max": 0.863834, + "nauc_ndcg_at_1_std": -0.321662, + "nauc_ndcg_at_1_diff1": 0.956427, + "nauc_ndcg_at_3_max": 0.848859, + "nauc_ndcg_at_3_std": -0.305765, + "nauc_ndcg_at_3_diff1": 0.947212, + "nauc_ndcg_at_5_max": 0.848859, + "nauc_ndcg_at_5_std": -0.305765, + "nauc_ndcg_at_5_diff1": 0.947212, + "nauc_ndcg_at_10_max": 0.848859, + "nauc_ndcg_at_10_std": -0.305765, + "nauc_ndcg_at_10_diff1": 0.947212, + "nauc_ndcg_at_20_max": 0.848859, + "nauc_ndcg_at_20_std": -0.305765, + "nauc_ndcg_at_20_diff1": 0.947212, + "nauc_ndcg_at_100_max": 0.848859, + "nauc_ndcg_at_100_std": -0.305765, + "nauc_ndcg_at_100_diff1": 0.947212, + "nauc_ndcg_at_1000_max": 0.848859, + "nauc_ndcg_at_1000_std": -0.305765, + "nauc_ndcg_at_1000_diff1": 0.947212, + "nauc_map_at_1_max": 0.863834, + "nauc_map_at_1_std": -0.321662, + "nauc_map_at_1_diff1": 0.956427, + "nauc_map_at_3_max": 0.849673, + "nauc_map_at_3_std": -0.306629, + "nauc_map_at_3_diff1": 0.947712, + "nauc_map_at_5_max": 0.849673, + "nauc_map_at_5_std": -0.306629, + "nauc_map_at_5_diff1": 0.947712, + "nauc_map_at_10_max": 0.849673, + "nauc_map_at_10_std": -0.306629, + "nauc_map_at_10_diff1": 0.947712, + "nauc_map_at_20_max": 0.849673, + "nauc_map_at_20_std": -0.306629, + "nauc_map_at_20_diff1": 0.947712, + "nauc_map_at_100_max": 0.849673, + "nauc_map_at_100_std": -0.306629, + "nauc_map_at_100_diff1": 0.947712, + "nauc_map_at_1000_max": 0.849673, + "nauc_map_at_1000_std": -0.306629, + "nauc_map_at_1000_diff1": 0.947712, + "nauc_recall_at_1_max": 0.863834, + "nauc_recall_at_1_std": -0.321662, + "nauc_recall_at_1_diff1": 0.956427, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.863834, + "nauc_precision_at_1_std": -0.321662, + "nauc_precision_at_1_diff1": 0.956427, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.863834, + "nauc_cv_recall_at_1_std": -0.321662, + "nauc_cv_recall_at_1_diff1": 0.956427, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.863834, + "nauc_mrr_at_1_std": -0.321662, + "nauc_mrr_at_1_diff1": 0.956427, + "nauc_mrr_at_3_max": 0.849673, + "nauc_mrr_at_3_std": -0.306629, + "nauc_mrr_at_3_diff1": 0.947712, + "nauc_mrr_at_5_max": 0.849673, + "nauc_mrr_at_5_std": -0.306629, + "nauc_mrr_at_5_diff1": 0.947712, + "nauc_mrr_at_10_max": 0.849673, + "nauc_mrr_at_10_std": -0.306629, + "nauc_mrr_at_10_diff1": 0.947712, + "nauc_mrr_at_20_max": 0.849673, + "nauc_mrr_at_20_std": -0.306629, + "nauc_mrr_at_20_diff1": 0.947712, + "nauc_mrr_at_100_max": 0.849673, + "nauc_mrr_at_100_std": -0.306629, + "nauc_mrr_at_100_diff1": 0.947712, + "nauc_mrr_at_1000_max": 0.849673, + "nauc_mrr_at_1000_std": -0.306629, + "nauc_mrr_at_1000_diff1": 0.947712, + "main_score": 0.98762, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 104.08194136619568, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..56c1aaa320 --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", + "task_name": "VidoreTabfquadRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.83571, + "ndcg_at_3": 0.88427, + "ndcg_at_5": 0.89011, + "ndcg_at_10": 0.89384, + "ndcg_at_20": 0.90372, + "ndcg_at_100": 0.90727, + "ndcg_at_1000": 0.90727, + "map_at_1": 0.83571, + "map_at_3": 0.87262, + "map_at_5": 0.87583, + "map_at_10": 0.87753, + "map_at_20": 0.88021, + "map_at_100": 0.88079, + "map_at_1000": 0.88079, + "recall_at_1": 0.83571, + "recall_at_3": 0.91786, + "recall_at_5": 0.93214, + "recall_at_10": 0.94286, + "recall_at_20": 0.98214, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.83571, + "cv_recall_at_3": 0.91786, + "cv_recall_at_5": 0.93214, + "cv_recall_at_10": 0.94286, + "cv_recall_at_20": 0.98214, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.83571, + "precision_at_3": 0.30595, + "precision_at_5": 0.18643, + "precision_at_10": 0.09429, + "precision_at_20": 0.04911, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.835714, + "mrr_at_3": 0.872619, + "mrr_at_5": 0.875833, + "mrr_at_10": 0.877534, + "mrr_at_20": 0.880211, + "mrr_at_100": 0.880789, + "mrr_at_1000": 0.880789, + "nauc_ndcg_at_1_max": 0.540431, + "nauc_ndcg_at_1_std": 0.211366, + "nauc_ndcg_at_1_diff1": 0.921662, + "nauc_ndcg_at_3_max": 0.611398, + "nauc_ndcg_at_3_std": 0.279481, + "nauc_ndcg_at_3_diff1": 0.88898, + "nauc_ndcg_at_5_max": 0.608219, + "nauc_ndcg_at_5_std": 0.298764, + "nauc_ndcg_at_5_diff1": 0.886908, + "nauc_ndcg_at_10_max": 0.594003, + "nauc_ndcg_at_10_std": 0.278149, + "nauc_ndcg_at_10_diff1": 0.886036, + "nauc_ndcg_at_20_max": 0.602482, + "nauc_ndcg_at_20_std": 0.287726, + "nauc_ndcg_at_20_diff1": 0.898444, + "nauc_ndcg_at_100_max": 0.590082, + "nauc_ndcg_at_100_std": 0.269435, + "nauc_ndcg_at_100_diff1": 0.897493, + "nauc_ndcg_at_1000_max": 0.590082, + "nauc_ndcg_at_1000_std": 0.269435, + "nauc_ndcg_at_1000_diff1": 0.897493, + "nauc_map_at_1_max": 0.540431, + "nauc_map_at_1_std": 0.211366, + "nauc_map_at_1_diff1": 0.921662, + "nauc_map_at_3_max": 0.59183, + "nauc_map_at_3_std": 0.261538, + "nauc_map_at_3_diff1": 0.898671, + "nauc_map_at_5_max": 0.589435, + "nauc_map_at_5_std": 0.269644, + "nauc_map_at_5_diff1": 0.898023, + "nauc_map_at_10_max": 0.583523, + "nauc_map_at_10_std": 0.260987, + "nauc_map_at_10_diff1": 0.897872, + "nauc_map_at_20_max": 0.585525, + "nauc_map_at_20_std": 0.263158, + "nauc_map_at_20_diff1": 0.900746, + "nauc_map_at_100_max": 0.583912, + "nauc_map_at_100_std": 0.260864, + "nauc_map_at_100_diff1": 0.900586, + "nauc_map_at_1000_max": 0.583912, + "nauc_map_at_1000_std": 0.260864, + "nauc_map_at_1000_diff1": 0.900586, + "nauc_recall_at_1_max": 0.540431, + "nauc_recall_at_1_std": 0.211366, + "nauc_recall_at_1_diff1": 0.921662, + "nauc_recall_at_3_max": 0.694658, + "nauc_recall_at_3_std": 0.355255, + "nauc_recall_at_3_diff1": 0.847278, + "nauc_recall_at_5_max": 0.702246, + "nauc_recall_at_5_std": 0.451668, + "nauc_recall_at_5_diff1": 0.829746, + "nauc_recall_at_10_max": 0.646417, + "nauc_recall_at_10_std": 0.374387, + "nauc_recall_at_10_diff1": 0.814163, + "nauc_recall_at_20_max": 0.918301, + "nauc_recall_at_20_std": 0.760504, + "nauc_recall_at_20_diff1": 0.918301, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.540431, + "nauc_precision_at_1_std": 0.211366, + "nauc_precision_at_1_diff1": 0.921662, + "nauc_precision_at_3_max": 0.694658, + "nauc_precision_at_3_std": 0.355255, + "nauc_precision_at_3_diff1": 0.847278, + "nauc_precision_at_5_max": 0.702246, + "nauc_precision_at_5_std": 0.451668, + "nauc_precision_at_5_diff1": 0.829746, + "nauc_precision_at_10_max": 0.646417, + "nauc_precision_at_10_std": 0.374387, + "nauc_precision_at_10_diff1": 0.814163, + "nauc_precision_at_20_max": 0.918301, + "nauc_precision_at_20_std": 0.760504, + "nauc_precision_at_20_diff1": 0.918301, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.540431, + "nauc_cv_recall_at_1_std": 0.211366, + "nauc_cv_recall_at_1_diff1": 0.921662, + "nauc_cv_recall_at_3_max": 0.694658, + "nauc_cv_recall_at_3_std": 0.355255, + "nauc_cv_recall_at_3_diff1": 0.847278, + "nauc_cv_recall_at_5_max": 0.702246, + "nauc_cv_recall_at_5_std": 0.451668, + "nauc_cv_recall_at_5_diff1": 0.829746, + "nauc_cv_recall_at_10_max": 0.646417, + "nauc_cv_recall_at_10_std": 0.374387, + "nauc_cv_recall_at_10_diff1": 0.814163, + "nauc_cv_recall_at_20_max": 0.918301, + "nauc_cv_recall_at_20_std": 0.760504, + "nauc_cv_recall_at_20_diff1": 0.918301, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.540431, + "nauc_mrr_at_1_std": 0.211366, + "nauc_mrr_at_1_diff1": 0.921662, + "nauc_mrr_at_3_max": 0.59183, + "nauc_mrr_at_3_std": 0.261538, + "nauc_mrr_at_3_diff1": 0.898671, + "nauc_mrr_at_5_max": 0.589435, + "nauc_mrr_at_5_std": 0.269644, + "nauc_mrr_at_5_diff1": 0.898023, + "nauc_mrr_at_10_max": 0.583523, + "nauc_mrr_at_10_std": 0.260987, + "nauc_mrr_at_10_diff1": 0.897872, + "nauc_mrr_at_20_max": 0.585525, + "nauc_mrr_at_20_std": 0.263158, + "nauc_mrr_at_20_diff1": 0.900746, + "nauc_mrr_at_100_max": 0.583912, + "nauc_mrr_at_100_std": 0.260864, + "nauc_mrr_at_100_diff1": 0.900586, + "nauc_mrr_at_1000_max": 0.583912, + "nauc_mrr_at_1000_std": 0.260864, + "nauc_mrr_at_1000_diff1": 0.900586, + "main_score": 0.89011, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.446826219558716, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..16d0c55970 --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", + "task_name": "VidoreTatdqaRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.71264, + "ndcg_at_3": 0.80115, + "ndcg_at_5": 0.82228, + "ndcg_at_10": 0.83405, + "ndcg_at_20": 0.83797, + "ndcg_at_100": 0.84367, + "ndcg_at_1000": 0.84458, + "map_at_1": 0.71142, + "map_at_3": 0.77972, + "map_at_5": 0.79155, + "map_at_10": 0.79658, + "map_at_20": 0.79772, + "map_at_100": 0.79853, + "map_at_1000": 0.79859, + "recall_at_1": 0.71142, + "recall_at_3": 0.86239, + "recall_at_5": 0.91312, + "recall_at_10": 0.94872, + "recall_at_20": 0.96367, + "recall_at_100": 0.99386, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.71264, + "cv_recall_at_3": 0.8633, + "cv_recall_at_5": 0.91434, + "cv_recall_at_10": 0.94957, + "cv_recall_at_20": 0.96416, + "cv_recall_at_100": 0.99453, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.71264, + "precision_at_3": 0.28838, + "precision_at_5": 0.18323, + "precision_at_10": 0.09532, + "precision_at_20": 0.04845, + "precision_at_100": 0.01001, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.712637, + "mrr_at_3": 0.780275, + "mrr_at_5": 0.792183, + "mrr_at_10": 0.797053, + "mrr_at_20": 0.798151, + "mrr_at_100": 0.798958, + "mrr_at_1000": 0.798995, + "nauc_ndcg_at_1_max": 0.336093, + "nauc_ndcg_at_1_std": -0.052821, + "nauc_ndcg_at_1_diff1": 0.82155, + "nauc_ndcg_at_3_max": 0.361247, + "nauc_ndcg_at_3_std": 0.003835, + "nauc_ndcg_at_3_diff1": 0.802609, + "nauc_ndcg_at_5_max": 0.361896, + "nauc_ndcg_at_5_std": 0.010307, + "nauc_ndcg_at_5_diff1": 0.793729, + "nauc_ndcg_at_10_max": 0.367548, + "nauc_ndcg_at_10_std": 0.020981, + "nauc_ndcg_at_10_diff1": 0.793492, + "nauc_ndcg_at_20_max": 0.363088, + "nauc_ndcg_at_20_std": 0.016476, + "nauc_ndcg_at_20_diff1": 0.794075, + "nauc_ndcg_at_100_max": 0.360184, + "nauc_ndcg_at_100_std": 0.003653, + "nauc_ndcg_at_100_diff1": 0.800274, + "nauc_ndcg_at_1000_max": 0.358014, + "nauc_ndcg_at_1000_std": -0.000404, + "nauc_ndcg_at_1000_diff1": 0.800622, + "nauc_map_at_1_max": 0.332049, + "nauc_map_at_1_std": -0.055301, + "nauc_map_at_1_diff1": 0.822104, + "nauc_map_at_3_max": 0.352538, + "nauc_map_at_3_std": -0.014109, + "nauc_map_at_3_diff1": 0.807133, + "nauc_map_at_5_max": 0.352559, + "nauc_map_at_5_std": -0.012138, + "nauc_map_at_5_diff1": 0.803032, + "nauc_map_at_10_max": 0.354733, + "nauc_map_at_10_std": -0.008127, + "nauc_map_at_10_diff1": 0.803219, + "nauc_map_at_20_max": 0.353637, + "nauc_map_at_20_std": -0.00931, + "nauc_map_at_20_diff1": 0.803427, + "nauc_map_at_100_max": 0.353357, + "nauc_map_at_100_std": -0.010763, + "nauc_map_at_100_diff1": 0.804241, + "nauc_map_at_1000_max": 0.35324, + "nauc_map_at_1000_std": -0.010979, + "nauc_map_at_1000_diff1": 0.804249, + "nauc_recall_at_1_max": 0.332049, + "nauc_recall_at_1_std": -0.055301, + "nauc_recall_at_1_diff1": 0.822104, + "nauc_recall_at_3_max": 0.395506, + "nauc_recall_at_3_std": 0.077991, + "nauc_recall_at_3_diff1": 0.784197, + "nauc_recall_at_5_max": 0.415097, + "nauc_recall_at_5_std": 0.149061, + "nauc_recall_at_5_diff1": 0.734523, + "nauc_recall_at_10_max": 0.49689, + "nauc_recall_at_10_std": 0.326782, + "nauc_recall_at_10_diff1": 0.692597, + "nauc_recall_at_20_max": 0.480633, + "nauc_recall_at_20_std": 0.382144, + "nauc_recall_at_20_diff1": 0.660643, + "nauc_recall_at_100_max": 0.674711, + "nauc_recall_at_100_std": 0.594437, + "nauc_recall_at_100_diff1": 0.732165, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.336093, + "nauc_precision_at_1_std": -0.052821, + "nauc_precision_at_1_diff1": 0.82155, + "nauc_precision_at_3_max": 0.392045, + "nauc_precision_at_3_std": 0.081528, + "nauc_precision_at_3_diff1": 0.754713, + "nauc_precision_at_5_max": 0.408313, + "nauc_precision_at_5_std": 0.149827, + "nauc_precision_at_5_diff1": 0.691089, + "nauc_precision_at_10_max": 0.44494, + "nauc_precision_at_10_std": 0.295184, + "nauc_precision_at_10_diff1": 0.560352, + "nauc_precision_at_20_max": 0.373511, + "nauc_precision_at_20_std": 0.29862, + "nauc_precision_at_20_diff1": 0.438984, + "nauc_precision_at_100_max": 0.125301, + "nauc_precision_at_100_std": 0.099448, + "nauc_precision_at_100_diff1": -0.09113, + "nauc_precision_at_1000_max": -0.292163, + "nauc_precision_at_1000_std": -0.313911, + "nauc_precision_at_1000_diff1": -0.531096, + "nauc_cv_recall_at_1_max": 0.336093, + "nauc_cv_recall_at_1_std": -0.052821, + "nauc_cv_recall_at_1_diff1": 0.82155, + "nauc_cv_recall_at_3_max": 0.398559, + "nauc_cv_recall_at_3_std": 0.081356, + "nauc_cv_recall_at_3_diff1": 0.783326, + "nauc_cv_recall_at_5_max": 0.418947, + "nauc_cv_recall_at_5_std": 0.15209, + "nauc_cv_recall_at_5_diff1": 0.732802, + "nauc_cv_recall_at_10_max": 0.491495, + "nauc_cv_recall_at_10_std": 0.319294, + "nauc_cv_recall_at_10_diff1": 0.687412, + "nauc_cv_recall_at_20_max": 0.475364, + "nauc_cv_recall_at_20_std": 0.373766, + "nauc_cv_recall_at_20_diff1": 0.656042, + "nauc_cv_recall_at_100_max": 0.650935, + "nauc_cv_recall_at_100_std": 0.544868, + "nauc_cv_recall_at_100_diff1": 0.699429, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.336093, + "nauc_mrr_at_1_std": -0.052821, + "nauc_mrr_at_1_diff1": 0.82155, + "nauc_mrr_at_3_max": 0.355069, + "nauc_mrr_at_3_std": -0.011806, + "nauc_mrr_at_3_diff1": 0.806862, + "nauc_mrr_at_5_max": 0.355023, + "nauc_mrr_at_5_std": -0.01011, + "nauc_mrr_at_5_diff1": 0.802651, + "nauc_mrr_at_10_max": 0.356742, + "nauc_mrr_at_10_std": -0.00658, + "nauc_mrr_at_10_diff1": 0.802903, + "nauc_mrr_at_20_max": 0.355766, + "nauc_mrr_at_20_std": -0.007634, + "nauc_mrr_at_20_diff1": 0.803182, + "nauc_mrr_at_100_max": 0.355494, + "nauc_mrr_at_100_std": -0.009084, + "nauc_mrr_at_100_diff1": 0.803996, + "nauc_mrr_at_1000_max": 0.355427, + "nauc_mrr_at_1000_std": -0.009202, + "nauc_mrr_at_1000_diff1": 0.804023, + "main_score": 0.82228, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 40.89088416099548, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json new file mode 100644 index 0000000000..29c6fe6d4c --- /dev/null +++ b/results/vidore__colqwen2-v1.0/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json @@ -0,0 +1 @@ +{"name": "vidore/colqwen2-v1.0", "revision": "530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f", "release_date": "2025-11-03", "languages": ["eng-Latn"], "n_parameters": 2210000000, "memory_usage_mb": 7200.0, "max_tokens": 32768.0, "embed_dim": 128, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/illuin-tech/colpali", "public_training_data": "https://huggingface.co/datasets/vidore/colpali_train_set", "framework": ["ColPali"], "reference": "https://huggingface.co/vidore/colqwen2-v1.0", "similarity_fn_name": "max_sim", "use_instructions": true, "training_datasets": {"DocVQA": ["train"], "InfoVQA": ["train"], "TATDQA": ["train"], "arXivQA": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "ColQwen2Wrapper"} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..ad68a9e2e9 --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.55625, + "ndcg_at_3": 0.57787, + "ndcg_at_5": 0.59261, + "ndcg_at_10": 0.63213, + "ndcg_at_20": 0.65865, + "ndcg_at_100": 0.68978, + "ndcg_at_1000": 0.70802, + "map_at_1": 0.34824, + "map_at_3": 0.46779, + "map_at_5": 0.50656, + "map_at_10": 0.54313, + "map_at_20": 0.55743, + "map_at_100": 0.56783, + "map_at_1000": 0.56963, + "recall_at_1": 0.34824, + "recall_at_3": 0.53691, + "recall_at_5": 0.61213, + "recall_at_10": 0.72854, + "recall_at_20": 0.80441, + "recall_at_100": 0.90111, + "recall_at_1000": 0.9902, + "cv_recall_at_1": 0.55625, + "cv_recall_at_3": 0.79375, + "cv_recall_at_5": 0.825, + "cv_recall_at_10": 0.9125, + "cv_recall_at_20": 0.94375, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 0.99375, + "precision_at_1": 0.55625, + "precision_at_3": 0.35833, + "precision_at_5": 0.26875, + "precision_at_10": 0.17563, + "precision_at_20": 0.10437, + "precision_at_100": 0.02669, + "precision_at_1000": 0.00318, + "mrr_at_1": 0.55625, + "mrr_at_3": 0.663542, + "mrr_at_5": 0.670417, + "mrr_at_10": 0.682636, + "mrr_at_20": 0.68502, + "mrr_at_100": 0.686095, + "mrr_at_1000": 0.686163, + "nauc_ndcg_at_1_max": 0.282699, + "nauc_ndcg_at_1_std": -0.071331, + "nauc_ndcg_at_1_diff1": 0.458556, + "nauc_ndcg_at_3_max": 0.301487, + "nauc_ndcg_at_3_std": 0.005627, + "nauc_ndcg_at_3_diff1": 0.351214, + "nauc_ndcg_at_5_max": 0.344632, + "nauc_ndcg_at_5_std": 0.02051, + "nauc_ndcg_at_5_diff1": 0.381519, + "nauc_ndcg_at_10_max": 0.354254, + "nauc_ndcg_at_10_std": 0.031082, + "nauc_ndcg_at_10_diff1": 0.359752, + "nauc_ndcg_at_20_max": 0.336989, + "nauc_ndcg_at_20_std": 0.024277, + "nauc_ndcg_at_20_diff1": 0.355952, + "nauc_ndcg_at_100_max": 0.322681, + "nauc_ndcg_at_100_std": 0.027238, + "nauc_ndcg_at_100_diff1": 0.335388, + "nauc_ndcg_at_1000_max": 0.303067, + "nauc_ndcg_at_1000_std": -0.001296, + "nauc_ndcg_at_1000_diff1": 0.338496, + "nauc_map_at_1_max": 0.352229, + "nauc_map_at_1_std": 0.059553, + "nauc_map_at_1_diff1": 0.537682, + "nauc_map_at_3_max": 0.35978, + "nauc_map_at_3_std": 0.063677, + "nauc_map_at_3_diff1": 0.414469, + "nauc_map_at_5_max": 0.359916, + "nauc_map_at_5_std": 0.037398, + "nauc_map_at_5_diff1": 0.416871, + "nauc_map_at_10_max": 0.347484, + "nauc_map_at_10_std": 0.027673, + "nauc_map_at_10_diff1": 0.38759, + "nauc_map_at_20_max": 0.342234, + "nauc_map_at_20_std": 0.025568, + "nauc_map_at_20_diff1": 0.382529, + "nauc_map_at_100_max": 0.342366, + "nauc_map_at_100_std": 0.029632, + "nauc_map_at_100_diff1": 0.374696, + "nauc_map_at_1000_max": 0.340529, + "nauc_map_at_1000_std": 0.02737, + "nauc_map_at_1000_diff1": 0.374234, + "nauc_recall_at_1_max": 0.352229, + "nauc_recall_at_1_std": 0.059553, + "nauc_recall_at_1_diff1": 0.537682, + "nauc_recall_at_3_max": 0.334124, + "nauc_recall_at_3_std": 0.081142, + "nauc_recall_at_3_diff1": 0.344109, + "nauc_recall_at_5_max": 0.361225, + "nauc_recall_at_5_std": 0.052151, + "nauc_recall_at_5_diff1": 0.368469, + "nauc_recall_at_10_max": 0.372786, + "nauc_recall_at_10_std": 0.093378, + "nauc_recall_at_10_diff1": 0.263392, + "nauc_recall_at_20_max": 0.329104, + "nauc_recall_at_20_std": 0.090878, + "nauc_recall_at_20_diff1": 0.234988, + "nauc_recall_at_100_max": 0.258514, + "nauc_recall_at_100_std": 0.174586, + "nauc_recall_at_100_diff1": 0.098913, + "nauc_recall_at_1000_max": -0.926644, + "nauc_recall_at_1000_std": -0.965957, + "nauc_recall_at_1000_diff1": 0.431567, + "nauc_precision_at_1_max": 0.282699, + "nauc_precision_at_1_std": -0.071331, + "nauc_precision_at_1_diff1": 0.458556, + "nauc_precision_at_3_max": 0.030821, + "nauc_precision_at_3_std": -0.085732, + "nauc_precision_at_3_diff1": -0.083835, + "nauc_precision_at_5_max": -0.014957, + "nauc_precision_at_5_std": -0.111676, + "nauc_precision_at_5_diff1": -0.128236, + "nauc_precision_at_10_max": -0.09189, + "nauc_precision_at_10_std": -0.103043, + "nauc_precision_at_10_diff1": -0.2453, + "nauc_precision_at_20_max": -0.144133, + "nauc_precision_at_20_std": -0.079547, + "nauc_precision_at_20_diff1": -0.295882, + "nauc_precision_at_100_max": -0.209518, + "nauc_precision_at_100_std": -0.061008, + "nauc_precision_at_100_diff1": -0.370103, + "nauc_precision_at_1000_max": -0.295281, + "nauc_precision_at_1000_std": -0.147, + "nauc_precision_at_1000_diff1": -0.396784, + "nauc_cv_recall_at_1_max": 0.282699, + "nauc_cv_recall_at_1_std": -0.071331, + "nauc_cv_recall_at_1_diff1": 0.458556, + "nauc_cv_recall_at_3_max": 0.256253, + "nauc_cv_recall_at_3_std": -0.013351, + "nauc_cv_recall_at_3_diff1": 0.240051, + "nauc_cv_recall_at_5_max": 0.301876, + "nauc_cv_recall_at_5_std": 0.025099, + "nauc_cv_recall_at_5_diff1": 0.227709, + "nauc_cv_recall_at_10_max": 0.423403, + "nauc_cv_recall_at_10_std": 0.211518, + "nauc_cv_recall_at_10_diff1": 0.082333, + "nauc_cv_recall_at_20_max": 0.274977, + "nauc_cv_recall_at_20_std": 0.146851, + "nauc_cv_recall_at_20_diff1": -0.006328, + "nauc_cv_recall_at_100_max": -0.236345, + "nauc_cv_recall_at_100_std": -0.089286, + "nauc_cv_recall_at_100_diff1": -0.347806, + "nauc_cv_recall_at_1000_max": -1.739963, + "nauc_cv_recall_at_1000_std": -1.739963, + "nauc_cv_recall_at_1000_diff1": 0.554155, + "nauc_mrr_at_1_max": 0.282699, + "nauc_mrr_at_1_std": -0.071331, + "nauc_mrr_at_1_diff1": 0.458556, + "nauc_mrr_at_3_max": 0.272317, + "nauc_mrr_at_3_std": -0.059512, + "nauc_mrr_at_3_diff1": 0.381157, + "nauc_mrr_at_5_max": 0.277418, + "nauc_mrr_at_5_std": -0.055943, + "nauc_mrr_at_5_diff1": 0.381297, + "nauc_mrr_at_10_max": 0.281939, + "nauc_mrr_at_10_std": -0.050267, + "nauc_mrr_at_10_diff1": 0.379326, + "nauc_mrr_at_20_max": 0.277773, + "nauc_mrr_at_20_std": -0.053459, + "nauc_mrr_at_20_diff1": 0.380495, + "nauc_mrr_at_100_max": 0.276122, + "nauc_mrr_at_100_std": -0.054804, + "nauc_mrr_at_100_diff1": 0.381054, + "nauc_mrr_at_1000_max": 0.276093, + "nauc_mrr_at_1000_std": -0.054913, + "nauc_mrr_at_1000_diff1": 0.381273, + "main_score": 0.59261, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.55, + "ndcg_at_3": 0.5639, + "ndcg_at_5": 0.5935, + "ndcg_at_10": 0.623, + "ndcg_at_20": 0.65164, + "ndcg_at_100": 0.68365, + "ndcg_at_1000": 0.70037, + "map_at_1": 0.33507, + "map_at_3": 0.4513, + "map_at_5": 0.49912, + "map_at_10": 0.53238, + "map_at_20": 0.54716, + "map_at_100": 0.55762, + "map_at_1000": 0.55913, + "recall_at_1": 0.33507, + "recall_at_3": 0.52294, + "recall_at_5": 0.62665, + "recall_at_10": 0.72141, + "recall_at_20": 0.8072, + "recall_at_100": 0.90854, + "recall_at_1000": 0.9902, + "cv_recall_at_1": 0.55, + "cv_recall_at_3": 0.775, + "cv_recall_at_5": 0.8375, + "cv_recall_at_10": 0.8875, + "cv_recall_at_20": 0.9375, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 0.99375, + "precision_at_1": 0.55, + "precision_at_3": 0.35208, + "precision_at_5": 0.28, + "precision_at_10": 0.17625, + "precision_at_20": 0.10406, + "precision_at_100": 0.02675, + "precision_at_1000": 0.00318, + "mrr_at_1": 0.55, + "mrr_at_3": 0.651042, + "mrr_at_5": 0.665417, + "mrr_at_10": 0.67246, + "mrr_at_20": 0.676259, + "mrr_at_100": 0.67745, + "mrr_at_1000": 0.677517, + "nauc_ndcg_at_1_max": 0.254625, + "nauc_ndcg_at_1_std": -0.011743, + "nauc_ndcg_at_1_diff1": 0.498648, + "nauc_ndcg_at_3_max": 0.258181, + "nauc_ndcg_at_3_std": -0.009664, + "nauc_ndcg_at_3_diff1": 0.354837, + "nauc_ndcg_at_5_max": 0.309558, + "nauc_ndcg_at_5_std": 0.044873, + "nauc_ndcg_at_5_diff1": 0.378367, + "nauc_ndcg_at_10_max": 0.311019, + "nauc_ndcg_at_10_std": 0.019366, + "nauc_ndcg_at_10_diff1": 0.395391, + "nauc_ndcg_at_20_max": 0.29302, + "nauc_ndcg_at_20_std": 0.012774, + "nauc_ndcg_at_20_diff1": 0.362713, + "nauc_ndcg_at_100_max": 0.27369, + "nauc_ndcg_at_100_std": 0.019691, + "nauc_ndcg_at_100_diff1": 0.367319, + "nauc_ndcg_at_1000_max": 0.254104, + "nauc_ndcg_at_1000_std": 0.002535, + "nauc_ndcg_at_1000_diff1": 0.369858, + "nauc_map_at_1_max": 0.308894, + "nauc_map_at_1_std": 0.091242, + "nauc_map_at_1_diff1": 0.523731, + "nauc_map_at_3_max": 0.305418, + "nauc_map_at_3_std": 0.048928, + "nauc_map_at_3_diff1": 0.399236, + "nauc_map_at_5_max": 0.306964, + "nauc_map_at_5_std": 0.041485, + "nauc_map_at_5_diff1": 0.405704, + "nauc_map_at_10_max": 0.297976, + "nauc_map_at_10_std": 0.018053, + "nauc_map_at_10_diff1": 0.404633, + "nauc_map_at_20_max": 0.290869, + "nauc_map_at_20_std": 0.017515, + "nauc_map_at_20_diff1": 0.390216, + "nauc_map_at_100_max": 0.28865, + "nauc_map_at_100_std": 0.021978, + "nauc_map_at_100_diff1": 0.388475, + "nauc_map_at_1000_max": 0.287068, + "nauc_map_at_1000_std": 0.021104, + "nauc_map_at_1000_diff1": 0.388059, + "nauc_recall_at_1_max": 0.308894, + "nauc_recall_at_1_std": 0.091242, + "nauc_recall_at_1_diff1": 0.523731, + "nauc_recall_at_3_max": 0.285038, + "nauc_recall_at_3_std": 0.024242, + "nauc_recall_at_3_diff1": 0.294888, + "nauc_recall_at_5_max": 0.34582, + "nauc_recall_at_5_std": 0.078329, + "nauc_recall_at_5_diff1": 0.309435, + "nauc_recall_at_10_max": 0.334405, + "nauc_recall_at_10_std": 0.017771, + "nauc_recall_at_10_diff1": 0.293776, + "nauc_recall_at_20_max": 0.308511, + "nauc_recall_at_20_std": 0.040502, + "nauc_recall_at_20_diff1": 0.139109, + "nauc_recall_at_100_max": 0.212304, + "nauc_recall_at_100_std": 0.080281, + "nauc_recall_at_100_diff1": 0.117304, + "nauc_recall_at_1000_max": -0.918785, + "nauc_recall_at_1000_std": -0.941846, + "nauc_recall_at_1000_diff1": 0.409131, + "nauc_precision_at_1_max": 0.254625, + "nauc_precision_at_1_std": -0.011743, + "nauc_precision_at_1_diff1": 0.498648, + "nauc_precision_at_3_max": 0.038902, + "nauc_precision_at_3_std": -0.104041, + "nauc_precision_at_3_diff1": -0.02668, + "nauc_precision_at_5_max": -0.018877, + "nauc_precision_at_5_std": -0.079701, + "nauc_precision_at_5_diff1": -0.070958, + "nauc_precision_at_10_max": -0.085581, + "nauc_precision_at_10_std": -0.115815, + "nauc_precision_at_10_diff1": -0.116898, + "nauc_precision_at_20_max": -0.150722, + "nauc_precision_at_20_std": -0.116765, + "nauc_precision_at_20_diff1": -0.209833, + "nauc_precision_at_100_max": -0.209768, + "nauc_precision_at_100_std": -0.058238, + "nauc_precision_at_100_diff1": -0.250539, + "nauc_precision_at_1000_max": -0.295057, + "nauc_precision_at_1000_std": -0.115885, + "nauc_precision_at_1000_diff1": -0.310549, + "nauc_cv_recall_at_1_max": 0.254625, + "nauc_cv_recall_at_1_std": -0.011743, + "nauc_cv_recall_at_1_diff1": 0.498648, + "nauc_cv_recall_at_3_max": 0.182002, + "nauc_cv_recall_at_3_std": -0.144722, + "nauc_cv_recall_at_3_diff1": 0.303465, + "nauc_cv_recall_at_5_max": 0.382761, + "nauc_cv_recall_at_5_std": 0.060595, + "nauc_cv_recall_at_5_diff1": 0.24533, + "nauc_cv_recall_at_10_max": 0.350458, + "nauc_cv_recall_at_10_std": 0.039057, + "nauc_cv_recall_at_10_diff1": 0.279819, + "nauc_cv_recall_at_20_max": 0.351914, + "nauc_cv_recall_at_20_std": 0.15, + "nauc_cv_recall_at_20_diff1": 0.080719, + "nauc_cv_recall_at_100_max": -0.327848, + "nauc_cv_recall_at_100_std": -0.121965, + "nauc_cv_recall_at_100_diff1": -0.25957, + "nauc_cv_recall_at_1000_max": -1.739963, + "nauc_cv_recall_at_1000_std": -1.739963, + "nauc_cv_recall_at_1000_diff1": 0.554155, + "nauc_mrr_at_1_max": 0.254625, + "nauc_mrr_at_1_std": -0.011743, + "nauc_mrr_at_1_diff1": 0.498648, + "nauc_mrr_at_3_max": 0.237494, + "nauc_mrr_at_3_std": -0.044152, + "nauc_mrr_at_3_diff1": 0.432889, + "nauc_mrr_at_5_max": 0.269361, + "nauc_mrr_at_5_std": -0.009119, + "nauc_mrr_at_5_diff1": 0.429768, + "nauc_mrr_at_10_max": 0.264655, + "nauc_mrr_at_10_std": -0.012051, + "nauc_mrr_at_10_diff1": 0.436387, + "nauc_mrr_at_20_max": 0.262182, + "nauc_mrr_at_20_std": -0.011709, + "nauc_mrr_at_20_diff1": 0.434149, + "nauc_mrr_at_100_max": 0.259722, + "nauc_mrr_at_100_std": -0.013327, + "nauc_mrr_at_100_diff1": 0.435259, + "nauc_mrr_at_1000_max": 0.259727, + "nauc_mrr_at_1000_std": -0.013391, + "nauc_mrr_at_1000_diff1": 0.435462, + "main_score": 0.5935, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.625, + "ndcg_at_3": 0.62047, + "ndcg_at_5": 0.63949, + "ndcg_at_10": 0.67614, + "ndcg_at_20": 0.70036, + "ndcg_at_100": 0.72901, + "ndcg_at_1000": 0.74362, + "map_at_1": 0.38585, + "map_at_3": 0.50823, + "map_at_5": 0.54682, + "map_at_10": 0.5868, + "map_at_20": 0.60214, + "map_at_100": 0.61197, + "map_at_1000": 0.61367, + "recall_at_1": 0.38585, + "recall_at_3": 0.5627, + "recall_at_5": 0.65614, + "recall_at_10": 0.76838, + "recall_at_20": 0.83526, + "recall_at_100": 0.92605, + "recall_at_1000": 0.9902, + "cv_recall_at_1": 0.625, + "cv_recall_at_3": 0.80625, + "cv_recall_at_5": 0.8875, + "cv_recall_at_10": 0.93125, + "cv_recall_at_20": 0.94375, + "cv_recall_at_100": 0.9875, + "cv_recall_at_1000": 0.99375, + "precision_at_1": 0.625, + "precision_at_3": 0.3875, + "precision_at_5": 0.29, + "precision_at_10": 0.18688, + "precision_at_20": 0.10875, + "precision_at_100": 0.02725, + "precision_at_1000": 0.00318, + "mrr_at_1": 0.625, + "mrr_at_3": 0.70625, + "mrr_at_5": 0.724375, + "mrr_at_10": 0.730761, + "mrr_at_20": 0.73185, + "mrr_at_100": 0.733029, + "mrr_at_1000": 0.733042, + "nauc_ndcg_at_1_max": 0.256928, + "nauc_ndcg_at_1_std": -0.049156, + "nauc_ndcg_at_1_diff1": 0.373006, + "nauc_ndcg_at_3_max": 0.280662, + "nauc_ndcg_at_3_std": -0.003522, + "nauc_ndcg_at_3_diff1": 0.335188, + "nauc_ndcg_at_5_max": 0.303778, + "nauc_ndcg_at_5_std": -0.006561, + "nauc_ndcg_at_5_diff1": 0.343285, + "nauc_ndcg_at_10_max": 0.301121, + "nauc_ndcg_at_10_std": -0.026385, + "nauc_ndcg_at_10_diff1": 0.332909, + "nauc_ndcg_at_20_max": 0.295139, + "nauc_ndcg_at_20_std": -0.016577, + "nauc_ndcg_at_20_diff1": 0.316977, + "nauc_ndcg_at_100_max": 0.270526, + "nauc_ndcg_at_100_std": -0.018714, + "nauc_ndcg_at_100_diff1": 0.310591, + "nauc_ndcg_at_1000_max": 0.254242, + "nauc_ndcg_at_1000_std": -0.02782, + "nauc_ndcg_at_1000_diff1": 0.313277, + "nauc_map_at_1_max": 0.378726, + "nauc_map_at_1_std": 0.051862, + "nauc_map_at_1_diff1": 0.467363, + "nauc_map_at_3_max": 0.344408, + "nauc_map_at_3_std": 0.032164, + "nauc_map_at_3_diff1": 0.416043, + "nauc_map_at_5_max": 0.33425, + "nauc_map_at_5_std": 0.003247, + "nauc_map_at_5_diff1": 0.39382, + "nauc_map_at_10_max": 0.310014, + "nauc_map_at_10_std": -0.015927, + "nauc_map_at_10_diff1": 0.35597, + "nauc_map_at_20_max": 0.310261, + "nauc_map_at_20_std": -0.007017, + "nauc_map_at_20_diff1": 0.347094, + "nauc_map_at_100_max": 0.305569, + "nauc_map_at_100_std": -0.006694, + "nauc_map_at_100_diff1": 0.344066, + "nauc_map_at_1000_max": 0.304004, + "nauc_map_at_1000_std": -0.006996, + "nauc_map_at_1000_diff1": 0.3438, + "nauc_recall_at_1_max": 0.378726, + "nauc_recall_at_1_std": 0.051862, + "nauc_recall_at_1_diff1": 0.467363, + "nauc_recall_at_3_max": 0.326524, + "nauc_recall_at_3_std": 0.036467, + "nauc_recall_at_3_diff1": 0.3816, + "nauc_recall_at_5_max": 0.301483, + "nauc_recall_at_5_std": -0.019315, + "nauc_recall_at_5_diff1": 0.33144, + "nauc_recall_at_10_max": 0.280778, + "nauc_recall_at_10_std": -0.062676, + "nauc_recall_at_10_diff1": 0.249811, + "nauc_recall_at_20_max": 0.272016, + "nauc_recall_at_20_std": -0.018049, + "nauc_recall_at_20_diff1": 0.179459, + "nauc_recall_at_100_max": 0.116798, + "nauc_recall_at_100_std": -0.050218, + "nauc_recall_at_100_diff1": 0.143788, + "nauc_recall_at_1000_max": -0.977777, + "nauc_recall_at_1000_std": -1.031463, + "nauc_recall_at_1000_diff1": 0.780587, + "nauc_precision_at_1_max": 0.256928, + "nauc_precision_at_1_std": -0.049156, + "nauc_precision_at_1_diff1": 0.373006, + "nauc_precision_at_3_max": -0.036379, + "nauc_precision_at_3_std": -0.066196, + "nauc_precision_at_3_diff1": -0.086483, + "nauc_precision_at_5_max": -0.119445, + "nauc_precision_at_5_std": -0.084789, + "nauc_precision_at_5_diff1": -0.231529, + "nauc_precision_at_10_max": -0.176419, + "nauc_precision_at_10_std": -0.100078, + "nauc_precision_at_10_diff1": -0.306468, + "nauc_precision_at_20_max": -0.201617, + "nauc_precision_at_20_std": -0.053191, + "nauc_precision_at_20_diff1": -0.326134, + "nauc_precision_at_100_max": -0.275704, + "nauc_precision_at_100_std": -0.037361, + "nauc_precision_at_100_diff1": -0.353191, + "nauc_precision_at_1000_max": -0.336662, + "nauc_precision_at_1000_std": -0.057585, + "nauc_precision_at_1000_diff1": -0.367925, + "nauc_cv_recall_at_1_max": 0.256928, + "nauc_cv_recall_at_1_std": -0.049156, + "nauc_cv_recall_at_1_diff1": 0.373006, + "nauc_cv_recall_at_3_max": 0.271872, + "nauc_cv_recall_at_3_std": -0.015, + "nauc_cv_recall_at_3_diff1": 0.304717, + "nauc_cv_recall_at_5_max": 0.162965, + "nauc_cv_recall_at_5_std": -0.059612, + "nauc_cv_recall_at_5_diff1": 0.182256, + "nauc_cv_recall_at_10_max": 0.267167, + "nauc_cv_recall_at_10_std": -0.019438, + "nauc_cv_recall_at_10_diff1": 0.146762, + "nauc_cv_recall_at_20_max": 0.166044, + "nauc_cv_recall_at_20_std": -0.051043, + "nauc_cv_recall_at_20_diff1": 0.228343, + "nauc_cv_recall_at_100_max": -1.445845, + "nauc_cv_recall_at_100_std": -0.690943, + "nauc_cv_recall_at_100_diff1": -0.075864, + "nauc_cv_recall_at_1000_max": -1.739963, + "nauc_cv_recall_at_1000_std": -1.739963, + "nauc_cv_recall_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.256928, + "nauc_mrr_at_1_std": -0.049156, + "nauc_mrr_at_1_diff1": 0.373006, + "nauc_mrr_at_3_max": 0.262964, + "nauc_mrr_at_3_std": -0.027456, + "nauc_mrr_at_3_diff1": 0.350707, + "nauc_mrr_at_5_max": 0.247096, + "nauc_mrr_at_5_std": -0.036724, + "nauc_mrr_at_5_diff1": 0.342818, + "nauc_mrr_at_10_max": 0.255193, + "nauc_mrr_at_10_std": -0.03376, + "nauc_mrr_at_10_diff1": 0.345594, + "nauc_mrr_at_20_max": 0.252933, + "nauc_mrr_at_20_std": -0.034518, + "nauc_mrr_at_20_diff1": 0.34826, + "nauc_mrr_at_100_max": 0.251446, + "nauc_mrr_at_100_std": -0.034706, + "nauc_mrr_at_100_diff1": 0.348213, + "nauc_mrr_at_1000_max": 0.251526, + "nauc_mrr_at_1000_std": -0.034729, + "nauc_mrr_at_1000_diff1": 0.348299, + "main_score": 0.63949, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.56875, + "ndcg_at_3": 0.58326, + "ndcg_at_5": 0.60235, + "ndcg_at_10": 0.63356, + "ndcg_at_20": 0.65965, + "ndcg_at_100": 0.69139, + "ndcg_at_1000": 0.70878, + "map_at_1": 0.34836, + "map_at_3": 0.46931, + "map_at_5": 0.50883, + "map_at_10": 0.54309, + "map_at_20": 0.55749, + "map_at_100": 0.56703, + "map_at_1000": 0.5687, + "recall_at_1": 0.34836, + "recall_at_3": 0.54019, + "recall_at_5": 0.62843, + "recall_at_10": 0.72474, + "recall_at_20": 0.80385, + "recall_at_100": 0.91113, + "recall_at_1000": 0.98998, + "cv_recall_at_1": 0.56875, + "cv_recall_at_3": 0.78125, + "cv_recall_at_5": 0.85, + "cv_recall_at_10": 0.89375, + "cv_recall_at_20": 0.9375, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 0.99375, + "precision_at_1": 0.56875, + "precision_at_3": 0.3625, + "precision_at_5": 0.27625, + "precision_at_10": 0.17563, + "precision_at_20": 0.1025, + "precision_at_100": 0.02612, + "precision_at_1000": 0.00317, + "mrr_at_1": 0.56875, + "mrr_at_3": 0.666667, + "mrr_at_5": 0.682604, + "mrr_at_10": 0.689033, + "mrr_at_20": 0.691687, + "mrr_at_100": 0.69288, + "mrr_at_1000": 0.692919, + "nauc_ndcg_at_1_max": 0.239827, + "nauc_ndcg_at_1_std": -0.048449, + "nauc_ndcg_at_1_diff1": 0.430497, + "nauc_ndcg_at_3_max": 0.308249, + "nauc_ndcg_at_3_std": 0.003112, + "nauc_ndcg_at_3_diff1": 0.37519, + "nauc_ndcg_at_5_max": 0.304068, + "nauc_ndcg_at_5_std": 0.005541, + "nauc_ndcg_at_5_diff1": 0.367657, + "nauc_ndcg_at_10_max": 0.313365, + "nauc_ndcg_at_10_std": 0.012521, + "nauc_ndcg_at_10_diff1": 0.365438, + "nauc_ndcg_at_20_max": 0.310854, + "nauc_ndcg_at_20_std": 0.025785, + "nauc_ndcg_at_20_diff1": 0.363729, + "nauc_ndcg_at_100_max": 0.295656, + "nauc_ndcg_at_100_std": 0.018959, + "nauc_ndcg_at_100_diff1": 0.3517, + "nauc_ndcg_at_1000_max": 0.277664, + "nauc_ndcg_at_1000_std": -0.007944, + "nauc_ndcg_at_1000_diff1": 0.356103, + "nauc_map_at_1_max": 0.344472, + "nauc_map_at_1_std": 0.092112, + "nauc_map_at_1_diff1": 0.496707, + "nauc_map_at_3_max": 0.352272, + "nauc_map_at_3_std": 0.054046, + "nauc_map_at_3_diff1": 0.432269, + "nauc_map_at_5_max": 0.331466, + "nauc_map_at_5_std": 0.024337, + "nauc_map_at_5_diff1": 0.408101, + "nauc_map_at_10_max": 0.317762, + "nauc_map_at_10_std": 0.019378, + "nauc_map_at_10_diff1": 0.388551, + "nauc_map_at_20_max": 0.316772, + "nauc_map_at_20_std": 0.022206, + "nauc_map_at_20_diff1": 0.385755, + "nauc_map_at_100_max": 0.314873, + "nauc_map_at_100_std": 0.023725, + "nauc_map_at_100_diff1": 0.379025, + "nauc_map_at_1000_max": 0.313365, + "nauc_map_at_1000_std": 0.022071, + "nauc_map_at_1000_diff1": 0.378654, + "nauc_recall_at_1_max": 0.344472, + "nauc_recall_at_1_std": 0.092112, + "nauc_recall_at_1_diff1": 0.496707, + "nauc_recall_at_3_max": 0.36148, + "nauc_recall_at_3_std": 0.066232, + "nauc_recall_at_3_diff1": 0.387958, + "nauc_recall_at_5_max": 0.281002, + "nauc_recall_at_5_std": -0.004939, + "nauc_recall_at_5_diff1": 0.324787, + "nauc_recall_at_10_max": 0.28365, + "nauc_recall_at_10_std": -0.000198, + "nauc_recall_at_10_diff1": 0.272052, + "nauc_recall_at_20_max": 0.272377, + "nauc_recall_at_20_std": 0.087722, + "nauc_recall_at_20_diff1": 0.23924, + "nauc_recall_at_100_max": 0.173835, + "nauc_recall_at_100_std": 0.096527, + "nauc_recall_at_100_diff1": 0.13806, + "nauc_recall_at_1000_max": -0.862942, + "nauc_recall_at_1000_std": -0.885078, + "nauc_recall_at_1000_diff1": 0.643401, + "nauc_precision_at_1_max": 0.239827, + "nauc_precision_at_1_std": -0.048449, + "nauc_precision_at_1_diff1": 0.430497, + "nauc_precision_at_3_max": 0.062603, + "nauc_precision_at_3_std": -0.067054, + "nauc_precision_at_3_diff1": -0.016068, + "nauc_precision_at_5_max": -0.042909, + "nauc_precision_at_5_std": -0.098064, + "nauc_precision_at_5_diff1": -0.144658, + "nauc_precision_at_10_max": -0.112818, + "nauc_precision_at_10_std": -0.086395, + "nauc_precision_at_10_diff1": -0.212474, + "nauc_precision_at_20_max": -0.131387, + "nauc_precision_at_20_std": -0.045441, + "nauc_precision_at_20_diff1": -0.257585, + "nauc_precision_at_100_max": -0.200281, + "nauc_precision_at_100_std": -0.053396, + "nauc_precision_at_100_diff1": -0.314083, + "nauc_precision_at_1000_max": -0.299338, + "nauc_precision_at_1000_std": -0.163486, + "nauc_precision_at_1000_diff1": -0.325216, + "nauc_cv_recall_at_1_max": 0.239827, + "nauc_cv_recall_at_1_std": -0.048449, + "nauc_cv_recall_at_1_diff1": 0.430497, + "nauc_cv_recall_at_3_max": 0.383771, + "nauc_cv_recall_at_3_std": 0.074201, + "nauc_cv_recall_at_3_diff1": 0.350135, + "nauc_cv_recall_at_5_max": 0.246583, + "nauc_cv_recall_at_5_std": -0.010352, + "nauc_cv_recall_at_5_diff1": 0.31252, + "nauc_cv_recall_at_10_max": 0.346488, + "nauc_cv_recall_at_10_std": 0.051503, + "nauc_cv_recall_at_10_diff1": 0.246001, + "nauc_cv_recall_at_20_max": 0.218394, + "nauc_cv_recall_at_20_std": 0.238702, + "nauc_cv_recall_at_20_diff1": 0.079785, + "nauc_cv_recall_at_100_max": -0.919234, + "nauc_cv_recall_at_100_std": -0.5831, + "nauc_cv_recall_at_100_diff1": -0.387021, + "nauc_cv_recall_at_1000_max": -1.739963, + "nauc_cv_recall_at_1000_std": -1.739963, + "nauc_cv_recall_at_1000_diff1": 0.554155, + "nauc_mrr_at_1_max": 0.239827, + "nauc_mrr_at_1_std": -0.048449, + "nauc_mrr_at_1_diff1": 0.430497, + "nauc_mrr_at_3_max": 0.296881, + "nauc_mrr_at_3_std": -0.009723, + "nauc_mrr_at_3_diff1": 0.40848, + "nauc_mrr_at_5_max": 0.27558, + "nauc_mrr_at_5_std": -0.024476, + "nauc_mrr_at_5_diff1": 0.407256, + "nauc_mrr_at_10_max": 0.281199, + "nauc_mrr_at_10_std": -0.022466, + "nauc_mrr_at_10_diff1": 0.405455, + "nauc_mrr_at_20_max": 0.278687, + "nauc_mrr_at_20_std": -0.020924, + "nauc_mrr_at_20_diff1": 0.40467, + "nauc_mrr_at_100_max": 0.276411, + "nauc_mrr_at_100_std": -0.024288, + "nauc_mrr_at_100_diff1": 0.405164, + "nauc_mrr_at_1000_max": 0.276492, + "nauc_mrr_at_1000_std": -0.024311, + "nauc_mrr_at_1000_diff1": 0.405357, + "main_score": 0.60235, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 346.1857535839081, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..c45b3e7cbf --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.64103, + "ndcg_at_3": 0.65452, + "ndcg_at_5": 0.67055, + "ndcg_at_10": 0.69183, + "ndcg_at_20": 0.71082, + "ndcg_at_100": 0.73538, + "ndcg_at_1000": 0.73956, + "map_at_1": 0.43365, + "map_at_3": 0.55705, + "map_at_5": 0.59763, + "map_at_10": 0.61823, + "map_at_20": 0.62774, + "map_at_100": 0.6353, + "map_at_1000": 0.63576, + "recall_at_1": 0.43365, + "recall_at_3": 0.64455, + "recall_at_5": 0.71892, + "recall_at_10": 0.7774, + "recall_at_20": 0.83514, + "recall_at_100": 0.92491, + "recall_at_1000": 0.94216, + "cv_recall_at_1": 0.65385, + "cv_recall_at_3": 0.84615, + "cv_recall_at_5": 0.90385, + "cv_recall_at_10": 0.92308, + "cv_recall_at_20": 0.94231, + "cv_recall_at_100": 0.98077, + "cv_recall_at_1000": 0.98077, + "precision_at_1": 0.65385, + "precision_at_3": 0.37179, + "precision_at_5": 0.27308, + "precision_at_10": 0.16154, + "precision_at_20": 0.09231, + "precision_at_100": 0.02192, + "precision_at_1000": 0.00233, + "mrr_at_1": 0.653846, + "mrr_at_3": 0.737179, + "mrr_at_5": 0.750641, + "mrr_at_10": 0.753388, + "mrr_at_20": 0.754868, + "mrr_at_100": 0.756289, + "mrr_at_1000": 0.756289, + "nauc_ndcg_at_1_max": 0.228611, + "nauc_ndcg_at_1_std": 0.206577, + "nauc_ndcg_at_1_diff1": 0.449699, + "nauc_ndcg_at_3_max": 0.287347, + "nauc_ndcg_at_3_std": 0.180988, + "nauc_ndcg_at_3_diff1": 0.423325, + "nauc_ndcg_at_5_max": 0.304868, + "nauc_ndcg_at_5_std": 0.148931, + "nauc_ndcg_at_5_diff1": 0.416591, + "nauc_ndcg_at_10_max": 0.255742, + "nauc_ndcg_at_10_std": 0.076786, + "nauc_ndcg_at_10_diff1": 0.381101, + "nauc_ndcg_at_20_max": 0.292227, + "nauc_ndcg_at_20_std": 0.080682, + "nauc_ndcg_at_20_diff1": 0.403174, + "nauc_ndcg_at_100_max": 0.324567, + "nauc_ndcg_at_100_std": 0.140043, + "nauc_ndcg_at_100_diff1": 0.396557, + "nauc_ndcg_at_1000_max": 0.312557, + "nauc_ndcg_at_1000_std": 0.127703, + "nauc_ndcg_at_1000_diff1": 0.384177, + "nauc_map_at_1_max": 0.393093, + "nauc_map_at_1_std": 0.24341, + "nauc_map_at_1_diff1": 0.697199, + "nauc_map_at_3_max": 0.377209, + "nauc_map_at_3_std": 0.18066, + "nauc_map_at_3_diff1": 0.570345, + "nauc_map_at_5_max": 0.391885, + "nauc_map_at_5_std": 0.18319, + "nauc_map_at_5_diff1": 0.497874, + "nauc_map_at_10_max": 0.353497, + "nauc_map_at_10_std": 0.164183, + "nauc_map_at_10_diff1": 0.465701, + "nauc_map_at_20_max": 0.367652, + "nauc_map_at_20_std": 0.171965, + "nauc_map_at_20_diff1": 0.468257, + "nauc_map_at_100_max": 0.37326, + "nauc_map_at_100_std": 0.187625, + "nauc_map_at_100_diff1": 0.465748, + "nauc_map_at_1000_max": 0.372436, + "nauc_map_at_1000_std": 0.186837, + "nauc_map_at_1000_diff1": 0.464889, + "nauc_recall_at_1_max": 0.393093, + "nauc_recall_at_1_std": 0.24341, + "nauc_recall_at_1_diff1": 0.697199, + "nauc_recall_at_3_max": 0.279751, + "nauc_recall_at_3_std": 0.086179, + "nauc_recall_at_3_diff1": 0.426984, + "nauc_recall_at_5_max": 0.273206, + "nauc_recall_at_5_std": 0.01224, + "nauc_recall_at_5_diff1": 0.339599, + "nauc_recall_at_10_max": 0.117991, + "nauc_recall_at_10_std": -0.161397, + "nauc_recall_at_10_diff1": 0.204327, + "nauc_recall_at_20_max": 0.219795, + "nauc_recall_at_20_std": -0.220846, + "nauc_recall_at_20_diff1": 0.294881, + "nauc_recall_at_100_max": 0.623075, + "nauc_recall_at_100_std": 0.1071, + "nauc_recall_at_100_diff1": 0.160876, + "nauc_recall_at_1000_max": 0.539436, + "nauc_recall_at_1000_std": -0.079603, + "nauc_recall_at_1000_diff1": -0.062313, + "nauc_precision_at_1_max": 0.218244, + "nauc_precision_at_1_std": 0.264148, + "nauc_precision_at_1_diff1": 0.468567, + "nauc_precision_at_3_max": -0.073866, + "nauc_precision_at_3_std": 0.005169, + "nauc_precision_at_3_diff1": -0.218957, + "nauc_precision_at_5_max": -0.037932, + "nauc_precision_at_5_std": 0.047126, + "nauc_precision_at_5_diff1": -0.367901, + "nauc_precision_at_10_max": -0.201914, + "nauc_precision_at_10_std": -0.046272, + "nauc_precision_at_10_diff1": -0.463369, + "nauc_precision_at_20_max": -0.136089, + "nauc_precision_at_20_std": -0.001904, + "nauc_precision_at_20_diff1": -0.465825, + "nauc_precision_at_100_max": -0.130277, + "nauc_precision_at_100_std": 0.074016, + "nauc_precision_at_100_diff1": -0.448826, + "nauc_precision_at_1000_max": -0.175889, + "nauc_precision_at_1000_std": 0.012852, + "nauc_precision_at_1000_diff1": -0.464492, + "nauc_cv_recall_at_1_max": 0.218244, + "nauc_cv_recall_at_1_std": 0.264148, + "nauc_cv_recall_at_1_diff1": 0.468567, + "nauc_cv_recall_at_3_max": -0.064597, + "nauc_cv_recall_at_3_std": -0.077389, + "nauc_cv_recall_at_3_diff1": 0.173084, + "nauc_cv_recall_at_5_max": -0.032864, + "nauc_cv_recall_at_5_std": -0.195862, + "nauc_cv_recall_at_5_diff1": 0.073038, + "nauc_cv_recall_at_10_max": -0.258539, + "nauc_cv_recall_at_10_std": -0.462287, + "nauc_cv_recall_at_10_diff1": -0.089747, + "nauc_cv_recall_at_20_max": -0.288149, + "nauc_cv_recall_at_20_std": -0.735349, + "nauc_cv_recall_at_20_diff1": 0.268279, + "nauc_cv_recall_at_100_max": 0.554244, + "nauc_cv_recall_at_100_std": -0.552063, + "nauc_cv_recall_at_100_diff1": -0.552063, + "nauc_cv_recall_at_1000_max": 0.554244, + "nauc_cv_recall_at_1000_std": -0.552063, + "nauc_cv_recall_at_1000_diff1": -0.552063, + "nauc_mrr_at_1_max": 0.218244, + "nauc_mrr_at_1_std": 0.264148, + "nauc_mrr_at_1_diff1": 0.468567, + "nauc_mrr_at_3_max": 0.127601, + "nauc_mrr_at_3_std": 0.181533, + "nauc_mrr_at_3_diff1": 0.386192, + "nauc_mrr_at_5_max": 0.145187, + "nauc_mrr_at_5_std": 0.187188, + "nauc_mrr_at_5_diff1": 0.385175, + "nauc_mrr_at_10_max": 0.135764, + "nauc_mrr_at_10_std": 0.178311, + "nauc_mrr_at_10_diff1": 0.380767, + "nauc_mrr_at_20_max": 0.137918, + "nauc_mrr_at_20_std": 0.177051, + "nauc_mrr_at_20_diff1": 0.391658, + "nauc_mrr_at_100_max": 0.142773, + "nauc_mrr_at_100_std": 0.183078, + "nauc_mrr_at_100_diff1": 0.389981, + "nauc_mrr_at_1000_max": 0.142773, + "nauc_mrr_at_1000_std": 0.183078, + "nauc_mrr_at_1000_diff1": 0.389981, + "main_score": 0.67055, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 223.7238359451294, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..7a7a3a43cc --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.61404, + "ndcg_at_3": 0.53826, + "ndcg_at_5": 0.58503, + "ndcg_at_10": 0.62429, + "ndcg_at_20": 0.6473, + "ndcg_at_100": 0.67953, + "ndcg_at_1000": 0.67953, + "map_at_1": 0.27928, + "map_at_3": 0.39492, + "map_at_5": 0.45416, + "map_at_10": 0.49585, + "map_at_20": 0.51377, + "map_at_100": 0.53188, + "map_at_1000": 0.53188, + "recall_at_1": 0.27928, + "recall_at_3": 0.45399, + "recall_at_5": 0.60973, + "recall_at_10": 0.7429, + "recall_at_20": 0.81631, + "recall_at_100": 0.90455, + "recall_at_1000": 0.90455, + "cv_recall_at_1": 0.61404, + "cv_recall_at_3": 0.77193, + "cv_recall_at_5": 0.89474, + "cv_recall_at_10": 0.92982, + "cv_recall_at_20": 0.96491, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.61404, + "precision_at_3": 0.35673, + "precision_at_5": 0.29825, + "precision_at_10": 0.2, + "precision_at_20": 0.12193, + "precision_at_100": 0.03421, + "precision_at_1000": 0.00342, + "mrr_at_1": 0.614035, + "mrr_at_3": 0.684211, + "mrr_at_5": 0.710526, + "mrr_at_10": 0.714669, + "mrr_at_20": 0.717433, + "mrr_at_100": 0.717934, + "mrr_at_1000": 0.717934, + "nauc_ndcg_at_1_max": -0.377052, + "nauc_ndcg_at_1_std": -0.14928, + "nauc_ndcg_at_1_diff1": 0.095975, + "nauc_ndcg_at_3_max": -0.113324, + "nauc_ndcg_at_3_std": -0.045931, + "nauc_ndcg_at_3_diff1": 0.05252, + "nauc_ndcg_at_5_max": -0.058534, + "nauc_ndcg_at_5_std": -0.086882, + "nauc_ndcg_at_5_diff1": 0.056297, + "nauc_ndcg_at_10_max": -0.010112, + "nauc_ndcg_at_10_std": -0.046845, + "nauc_ndcg_at_10_diff1": 0.13622, + "nauc_ndcg_at_20_max": -0.056124, + "nauc_ndcg_at_20_std": -0.070431, + "nauc_ndcg_at_20_diff1": 0.114764, + "nauc_ndcg_at_100_max": -0.091738, + "nauc_ndcg_at_100_std": -0.004055, + "nauc_ndcg_at_100_diff1": 0.082325, + "nauc_ndcg_at_1000_max": -0.091738, + "nauc_ndcg_at_1000_std": -0.004055, + "nauc_ndcg_at_1000_diff1": 0.082325, + "nauc_map_at_1_max": -0.060104, + "nauc_map_at_1_std": -0.040774, + "nauc_map_at_1_diff1": 0.013367, + "nauc_map_at_3_max": -0.001827, + "nauc_map_at_3_std": -0.048965, + "nauc_map_at_3_diff1": 0.005379, + "nauc_map_at_5_max": 0.025223, + "nauc_map_at_5_std": -0.06179, + "nauc_map_at_5_diff1": 0.054264, + "nauc_map_at_10_max": 0.014204, + "nauc_map_at_10_std": -0.049967, + "nauc_map_at_10_diff1": 0.121885, + "nauc_map_at_20_max": -0.024409, + "nauc_map_at_20_std": -0.062757, + "nauc_map_at_20_diff1": 0.104975, + "nauc_map_at_100_max": -0.044584, + "nauc_map_at_100_std": -0.026269, + "nauc_map_at_100_diff1": 0.075002, + "nauc_map_at_1000_max": -0.044584, + "nauc_map_at_1000_std": -0.026269, + "nauc_map_at_1000_diff1": 0.075002, + "nauc_recall_at_1_max": -0.060104, + "nauc_recall_at_1_std": -0.040774, + "nauc_recall_at_1_diff1": 0.013367, + "nauc_recall_at_3_max": 0.092138, + "nauc_recall_at_3_std": -0.047692, + "nauc_recall_at_3_diff1": 0.002027, + "nauc_recall_at_5_max": 0.243053, + "nauc_recall_at_5_std": -0.076664, + "nauc_recall_at_5_diff1": 0.053786, + "nauc_recall_at_10_max": 0.352707, + "nauc_recall_at_10_std": 0.0015, + "nauc_recall_at_10_diff1": 0.23627, + "nauc_recall_at_20_max": 0.280197, + "nauc_recall_at_20_std": -0.027586, + "nauc_recall_at_20_diff1": 0.169417, + "nauc_recall_at_100_max": 0.384314, + "nauc_recall_at_100_std": 0.283506, + "nauc_recall_at_100_diff1": 0.202821, + "nauc_recall_at_1000_max": 0.384314, + "nauc_recall_at_1000_std": 0.283506, + "nauc_recall_at_1000_diff1": 0.202821, + "nauc_precision_at_1_max": -0.377052, + "nauc_precision_at_1_std": -0.14928, + "nauc_precision_at_1_diff1": 0.095975, + "nauc_precision_at_3_max": -0.222945, + "nauc_precision_at_3_std": -0.042316, + "nauc_precision_at_3_diff1": 0.106604, + "nauc_precision_at_5_max": -0.1839, + "nauc_precision_at_5_std": -0.054152, + "nauc_precision_at_5_diff1": 0.131962, + "nauc_precision_at_10_max": -0.199645, + "nauc_precision_at_10_std": 0.029417, + "nauc_precision_at_10_diff1": 0.126076, + "nauc_precision_at_20_max": -0.257662, + "nauc_precision_at_20_std": 0.060531, + "nauc_precision_at_20_diff1": -0.017253, + "nauc_precision_at_100_max": -0.235102, + "nauc_precision_at_100_std": 0.206253, + "nauc_precision_at_100_diff1": -0.168616, + "nauc_precision_at_1000_max": -0.235102, + "nauc_precision_at_1000_std": 0.206253, + "nauc_precision_at_1000_diff1": -0.168616, + "nauc_cv_recall_at_1_max": -0.377052, + "nauc_cv_recall_at_1_std": -0.14928, + "nauc_cv_recall_at_1_diff1": 0.095975, + "nauc_cv_recall_at_3_max": -0.315581, + "nauc_cv_recall_at_3_std": -0.125419, + "nauc_cv_recall_at_3_diff1": 0.10322, + "nauc_cv_recall_at_5_max": -0.100346, + "nauc_cv_recall_at_5_std": -0.14489, + "nauc_cv_recall_at_5_diff1": -0.065771, + "nauc_cv_recall_at_10_max": 0.322681, + "nauc_cv_recall_at_10_std": 0.182975, + "nauc_cv_recall_at_10_diff1": 0.009542, + "nauc_cv_recall_at_20_max": 0.795622, + "nauc_cv_recall_at_20_std": 0.795622, + "nauc_cv_recall_at_20_diff1": -0.521209, + "nauc_cv_recall_at_100_max": 0.868512, + "nauc_cv_recall_at_100_std": 0.722732, + "nauc_cv_recall_at_100_diff1": 0.126616, + "nauc_cv_recall_at_1000_max": 0.868512, + "nauc_cv_recall_at_1000_std": 0.722732, + "nauc_cv_recall_at_1000_diff1": 0.126616, + "nauc_mrr_at_1_max": -0.377052, + "nauc_mrr_at_1_std": -0.14928, + "nauc_mrr_at_1_diff1": 0.095975, + "nauc_mrr_at_3_max": -0.336321, + "nauc_mrr_at_3_std": -0.151174, + "nauc_mrr_at_3_diff1": 0.103964, + "nauc_mrr_at_5_max": -0.310522, + "nauc_mrr_at_5_std": -0.150854, + "nauc_mrr_at_5_diff1": 0.089127, + "nauc_mrr_at_10_max": -0.298611, + "nauc_mrr_at_10_std": -0.138551, + "nauc_mrr_at_10_diff1": 0.094111, + "nauc_mrr_at_20_max": -0.30221, + "nauc_mrr_at_20_std": -0.137473, + "nauc_mrr_at_20_diff1": 0.088517, + "nauc_mrr_at_100_max": -0.304406, + "nauc_mrr_at_100_std": -0.139629, + "nauc_mrr_at_100_diff1": 0.091213, + "nauc_mrr_at_1000_max": -0.304406, + "nauc_mrr_at_1000_std": -0.139629, + "nauc_mrr_at_1000_diff1": 0.091213, + "main_score": 0.58503, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.54386, + "ndcg_at_3": 0.5296, + "ndcg_at_5": 0.56703, + "ndcg_at_10": 0.60852, + "ndcg_at_20": 0.6406, + "ndcg_at_100": 0.66545, + "ndcg_at_1000": 0.66545, + "map_at_1": 0.24419, + "map_at_3": 0.38493, + "map_at_5": 0.43588, + "map_at_10": 0.48166, + "map_at_20": 0.50227, + "map_at_100": 0.51861, + "map_at_1000": 0.51861, + "recall_at_1": 0.24419, + "recall_at_3": 0.47519, + "recall_at_5": 0.59906, + "recall_at_10": 0.74747, + "recall_at_20": 0.84169, + "recall_at_100": 0.90455, + "recall_at_1000": 0.90455, + "cv_recall_at_1": 0.54386, + "cv_recall_at_3": 0.77193, + "cv_recall_at_5": 0.87719, + "cv_recall_at_10": 0.94737, + "cv_recall_at_20": 0.98246, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.54386, + "precision_at_3": 0.36257, + "precision_at_5": 0.29825, + "precision_at_10": 0.2, + "precision_at_20": 0.12807, + "precision_at_100": 0.03421, + "precision_at_1000": 0.00342, + "mrr_at_1": 0.54386, + "mrr_at_3": 0.643275, + "mrr_at_5": 0.666959, + "mrr_at_10": 0.676754, + "mrr_at_20": 0.679357, + "mrr_at_100": 0.679357, + "mrr_at_1000": 0.679357, + "nauc_ndcg_at_1_max": -0.092224, + "nauc_ndcg_at_1_std": -0.047835, + "nauc_ndcg_at_1_diff1": 0.350879, + "nauc_ndcg_at_3_max": 0.059779, + "nauc_ndcg_at_3_std": 0.039031, + "nauc_ndcg_at_3_diff1": 0.216279, + "nauc_ndcg_at_5_max": 0.006658, + "nauc_ndcg_at_5_std": -0.079515, + "nauc_ndcg_at_5_diff1": 0.149826, + "nauc_ndcg_at_10_max": 0.048972, + "nauc_ndcg_at_10_std": -0.104296, + "nauc_ndcg_at_10_diff1": 0.284883, + "nauc_ndcg_at_20_max": 0.039302, + "nauc_ndcg_at_20_std": -0.072581, + "nauc_ndcg_at_20_diff1": 0.29725, + "nauc_ndcg_at_100_max": 0.070821, + "nauc_ndcg_at_100_std": 0.020096, + "nauc_ndcg_at_100_diff1": 0.255779, + "nauc_ndcg_at_1000_max": 0.070821, + "nauc_ndcg_at_1000_std": 0.020096, + "nauc_ndcg_at_1000_diff1": 0.255779, + "nauc_map_at_1_max": 0.12132, + "nauc_map_at_1_std": -0.092198, + "nauc_map_at_1_diff1": 0.186052, + "nauc_map_at_3_max": 0.145001, + "nauc_map_at_3_std": -0.021197, + "nauc_map_at_3_diff1": 0.213323, + "nauc_map_at_5_max": 0.103463, + "nauc_map_at_5_std": -0.079721, + "nauc_map_at_5_diff1": 0.187143, + "nauc_map_at_10_max": 0.086904, + "nauc_map_at_10_std": -0.085692, + "nauc_map_at_10_diff1": 0.23875, + "nauc_map_at_20_max": 0.066436, + "nauc_map_at_20_std": -0.080731, + "nauc_map_at_20_diff1": 0.237331, + "nauc_map_at_100_max": 0.068376, + "nauc_map_at_100_std": -0.046608, + "nauc_map_at_100_diff1": 0.212511, + "nauc_map_at_1000_max": 0.068376, + "nauc_map_at_1000_std": -0.046608, + "nauc_map_at_1000_diff1": 0.212511, + "nauc_recall_at_1_max": 0.12132, + "nauc_recall_at_1_std": -0.092198, + "nauc_recall_at_1_diff1": 0.186052, + "nauc_recall_at_3_max": 0.231981, + "nauc_recall_at_3_std": 0.077325, + "nauc_recall_at_3_diff1": 0.166508, + "nauc_recall_at_5_max": 0.126276, + "nauc_recall_at_5_std": -0.124838, + "nauc_recall_at_5_diff1": 0.040433, + "nauc_recall_at_10_max": 0.145689, + "nauc_recall_at_10_std": -0.224766, + "nauc_recall_at_10_diff1": 0.286227, + "nauc_recall_at_20_max": 0.127577, + "nauc_recall_at_20_std": -0.182353, + "nauc_recall_at_20_diff1": 0.371256, + "nauc_recall_at_100_max": 0.403229, + "nauc_recall_at_100_std": 0.282664, + "nauc_recall_at_100_diff1": 0.239005, + "nauc_recall_at_1000_max": 0.403229, + "nauc_recall_at_1000_std": 0.282664, + "nauc_recall_at_1000_diff1": 0.239005, + "nauc_precision_at_1_max": -0.092224, + "nauc_precision_at_1_std": -0.047835, + "nauc_precision_at_1_diff1": 0.350879, + "nauc_precision_at_3_max": -0.059412, + "nauc_precision_at_3_std": 0.091369, + "nauc_precision_at_3_diff1": 0.210844, + "nauc_precision_at_5_max": -0.19823, + "nauc_precision_at_5_std": -0.043696, + "nauc_precision_at_5_diff1": 0.037987, + "nauc_precision_at_10_max": -0.221887, + "nauc_precision_at_10_std": -0.042803, + "nauc_precision_at_10_diff1": 0.061514, + "nauc_precision_at_20_max": -0.226346, + "nauc_precision_at_20_std": 0.053067, + "nauc_precision_at_20_diff1": -0.05147, + "nauc_precision_at_100_max": -0.118773, + "nauc_precision_at_100_std": 0.202534, + "nauc_precision_at_100_diff1": -0.170847, + "nauc_precision_at_1000_max": -0.118773, + "nauc_precision_at_1000_std": 0.202534, + "nauc_precision_at_1000_diff1": -0.170847, + "nauc_cv_recall_at_1_max": -0.092224, + "nauc_cv_recall_at_1_std": -0.047835, + "nauc_cv_recall_at_1_diff1": 0.350879, + "nauc_cv_recall_at_3_max": 0.038858, + "nauc_cv_recall_at_3_std": 0.152097, + "nauc_cv_recall_at_3_diff1": 0.423875, + "nauc_cv_recall_at_5_max": 0.021395, + "nauc_cv_recall_at_5_std": 0.064009, + "nauc_cv_recall_at_5_diff1": 0.129774, + "nauc_cv_recall_at_10_max": 0.084855, + "nauc_cv_recall_at_10_std": -0.167164, + "nauc_cv_recall_at_10_diff1": 0.22337, + "nauc_cv_recall_at_20_max": 0.868512, + "nauc_cv_recall_at_20_std": 0.868512, + "nauc_cv_recall_at_20_diff1": -0.164944, + "nauc_cv_recall_at_100_max": 0.868512, + "nauc_cv_recall_at_100_std": 0.868512, + "nauc_cv_recall_at_100_diff1": -0.164944, + "nauc_cv_recall_at_1000_max": 0.868512, + "nauc_cv_recall_at_1000_std": 0.868512, + "nauc_cv_recall_at_1000_diff1": -0.164944, + "nauc_mrr_at_1_max": -0.092224, + "nauc_mrr_at_1_std": -0.047835, + "nauc_mrr_at_1_diff1": 0.350879, + "nauc_mrr_at_3_max": -0.047316, + "nauc_mrr_at_3_std": 0.020723, + "nauc_mrr_at_3_diff1": 0.370192, + "nauc_mrr_at_5_max": -0.049607, + "nauc_mrr_at_5_std": 0.006832, + "nauc_mrr_at_5_diff1": 0.334298, + "nauc_mrr_at_10_max": -0.049571, + "nauc_mrr_at_10_std": -0.002576, + "nauc_mrr_at_10_diff1": 0.350332, + "nauc_mrr_at_20_max": -0.046553, + "nauc_mrr_at_20_std": 0.004943, + "nauc_mrr_at_20_diff1": 0.349406, + "nauc_mrr_at_100_max": -0.046553, + "nauc_mrr_at_100_std": 0.004943, + "nauc_mrr_at_100_diff1": 0.349406, + "nauc_mrr_at_1000_max": -0.046553, + "nauc_mrr_at_1000_std": 0.004943, + "nauc_mrr_at_1000_diff1": 0.349406, + "main_score": 0.56703, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.59649, + "ndcg_at_3": 0.5542, + "ndcg_at_5": 0.57668, + "ndcg_at_10": 0.6298, + "ndcg_at_20": 0.65098, + "ndcg_at_100": 0.68372, + "ndcg_at_1000": 0.68372, + "map_at_1": 0.29683, + "map_at_3": 0.41332, + "map_at_5": 0.45442, + "map_at_10": 0.50628, + "map_at_20": 0.52251, + "map_at_100": 0.54052, + "map_at_1000": 0.54052, + "recall_at_1": 0.29683, + "recall_at_3": 0.47957, + "recall_at_5": 0.58488, + "recall_at_10": 0.74768, + "recall_at_20": 0.80934, + "recall_at_100": 0.90455, + "recall_at_1000": 0.90455, + "cv_recall_at_1": 0.59649, + "cv_recall_at_3": 0.80702, + "cv_recall_at_5": 0.87719, + "cv_recall_at_10": 0.94737, + "cv_recall_at_20": 0.96491, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.59649, + "precision_at_3": 0.36257, + "precision_at_5": 0.27719, + "precision_at_10": 0.19825, + "precision_at_20": 0.12193, + "precision_at_100": 0.03421, + "precision_at_1000": 0.00342, + "mrr_at_1": 0.596491, + "mrr_at_3": 0.692982, + "mrr_at_5": 0.707895, + "mrr_at_10": 0.716541, + "mrr_at_20": 0.717794, + "mrr_at_100": 0.718145, + "mrr_at_1000": 0.718145, + "nauc_ndcg_at_1_max": -0.114947, + "nauc_ndcg_at_1_std": -0.114897, + "nauc_ndcg_at_1_diff1": 0.484341, + "nauc_ndcg_at_3_max": 0.141569, + "nauc_ndcg_at_3_std": 0.072892, + "nauc_ndcg_at_3_diff1": 0.252146, + "nauc_ndcg_at_5_max": 0.147571, + "nauc_ndcg_at_5_std": 0.052805, + "nauc_ndcg_at_5_diff1": 0.236896, + "nauc_ndcg_at_10_max": 0.176786, + "nauc_ndcg_at_10_std": 0.058788, + "nauc_ndcg_at_10_diff1": 0.246236, + "nauc_ndcg_at_20_max": 0.134518, + "nauc_ndcg_at_20_std": 0.023963, + "nauc_ndcg_at_20_diff1": 0.253393, + "nauc_ndcg_at_100_max": 0.120137, + "nauc_ndcg_at_100_std": 0.113863, + "nauc_ndcg_at_100_diff1": 0.21433, + "nauc_ndcg_at_1000_max": 0.120137, + "nauc_ndcg_at_1000_std": 0.113863, + "nauc_ndcg_at_1000_diff1": 0.21433, + "nauc_map_at_1_max": 0.124189, + "nauc_map_at_1_std": 0.008542, + "nauc_map_at_1_diff1": 0.186145, + "nauc_map_at_3_max": 0.21123, + "nauc_map_at_3_std": 0.059584, + "nauc_map_at_3_diff1": 0.180016, + "nauc_map_at_5_max": 0.231418, + "nauc_map_at_5_std": 0.063536, + "nauc_map_at_5_diff1": 0.218519, + "nauc_map_at_10_max": 0.220009, + "nauc_map_at_10_std": 0.050434, + "nauc_map_at_10_diff1": 0.242241, + "nauc_map_at_20_max": 0.189825, + "nauc_map_at_20_std": 0.043869, + "nauc_map_at_20_diff1": 0.228177, + "nauc_map_at_100_max": 0.173251, + "nauc_map_at_100_std": 0.0883, + "nauc_map_at_100_diff1": 0.19844, + "nauc_map_at_1000_max": 0.173251, + "nauc_map_at_1000_std": 0.0883, + "nauc_map_at_1000_diff1": 0.19844, + "nauc_recall_at_1_max": 0.124189, + "nauc_recall_at_1_std": 0.008542, + "nauc_recall_at_1_diff1": 0.186145, + "nauc_recall_at_3_max": 0.28357, + "nauc_recall_at_3_std": 0.086397, + "nauc_recall_at_3_diff1": 0.104036, + "nauc_recall_at_5_max": 0.320237, + "nauc_recall_at_5_std": 0.073689, + "nauc_recall_at_5_diff1": 0.164711, + "nauc_recall_at_10_max": 0.307919, + "nauc_recall_at_10_std": -0.027351, + "nauc_recall_at_10_diff1": 0.127796, + "nauc_recall_at_20_max": 0.221739, + "nauc_recall_at_20_std": -0.106119, + "nauc_recall_at_20_diff1": 0.117824, + "nauc_recall_at_100_max": 0.407165, + "nauc_recall_at_100_std": 0.321886, + "nauc_recall_at_100_diff1": -0.151814, + "nauc_recall_at_1000_max": 0.407165, + "nauc_recall_at_1000_std": 0.321886, + "nauc_recall_at_1000_diff1": -0.151814, + "nauc_precision_at_1_max": -0.114947, + "nauc_precision_at_1_std": -0.114897, + "nauc_precision_at_1_diff1": 0.484341, + "nauc_precision_at_3_max": -0.020587, + "nauc_precision_at_3_std": 0.056362, + "nauc_precision_at_3_diff1": 0.286949, + "nauc_precision_at_5_max": -0.095151, + "nauc_precision_at_5_std": 0.003358, + "nauc_precision_at_5_diff1": 0.233939, + "nauc_precision_at_10_max": -0.139927, + "nauc_precision_at_10_std": 0.068511, + "nauc_precision_at_10_diff1": 0.063852, + "nauc_precision_at_20_max": -0.245354, + "nauc_precision_at_20_std": 0.082413, + "nauc_precision_at_20_diff1": -0.072759, + "nauc_precision_at_100_max": -0.252362, + "nauc_precision_at_100_std": 0.258976, + "nauc_precision_at_100_diff1": -0.221202, + "nauc_precision_at_1000_max": -0.252362, + "nauc_precision_at_1000_std": 0.258976, + "nauc_precision_at_1000_diff1": -0.221202, + "nauc_cv_recall_at_1_max": -0.114947, + "nauc_cv_recall_at_1_std": -0.114897, + "nauc_cv_recall_at_1_diff1": 0.484341, + "nauc_cv_recall_at_3_max": 0.014216, + "nauc_cv_recall_at_3_std": 0.221295, + "nauc_cv_recall_at_3_diff1": 0.28051, + "nauc_cv_recall_at_5_max": 0.041382, + "nauc_cv_recall_at_5_std": 0.263319, + "nauc_cv_recall_at_5_diff1": 0.430566, + "nauc_cv_recall_at_10_max": 0.096647, + "nauc_cv_recall_at_10_std": 0.419551, + "nauc_cv_recall_at_10_diff1": 0.320557, + "nauc_cv_recall_at_20_max": 0.081662, + "nauc_cv_recall_at_20_std": 0.351784, + "nauc_cv_recall_at_20_diff1": 0.417528, + "nauc_cv_recall_at_100_max": 0.722732, + "nauc_cv_recall_at_100_std": 0.868512, + "nauc_cv_recall_at_100_diff1": -0.164944, + "nauc_cv_recall_at_1000_max": 0.722732, + "nauc_cv_recall_at_1000_std": 0.868512, + "nauc_cv_recall_at_1000_diff1": -0.164944, + "nauc_mrr_at_1_max": -0.114947, + "nauc_mrr_at_1_std": -0.114897, + "nauc_mrr_at_1_diff1": 0.484341, + "nauc_mrr_at_3_max": -0.055118, + "nauc_mrr_at_3_std": 0.006536, + "nauc_mrr_at_3_diff1": 0.403457, + "nauc_mrr_at_5_max": -0.051842, + "nauc_mrr_at_5_std": 0.001954, + "nauc_mrr_at_5_diff1": 0.426453, + "nauc_mrr_at_10_max": -0.051008, + "nauc_mrr_at_10_std": -0.003063, + "nauc_mrr_at_10_diff1": 0.424696, + "nauc_mrr_at_20_max": -0.051965, + "nauc_mrr_at_20_std": -0.006071, + "nauc_mrr_at_20_diff1": 0.426302, + "nauc_mrr_at_100_max": -0.051198, + "nauc_mrr_at_100_std": -0.005831, + "nauc_mrr_at_100_diff1": 0.425435, + "nauc_mrr_at_1000_max": -0.051198, + "nauc_mrr_at_1000_std": -0.005831, + "nauc_mrr_at_1000_diff1": 0.425435, + "main_score": 0.57668, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.5614, + "ndcg_at_3": 0.5034, + "ndcg_at_5": 0.54419, + "ndcg_at_10": 0.59298, + "ndcg_at_20": 0.62628, + "ndcg_at_100": 0.65693, + "ndcg_at_1000": 0.65693, + "map_at_1": 0.26759, + "map_at_3": 0.37585, + "map_at_5": 0.42452, + "map_at_10": 0.47037, + "map_at_20": 0.49233, + "map_at_100": 0.50947, + "map_at_1000": 0.50947, + "recall_at_1": 0.26759, + "recall_at_3": 0.442, + "recall_at_5": 0.55261, + "recall_at_10": 0.70591, + "recall_at_20": 0.81621, + "recall_at_100": 0.90455, + "recall_at_1000": 0.90455, + "cv_recall_at_1": 0.5614, + "cv_recall_at_3": 0.7193, + "cv_recall_at_5": 0.82456, + "cv_recall_at_10": 0.89474, + "cv_recall_at_20": 0.96491, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 0.98246, + "precision_at_1": 0.5614, + "precision_at_3": 0.32749, + "precision_at_5": 0.2807, + "precision_at_10": 0.2, + "precision_at_20": 0.12456, + "precision_at_100": 0.03421, + "precision_at_1000": 0.00342, + "mrr_at_1": 0.561404, + "mrr_at_3": 0.631579, + "mrr_at_5": 0.65614, + "mrr_at_10": 0.665713, + "mrr_at_20": 0.671618, + "mrr_at_100": 0.672079, + "mrr_at_1000": 0.672079, + "nauc_ndcg_at_1_max": -0.016325, + "nauc_ndcg_at_1_std": -0.14001, + "nauc_ndcg_at_1_diff1": 0.399107, + "nauc_ndcg_at_3_max": 0.124999, + "nauc_ndcg_at_3_std": -0.067006, + "nauc_ndcg_at_3_diff1": 0.237167, + "nauc_ndcg_at_5_max": 0.139707, + "nauc_ndcg_at_5_std": -0.031052, + "nauc_ndcg_at_5_diff1": 0.234746, + "nauc_ndcg_at_10_max": 0.137091, + "nauc_ndcg_at_10_std": -0.050406, + "nauc_ndcg_at_10_diff1": 0.390236, + "nauc_ndcg_at_20_max": 0.122446, + "nauc_ndcg_at_20_std": -0.044703, + "nauc_ndcg_at_20_diff1": 0.365239, + "nauc_ndcg_at_100_max": 0.113475, + "nauc_ndcg_at_100_std": 0.021053, + "nauc_ndcg_at_100_diff1": 0.311824, + "nauc_ndcg_at_1000_max": 0.113475, + "nauc_ndcg_at_1000_std": 0.021053, + "nauc_ndcg_at_1000_diff1": 0.311824, + "nauc_map_at_1_max": 0.01339, + "nauc_map_at_1_std": -0.067946, + "nauc_map_at_1_diff1": 0.280659, + "nauc_map_at_3_max": 0.085156, + "nauc_map_at_3_std": -0.075814, + "nauc_map_at_3_diff1": 0.225273, + "nauc_map_at_5_max": 0.112613, + "nauc_map_at_5_std": -0.057962, + "nauc_map_at_5_diff1": 0.236008, + "nauc_map_at_10_max": 0.113305, + "nauc_map_at_10_std": -0.070399, + "nauc_map_at_10_diff1": 0.313832, + "nauc_map_at_20_max": 0.107307, + "nauc_map_at_20_std": -0.060258, + "nauc_map_at_20_diff1": 0.309605, + "nauc_map_at_100_max": 0.100127, + "nauc_map_at_100_std": -0.029917, + "nauc_map_at_100_diff1": 0.290294, + "nauc_map_at_1000_max": 0.100127, + "nauc_map_at_1000_std": -0.029917, + "nauc_map_at_1000_diff1": 0.290294, + "nauc_recall_at_1_max": 0.01339, + "nauc_recall_at_1_std": -0.067946, + "nauc_recall_at_1_diff1": 0.280659, + "nauc_recall_at_3_max": 0.176945, + "nauc_recall_at_3_std": -0.006026, + "nauc_recall_at_3_diff1": 0.129627, + "nauc_recall_at_5_max": 0.228259, + "nauc_recall_at_5_std": -0.018017, + "nauc_recall_at_5_diff1": 0.159646, + "nauc_recall_at_10_max": 0.212827, + "nauc_recall_at_10_std": -0.089216, + "nauc_recall_at_10_diff1": 0.566947, + "nauc_recall_at_20_max": 0.214208, + "nauc_recall_at_20_std": -0.050267, + "nauc_recall_at_20_diff1": 0.47785, + "nauc_recall_at_100_max": 0.250948, + "nauc_recall_at_100_std": 0.285721, + "nauc_recall_at_100_diff1": 0.224867, + "nauc_recall_at_1000_max": 0.250948, + "nauc_recall_at_1000_std": 0.285721, + "nauc_recall_at_1000_diff1": 0.224867, + "nauc_precision_at_1_max": -0.016325, + "nauc_precision_at_1_std": -0.14001, + "nauc_precision_at_1_diff1": 0.399107, + "nauc_precision_at_3_max": 0.073823, + "nauc_precision_at_3_std": -0.099318, + "nauc_precision_at_3_diff1": 0.082554, + "nauc_precision_at_5_max": 0.109566, + "nauc_precision_at_5_std": 0.053649, + "nauc_precision_at_5_diff1": 0.074461, + "nauc_precision_at_10_max": 0.062934, + "nauc_precision_at_10_std": 0.089535, + "nauc_precision_at_10_diff1": 0.158651, + "nauc_precision_at_20_max": 0.002901, + "nauc_precision_at_20_std": 0.144324, + "nauc_precision_at_20_diff1": 0.049004, + "nauc_precision_at_100_max": -0.053766, + "nauc_precision_at_100_std": 0.253695, + "nauc_precision_at_100_diff1": -0.114112, + "nauc_precision_at_1000_max": -0.053766, + "nauc_precision_at_1000_std": 0.253695, + "nauc_precision_at_1000_diff1": -0.114112, + "nauc_cv_recall_at_1_max": -0.016325, + "nauc_cv_recall_at_1_std": -0.14001, + "nauc_cv_recall_at_1_diff1": 0.399107, + "nauc_cv_recall_at_3_max": 0.206909, + "nauc_cv_recall_at_3_std": 0.04329, + "nauc_cv_recall_at_3_diff1": 0.260676, + "nauc_cv_recall_at_5_max": 0.243712, + "nauc_cv_recall_at_5_std": 0.11575, + "nauc_cv_recall_at_5_diff1": 0.211526, + "nauc_cv_recall_at_10_max": 0.31778, + "nauc_cv_recall_at_10_std": 0.233029, + "nauc_cv_recall_at_10_diff1": 0.743384, + "nauc_cv_recall_at_20_max": 0.795622, + "nauc_cv_recall_at_20_std": 0.722732, + "nauc_cv_recall_at_20_diff1": 0.722732, + "nauc_cv_recall_at_100_max": 0.722732, + "nauc_cv_recall_at_100_std": 0.722732, + "nauc_cv_recall_at_100_diff1": 0.722732, + "nauc_cv_recall_at_1000_max": 0.722732, + "nauc_cv_recall_at_1000_std": 0.722732, + "nauc_cv_recall_at_1000_diff1": 0.722732, + "nauc_mrr_at_1_max": -0.016325, + "nauc_mrr_at_1_std": -0.14001, + "nauc_mrr_at_1_diff1": 0.399107, + "nauc_mrr_at_3_max": 0.060212, + "nauc_mrr_at_3_std": -0.091618, + "nauc_mrr_at_3_diff1": 0.358018, + "nauc_mrr_at_5_max": 0.057882, + "nauc_mrr_at_5_std": -0.084858, + "nauc_mrr_at_5_diff1": 0.355708, + "nauc_mrr_at_10_max": 0.056581, + "nauc_mrr_at_10_std": -0.08127, + "nauc_mrr_at_10_diff1": 0.383461, + "nauc_mrr_at_20_max": 0.055498, + "nauc_mrr_at_20_std": -0.083261, + "nauc_mrr_at_20_diff1": 0.375087, + "nauc_mrr_at_100_max": 0.054049, + "nauc_mrr_at_100_std": -0.084697, + "nauc_mrr_at_100_diff1": 0.374468, + "nauc_mrr_at_1000_max": 0.054049, + "nauc_mrr_at_1000_std": -0.084697, + "nauc_mrr_at_1000_diff1": 0.374468, + "main_score": 0.54419, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 885.0184173583984, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..b9c3809a3d --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.55172, + "ndcg_at_3": 0.56662, + "ndcg_at_5": 0.52993, + "ndcg_at_10": 0.52908, + "ndcg_at_20": 0.55629, + "ndcg_at_100": 0.67428, + "ndcg_at_1000": 0.70862, + "map_at_1": 0.07656, + "map_at_3": 0.1792, + "map_at_5": 0.21917, + "map_at_10": 0.2895, + "map_at_20": 0.34436, + "map_at_100": 0.43764, + "map_at_1000": 0.4533, + "recall_at_1": 0.07656, + "recall_at_3": 0.23012, + "recall_at_5": 0.29378, + "recall_at_10": 0.42487, + "recall_at_20": 0.5606, + "recall_at_100": 0.9072, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.55172, + "cv_recall_at_3": 0.86207, + "cv_recall_at_5": 0.89655, + "cv_recall_at_10": 0.93103, + "cv_recall_at_20": 0.94828, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.55172, + "precision_at_3": 0.52874, + "precision_at_5": 0.45862, + "precision_at_10": 0.39138, + "precision_at_20": 0.3, + "precision_at_100": 0.12759, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.551724, + "mrr_at_3": 0.698276, + "mrr_at_5": 0.705172, + "mrr_at_10": 0.709962, + "mrr_at_20": 0.711398, + "mrr_at_100": 0.713642, + "mrr_at_1000": 0.713642, + "nauc_ndcg_at_1_max": -0.13945, + "nauc_ndcg_at_1_std": -0.254139, + "nauc_ndcg_at_1_diff1": 0.123838, + "nauc_ndcg_at_3_max": 0.012805, + "nauc_ndcg_at_3_std": -0.127767, + "nauc_ndcg_at_3_diff1": 0.119879, + "nauc_ndcg_at_5_max": -0.074053, + "nauc_ndcg_at_5_std": -0.214036, + "nauc_ndcg_at_5_diff1": 0.222478, + "nauc_ndcg_at_10_max": -0.008841, + "nauc_ndcg_at_10_std": -0.138904, + "nauc_ndcg_at_10_diff1": 0.118799, + "nauc_ndcg_at_20_max": 0.05861, + "nauc_ndcg_at_20_std": -0.053653, + "nauc_ndcg_at_20_diff1": 0.049375, + "nauc_ndcg_at_100_max": 0.024053, + "nauc_ndcg_at_100_std": -0.07898, + "nauc_ndcg_at_100_diff1": 0.078993, + "nauc_ndcg_at_1000_max": -0.030748, + "nauc_ndcg_at_1000_std": -0.139754, + "nauc_ndcg_at_1000_diff1": 0.108056, + "nauc_map_at_1_max": -0.189739, + "nauc_map_at_1_std": -0.224906, + "nauc_map_at_1_diff1": 0.244719, + "nauc_map_at_3_max": -0.037096, + "nauc_map_at_3_std": -0.129389, + "nauc_map_at_3_diff1": 0.094903, + "nauc_map_at_5_max": -0.034088, + "nauc_map_at_5_std": -0.143952, + "nauc_map_at_5_diff1": 0.136838, + "nauc_map_at_10_max": 0.073555, + "nauc_map_at_10_std": -0.058209, + "nauc_map_at_10_diff1": 0.060227, + "nauc_map_at_20_max": 0.098204, + "nauc_map_at_20_std": -0.03754, + "nauc_map_at_20_diff1": 0.091559, + "nauc_map_at_100_max": 0.017701, + "nauc_map_at_100_std": -0.093303, + "nauc_map_at_100_diff1": 0.096803, + "nauc_map_at_1000_max": -0.011396, + "nauc_map_at_1000_std": -0.115707, + "nauc_map_at_1000_diff1": 0.101345, + "nauc_recall_at_1_max": -0.189739, + "nauc_recall_at_1_std": -0.224906, + "nauc_recall_at_1_diff1": 0.244719, + "nauc_recall_at_3_max": -0.040661, + "nauc_recall_at_3_std": -0.109397, + "nauc_recall_at_3_diff1": 0.009552, + "nauc_recall_at_5_max": -0.074634, + "nauc_recall_at_5_std": -0.161324, + "nauc_recall_at_5_diff1": 0.087517, + "nauc_recall_at_10_max": 0.107137, + "nauc_recall_at_10_std": -0.003976, + "nauc_recall_at_10_diff1": -0.032272, + "nauc_recall_at_20_max": 0.203911, + "nauc_recall_at_20_std": 0.08092, + "nauc_recall_at_20_diff1": -0.031434, + "nauc_recall_at_100_max": 0.288257, + "nauc_recall_at_100_std": 0.289076, + "nauc_recall_at_100_diff1": -0.070682, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.13945, + "nauc_precision_at_1_std": -0.254139, + "nauc_precision_at_1_diff1": 0.123838, + "nauc_precision_at_3_max": 0.116545, + "nauc_precision_at_3_std": -0.020362, + "nauc_precision_at_3_diff1": 0.051411, + "nauc_precision_at_5_max": 0.012211, + "nauc_precision_at_5_std": -0.113515, + "nauc_precision_at_5_diff1": 0.165787, + "nauc_precision_at_10_max": 0.084352, + "nauc_precision_at_10_std": 0.007751, + "nauc_precision_at_10_diff1": 0.048613, + "nauc_precision_at_20_max": 0.018801, + "nauc_precision_at_20_std": -0.017531, + "nauc_precision_at_20_diff1": 0.072917, + "nauc_precision_at_100_max": -0.18297, + "nauc_precision_at_100_std": -0.143694, + "nauc_precision_at_100_diff1": 0.02561, + "nauc_precision_at_1000_max": -0.20206, + "nauc_precision_at_1000_std": -0.168427, + "nauc_precision_at_1000_diff1": 0.0502, + "nauc_cv_recall_at_1_max": -0.13945, + "nauc_cv_recall_at_1_std": -0.254139, + "nauc_cv_recall_at_1_diff1": 0.123838, + "nauc_cv_recall_at_3_max": 0.003736, + "nauc_cv_recall_at_3_std": -0.092728, + "nauc_cv_recall_at_3_diff1": -0.238373, + "nauc_cv_recall_at_5_max": -0.232314, + "nauc_cv_recall_at_5_std": -0.452915, + "nauc_cv_recall_at_5_diff1": 0.168343, + "nauc_cv_recall_at_10_max": 0.122646, + "nauc_cv_recall_at_10_std": -0.208256, + "nauc_cv_recall_at_10_diff1": -0.01736, + "nauc_cv_recall_at_20_max": 0.043843, + "nauc_cv_recall_at_20_std": -0.220036, + "nauc_cv_recall_at_20_diff1": 0.034492, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.13945, + "nauc_mrr_at_1_std": -0.254139, + "nauc_mrr_at_1_diff1": 0.123838, + "nauc_mrr_at_3_max": -0.098517, + "nauc_mrr_at_3_std": -0.210881, + "nauc_mrr_at_3_diff1": 0.020256, + "nauc_mrr_at_5_max": -0.122021, + "nauc_mrr_at_5_std": -0.245774, + "nauc_mrr_at_5_diff1": 0.062865, + "nauc_mrr_at_10_max": -0.108089, + "nauc_mrr_at_10_std": -0.23443, + "nauc_mrr_at_10_diff1": 0.05365, + "nauc_mrr_at_20_max": -0.111038, + "nauc_mrr_at_20_std": -0.234818, + "nauc_mrr_at_20_diff1": 0.05508, + "nauc_mrr_at_100_max": -0.113428, + "nauc_mrr_at_100_std": -0.235616, + "nauc_mrr_at_100_diff1": 0.05572, + "nauc_mrr_at_1000_max": -0.113428, + "nauc_mrr_at_1000_std": -0.235616, + "nauc_mrr_at_1000_diff1": 0.05572, + "main_score": 0.52993, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.58621, + "ndcg_at_3": 0.59578, + "ndcg_at_5": 0.57123, + "ndcg_at_10": 0.55393, + "ndcg_at_20": 0.575, + "ndcg_at_100": 0.68894, + "ndcg_at_1000": 0.72468, + "map_at_1": 0.09427, + "map_at_3": 0.19307, + "map_at_5": 0.24272, + "map_at_10": 0.30662, + "map_at_20": 0.36038, + "map_at_100": 0.45204, + "map_at_1000": 0.46897, + "recall_at_1": 0.09427, + "recall_at_3": 0.23923, + "recall_at_5": 0.31242, + "recall_at_10": 0.44054, + "recall_at_20": 0.56536, + "recall_at_100": 0.90676, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.58621, + "cv_recall_at_3": 0.87931, + "cv_recall_at_5": 0.91379, + "cv_recall_at_10": 0.98276, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.58621, + "precision_at_3": 0.55747, + "precision_at_5": 0.5, + "precision_at_10": 0.40172, + "precision_at_20": 0.30172, + "precision_at_100": 0.12603, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.586207, + "mrr_at_3": 0.718391, + "mrr_at_5": 0.727011, + "mrr_at_10": 0.736227, + "mrr_at_20": 0.736227, + "mrr_at_100": 0.736681, + "mrr_at_1000": 0.736681, + "nauc_ndcg_at_1_max": 0.000856, + "nauc_ndcg_at_1_std": -0.098766, + "nauc_ndcg_at_1_diff1": 0.035971, + "nauc_ndcg_at_3_max": -0.100107, + "nauc_ndcg_at_3_std": -0.21168, + "nauc_ndcg_at_3_diff1": 0.227953, + "nauc_ndcg_at_5_max": -0.071474, + "nauc_ndcg_at_5_std": -0.148046, + "nauc_ndcg_at_5_diff1": 0.275856, + "nauc_ndcg_at_10_max": -0.097041, + "nauc_ndcg_at_10_std": -0.173456, + "nauc_ndcg_at_10_diff1": 0.182833, + "nauc_ndcg_at_20_max": -0.042543, + "nauc_ndcg_at_20_std": -0.127991, + "nauc_ndcg_at_20_diff1": 0.060441, + "nauc_ndcg_at_100_max": -0.072567, + "nauc_ndcg_at_100_std": -0.118894, + "nauc_ndcg_at_100_diff1": 0.074303, + "nauc_ndcg_at_1000_max": -0.091097, + "nauc_ndcg_at_1000_std": -0.124763, + "nauc_ndcg_at_1000_diff1": 0.139007, + "nauc_map_at_1_max": -0.18154, + "nauc_map_at_1_std": -0.187879, + "nauc_map_at_1_diff1": 0.047645, + "nauc_map_at_3_max": -0.125321, + "nauc_map_at_3_std": -0.238588, + "nauc_map_at_3_diff1": 0.142996, + "nauc_map_at_5_max": -0.046681, + "nauc_map_at_5_std": -0.180135, + "nauc_map_at_5_diff1": 0.152557, + "nauc_map_at_10_max": -0.032923, + "nauc_map_at_10_std": -0.180612, + "nauc_map_at_10_diff1": 0.109353, + "nauc_map_at_20_max": -0.017334, + "nauc_map_at_20_std": -0.164175, + "nauc_map_at_20_diff1": 0.086686, + "nauc_map_at_100_max": -0.048404, + "nauc_map_at_100_std": -0.147821, + "nauc_map_at_100_diff1": 0.075272, + "nauc_map_at_1000_max": -0.062573, + "nauc_map_at_1000_std": -0.146646, + "nauc_map_at_1000_diff1": 0.082328, + "nauc_recall_at_1_max": -0.18154, + "nauc_recall_at_1_std": -0.187879, + "nauc_recall_at_1_diff1": 0.047645, + "nauc_recall_at_3_max": -0.149524, + "nauc_recall_at_3_std": -0.261787, + "nauc_recall_at_3_diff1": 0.205966, + "nauc_recall_at_5_max": -0.044619, + "nauc_recall_at_5_std": -0.152916, + "nauc_recall_at_5_diff1": 0.197594, + "nauc_recall_at_10_max": -0.024522, + "nauc_recall_at_10_std": -0.151521, + "nauc_recall_at_10_diff1": 0.062059, + "nauc_recall_at_20_max": 0.008684, + "nauc_recall_at_20_std": -0.14672, + "nauc_recall_at_20_diff1": 0.00742, + "nauc_recall_at_100_max": 0.081909, + "nauc_recall_at_100_std": 0.060757, + "nauc_recall_at_100_diff1": -0.25447, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.000856, + "nauc_precision_at_1_std": -0.098766, + "nauc_precision_at_1_diff1": 0.035971, + "nauc_precision_at_3_max": -0.026628, + "nauc_precision_at_3_std": -0.13259, + "nauc_precision_at_3_diff1": 0.173845, + "nauc_precision_at_5_max": 0.062327, + "nauc_precision_at_5_std": 0.006082, + "nauc_precision_at_5_diff1": 0.195548, + "nauc_precision_at_10_max": -0.001504, + "nauc_precision_at_10_std": 0.002767, + "nauc_precision_at_10_diff1": 0.088422, + "nauc_precision_at_20_max": -0.005238, + "nauc_precision_at_20_std": 0.035376, + "nauc_precision_at_20_diff1": -0.001608, + "nauc_precision_at_100_max": -0.088987, + "nauc_precision_at_100_std": 0.023434, + "nauc_precision_at_100_diff1": 0.008017, + "nauc_precision_at_1000_max": -0.100728, + "nauc_precision_at_1000_std": 0.004216, + "nauc_precision_at_1000_diff1": 0.032449, + "nauc_cv_recall_at_1_max": 0.000856, + "nauc_cv_recall_at_1_std": -0.098766, + "nauc_cv_recall_at_1_diff1": 0.035971, + "nauc_cv_recall_at_3_max": -0.678896, + "nauc_cv_recall_at_3_std": -0.47564, + "nauc_cv_recall_at_3_diff1": 0.700211, + "nauc_cv_recall_at_5_max": -0.589192, + "nauc_cv_recall_at_5_std": -0.382914, + "nauc_cv_recall_at_5_diff1": 0.76341, + "nauc_cv_recall_at_10_max": 0.123761, + "nauc_cv_recall_at_10_std": 0.123761, + "nauc_cv_recall_at_10_diff1": 0.720441, + "nauc_cv_recall_at_20_max": 0.123761, + "nauc_cv_recall_at_20_std": 0.123761, + "nauc_cv_recall_at_20_diff1": 0.720441, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.000856, + "nauc_mrr_at_1_std": -0.098766, + "nauc_mrr_at_1_diff1": 0.035971, + "nauc_mrr_at_3_max": -0.168725, + "nauc_mrr_at_3_std": -0.206662, + "nauc_mrr_at_3_diff1": 0.198368, + "nauc_mrr_at_5_max": -0.141641, + "nauc_mrr_at_5_std": -0.187993, + "nauc_mrr_at_5_diff1": 0.186355, + "nauc_mrr_at_10_max": -0.11694, + "nauc_mrr_at_10_std": -0.177253, + "nauc_mrr_at_10_diff1": 0.162288, + "nauc_mrr_at_20_max": -0.11694, + "nauc_mrr_at_20_std": -0.177253, + "nauc_mrr_at_20_diff1": 0.162288, + "nauc_mrr_at_100_max": -0.117452, + "nauc_mrr_at_100_std": -0.177893, + "nauc_mrr_at_100_diff1": 0.161102, + "nauc_mrr_at_1000_max": -0.117452, + "nauc_mrr_at_1000_std": -0.177893, + "nauc_mrr_at_1000_diff1": 0.161102, + "main_score": 0.57123, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.63793, + "ndcg_at_3": 0.61498, + "ndcg_at_5": 0.59105, + "ndcg_at_10": 0.58375, + "ndcg_at_20": 0.61507, + "ndcg_at_100": 0.70934, + "ndcg_at_1000": 0.73935, + "map_at_1": 0.0779, + "map_at_3": 0.19514, + "map_at_5": 0.25171, + "map_at_10": 0.32757, + "map_at_20": 0.39657, + "map_at_100": 0.47946, + "map_at_1000": 0.49522, + "recall_at_1": 0.0779, + "recall_at_3": 0.24119, + "recall_at_5": 0.32904, + "recall_at_10": 0.4642, + "recall_at_20": 0.62843, + "recall_at_100": 0.92393, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.63793, + "cv_recall_at_3": 0.86207, + "cv_recall_at_5": 0.93103, + "cv_recall_at_10": 0.96552, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.63793, + "precision_at_3": 0.56897, + "precision_at_5": 0.52069, + "precision_at_10": 0.43448, + "precision_at_20": 0.33103, + "precision_at_100": 0.12983, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.637931, + "mrr_at_3": 0.744253, + "mrr_at_5": 0.75977, + "mrr_at_10": 0.765517, + "mrr_at_20": 0.766595, + "mrr_at_100": 0.767344, + "mrr_at_1000": 0.767344, + "nauc_ndcg_at_1_max": 0.194061, + "nauc_ndcg_at_1_std": 0.242006, + "nauc_ndcg_at_1_diff1": 0.159659, + "nauc_ndcg_at_3_max": 0.003009, + "nauc_ndcg_at_3_std": 0.110287, + "nauc_ndcg_at_3_diff1": -0.056492, + "nauc_ndcg_at_5_max": -0.013274, + "nauc_ndcg_at_5_std": 0.09112, + "nauc_ndcg_at_5_diff1": 0.01332, + "nauc_ndcg_at_10_max": 0.046758, + "nauc_ndcg_at_10_std": 0.089895, + "nauc_ndcg_at_10_diff1": 0.033046, + "nauc_ndcg_at_20_max": 0.134713, + "nauc_ndcg_at_20_std": 0.135315, + "nauc_ndcg_at_20_diff1": -0.103912, + "nauc_ndcg_at_100_max": 0.102413, + "nauc_ndcg_at_100_std": 0.152723, + "nauc_ndcg_at_100_diff1": -0.189935, + "nauc_ndcg_at_1000_max": 0.079152, + "nauc_ndcg_at_1000_std": 0.145956, + "nauc_ndcg_at_1000_diff1": -0.082405, + "nauc_map_at_1_max": -0.103327, + "nauc_map_at_1_std": -0.108751, + "nauc_map_at_1_diff1": -0.211651, + "nauc_map_at_3_max": -0.096024, + "nauc_map_at_3_std": -0.10239, + "nauc_map_at_3_diff1": -0.179096, + "nauc_map_at_5_max": -0.073032, + "nauc_map_at_5_std": -0.094636, + "nauc_map_at_5_diff1": -0.135056, + "nauc_map_at_10_max": 0.009519, + "nauc_map_at_10_std": -0.046008, + "nauc_map_at_10_diff1": -0.100307, + "nauc_map_at_20_max": 0.067004, + "nauc_map_at_20_std": 0.003899, + "nauc_map_at_20_diff1": -0.139232, + "nauc_map_at_100_max": 0.053054, + "nauc_map_at_100_std": 0.0834, + "nauc_map_at_100_diff1": -0.187585, + "nauc_map_at_1000_max": 0.03404, + "nauc_map_at_1000_std": 0.091824, + "nauc_map_at_1000_diff1": -0.173712, + "nauc_recall_at_1_max": -0.103327, + "nauc_recall_at_1_std": -0.108751, + "nauc_recall_at_1_diff1": -0.211651, + "nauc_recall_at_3_max": -0.109513, + "nauc_recall_at_3_std": -0.146893, + "nauc_recall_at_3_diff1": -0.12839, + "nauc_recall_at_5_max": -0.114406, + "nauc_recall_at_5_std": -0.177811, + "nauc_recall_at_5_diff1": -0.059938, + "nauc_recall_at_10_max": 0.035983, + "nauc_recall_at_10_std": -0.088592, + "nauc_recall_at_10_diff1": -0.009115, + "nauc_recall_at_20_max": 0.17223, + "nauc_recall_at_20_std": -0.000316, + "nauc_recall_at_20_diff1": -0.165141, + "nauc_recall_at_100_max": 0.169329, + "nauc_recall_at_100_std": 0.116986, + "nauc_recall_at_100_diff1": -0.594093, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.194061, + "nauc_precision_at_1_std": 0.242006, + "nauc_precision_at_1_diff1": 0.159659, + "nauc_precision_at_3_max": 0.046983, + "nauc_precision_at_3_std": 0.176782, + "nauc_precision_at_3_diff1": -0.066478, + "nauc_precision_at_5_max": 0.066243, + "nauc_precision_at_5_std": 0.185362, + "nauc_precision_at_5_diff1": 0.041136, + "nauc_precision_at_10_max": 0.14604, + "nauc_precision_at_10_std": 0.203298, + "nauc_precision_at_10_diff1": 0.124301, + "nauc_precision_at_20_max": 0.129679, + "nauc_precision_at_20_std": 0.21553, + "nauc_precision_at_20_diff1": 0.008206, + "nauc_precision_at_100_max": -0.040693, + "nauc_precision_at_100_std": 0.150691, + "nauc_precision_at_100_diff1": -0.003701, + "nauc_precision_at_1000_max": -0.065004, + "nauc_precision_at_1000_std": 0.134248, + "nauc_precision_at_1000_diff1": 0.024678, + "nauc_cv_recall_at_1_max": 0.194061, + "nauc_cv_recall_at_1_std": 0.242006, + "nauc_cv_recall_at_1_diff1": 0.159659, + "nauc_cv_recall_at_3_max": 0.281738, + "nauc_cv_recall_at_3_std": 0.354176, + "nauc_cv_recall_at_3_diff1": 0.009395, + "nauc_cv_recall_at_5_max": 0.068345, + "nauc_cv_recall_at_5_std": 0.171385, + "nauc_cv_recall_at_5_diff1": -0.164158, + "nauc_cv_recall_at_10_max": -0.211243, + "nauc_cv_recall_at_10_std": -0.294456, + "nauc_cv_recall_at_10_diff1": 0.613917, + "nauc_cv_recall_at_20_max": 0.720441, + "nauc_cv_recall_at_20_std": 0.554013, + "nauc_cv_recall_at_20_diff1": 0.359055, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.194061, + "nauc_mrr_at_1_std": 0.242006, + "nauc_mrr_at_1_diff1": 0.159659, + "nauc_mrr_at_3_max": 0.227267, + "nauc_mrr_at_3_std": 0.274589, + "nauc_mrr_at_3_diff1": 0.124338, + "nauc_mrr_at_5_max": 0.207584, + "nauc_mrr_at_5_std": 0.255735, + "nauc_mrr_at_5_diff1": 0.114695, + "nauc_mrr_at_10_max": 0.203518, + "nauc_mrr_at_10_std": 0.244683, + "nauc_mrr_at_10_diff1": 0.145314, + "nauc_mrr_at_20_max": 0.210872, + "nauc_mrr_at_20_std": 0.252262, + "nauc_mrr_at_20_diff1": 0.141363, + "nauc_mrr_at_100_max": 0.208928, + "nauc_mrr_at_100_std": 0.251111, + "nauc_mrr_at_100_diff1": 0.140533, + "nauc_mrr_at_1000_max": 0.208928, + "nauc_mrr_at_1000_std": 0.251111, + "nauc_mrr_at_1000_diff1": 0.140533, + "main_score": 0.59105, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.60345, + "ndcg_at_3": 0.59157, + "ndcg_at_5": 0.56539, + "ndcg_at_10": 0.53962, + "ndcg_at_20": 0.57778, + "ndcg_at_100": 0.68091, + "ndcg_at_1000": 0.72124, + "map_at_1": 0.08125, + "map_at_3": 0.18623, + "map_at_5": 0.24229, + "map_at_10": 0.30623, + "map_at_20": 0.36571, + "map_at_100": 0.44706, + "map_at_1000": 0.46505, + "recall_at_1": 0.08125, + "recall_at_3": 0.23472, + "recall_at_5": 0.31417, + "recall_at_10": 0.4318, + "recall_at_20": 0.59276, + "recall_at_100": 0.89549, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.60345, + "cv_recall_at_3": 0.86207, + "cv_recall_at_5": 0.93103, + "cv_recall_at_10": 0.96552, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.60345, + "precision_at_3": 0.55172, + "precision_at_5": 0.5, + "precision_at_10": 0.38621, + "precision_at_20": 0.3, + "precision_at_100": 0.12328, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.603448, + "mrr_at_3": 0.721264, + "mrr_at_5": 0.737644, + "mrr_at_10": 0.742672, + "mrr_at_20": 0.745317, + "mrr_at_100": 0.745317, + "mrr_at_1000": 0.745317, + "nauc_ndcg_at_1_max": 0.104166, + "nauc_ndcg_at_1_std": 0.138854, + "nauc_ndcg_at_1_diff1": 0.092541, + "nauc_ndcg_at_3_max": 0.050067, + "nauc_ndcg_at_3_std": 0.137106, + "nauc_ndcg_at_3_diff1": 0.043122, + "nauc_ndcg_at_5_max": 0.091989, + "nauc_ndcg_at_5_std": 0.164945, + "nauc_ndcg_at_5_diff1": 0.050203, + "nauc_ndcg_at_10_max": 0.094616, + "nauc_ndcg_at_10_std": 0.190274, + "nauc_ndcg_at_10_diff1": -0.03391, + "nauc_ndcg_at_20_max": 0.094185, + "nauc_ndcg_at_20_std": 0.202324, + "nauc_ndcg_at_20_diff1": -0.073236, + "nauc_ndcg_at_100_max": 0.038189, + "nauc_ndcg_at_100_std": 0.150077, + "nauc_ndcg_at_100_diff1": -0.084118, + "nauc_ndcg_at_1000_max": 0.079649, + "nauc_ndcg_at_1000_std": 0.217489, + "nauc_ndcg_at_1000_diff1": 0.014798, + "nauc_map_at_1_max": -0.179877, + "nauc_map_at_1_std": -0.209588, + "nauc_map_at_1_diff1": 0.305351, + "nauc_map_at_3_max": -0.11668, + "nauc_map_at_3_std": -0.163011, + "nauc_map_at_3_diff1": 0.06549, + "nauc_map_at_5_max": -0.064036, + "nauc_map_at_5_std": -0.097233, + "nauc_map_at_5_diff1": 0.012776, + "nauc_map_at_10_max": -0.006609, + "nauc_map_at_10_std": -0.042655, + "nauc_map_at_10_diff1": -0.078878, + "nauc_map_at_20_max": 0.014385, + "nauc_map_at_20_std": -0.006864, + "nauc_map_at_20_diff1": -0.060256, + "nauc_map_at_100_max": 0.014215, + "nauc_map_at_100_std": 0.101036, + "nauc_map_at_100_diff1": -0.083252, + "nauc_map_at_1000_max": 0.026103, + "nauc_map_at_1000_std": 0.155038, + "nauc_map_at_1000_diff1": -0.068024, + "nauc_recall_at_1_max": -0.179877, + "nauc_recall_at_1_std": -0.209588, + "nauc_recall_at_1_diff1": 0.305351, + "nauc_recall_at_3_max": -0.095585, + "nauc_recall_at_3_std": -0.171299, + "nauc_recall_at_3_diff1": 0.033364, + "nauc_recall_at_5_max": -0.041874, + "nauc_recall_at_5_std": -0.104901, + "nauc_recall_at_5_diff1": -0.012248, + "nauc_recall_at_10_max": -0.013782, + "nauc_recall_at_10_std": -0.087164, + "nauc_recall_at_10_diff1": -0.154722, + "nauc_recall_at_20_max": 0.018438, + "nauc_recall_at_20_std": -0.0833, + "nauc_recall_at_20_diff1": -0.15601, + "nauc_recall_at_100_max": -0.133375, + "nauc_recall_at_100_std": -0.130248, + "nauc_recall_at_100_diff1": -0.390272, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.104166, + "nauc_precision_at_1_std": 0.138854, + "nauc_precision_at_1_diff1": 0.092541, + "nauc_precision_at_3_max": 0.060665, + "nauc_precision_at_3_std": 0.207123, + "nauc_precision_at_3_diff1": -0.045908, + "nauc_precision_at_5_max": 0.163356, + "nauc_precision_at_5_std": 0.282216, + "nauc_precision_at_5_diff1": -0.057401, + "nauc_precision_at_10_max": 0.208324, + "nauc_precision_at_10_std": 0.361727, + "nauc_precision_at_10_diff1": -0.112208, + "nauc_precision_at_20_max": 0.151973, + "nauc_precision_at_20_std": 0.371252, + "nauc_precision_at_20_diff1": -0.034508, + "nauc_precision_at_100_max": 0.060601, + "nauc_precision_at_100_std": 0.390833, + "nauc_precision_at_100_diff1": 0.021369, + "nauc_precision_at_1000_max": 0.076272, + "nauc_precision_at_1000_std": 0.41123, + "nauc_precision_at_1000_diff1": 0.056279, + "nauc_cv_recall_at_1_max": 0.104166, + "nauc_cv_recall_at_1_std": 0.138854, + "nauc_cv_recall_at_1_diff1": 0.092541, + "nauc_cv_recall_at_3_max": 0.383172, + "nauc_cv_recall_at_3_std": 0.494899, + "nauc_cv_recall_at_3_diff1": 0.097447, + "nauc_cv_recall_at_5_max": 0.313542, + "nauc_cv_recall_at_5_std": 0.566748, + "nauc_cv_recall_at_5_diff1": 0.44384, + "nauc_cv_recall_at_10_max": -0.010143, + "nauc_cv_recall_at_10_std": 0.338887, + "nauc_cv_recall_at_10_diff1": 0.093071, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.104166, + "nauc_mrr_at_1_std": 0.138854, + "nauc_mrr_at_1_diff1": 0.092541, + "nauc_mrr_at_3_max": 0.202266, + "nauc_mrr_at_3_std": 0.250957, + "nauc_mrr_at_3_diff1": 0.086268, + "nauc_mrr_at_5_max": 0.186012, + "nauc_mrr_at_5_std": 0.24046, + "nauc_mrr_at_5_diff1": 0.115958, + "nauc_mrr_at_10_max": 0.175067, + "nauc_mrr_at_10_std": 0.227614, + "nauc_mrr_at_10_diff1": 0.10017, + "nauc_mrr_at_20_max": 0.178713, + "nauc_mrr_at_20_std": 0.226715, + "nauc_mrr_at_20_diff1": 0.100879, + "nauc_mrr_at_100_max": 0.178713, + "nauc_mrr_at_100_std": 0.226715, + "nauc_mrr_at_100_diff1": 0.100879, + "nauc_mrr_at_1000_max": 0.178713, + "nauc_mrr_at_1000_std": 0.226715, + "nauc_mrr_at_1000_diff1": 0.100879, + "main_score": 0.56539, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 147.54693341255188, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..890504211c --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", + "task_name": "VidoreArxivQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.854, + "ndcg_at_3": 0.88393, + "ndcg_at_5": 0.89124, + "ndcg_at_10": 0.90087, + "ndcg_at_20": 0.907, + "ndcg_at_100": 0.91018, + "ndcg_at_1000": 0.91092, + "map_at_1": 0.854, + "map_at_3": 0.87633, + "map_at_5": 0.88033, + "map_at_10": 0.88427, + "map_at_20": 0.88598, + "map_at_100": 0.88652, + "map_at_1000": 0.88654, + "recall_at_1": 0.854, + "recall_at_3": 0.906, + "recall_at_5": 0.924, + "recall_at_10": 0.954, + "recall_at_20": 0.978, + "recall_at_100": 0.994, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.854, + "cv_recall_at_3": 0.906, + "cv_recall_at_5": 0.924, + "cv_recall_at_10": 0.954, + "cv_recall_at_20": 0.978, + "cv_recall_at_100": 0.994, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.854, + "precision_at_3": 0.302, + "precision_at_5": 0.1848, + "precision_at_10": 0.0954, + "precision_at_20": 0.0489, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "mrr_at_1": 0.854, + "mrr_at_3": 0.876333, + "mrr_at_5": 0.880333, + "mrr_at_10": 0.884274, + "mrr_at_20": 0.885981, + "mrr_at_100": 0.886515, + "mrr_at_1000": 0.886537, + "nauc_ndcg_at_1_max": 0.808499, + "nauc_ndcg_at_1_std": 0.083648, + "nauc_ndcg_at_1_diff1": 0.943305, + "nauc_ndcg_at_3_max": 0.811782, + "nauc_ndcg_at_3_std": 0.104489, + "nauc_ndcg_at_3_diff1": 0.928494, + "nauc_ndcg_at_5_max": 0.802742, + "nauc_ndcg_at_5_std": 0.075222, + "nauc_ndcg_at_5_diff1": 0.925595, + "nauc_ndcg_at_10_max": 0.796756, + "nauc_ndcg_at_10_std": 0.048242, + "nauc_ndcg_at_10_diff1": 0.929898, + "nauc_ndcg_at_20_max": 0.79902, + "nauc_ndcg_at_20_std": 0.072761, + "nauc_ndcg_at_20_diff1": 0.929745, + "nauc_ndcg_at_100_max": 0.796629, + "nauc_ndcg_at_100_std": 0.078642, + "nauc_ndcg_at_100_diff1": 0.931001, + "nauc_ndcg_at_1000_max": 0.800447, + "nauc_ndcg_at_1000_std": 0.07789, + "nauc_ndcg_at_1000_diff1": 0.931564, + "nauc_map_at_1_max": 0.808499, + "nauc_map_at_1_std": 0.083648, + "nauc_map_at_1_diff1": 0.943305, + "nauc_map_at_3_max": 0.810501, + "nauc_map_at_3_std": 0.101965, + "nauc_map_at_3_diff1": 0.932604, + "nauc_map_at_5_max": 0.805894, + "nauc_map_at_5_std": 0.087195, + "nauc_map_at_5_diff1": 0.931242, + "nauc_map_at_10_max": 0.804252, + "nauc_map_at_10_std": 0.07741, + "nauc_map_at_10_diff1": 0.933005, + "nauc_map_at_20_max": 0.804763, + "nauc_map_at_20_std": 0.083179, + "nauc_map_at_20_diff1": 0.933052, + "nauc_map_at_100_max": 0.804455, + "nauc_map_at_100_std": 0.083826, + "nauc_map_at_100_diff1": 0.933285, + "nauc_map_at_1000_max": 0.804494, + "nauc_map_at_1000_std": 0.083745, + "nauc_map_at_1000_diff1": 0.933288, + "nauc_recall_at_1_max": 0.808499, + "nauc_recall_at_1_std": 0.083648, + "nauc_recall_at_1_diff1": 0.943305, + "nauc_recall_at_3_max": 0.816705, + "nauc_recall_at_3_std": 0.112055, + "nauc_recall_at_3_diff1": 0.913334, + "nauc_recall_at_5_max": 0.787053, + "nauc_recall_at_5_std": 0.013514, + "nauc_recall_at_5_diff1": 0.900118, + "nauc_recall_at_10_max": 0.733203, + "nauc_recall_at_10_std": -0.193643, + "nauc_recall_at_10_diff1": 0.910993, + "nauc_recall_at_20_max": 0.704864, + "nauc_recall_at_20_std": -0.053349, + "nauc_recall_at_20_diff1": 0.886682, + "nauc_recall_at_100_max": 0.282758, + "nauc_recall_at_100_std": 0.086679, + "nauc_recall_at_100_diff1": 0.851385, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.808499, + "nauc_precision_at_1_std": 0.083648, + "nauc_precision_at_1_diff1": 0.943305, + "nauc_precision_at_3_max": 0.816705, + "nauc_precision_at_3_std": 0.112055, + "nauc_precision_at_3_diff1": 0.913334, + "nauc_precision_at_5_max": 0.787053, + "nauc_precision_at_5_std": 0.013514, + "nauc_precision_at_5_diff1": 0.900118, + "nauc_precision_at_10_max": 0.733203, + "nauc_precision_at_10_std": -0.193643, + "nauc_precision_at_10_diff1": 0.910993, + "nauc_precision_at_20_max": 0.704864, + "nauc_precision_at_20_std": -0.053349, + "nauc_precision_at_20_diff1": 0.886682, + "nauc_precision_at_100_max": 0.282758, + "nauc_precision_at_100_std": 0.086679, + "nauc_precision_at_100_diff1": 0.851385, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_cv_recall_at_1_max": 0.808499, + "nauc_cv_recall_at_1_std": 0.083648, + "nauc_cv_recall_at_1_diff1": 0.943305, + "nauc_cv_recall_at_3_max": 0.816705, + "nauc_cv_recall_at_3_std": 0.112055, + "nauc_cv_recall_at_3_diff1": 0.913334, + "nauc_cv_recall_at_5_max": 0.787053, + "nauc_cv_recall_at_5_std": 0.013514, + "nauc_cv_recall_at_5_diff1": 0.900118, + "nauc_cv_recall_at_10_max": 0.733203, + "nauc_cv_recall_at_10_std": -0.193643, + "nauc_cv_recall_at_10_diff1": 0.910993, + "nauc_cv_recall_at_20_max": 0.704864, + "nauc_cv_recall_at_20_std": -0.053349, + "nauc_cv_recall_at_20_diff1": 0.886682, + "nauc_cv_recall_at_100_max": 0.282758, + "nauc_cv_recall_at_100_std": 0.086679, + "nauc_cv_recall_at_100_diff1": 0.851385, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.808499, + "nauc_mrr_at_1_std": 0.083648, + "nauc_mrr_at_1_diff1": 0.943305, + "nauc_mrr_at_3_max": 0.810501, + "nauc_mrr_at_3_std": 0.101965, + "nauc_mrr_at_3_diff1": 0.932604, + "nauc_mrr_at_5_max": 0.805894, + "nauc_mrr_at_5_std": 0.087195, + "nauc_mrr_at_5_diff1": 0.931242, + "nauc_mrr_at_10_max": 0.804252, + "nauc_mrr_at_10_std": 0.07741, + "nauc_mrr_at_10_diff1": 0.933005, + "nauc_mrr_at_20_max": 0.804763, + "nauc_mrr_at_20_std": 0.083179, + "nauc_mrr_at_20_diff1": 0.933052, + "nauc_mrr_at_100_max": 0.804455, + "nauc_mrr_at_100_std": 0.083826, + "nauc_mrr_at_100_diff1": 0.933285, + "nauc_mrr_at_1000_max": 0.804494, + "nauc_mrr_at_1000_std": 0.083745, + "nauc_mrr_at_1000_diff1": 0.933288, + "main_score": 0.89124, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 55.49970769882202, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..2569d862c8 --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", + "task_name": "VidoreDocVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.55432, + "ndcg_at_3": 0.6159, + "ndcg_at_5": 0.6351, + "ndcg_at_10": 0.65954, + "ndcg_at_20": 0.66631, + "ndcg_at_100": 0.68626, + "ndcg_at_1000": 0.69834, + "map_at_1": 0.54682, + "map_at_3": 0.59713, + "map_at_5": 0.60888, + "map_at_10": 0.61914, + "map_at_20": 0.62136, + "map_at_100": 0.62407, + "map_at_1000": 0.62458, + "recall_at_1": 0.54682, + "recall_at_3": 0.6585, + "recall_at_5": 0.70506, + "recall_at_10": 0.78041, + "recall_at_20": 0.80617, + "recall_at_100": 0.91229, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.55432, + "cv_recall_at_3": 0.66962, + "cv_recall_at_5": 0.71397, + "cv_recall_at_10": 0.78936, + "cv_recall_at_20": 0.81153, + "cv_recall_at_100": 0.92018, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.55432, + "precision_at_3": 0.22469, + "precision_at_5": 0.14501, + "precision_at_10": 0.08115, + "precision_at_20": 0.04224, + "precision_at_100": 0.00976, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.554324, + "mrr_at_3": 0.605691, + "mrr_at_5": 0.616223, + "mrr_at_10": 0.626027, + "mrr_at_20": 0.627495, + "mrr_at_100": 0.629951, + "mrr_at_1000": 0.630331, + "nauc_ndcg_at_1_max": 0.304797, + "nauc_ndcg_at_1_std": 0.211033, + "nauc_ndcg_at_1_diff1": 0.916703, + "nauc_ndcg_at_3_max": 0.253684, + "nauc_ndcg_at_3_std": 0.197645, + "nauc_ndcg_at_3_diff1": 0.90001, + "nauc_ndcg_at_5_max": 0.237987, + "nauc_ndcg_at_5_std": 0.19665, + "nauc_ndcg_at_5_diff1": 0.894517, + "nauc_ndcg_at_10_max": 0.205359, + "nauc_ndcg_at_10_std": 0.169323, + "nauc_ndcg_at_10_diff1": 0.901275, + "nauc_ndcg_at_20_max": 0.199465, + "nauc_ndcg_at_20_std": 0.16372, + "nauc_ndcg_at_20_diff1": 0.901666, + "nauc_ndcg_at_100_max": 0.188775, + "nauc_ndcg_at_100_std": 0.147998, + "nauc_ndcg_at_100_diff1": 0.899607, + "nauc_ndcg_at_1000_max": 0.217281, + "nauc_ndcg_at_1000_std": 0.171259, + "nauc_ndcg_at_1000_diff1": 0.900723, + "nauc_map_at_1_max": 0.315867, + "nauc_map_at_1_std": 0.22286, + "nauc_map_at_1_diff1": 0.927171, + "nauc_map_at_3_max": 0.272334, + "nauc_map_at_3_std": 0.206587, + "nauc_map_at_3_diff1": 0.90843, + "nauc_map_at_5_max": 0.262822, + "nauc_map_at_5_std": 0.205714, + "nauc_map_at_5_diff1": 0.904709, + "nauc_map_at_10_max": 0.250485, + "nauc_map_at_10_std": 0.195318, + "nauc_map_at_10_diff1": 0.907192, + "nauc_map_at_20_max": 0.248737, + "nauc_map_at_20_std": 0.193834, + "nauc_map_at_20_diff1": 0.906888, + "nauc_map_at_100_max": 0.24693, + "nauc_map_at_100_std": 0.191487, + "nauc_map_at_100_diff1": 0.90663, + "nauc_map_at_1000_max": 0.247869, + "nauc_map_at_1000_std": 0.192217, + "nauc_map_at_1000_diff1": 0.906666, + "nauc_recall_at_1_max": 0.315867, + "nauc_recall_at_1_std": 0.22286, + "nauc_recall_at_1_diff1": 0.927171, + "nauc_recall_at_3_max": 0.215122, + "nauc_recall_at_3_std": 0.188111, + "nauc_recall_at_3_diff1": 0.886511, + "nauc_recall_at_5_max": 0.165545, + "nauc_recall_at_5_std": 0.179072, + "nauc_recall_at_5_diff1": 0.866856, + "nauc_recall_at_10_max": 0.012693, + "nauc_recall_at_10_std": 0.058214, + "nauc_recall_at_10_diff1": 0.890162, + "nauc_recall_at_20_max": -0.042708, + "nauc_recall_at_20_std": 0.011549, + "nauc_recall_at_20_diff1": 0.892559, + "nauc_recall_at_100_max": -0.421773, + "nauc_recall_at_100_std": -0.3574, + "nauc_recall_at_100_diff1": 0.861732, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.304797, + "nauc_precision_at_1_std": 0.211033, + "nauc_precision_at_1_diff1": 0.916703, + "nauc_precision_at_3_max": 0.178051, + "nauc_precision_at_3_std": 0.157872, + "nauc_precision_at_3_diff1": 0.838865, + "nauc_precision_at_5_max": 0.111434, + "nauc_precision_at_5_std": 0.13993, + "nauc_precision_at_5_diff1": 0.771034, + "nauc_precision_at_10_max": -0.081416, + "nauc_precision_at_10_std": -0.015102, + "nauc_precision_at_10_diff1": 0.669798, + "nauc_precision_at_20_max": -0.148121, + "nauc_precision_at_20_std": -0.07406, + "nauc_precision_at_20_diff1": 0.566753, + "nauc_precision_at_100_max": -0.414213, + "nauc_precision_at_100_std": -0.338991, + "nauc_precision_at_100_diff1": 0.139012, + "nauc_precision_at_1000_max": -0.278407, + "nauc_precision_at_1000_std": -0.233596, + "nauc_precision_at_1000_diff1": -0.474455, + "nauc_cv_recall_at_1_max": 0.304797, + "nauc_cv_recall_at_1_std": 0.211033, + "nauc_cv_recall_at_1_diff1": 0.916703, + "nauc_cv_recall_at_3_max": 0.19584, + "nauc_cv_recall_at_3_std": 0.175095, + "nauc_cv_recall_at_3_diff1": 0.876367, + "nauc_cv_recall_at_5_max": 0.14541, + "nauc_cv_recall_at_5_std": 0.1609, + "nauc_cv_recall_at_5_diff1": 0.858425, + "nauc_cv_recall_at_10_max": -0.017745, + "nauc_cv_recall_at_10_std": 0.036452, + "nauc_cv_recall_at_10_diff1": 0.880532, + "nauc_cv_recall_at_20_max": -0.062019, + "nauc_cv_recall_at_20_std": -0.005412, + "nauc_cv_recall_at_20_diff1": 0.888739, + "nauc_cv_recall_at_100_max": -0.493081, + "nauc_cv_recall_at_100_std": -0.423331, + "nauc_cv_recall_at_100_diff1": 0.860925, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.304797, + "nauc_mrr_at_1_std": 0.211033, + "nauc_mrr_at_1_diff1": 0.916703, + "nauc_mrr_at_3_max": 0.259293, + "nauc_mrr_at_3_std": 0.194815, + "nauc_mrr_at_3_diff1": 0.897732, + "nauc_mrr_at_5_max": 0.250724, + "nauc_mrr_at_5_std": 0.192829, + "nauc_mrr_at_5_diff1": 0.894297, + "nauc_mrr_at_10_max": 0.239181, + "nauc_mrr_at_10_std": 0.183688, + "nauc_mrr_at_10_diff1": 0.896967, + "nauc_mrr_at_20_max": 0.239047, + "nauc_mrr_at_20_std": 0.183136, + "nauc_mrr_at_20_diff1": 0.89733, + "nauc_mrr_at_100_max": 0.237643, + "nauc_mrr_at_100_std": 0.18116, + "nauc_mrr_at_100_diff1": 0.897267, + "nauc_mrr_at_1000_max": 0.238627, + "nauc_mrr_at_1000_std": 0.181962, + "nauc_mrr_at_1000_diff1": 0.897316, + "main_score": 0.6351, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 62.05435061454773, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..86db20a871 --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", + "task_name": "VidoreInfoVQARetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.88664, + "ndcg_at_3": 0.91786, + "ndcg_at_5": 0.92601, + "ndcg_at_10": 0.92871, + "ndcg_at_20": 0.93205, + "ndcg_at_100": 0.93485, + "ndcg_at_1000": 0.93584, + "map_at_1": 0.88664, + "map_at_3": 0.91059, + "map_at_5": 0.91535, + "map_at_10": 0.91646, + "map_at_20": 0.91743, + "map_at_100": 0.91781, + "map_at_1000": 0.91785, + "recall_at_1": 0.88664, + "recall_at_3": 0.93826, + "recall_at_5": 0.95749, + "recall_at_10": 0.96498, + "recall_at_20": 0.97753, + "recall_at_100": 0.99251, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.88664, + "cv_recall_at_3": 0.93927, + "cv_recall_at_5": 0.95749, + "cv_recall_at_10": 0.96761, + "cv_recall_at_20": 0.97976, + "cv_recall_at_100": 0.99393, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.88664, + "precision_at_3": 0.31309, + "precision_at_5": 0.1919, + "precision_at_10": 0.09696, + "precision_at_20": 0.04919, + "precision_at_100": 0.01002, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.88664, + "mrr_at_3": 0.910931, + "mrr_at_5": 0.915283, + "mrr_at_10": 0.916774, + "mrr_at_20": 0.917681, + "mrr_at_100": 0.918003, + "mrr_at_1000": 0.918024, + "nauc_ndcg_at_1_max": 0.636802, + "nauc_ndcg_at_1_std": -0.076489, + "nauc_ndcg_at_1_diff1": 0.94664, + "nauc_ndcg_at_3_max": 0.702148, + "nauc_ndcg_at_3_std": 0.016928, + "nauc_ndcg_at_3_diff1": 0.945049, + "nauc_ndcg_at_5_max": 0.719313, + "nauc_ndcg_at_5_std": 0.046593, + "nauc_ndcg_at_5_diff1": 0.951601, + "nauc_ndcg_at_10_max": 0.711251, + "nauc_ndcg_at_10_std": 0.03622, + "nauc_ndcg_at_10_diff1": 0.951385, + "nauc_ndcg_at_20_max": 0.707015, + "nauc_ndcg_at_20_std": 0.050801, + "nauc_ndcg_at_20_diff1": 0.950317, + "nauc_ndcg_at_100_max": 0.695699, + "nauc_ndcg_at_100_std": 0.025328, + "nauc_ndcg_at_100_diff1": 0.948644, + "nauc_ndcg_at_1000_max": 0.690943, + "nauc_ndcg_at_1000_std": 0.014122, + "nauc_ndcg_at_1000_diff1": 0.94904, + "nauc_map_at_1_max": 0.636802, + "nauc_map_at_1_std": -0.076489, + "nauc_map_at_1_diff1": 0.94664, + "nauc_map_at_3_max": 0.677653, + "nauc_map_at_3_std": -0.01824, + "nauc_map_at_3_diff1": 0.94558, + "nauc_map_at_5_max": 0.68578, + "nauc_map_at_5_std": -0.002312, + "nauc_map_at_5_diff1": 0.948846, + "nauc_map_at_10_max": 0.682633, + "nauc_map_at_10_std": -0.005499, + "nauc_map_at_10_diff1": 0.948667, + "nauc_map_at_20_max": 0.681329, + "nauc_map_at_20_std": -0.002295, + "nauc_map_at_20_diff1": 0.948377, + "nauc_map_at_100_max": 0.679976, + "nauc_map_at_100_std": -0.005376, + "nauc_map_at_100_diff1": 0.948223, + "nauc_map_at_1000_max": 0.679801, + "nauc_map_at_1000_std": -0.005801, + "nauc_map_at_1000_diff1": 0.94824, + "nauc_recall_at_1_max": 0.636802, + "nauc_recall_at_1_std": -0.076489, + "nauc_recall_at_1_diff1": 0.94664, + "nauc_recall_at_3_max": 0.803514, + "nauc_recall_at_3_std": 0.157285, + "nauc_recall_at_3_diff1": 0.94325, + "nauc_recall_at_5_max": 0.91501, + "nauc_recall_at_5_std": 0.327838, + "nauc_recall_at_5_diff1": 0.968904, + "nauc_recall_at_10_max": 0.911932, + "nauc_recall_at_10_std": 0.330458, + "nauc_recall_at_10_diff1": 0.971313, + "nauc_recall_at_20_max": 0.976468, + "nauc_recall_at_20_std": 0.663318, + "nauc_recall_at_20_diff1": 0.969409, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.738172, + "nauc_recall_at_100_diff1": 0.922345, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.636802, + "nauc_precision_at_1_std": -0.076489, + "nauc_precision_at_1_diff1": 0.94664, + "nauc_precision_at_3_max": 0.81094, + "nauc_precision_at_3_std": 0.179083, + "nauc_precision_at_3_diff1": 0.942304, + "nauc_precision_at_5_max": 0.829123, + "nauc_precision_at_5_std": 0.353295, + "nauc_precision_at_5_diff1": 0.850655, + "nauc_precision_at_10_max": 0.795692, + "nauc_precision_at_10_std": 0.323344, + "nauc_precision_at_10_diff1": 0.823292, + "nauc_precision_at_20_max": 0.621389, + "nauc_precision_at_20_std": 0.495852, + "nauc_precision_at_20_diff1": 0.583468, + "nauc_precision_at_100_max": -0.035887, + "nauc_precision_at_100_std": 0.066676, + "nauc_precision_at_100_diff1": -0.087162, + "nauc_precision_at_1000_max": -0.512143, + "nauc_precision_at_1000_std": -0.305529, + "nauc_precision_at_1000_diff1": -0.523579, + "nauc_cv_recall_at_1_max": 0.636802, + "nauc_cv_recall_at_1_std": -0.076489, + "nauc_cv_recall_at_1_diff1": 0.94664, + "nauc_cv_recall_at_3_max": 0.81094, + "nauc_cv_recall_at_3_std": 0.179083, + "nauc_cv_recall_at_3_diff1": 0.942304, + "nauc_cv_recall_at_5_max": 0.91501, + "nauc_cv_recall_at_5_std": 0.327838, + "nauc_cv_recall_at_5_diff1": 0.968904, + "nauc_cv_recall_at_10_max": 0.904776, + "nauc_cv_recall_at_10_std": 0.286669, + "nauc_cv_recall_at_10_diff1": 0.975512, + "nauc_cv_recall_at_20_max": 0.97388, + "nauc_cv_recall_at_20_std": 0.640649, + "nauc_cv_recall_at_20_diff1": 0.97388, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 0.707553, + "nauc_cv_recall_at_100_diff1": 0.912932, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.636802, + "nauc_mrr_at_1_std": -0.076489, + "nauc_mrr_at_1_diff1": 0.94664, + "nauc_mrr_at_3_max": 0.678877, + "nauc_mrr_at_3_std": -0.013905, + "nauc_mrr_at_3_diff1": 0.945372, + "nauc_mrr_at_5_max": 0.685516, + "nauc_mrr_at_5_std": -0.003237, + "nauc_mrr_at_5_diff1": 0.948887, + "nauc_mrr_at_10_max": 0.680893, + "nauc_mrr_at_10_std": -0.010502, + "nauc_mrr_at_10_diff1": 0.948898, + "nauc_mrr_at_20_max": 0.679828, + "nauc_mrr_at_20_std": -0.006619, + "nauc_mrr_at_20_diff1": 0.948545, + "nauc_mrr_at_100_max": 0.678671, + "nauc_mrr_at_100_std": -0.009164, + "nauc_mrr_at_100_diff1": 0.948341, + "nauc_mrr_at_1000_max": 0.67859, + "nauc_mrr_at_1000_std": -0.009332, + "nauc_mrr_at_1000_diff1": 0.948348, + "main_score": 0.92601, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 58.656922340393066, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..ad584a0d78 --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "84a382e05c4473fed9cff2bbae95fe2379416117", + "task_name": "VidoreShiftProjectRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.8, + "ndcg_at_3": 0.87178, + "ndcg_at_5": 0.8804, + "ndcg_at_10": 0.89007, + "ndcg_at_20": 0.8927, + "ndcg_at_100": 0.89695, + "ndcg_at_1000": 0.89695, + "map_at_1": 0.8, + "map_at_3": 0.855, + "map_at_5": 0.86, + "map_at_10": 0.86397, + "map_at_20": 0.86474, + "map_at_100": 0.86554, + "map_at_1000": 0.86554, + "recall_at_1": 0.8, + "recall_at_3": 0.92, + "recall_at_5": 0.94, + "recall_at_10": 0.97, + "recall_at_20": 0.98, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.8, + "cv_recall_at_3": 0.92, + "cv_recall_at_5": 0.94, + "cv_recall_at_10": 0.97, + "cv_recall_at_20": 0.98, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.8, + "precision_at_3": 0.30667, + "precision_at_5": 0.188, + "precision_at_10": 0.097, + "precision_at_20": 0.049, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.8, + "mrr_at_3": 0.855, + "mrr_at_5": 0.86, + "mrr_at_10": 0.863968, + "mrr_at_20": 0.864737, + "mrr_at_100": 0.865537, + "mrr_at_1000": 0.865537, + "nauc_ndcg_at_1_max": -0.334091, + "nauc_ndcg_at_1_std": -0.674076, + "nauc_ndcg_at_1_diff1": 0.757318, + "nauc_ndcg_at_3_max": -0.131854, + "nauc_ndcg_at_3_std": -0.511342, + "nauc_ndcg_at_3_diff1": 0.805431, + "nauc_ndcg_at_5_max": -0.212009, + "nauc_ndcg_at_5_std": -0.604053, + "nauc_ndcg_at_5_diff1": 0.807562, + "nauc_ndcg_at_10_max": -0.233541, + "nauc_ndcg_at_10_std": -0.58469, + "nauc_ndcg_at_10_diff1": 0.801937, + "nauc_ndcg_at_20_max": -0.261854, + "nauc_ndcg_at_20_std": -0.608894, + "nauc_ndcg_at_20_diff1": 0.796852, + "nauc_ndcg_at_100_max": -0.248983, + "nauc_ndcg_at_100_std": -0.607566, + "nauc_ndcg_at_100_diff1": 0.793714, + "nauc_ndcg_at_1000_max": -0.248983, + "nauc_ndcg_at_1000_std": -0.607566, + "nauc_ndcg_at_1000_diff1": 0.793714, + "nauc_map_at_1_max": -0.334091, + "nauc_map_at_1_std": -0.674076, + "nauc_map_at_1_diff1": 0.757318, + "nauc_map_at_3_max": -0.202601, + "nauc_map_at_3_std": -0.566943, + "nauc_map_at_3_diff1": 0.793802, + "nauc_map_at_5_max": -0.245276, + "nauc_map_at_5_std": -0.615318, + "nauc_map_at_5_diff1": 0.79443, + "nauc_map_at_10_max": -0.253323, + "nauc_map_at_10_std": -0.609686, + "nauc_map_at_10_diff1": 0.791952, + "nauc_map_at_20_max": -0.260043, + "nauc_map_at_20_std": -0.615479, + "nauc_map_at_20_diff1": 0.790706, + "nauc_map_at_100_max": -0.257428, + "nauc_map_at_100_std": -0.614635, + "nauc_map_at_100_diff1": 0.790214, + "nauc_map_at_1000_max": -0.257428, + "nauc_map_at_1000_std": -0.614635, + "nauc_map_at_1000_diff1": 0.790214, + "nauc_recall_at_1_max": -0.334091, + "nauc_recall_at_1_std": -0.674076, + "nauc_recall_at_1_diff1": 0.757318, + "nauc_recall_at_3_max": 0.224381, + "nauc_recall_at_3_std": -0.23226, + "nauc_recall_at_3_diff1": 0.860528, + "nauc_recall_at_5_max": -0.012372, + "nauc_recall_at_5_std": -0.55042, + "nauc_recall_at_5_diff1": 0.888344, + "nauc_recall_at_10_max": -0.049486, + "nauc_recall_at_10_std": -0.27591, + "nauc_recall_at_10_diff1": 0.912854, + "nauc_recall_at_20_max": -0.50887, + "nauc_recall_at_20_std": -0.592904, + "nauc_recall_at_20_diff1": 0.869281, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.334091, + "nauc_precision_at_1_std": -0.674076, + "nauc_precision_at_1_diff1": 0.757318, + "nauc_precision_at_3_max": 0.224381, + "nauc_precision_at_3_std": -0.23226, + "nauc_precision_at_3_diff1": 0.860528, + "nauc_precision_at_5_max": -0.012372, + "nauc_precision_at_5_std": -0.55042, + "nauc_precision_at_5_diff1": 0.888344, + "nauc_precision_at_10_max": -0.049486, + "nauc_precision_at_10_std": -0.27591, + "nauc_precision_at_10_diff1": 0.912854, + "nauc_precision_at_20_max": -0.50887, + "nauc_precision_at_20_std": -0.592904, + "nauc_precision_at_20_diff1": 0.869281, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": -0.334091, + "nauc_cv_recall_at_1_std": -0.674076, + "nauc_cv_recall_at_1_diff1": 0.757318, + "nauc_cv_recall_at_3_max": 0.224381, + "nauc_cv_recall_at_3_std": -0.23226, + "nauc_cv_recall_at_3_diff1": 0.860528, + "nauc_cv_recall_at_5_max": -0.012372, + "nauc_cv_recall_at_5_std": -0.55042, + "nauc_cv_recall_at_5_diff1": 0.888344, + "nauc_cv_recall_at_10_max": -0.049486, + "nauc_cv_recall_at_10_std": -0.27591, + "nauc_cv_recall_at_10_diff1": 0.912854, + "nauc_cv_recall_at_20_max": -0.50887, + "nauc_cv_recall_at_20_std": -0.592904, + "nauc_cv_recall_at_20_diff1": 0.869281, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.334091, + "nauc_mrr_at_1_std": -0.674076, + "nauc_mrr_at_1_diff1": 0.757318, + "nauc_mrr_at_3_max": -0.202601, + "nauc_mrr_at_3_std": -0.566943, + "nauc_mrr_at_3_diff1": 0.793802, + "nauc_mrr_at_5_max": -0.245276, + "nauc_mrr_at_5_std": -0.615318, + "nauc_mrr_at_5_diff1": 0.79443, + "nauc_mrr_at_10_max": -0.253323, + "nauc_mrr_at_10_std": -0.609686, + "nauc_mrr_at_10_diff1": 0.791952, + "nauc_mrr_at_20_max": -0.260043, + "nauc_mrr_at_20_std": -0.615479, + "nauc_mrr_at_20_diff1": 0.790706, + "nauc_mrr_at_100_max": -0.257428, + "nauc_mrr_at_100_std": -0.614635, + "nauc_mrr_at_100_diff1": 0.790214, + "nauc_mrr_at_1000_max": -0.257428, + "nauc_mrr_at_1000_std": -0.614635, + "nauc_mrr_at_1000_diff1": 0.790214, + "main_score": 0.8804, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 113.57593250274658, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..507534104c --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", + "task_name": "VidoreSyntheticDocQAAIRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.99, + "ndcg_at_3": 0.99631, + "ndcg_at_5": 0.99631, + "ndcg_at_10": 0.99631, + "ndcg_at_20": 0.99631, + "ndcg_at_100": 0.99631, + "ndcg_at_1000": 0.99631, + "map_at_1": 0.99, + "map_at_3": 0.995, + "map_at_5": 0.995, + "map_at_10": 0.995, + "map_at_20": 0.995, + "map_at_100": 0.995, + "map_at_1000": 0.995, + "recall_at_1": 0.99, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.99, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.99, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.99, + "mrr_at_3": 0.995, + "mrr_at_5": 0.995, + "mrr_at_10": 0.995, + "mrr_at_20": 0.995, + "mrr_at_100": 0.995, + "mrr_at_1000": 0.995, + "nauc_ndcg_at_1_max": 0.869281, + "nauc_ndcg_at_1_std": -0.563492, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.869281, + "nauc_ndcg_at_3_std": -0.563492, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.869281, + "nauc_ndcg_at_5_std": -0.563492, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.869281, + "nauc_ndcg_at_10_std": -0.563492, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.869281, + "nauc_ndcg_at_20_std": -0.563492, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.869281, + "nauc_ndcg_at_100_std": -0.563492, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.869281, + "nauc_ndcg_at_1000_std": -0.563492, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.869281, + "nauc_map_at_1_std": -0.563492, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.869281, + "nauc_map_at_3_std": -0.563492, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.869281, + "nauc_map_at_5_std": -0.563492, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.869281, + "nauc_map_at_10_std": -0.563492, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.869281, + "nauc_map_at_20_std": -0.563492, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.869281, + "nauc_map_at_100_std": -0.563492, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.869281, + "nauc_map_at_1000_std": -0.563492, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.869281, + "nauc_recall_at_1_std": -0.563492, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.869281, + "nauc_precision_at_1_std": -0.563492, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.869281, + "nauc_cv_recall_at_1_std": -0.563492, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.869281, + "nauc_mrr_at_1_std": -0.563492, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.869281, + "nauc_mrr_at_3_std": -0.563492, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.869281, + "nauc_mrr_at_5_std": -0.563492, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.869281, + "nauc_mrr_at_10_std": -0.563492, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.869281, + "nauc_mrr_at_20_std": -0.563492, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.869281, + "nauc_mrr_at_100_std": -0.563492, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.869281, + "nauc_mrr_at_1000_std": -0.563492, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.99631, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 115.69474864006042, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..95917e94dd --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", + "task_name": "VidoreSyntheticDocQAEnergyRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.93, + "ndcg_at_3": 0.94893, + "ndcg_at_5": 0.95754, + "ndcg_at_10": 0.95754, + "ndcg_at_20": 0.96295, + "ndcg_at_100": 0.96295, + "ndcg_at_1000": 0.96295, + "map_at_1": 0.93, + "map_at_3": 0.945, + "map_at_5": 0.95, + "map_at_10": 0.95, + "map_at_20": 0.95167, + "map_at_100": 0.95167, + "map_at_1000": 0.95167, + "recall_at_1": 0.93, + "recall_at_3": 0.96, + "recall_at_5": 0.98, + "recall_at_10": 0.98, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.93, + "cv_recall_at_3": 0.96, + "cv_recall_at_5": 0.98, + "cv_recall_at_10": 0.98, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.93, + "precision_at_3": 0.32, + "precision_at_5": 0.196, + "precision_at_10": 0.098, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.93, + "mrr_at_3": 0.945, + "mrr_at_5": 0.95, + "mrr_at_10": 0.95, + "mrr_at_20": 0.951667, + "mrr_at_100": 0.951667, + "mrr_at_1000": 0.951667, + "nauc_ndcg_at_1_max": 0.394825, + "nauc_ndcg_at_1_std": -0.312592, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.559491, + "nauc_ndcg_at_3_std": -0.371172, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.51535, + "nauc_ndcg_at_5_std": -0.465413, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.51535, + "nauc_ndcg_at_10_std": -0.465413, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.491474, + "nauc_ndcg_at_20_std": -0.393906, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.491474, + "nauc_ndcg_at_100_std": -0.393906, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.491474, + "nauc_ndcg_at_1000_std": -0.393906, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.394825, + "nauc_map_at_1_std": -0.312592, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.516, + "nauc_map_at_3_std": -0.3557, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.489893, + "nauc_map_at_5_std": -0.400607, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.489893, + "nauc_map_at_10_std": -0.400607, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.48337, + "nauc_map_at_20_std": -0.381468, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.48337, + "nauc_map_at_100_std": -0.381468, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.48337, + "nauc_map_at_1000_std": -0.381468, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.394825, + "nauc_recall_at_1_std": -0.312592, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 0.728058, + "nauc_recall_at_3_std": -0.431139, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 0.679038, + "nauc_recall_at_5_std": -0.955649, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 0.679038, + "nauc_recall_at_10_std": -0.955649, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.394825, + "nauc_precision_at_1_std": -0.312592, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 0.728058, + "nauc_precision_at_3_std": -0.431139, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 0.679038, + "nauc_precision_at_5_std": -0.955649, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 0.679038, + "nauc_precision_at_10_std": -0.955649, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.394825, + "nauc_cv_recall_at_1_std": -0.312592, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": 0.728058, + "nauc_cv_recall_at_3_std": -0.431139, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 0.679038, + "nauc_cv_recall_at_5_std": -0.955649, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": 0.679038, + "nauc_cv_recall_at_10_std": -0.955649, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.394825, + "nauc_mrr_at_1_std": -0.312592, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.516, + "nauc_mrr_at_3_std": -0.3557, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.489893, + "nauc_mrr_at_5_std": -0.400607, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.489893, + "nauc_mrr_at_10_std": -0.400607, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.48337, + "nauc_mrr_at_20_std": -0.381468, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.48337, + "nauc_mrr_at_100_std": -0.381468, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.48337, + "nauc_mrr_at_1000_std": -0.381468, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.95754, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 119.98192834854126, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..1d725cee55 --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b4909afa930f81282fd20601e860668073ad02aa", + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.92, + "ndcg_at_3": 0.95786, + "ndcg_at_5": 0.96559, + "ndcg_at_10": 0.96559, + "ndcg_at_20": 0.96559, + "ndcg_at_100": 0.96559, + "ndcg_at_1000": 0.96559, + "map_at_1": 0.92, + "map_at_3": 0.95, + "map_at_5": 0.954, + "map_at_10": 0.954, + "map_at_20": 0.954, + "map_at_100": 0.954, + "map_at_1000": 0.954, + "recall_at_1": 0.92, + "recall_at_3": 0.98, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.92, + "cv_recall_at_3": 0.98, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.92, + "precision_at_3": 0.32667, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.92, + "mrr_at_3": 0.95, + "mrr_at_5": 0.954, + "mrr_at_10": 0.954, + "mrr_at_20": 0.954, + "mrr_at_100": 0.954, + "mrr_at_1000": 0.954, + "nauc_ndcg_at_1_max": 0.837477, + "nauc_ndcg_at_1_std": 0.258695, + "nauc_ndcg_at_1_diff1": 0.95098, + "nauc_ndcg_at_3_max": 0.886139, + "nauc_ndcg_at_3_std": 0.384552, + "nauc_ndcg_at_3_diff1": 0.965658, + "nauc_ndcg_at_5_max": 0.860535, + "nauc_ndcg_at_5_std": 0.318332, + "nauc_ndcg_at_5_diff1": 0.957935, + "nauc_ndcg_at_10_max": 0.860535, + "nauc_ndcg_at_10_std": 0.318332, + "nauc_ndcg_at_10_diff1": 0.957935, + "nauc_ndcg_at_20_max": 0.860535, + "nauc_ndcg_at_20_std": 0.318332, + "nauc_ndcg_at_20_diff1": 0.957935, + "nauc_ndcg_at_100_max": 0.860535, + "nauc_ndcg_at_100_std": 0.318332, + "nauc_ndcg_at_100_diff1": 0.957935, + "nauc_ndcg_at_1000_max": 0.860535, + "nauc_ndcg_at_1000_std": 0.318332, + "nauc_ndcg_at_1000_diff1": 0.957935, + "nauc_map_at_1_max": 0.837477, + "nauc_map_at_1_std": 0.258695, + "nauc_map_at_1_diff1": 0.95098, + "nauc_map_at_3_max": 0.869981, + "nauc_map_at_3_std": 0.342764, + "nauc_map_at_3_diff1": 0.960784, + "nauc_map_at_5_max": 0.858675, + "nauc_map_at_5_std": 0.313523, + "nauc_map_at_5_diff1": 0.957374, + "nauc_map_at_10_max": 0.858675, + "nauc_map_at_10_std": 0.313523, + "nauc_map_at_10_diff1": 0.957374, + "nauc_map_at_20_max": 0.858675, + "nauc_map_at_20_std": 0.313523, + "nauc_map_at_20_diff1": 0.957374, + "nauc_map_at_100_max": 0.858675, + "nauc_map_at_100_std": 0.313523, + "nauc_map_at_100_diff1": 0.957374, + "nauc_map_at_1000_max": 0.858675, + "nauc_map_at_1000_std": 0.313523, + "nauc_map_at_1000_diff1": 0.957374, + "nauc_recall_at_1_max": 0.837477, + "nauc_recall_at_1_std": 0.258695, + "nauc_recall_at_1_diff1": 0.95098, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": 0.679038, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.837477, + "nauc_precision_at_1_std": 0.258695, + "nauc_precision_at_1_diff1": 0.95098, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 0.679038, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.837477, + "nauc_cv_recall_at_1_std": 0.258695, + "nauc_cv_recall_at_1_diff1": 0.95098, + "nauc_cv_recall_at_3_max": 1.0, + "nauc_cv_recall_at_3_std": 0.679038, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.837477, + "nauc_mrr_at_1_std": 0.258695, + "nauc_mrr_at_1_diff1": 0.95098, + "nauc_mrr_at_3_max": 0.869981, + "nauc_mrr_at_3_std": 0.342764, + "nauc_mrr_at_3_diff1": 0.960784, + "nauc_mrr_at_5_max": 0.858675, + "nauc_mrr_at_5_std": 0.313523, + "nauc_mrr_at_5_diff1": 0.957374, + "nauc_mrr_at_10_max": 0.858675, + "nauc_mrr_at_10_std": 0.313523, + "nauc_mrr_at_10_diff1": 0.957374, + "nauc_mrr_at_20_max": 0.858675, + "nauc_mrr_at_20_std": 0.313523, + "nauc_mrr_at_20_diff1": 0.957374, + "nauc_mrr_at_100_max": 0.858675, + "nauc_mrr_at_100_std": 0.313523, + "nauc_mrr_at_100_diff1": 0.957374, + "nauc_mrr_at_1000_max": 0.858675, + "nauc_mrr_at_1000_std": 0.313523, + "nauc_mrr_at_1000_diff1": 0.957374, + "main_score": 0.96559, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 110.69024658203125, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..451706b747 --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.95, + "ndcg_at_3": 0.98024, + "ndcg_at_5": 0.98024, + "ndcg_at_10": 0.98024, + "ndcg_at_20": 0.98024, + "ndcg_at_100": 0.98024, + "ndcg_at_1000": 0.98024, + "map_at_1": 0.95, + "map_at_3": 0.97333, + "map_at_5": 0.97333, + "map_at_10": 0.97333, + "map_at_20": 0.97333, + "map_at_100": 0.97333, + "map_at_1000": 0.97333, + "recall_at_1": 0.95, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.95, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.95, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.95, + "mrr_at_3": 0.973333, + "mrr_at_5": 0.973333, + "mrr_at_10": 0.973333, + "mrr_at_20": 0.973333, + "mrr_at_100": 0.973333, + "mrr_at_1000": 0.973333, + "nauc_ndcg_at_1_max": 0.726891, + "nauc_ndcg_at_1_std": -0.45761, + "nauc_ndcg_at_1_diff1": 0.973856, + "nauc_ndcg_at_3_max": 0.702457, + "nauc_ndcg_at_3_std": -0.503595, + "nauc_ndcg_at_3_diff1": 0.975588, + "nauc_ndcg_at_5_max": 0.702457, + "nauc_ndcg_at_5_std": -0.503595, + "nauc_ndcg_at_5_diff1": 0.975588, + "nauc_ndcg_at_10_max": 0.702457, + "nauc_ndcg_at_10_std": -0.503595, + "nauc_ndcg_at_10_diff1": 0.975588, + "nauc_ndcg_at_20_max": 0.702457, + "nauc_ndcg_at_20_std": -0.503595, + "nauc_ndcg_at_20_diff1": 0.975588, + "nauc_ndcg_at_100_max": 0.702457, + "nauc_ndcg_at_100_std": -0.503595, + "nauc_ndcg_at_100_diff1": 0.975588, + "nauc_ndcg_at_1000_max": 0.702457, + "nauc_ndcg_at_1000_std": -0.503595, + "nauc_ndcg_at_1000_diff1": 0.975588, + "nauc_map_at_1_max": 0.726891, + "nauc_map_at_1_std": -0.45761, + "nauc_map_at_1_diff1": 0.973856, + "nauc_map_at_3_max": 0.70384, + "nauc_map_at_3_std": -0.500992, + "nauc_map_at_3_diff1": 0.97549, + "nauc_map_at_5_max": 0.70384, + "nauc_map_at_5_std": -0.500992, + "nauc_map_at_5_diff1": 0.97549, + "nauc_map_at_10_max": 0.70384, + "nauc_map_at_10_std": -0.500992, + "nauc_map_at_10_diff1": 0.97549, + "nauc_map_at_20_max": 0.70384, + "nauc_map_at_20_std": -0.500992, + "nauc_map_at_20_diff1": 0.97549, + "nauc_map_at_100_max": 0.70384, + "nauc_map_at_100_std": -0.500992, + "nauc_map_at_100_diff1": 0.97549, + "nauc_map_at_1000_max": 0.70384, + "nauc_map_at_1000_std": -0.500992, + "nauc_map_at_1000_diff1": 0.97549, + "nauc_recall_at_1_max": 0.726891, + "nauc_recall_at_1_std": -0.45761, + "nauc_recall_at_1_diff1": 0.973856, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.726891, + "nauc_precision_at_1_std": -0.45761, + "nauc_precision_at_1_diff1": 0.973856, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.726891, + "nauc_cv_recall_at_1_std": -0.45761, + "nauc_cv_recall_at_1_diff1": 0.973856, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.726891, + "nauc_mrr_at_1_std": -0.45761, + "nauc_mrr_at_1_diff1": 0.973856, + "nauc_mrr_at_3_max": 0.70384, + "nauc_mrr_at_3_std": -0.500992, + "nauc_mrr_at_3_diff1": 0.97549, + "nauc_mrr_at_5_max": 0.70384, + "nauc_mrr_at_5_std": -0.500992, + "nauc_mrr_at_5_diff1": 0.97549, + "nauc_mrr_at_10_max": 0.70384, + "nauc_mrr_at_10_std": -0.500992, + "nauc_mrr_at_10_diff1": 0.97549, + "nauc_mrr_at_20_max": 0.70384, + "nauc_mrr_at_20_std": -0.500992, + "nauc_mrr_at_20_diff1": 0.97549, + "nauc_mrr_at_100_max": 0.70384, + "nauc_mrr_at_100_std": -0.500992, + "nauc_mrr_at_100_diff1": 0.97549, + "nauc_mrr_at_1000_max": 0.70384, + "nauc_mrr_at_1000_std": -0.500992, + "nauc_mrr_at_1000_diff1": 0.97549, + "main_score": 0.98024, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 111.92106890678406, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..4416e858e0 --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", + "task_name": "VidoreTabfquadRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.85357, + "ndcg_at_3": 0.90391, + "ndcg_at_5": 0.90821, + "ndcg_at_10": 0.91641, + "ndcg_at_20": 0.92002, + "ndcg_at_100": 0.92226, + "ndcg_at_1000": 0.92226, + "map_at_1": 0.85357, + "map_at_3": 0.89167, + "map_at_5": 0.89399, + "map_at_10": 0.89744, + "map_at_20": 0.89843, + "map_at_100": 0.89884, + "map_at_1000": 0.89884, + "recall_at_1": 0.85357, + "recall_at_3": 0.93929, + "recall_at_5": 0.95, + "recall_at_10": 0.975, + "recall_at_20": 0.98929, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.85357, + "cv_recall_at_3": 0.93929, + "cv_recall_at_5": 0.95, + "cv_recall_at_10": 0.975, + "cv_recall_at_20": 0.98929, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.85357, + "precision_at_3": 0.3131, + "precision_at_5": 0.19, + "precision_at_10": 0.0975, + "precision_at_20": 0.04946, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.853571, + "mrr_at_3": 0.891667, + "mrr_at_5": 0.893988, + "mrr_at_10": 0.897445, + "mrr_at_20": 0.898434, + "mrr_at_100": 0.898837, + "mrr_at_1000": 0.898837, + "nauc_ndcg_at_1_max": 0.444786, + "nauc_ndcg_at_1_std": 0.087715, + "nauc_ndcg_at_1_diff1": 0.920185, + "nauc_ndcg_at_3_max": 0.550982, + "nauc_ndcg_at_3_std": 0.226119, + "nauc_ndcg_at_3_diff1": 0.918926, + "nauc_ndcg_at_5_max": 0.564857, + "nauc_ndcg_at_5_std": 0.240278, + "nauc_ndcg_at_5_diff1": 0.915031, + "nauc_ndcg_at_10_max": 0.536659, + "nauc_ndcg_at_10_std": 0.214473, + "nauc_ndcg_at_10_diff1": 0.918748, + "nauc_ndcg_at_20_max": 0.530726, + "nauc_ndcg_at_20_std": 0.202895, + "nauc_ndcg_at_20_diff1": 0.917881, + "nauc_ndcg_at_100_max": 0.518139, + "nauc_ndcg_at_100_std": 0.181821, + "nauc_ndcg_at_100_diff1": 0.91678, + "nauc_ndcg_at_1000_max": 0.518139, + "nauc_ndcg_at_1000_std": 0.181821, + "nauc_ndcg_at_1000_diff1": 0.91678, + "nauc_map_at_1_max": 0.444786, + "nauc_map_at_1_std": 0.087715, + "nauc_map_at_1_diff1": 0.920185, + "nauc_map_at_3_max": 0.51742, + "nauc_map_at_3_std": 0.180593, + "nauc_map_at_3_diff1": 0.917406, + "nauc_map_at_5_max": 0.5226, + "nauc_map_at_5_std": 0.185461, + "nauc_map_at_5_diff1": 0.915544, + "nauc_map_at_10_max": 0.511779, + "nauc_map_at_10_std": 0.174514, + "nauc_map_at_10_diff1": 0.917081, + "nauc_map_at_20_max": 0.510276, + "nauc_map_at_20_std": 0.171547, + "nauc_map_at_20_diff1": 0.916753, + "nauc_map_at_100_max": 0.508454, + "nauc_map_at_100_std": 0.168481, + "nauc_map_at_100_diff1": 0.916609, + "nauc_map_at_1000_max": 0.508454, + "nauc_map_at_1000_std": 0.168481, + "nauc_map_at_1000_diff1": 0.916609, + "nauc_recall_at_1_max": 0.444786, + "nauc_recall_at_1_std": 0.087715, + "nauc_recall_at_1_diff1": 0.920185, + "nauc_recall_at_3_max": 0.718542, + "nauc_recall_at_3_std": 0.454715, + "nauc_recall_at_3_diff1": 0.927912, + "nauc_recall_at_5_max": 0.821262, + "nauc_recall_at_5_std": 0.573263, + "nauc_recall_at_5_diff1": 0.912465, + "nauc_recall_at_10_max": 0.790249, + "nauc_recall_at_10_std": 0.653928, + "nauc_recall_at_10_diff1": 0.941643, + "nauc_recall_at_20_max": 0.956427, + "nauc_recall_at_20_std": 0.912854, + "nauc_recall_at_20_diff1": 0.956427, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.444786, + "nauc_precision_at_1_std": 0.087715, + "nauc_precision_at_1_diff1": 0.920185, + "nauc_precision_at_3_max": 0.718542, + "nauc_precision_at_3_std": 0.454715, + "nauc_precision_at_3_diff1": 0.927912, + "nauc_precision_at_5_max": 0.821262, + "nauc_precision_at_5_std": 0.573263, + "nauc_precision_at_5_diff1": 0.912465, + "nauc_precision_at_10_max": 0.790249, + "nauc_precision_at_10_std": 0.653928, + "nauc_precision_at_10_diff1": 0.941643, + "nauc_precision_at_20_max": 0.956427, + "nauc_precision_at_20_std": 0.912854, + "nauc_precision_at_20_diff1": 0.956427, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.444786, + "nauc_cv_recall_at_1_std": 0.087715, + "nauc_cv_recall_at_1_diff1": 0.920185, + "nauc_cv_recall_at_3_max": 0.718542, + "nauc_cv_recall_at_3_std": 0.454715, + "nauc_cv_recall_at_3_diff1": 0.927912, + "nauc_cv_recall_at_5_max": 0.821262, + "nauc_cv_recall_at_5_std": 0.573263, + "nauc_cv_recall_at_5_diff1": 0.912465, + "nauc_cv_recall_at_10_max": 0.790249, + "nauc_cv_recall_at_10_std": 0.653928, + "nauc_cv_recall_at_10_diff1": 0.941643, + "nauc_cv_recall_at_20_max": 0.956427, + "nauc_cv_recall_at_20_std": 0.912854, + "nauc_cv_recall_at_20_diff1": 0.956427, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.444786, + "nauc_mrr_at_1_std": 0.087715, + "nauc_mrr_at_1_diff1": 0.920185, + "nauc_mrr_at_3_max": 0.51742, + "nauc_mrr_at_3_std": 0.180593, + "nauc_mrr_at_3_diff1": 0.917406, + "nauc_mrr_at_5_max": 0.5226, + "nauc_mrr_at_5_std": 0.185461, + "nauc_mrr_at_5_diff1": 0.915544, + "nauc_mrr_at_10_max": 0.511779, + "nauc_mrr_at_10_std": 0.174514, + "nauc_mrr_at_10_diff1": 0.917081, + "nauc_mrr_at_20_max": 0.510276, + "nauc_mrr_at_20_std": 0.171547, + "nauc_mrr_at_20_diff1": 0.916753, + "nauc_mrr_at_100_max": 0.508454, + "nauc_mrr_at_100_std": 0.168481, + "nauc_mrr_at_100_diff1": 0.916609, + "nauc_mrr_at_1000_max": 0.508454, + "nauc_mrr_at_1000_std": 0.168481, + "nauc_mrr_at_1000_diff1": 0.916609, + "main_score": 0.90821, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.908632278442383, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..4d31a780eb --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", + "task_name": "VidoreTatdqaRetrieval", + "mteb_version": "1.38.16", + "scores": { + "test": [ + { + "ndcg_at_1": 0.71446, + "ndcg_at_3": 0.80242, + "ndcg_at_5": 0.82111, + "ndcg_at_10": 0.83234, + "ndcg_at_20": 0.83733, + "ndcg_at_100": 0.84281, + "ndcg_at_1000": 0.84415, + "map_at_1": 0.71294, + "map_at_3": 0.78134, + "map_at_5": 0.79167, + "map_at_10": 0.79651, + "map_at_20": 0.79792, + "map_at_100": 0.79864, + "map_at_1000": 0.79872, + "recall_at_1": 0.71294, + "recall_at_3": 0.86239, + "recall_at_5": 0.90796, + "recall_at_10": 0.94168, + "recall_at_20": 0.96118, + "recall_at_100": 0.99083, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.71446, + "cv_recall_at_3": 0.8633, + "cv_recall_at_5": 0.90887, + "cv_recall_at_10": 0.94228, + "cv_recall_at_20": 0.96173, + "cv_recall_at_100": 0.99149, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.71446, + "precision_at_3": 0.28838, + "precision_at_5": 0.18214, + "precision_at_10": 0.09447, + "precision_at_20": 0.04827, + "precision_at_100": 0.00998, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.714459, + "mrr_at_3": 0.782098, + "mrr_at_5": 0.792426, + "mrr_at_10": 0.797166, + "mrr_at_20": 0.798514, + "mrr_at_100": 0.799229, + "mrr_at_1000": 0.79929, + "nauc_ndcg_at_1_max": 0.2737, + "nauc_ndcg_at_1_std": -0.093679, + "nauc_ndcg_at_1_diff1": 0.833389, + "nauc_ndcg_at_3_max": 0.303746, + "nauc_ndcg_at_3_std": -0.052607, + "nauc_ndcg_at_3_diff1": 0.802311, + "nauc_ndcg_at_5_max": 0.304482, + "nauc_ndcg_at_5_std": -0.030563, + "nauc_ndcg_at_5_diff1": 0.798978, + "nauc_ndcg_at_10_max": 0.305373, + "nauc_ndcg_at_10_std": -0.026656, + "nauc_ndcg_at_10_diff1": 0.800888, + "nauc_ndcg_at_20_max": 0.307321, + "nauc_ndcg_at_20_std": -0.033254, + "nauc_ndcg_at_20_diff1": 0.804519, + "nauc_ndcg_at_100_max": 0.302555, + "nauc_ndcg_at_100_std": -0.040164, + "nauc_ndcg_at_100_diff1": 0.809097, + "nauc_ndcg_at_1000_max": 0.299433, + "nauc_ndcg_at_1000_std": -0.046116, + "nauc_ndcg_at_1000_diff1": 0.808904, + "nauc_map_at_1_max": 0.271209, + "nauc_map_at_1_std": -0.094353, + "nauc_map_at_1_diff1": 0.834407, + "nauc_map_at_3_max": 0.293436, + "nauc_map_at_3_std": -0.06551, + "nauc_map_at_3_diff1": 0.811601, + "nauc_map_at_5_max": 0.293505, + "nauc_map_at_5_std": -0.055694, + "nauc_map_at_5_diff1": 0.810477, + "nauc_map_at_10_max": 0.293658, + "nauc_map_at_10_std": -0.055203, + "nauc_map_at_10_diff1": 0.8115, + "nauc_map_at_20_max": 0.294316, + "nauc_map_at_20_std": -0.05629, + "nauc_map_at_20_diff1": 0.812379, + "nauc_map_at_100_max": 0.293924, + "nauc_map_at_100_std": -0.056693, + "nauc_map_at_100_diff1": 0.812863, + "nauc_map_at_1000_max": 0.293765, + "nauc_map_at_1000_std": -0.05698, + "nauc_map_at_1000_diff1": 0.812847, + "nauc_recall_at_1_max": 0.271209, + "nauc_recall_at_1_std": -0.094353, + "nauc_recall_at_1_diff1": 0.834407, + "nauc_recall_at_3_max": 0.345294, + "nauc_recall_at_3_std": 0.000437, + "nauc_recall_at_3_diff1": 0.762385, + "nauc_recall_at_5_max": 0.364577, + "nauc_recall_at_5_std": 0.120726, + "nauc_recall_at_5_diff1": 0.729071, + "nauc_recall_at_10_max": 0.403502, + "nauc_recall_at_10_std": 0.240947, + "nauc_recall_at_10_diff1": 0.704198, + "nauc_recall_at_20_max": 0.478052, + "nauc_recall_at_20_std": 0.262748, + "nauc_recall_at_20_diff1": 0.713715, + "nauc_recall_at_100_max": 0.61556, + "nauc_recall_at_100_std": 0.570871, + "nauc_recall_at_100_diff1": 0.821175, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.2737, + "nauc_precision_at_1_std": -0.093679, + "nauc_precision_at_1_diff1": 0.833389, + "nauc_precision_at_3_max": 0.340456, + "nauc_precision_at_3_std": 0.002542, + "nauc_precision_at_3_diff1": 0.734067, + "nauc_precision_at_5_max": 0.35762, + "nauc_precision_at_5_std": 0.120862, + "nauc_precision_at_5_diff1": 0.690262, + "nauc_precision_at_10_max": 0.379089, + "nauc_precision_at_10_std": 0.227939, + "nauc_precision_at_10_diff1": 0.627841, + "nauc_precision_at_20_max": 0.41063, + "nauc_precision_at_20_std": 0.230869, + "nauc_precision_at_20_diff1": 0.551542, + "nauc_precision_at_100_max": 0.149778, + "nauc_precision_at_100_std": 0.146041, + "nauc_precision_at_100_diff1": 0.098599, + "nauc_precision_at_1000_max": -0.354982, + "nauc_precision_at_1000_std": -0.336123, + "nauc_precision_at_1000_diff1": -0.514356, + "nauc_cv_recall_at_1_max": 0.2737, + "nauc_cv_recall_at_1_std": -0.093679, + "nauc_cv_recall_at_1_diff1": 0.833389, + "nauc_cv_recall_at_3_max": 0.348432, + "nauc_cv_recall_at_3_std": 0.003175, + "nauc_cv_recall_at_3_diff1": 0.762861, + "nauc_cv_recall_at_5_max": 0.369765, + "nauc_cv_recall_at_5_std": 0.125894, + "nauc_cv_recall_at_5_diff1": 0.730424, + "nauc_cv_recall_at_10_max": 0.413985, + "nauc_cv_recall_at_10_std": 0.253538, + "nauc_cv_recall_at_10_diff1": 0.70515, + "nauc_cv_recall_at_20_max": 0.470595, + "nauc_cv_recall_at_20_std": 0.252216, + "nauc_cv_recall_at_20_diff1": 0.709625, + "nauc_cv_recall_at_100_max": 0.585354, + "nauc_cv_recall_at_100_std": 0.537153, + "nauc_cv_recall_at_100_diff1": 0.807125, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.2737, + "nauc_mrr_at_1_std": -0.093679, + "nauc_mrr_at_1_diff1": 0.833389, + "nauc_mrr_at_3_max": 0.295434, + "nauc_mrr_at_3_std": -0.063994, + "nauc_mrr_at_3_diff1": 0.811481, + "nauc_mrr_at_5_max": 0.295603, + "nauc_mrr_at_5_std": -0.054073, + "nauc_mrr_at_5_diff1": 0.810348, + "nauc_mrr_at_10_max": 0.296093, + "nauc_mrr_at_10_std": -0.053115, + "nauc_mrr_at_10_diff1": 0.811353, + "nauc_mrr_at_20_max": 0.296103, + "nauc_mrr_at_20_std": -0.054991, + "nauc_mrr_at_20_diff1": 0.81217, + "nauc_mrr_at_100_max": 0.295738, + "nauc_mrr_at_100_std": -0.055359, + "nauc_mrr_at_100_diff1": 0.81266, + "nauc_mrr_at_1000_max": 0.295648, + "nauc_mrr_at_1000_std": -0.055543, + "nauc_mrr_at_1000_diff1": 0.812661, + "main_score": 0.82111, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 44.65311527252197, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json new file mode 100644 index 0000000000..5480d1a3b8 --- /dev/null +++ b/results/vidore__colqwen2.5-v0.2/530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f/model_meta.json @@ -0,0 +1 @@ +{"name": "vidore/colqwen2.5-v0.2", "revision": "530094e83a40ca4edcb5c9e5ddfa61a4b5ea0d2f", "release_date": "2025-01-31", "languages": ["eng-Latn"], "n_parameters": 3000000000, "memory_usage_mb": 7200.0, "max_tokens": 128000.0, "embed_dim": 128, "license": "apache-2.0", "open_weights": true, "public_training_code": "https://github.com/illuin-tech/colpali", "public_training_data": "https://huggingface.co/datasets/vidore/colpali_train_set", "framework": ["ColPali"], "reference": "https://huggingface.co/vidore/colqwen2.5-v0.2", "similarity_fn_name": "max_sim", "use_instructions": true, "training_datasets": {"DocVQA": ["train"], "InfoVQA": ["train"], "TATDQA": ["train"], "arXivQA": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "ColQwen2_5Wrapper"} \ No newline at end of file