diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/Vidore2BioMedicalLecturesRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..e516137ff9 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.6125, + "ndcg_at_3": 0.62171, + "ndcg_at_5": 0.62897, + "ndcg_at_10": 0.65438, + "ndcg_at_20": 0.67796, + "ndcg_at_100": 0.71573, + "ndcg_at_1000": 0.73078, + "map_at_1": 0.35533, + "map_at_3": 0.50127, + "map_at_5": 0.53464, + "map_at_10": 0.56376, + "map_at_20": 0.57738, + "map_at_100": 0.58954, + "map_at_1000": 0.59101, + "recall_at_1": 0.35533, + "recall_at_3": 0.59221, + "recall_at_5": 0.65531, + "recall_at_10": 0.74033, + "recall_at_20": 0.80617, + "recall_at_100": 0.92705, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.6125, + "cv_recall_at_3": 0.8375, + "cv_recall_at_5": 0.875, + "cv_recall_at_10": 0.91875, + "cv_recall_at_20": 0.95625, + "cv_recall_at_100": 0.975, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.6125, + "precision_at_3": 0.39167, + "precision_at_5": 0.2825, + "precision_at_10": 0.17438, + "precision_at_20": 0.10188, + "precision_at_100": 0.02756, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.6125, + "mrr_at_3": 0.711458, + "mrr_at_5": 0.720208, + "mrr_at_10": 0.726414, + "mrr_at_20": 0.728746, + "mrr_at_100": 0.729437, + "mrr_at_1000": 0.729569, + "nauc_ndcg_at_1_max": 0.246306, + "nauc_ndcg_at_1_std": 0.056035, + "nauc_ndcg_at_1_diff1": 0.553911, + "nauc_ndcg_at_3_max": 0.317026, + "nauc_ndcg_at_3_std": 0.107582, + "nauc_ndcg_at_3_diff1": 0.440509, + "nauc_ndcg_at_5_max": 0.318938, + "nauc_ndcg_at_5_std": 0.101494, + "nauc_ndcg_at_5_diff1": 0.475691, + "nauc_ndcg_at_10_max": 0.330813, + "nauc_ndcg_at_10_std": 0.103872, + "nauc_ndcg_at_10_diff1": 0.492478, + "nauc_ndcg_at_20_max": 0.319181, + "nauc_ndcg_at_20_std": 0.112647, + "nauc_ndcg_at_20_diff1": 0.479269, + "nauc_ndcg_at_100_max": 0.329744, + "nauc_ndcg_at_100_std": 0.120409, + "nauc_ndcg_at_100_diff1": 0.463551, + "nauc_ndcg_at_1000_max": 0.317379, + "nauc_ndcg_at_1000_std": 0.098912, + "nauc_ndcg_at_1000_diff1": 0.46865, + "nauc_map_at_1_max": 0.277224, + "nauc_map_at_1_std": 0.113052, + "nauc_map_at_1_diff1": 0.643296, + "nauc_map_at_3_max": 0.316157, + "nauc_map_at_3_std": 0.132597, + "nauc_map_at_3_diff1": 0.53201, + "nauc_map_at_5_max": 0.316212, + "nauc_map_at_5_std": 0.114215, + "nauc_map_at_5_diff1": 0.522615, + "nauc_map_at_10_max": 0.306314, + "nauc_map_at_10_std": 0.093065, + "nauc_map_at_10_diff1": 0.511232, + "nauc_map_at_20_max": 0.303317, + "nauc_map_at_20_std": 0.093898, + "nauc_map_at_20_diff1": 0.502892, + "nauc_map_at_100_max": 0.308108, + "nauc_map_at_100_std": 0.098096, + "nauc_map_at_100_diff1": 0.497586, + "nauc_map_at_1000_max": 0.30752, + "nauc_map_at_1000_std": 0.096675, + "nauc_map_at_1000_diff1": 0.497395, + "nauc_recall_at_1_max": 0.277224, + "nauc_recall_at_1_std": 0.113052, + "nauc_recall_at_1_diff1": 0.643296, + "nauc_recall_at_3_max": 0.320749, + "nauc_recall_at_3_std": 0.159677, + "nauc_recall_at_3_diff1": 0.399613, + "nauc_recall_at_5_max": 0.312685, + "nauc_recall_at_5_std": 0.126089, + "nauc_recall_at_5_diff1": 0.382674, + "nauc_recall_at_10_max": 0.327558, + "nauc_recall_at_10_std": 0.11601, + "nauc_recall_at_10_diff1": 0.356044, + "nauc_recall_at_20_max": 0.295672, + "nauc_recall_at_20_std": 0.17281, + "nauc_recall_at_20_diff1": 0.311147, + "nauc_recall_at_100_max": 0.425883, + "nauc_recall_at_100_std": 0.348778, + "nauc_recall_at_100_diff1": 0.15205, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.246306, + "nauc_precision_at_1_std": 0.056035, + "nauc_precision_at_1_diff1": 0.553911, + "nauc_precision_at_3_max": 0.14163, + "nauc_precision_at_3_std": 0.041578, + "nauc_precision_at_3_diff1": -0.050821, + "nauc_precision_at_5_max": 0.053086, + "nauc_precision_at_5_std": -0.050368, + "nauc_precision_at_5_diff1": -0.114893, + "nauc_precision_at_10_max": -0.014381, + "nauc_precision_at_10_std": -0.090836, + "nauc_precision_at_10_diff1": -0.193268, + "nauc_precision_at_20_max": -0.070023, + "nauc_precision_at_20_std": -0.102489, + "nauc_precision_at_20_diff1": -0.275612, + "nauc_precision_at_100_max": -0.112928, + "nauc_precision_at_100_std": -0.110665, + "nauc_precision_at_100_diff1": -0.342323, + "nauc_precision_at_1000_max": -0.191239, + "nauc_precision_at_1000_std": -0.187467, + "nauc_precision_at_1000_diff1": -0.36721, + "nauc_cv_recall_at_1_max": 0.246306, + "nauc_cv_recall_at_1_std": 0.056035, + "nauc_cv_recall_at_1_diff1": 0.553911, + "nauc_cv_recall_at_3_max": 0.384746, + "nauc_cv_recall_at_3_std": 0.131816, + "nauc_cv_recall_at_3_diff1": 0.371701, + "nauc_cv_recall_at_5_max": 0.332023, + "nauc_cv_recall_at_5_std": 0.0279, + "nauc_cv_recall_at_5_diff1": 0.285043, + "nauc_cv_recall_at_10_max": 0.605078, + "nauc_cv_recall_at_10_std": 0.320872, + "nauc_cv_recall_at_10_diff1": 0.274761, + "nauc_cv_recall_at_20_max": 0.681939, + "nauc_cv_recall_at_20_std": 0.642924, + "nauc_cv_recall_at_20_diff1": 0.119848, + "nauc_cv_recall_at_100_max": 0.508754, + "nauc_cv_recall_at_100_std": 0.514006, + "nauc_cv_recall_at_100_diff1": -0.401377, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.246306, + "nauc_mrr_at_1_std": 0.056035, + "nauc_mrr_at_1_diff1": 0.553911, + "nauc_mrr_at_3_max": 0.299604, + "nauc_mrr_at_3_std": 0.0838, + "nauc_mrr_at_3_diff1": 0.503884, + "nauc_mrr_at_5_max": 0.290308, + "nauc_mrr_at_5_std": 0.070352, + "nauc_mrr_at_5_diff1": 0.498125, + "nauc_mrr_at_10_max": 0.304778, + "nauc_mrr_at_10_std": 0.087282, + "nauc_mrr_at_10_diff1": 0.503178, + "nauc_mrr_at_20_max": 0.302742, + "nauc_mrr_at_20_std": 0.088546, + "nauc_mrr_at_20_diff1": 0.503226, + "nauc_mrr_at_100_max": 0.300866, + "nauc_mrr_at_100_std": 0.086308, + "nauc_mrr_at_100_diff1": 0.502335, + "nauc_mrr_at_1000_max": 0.300749, + "nauc_mrr_at_1000_std": 0.086119, + "nauc_mrr_at_1000_diff1": 0.502631, + "main_score": 0.62897, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.5875, + "ndcg_at_3": 0.58916, + "ndcg_at_5": 0.61624, + "ndcg_at_10": 0.64247, + "ndcg_at_20": 0.66517, + "ndcg_at_100": 0.70586, + "ndcg_at_1000": 0.72057, + "map_at_1": 0.35866, + "map_at_3": 0.48335, + "map_at_5": 0.5225, + "map_at_10": 0.55239, + "map_at_20": 0.56536, + "map_at_100": 0.57793, + "map_at_1000": 0.5794, + "recall_at_1": 0.35866, + "recall_at_3": 0.55877, + "recall_at_5": 0.65227, + "recall_at_10": 0.73162, + "recall_at_20": 0.795, + "recall_at_100": 0.93281, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.5875, + "cv_recall_at_3": 0.78125, + "cv_recall_at_5": 0.86875, + "cv_recall_at_10": 0.91875, + "cv_recall_at_20": 0.94375, + "cv_recall_at_100": 0.9875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.5875, + "precision_at_3": 0.36042, + "precision_at_5": 0.2775, + "precision_at_10": 0.17375, + "precision_at_20": 0.10094, + "precision_at_100": 0.02731, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.5875, + "mrr_at_3": 0.673958, + "mrr_at_5": 0.694896, + "mrr_at_10": 0.701667, + "mrr_at_20": 0.703579, + "mrr_at_100": 0.704662, + "mrr_at_1000": 0.704701, + "nauc_ndcg_at_1_max": 0.259251, + "nauc_ndcg_at_1_std": 0.046652, + "nauc_ndcg_at_1_diff1": 0.518474, + "nauc_ndcg_at_3_max": 0.230654, + "nauc_ndcg_at_3_std": 0.075613, + "nauc_ndcg_at_3_diff1": 0.385742, + "nauc_ndcg_at_5_max": 0.275607, + "nauc_ndcg_at_5_std": 0.122743, + "nauc_ndcg_at_5_diff1": 0.422063, + "nauc_ndcg_at_10_max": 0.260189, + "nauc_ndcg_at_10_std": 0.093574, + "nauc_ndcg_at_10_diff1": 0.421057, + "nauc_ndcg_at_20_max": 0.256572, + "nauc_ndcg_at_20_std": 0.112435, + "nauc_ndcg_at_20_diff1": 0.407744, + "nauc_ndcg_at_100_max": 0.256096, + "nauc_ndcg_at_100_std": 0.118752, + "nauc_ndcg_at_100_diff1": 0.410332, + "nauc_ndcg_at_1000_max": 0.248393, + "nauc_ndcg_at_1000_std": 0.094016, + "nauc_ndcg_at_1000_diff1": 0.408247, + "nauc_map_at_1_max": 0.287097, + "nauc_map_at_1_std": 0.123404, + "nauc_map_at_1_diff1": 0.558079, + "nauc_map_at_3_max": 0.258308, + "nauc_map_at_3_std": 0.118079, + "nauc_map_at_3_diff1": 0.448087, + "nauc_map_at_5_max": 0.273625, + "nauc_map_at_5_std": 0.127213, + "nauc_map_at_5_diff1": 0.454543, + "nauc_map_at_10_max": 0.24939, + "nauc_map_at_10_std": 0.089414, + "nauc_map_at_10_diff1": 0.440984, + "nauc_map_at_20_max": 0.246447, + "nauc_map_at_20_std": 0.093625, + "nauc_map_at_20_diff1": 0.432614, + "nauc_map_at_100_max": 0.246858, + "nauc_map_at_100_std": 0.097426, + "nauc_map_at_100_diff1": 0.432766, + "nauc_map_at_1000_max": 0.246306, + "nauc_map_at_1000_std": 0.095883, + "nauc_map_at_1000_diff1": 0.432055, + "nauc_recall_at_1_max": 0.287097, + "nauc_recall_at_1_std": 0.123404, + "nauc_recall_at_1_diff1": 0.558079, + "nauc_recall_at_3_max": 0.238311, + "nauc_recall_at_3_std": 0.12674, + "nauc_recall_at_3_diff1": 0.336868, + "nauc_recall_at_5_max": 0.273601, + "nauc_recall_at_5_std": 0.176418, + "nauc_recall_at_5_diff1": 0.33013, + "nauc_recall_at_10_max": 0.218793, + "nauc_recall_at_10_std": 0.093159, + "nauc_recall_at_10_diff1": 0.281031, + "nauc_recall_at_20_max": 0.22195, + "nauc_recall_at_20_std": 0.176757, + "nauc_recall_at_20_diff1": 0.222003, + "nauc_recall_at_100_max": 0.22701, + "nauc_recall_at_100_std": 0.39827, + "nauc_recall_at_100_diff1": 0.128403, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.259251, + "nauc_precision_at_1_std": 0.046652, + "nauc_precision_at_1_diff1": 0.518474, + "nauc_precision_at_3_max": 0.01643, + "nauc_precision_at_3_std": -0.030138, + "nauc_precision_at_3_diff1": -0.031044, + "nauc_precision_at_5_max": -0.012401, + "nauc_precision_at_5_std": -0.039136, + "nauc_precision_at_5_diff1": -0.097853, + "nauc_precision_at_10_max": -0.093386, + "nauc_precision_at_10_std": -0.124851, + "nauc_precision_at_10_diff1": -0.168222, + "nauc_precision_at_20_max": -0.125772, + "nauc_precision_at_20_std": -0.107732, + "nauc_precision_at_20_diff1": -0.228774, + "nauc_precision_at_100_max": -0.158771, + "nauc_precision_at_100_std": -0.127817, + "nauc_precision_at_100_diff1": -0.279175, + "nauc_precision_at_1000_max": -0.220164, + "nauc_precision_at_1000_std": -0.215468, + "nauc_precision_at_1000_diff1": -0.34212, + "nauc_cv_recall_at_1_max": 0.259251, + "nauc_cv_recall_at_1_std": 0.046652, + "nauc_cv_recall_at_1_diff1": 0.518474, + "nauc_cv_recall_at_3_max": 0.333935, + "nauc_cv_recall_at_3_std": 0.093792, + "nauc_cv_recall_at_3_diff1": 0.395765, + "nauc_cv_recall_at_5_max": 0.213207, + "nauc_cv_recall_at_5_std": 0.071474, + "nauc_cv_recall_at_5_diff1": 0.226378, + "nauc_cv_recall_at_10_max": 0.225239, + "nauc_cv_recall_at_10_std": 0.099332, + "nauc_cv_recall_at_10_diff1": 0.088271, + "nauc_cv_recall_at_20_max": 0.378462, + "nauc_cv_recall_at_20_std": 0.444029, + "nauc_cv_recall_at_20_diff1": -0.05068, + "nauc_cv_recall_at_100_max": -0.367414, + "nauc_cv_recall_at_100_std": 0.613679, + "nauc_cv_recall_at_100_diff1": -1.739963, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.259251, + "nauc_mrr_at_1_std": 0.046652, + "nauc_mrr_at_1_diff1": 0.518474, + "nauc_mrr_at_3_max": 0.296415, + "nauc_mrr_at_3_std": 0.079662, + "nauc_mrr_at_3_diff1": 0.480709, + "nauc_mrr_at_5_max": 0.277587, + "nauc_mrr_at_5_std": 0.073158, + "nauc_mrr_at_5_diff1": 0.466682, + "nauc_mrr_at_10_max": 0.281977, + "nauc_mrr_at_10_std": 0.075884, + "nauc_mrr_at_10_diff1": 0.465705, + "nauc_mrr_at_20_max": 0.28461, + "nauc_mrr_at_20_std": 0.08111, + "nauc_mrr_at_20_diff1": 0.46595, + "nauc_mrr_at_100_max": 0.283374, + "nauc_mrr_at_100_std": 0.079833, + "nauc_mrr_at_100_diff1": 0.466279, + "nauc_mrr_at_1000_max": 0.283497, + "nauc_mrr_at_1000_std": 0.079723, + "nauc_mrr_at_1000_diff1": 0.466635, + "main_score": 0.61624, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.66875, + "ndcg_at_3": 0.64104, + "ndcg_at_5": 0.64524, + "ndcg_at_10": 0.68324, + "ndcg_at_20": 0.70793, + "ndcg_at_100": 0.74242, + "ndcg_at_1000": 0.75453, + "map_at_1": 0.39877, + "map_at_3": 0.52756, + "map_at_5": 0.55722, + "map_at_10": 0.59581, + "map_at_20": 0.6098, + "map_at_100": 0.62108, + "map_at_1000": 0.62244, + "recall_at_1": 0.39877, + "recall_at_3": 0.58442, + "recall_at_5": 0.64848, + "recall_at_10": 0.76855, + "recall_at_20": 0.83456, + "recall_at_100": 0.94927, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.66875, + "cv_recall_at_3": 0.8125, + "cv_recall_at_5": 0.85625, + "cv_recall_at_10": 0.925, + "cv_recall_at_20": 0.95625, + "cv_recall_at_100": 0.99375, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.66875, + "precision_at_3": 0.39375, + "precision_at_5": 0.2825, + "precision_at_10": 0.18125, + "precision_at_20": 0.10656, + "precision_at_100": 0.02775, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.66875, + "mrr_at_3": 0.73125, + "mrr_at_5": 0.741562, + "mrr_at_10": 0.750632, + "mrr_at_20": 0.752939, + "mrr_at_100": 0.753888, + "mrr_at_1000": 0.753896, + "nauc_ndcg_at_1_max": 0.408059, + "nauc_ndcg_at_1_std": 0.13832, + "nauc_ndcg_at_1_diff1": 0.47132, + "nauc_ndcg_at_3_max": 0.276079, + "nauc_ndcg_at_3_std": 0.018929, + "nauc_ndcg_at_3_diff1": 0.424511, + "nauc_ndcg_at_5_max": 0.278052, + "nauc_ndcg_at_5_std": 0.014432, + "nauc_ndcg_at_5_diff1": 0.44012, + "nauc_ndcg_at_10_max": 0.345828, + "nauc_ndcg_at_10_std": 0.105118, + "nauc_ndcg_at_10_diff1": 0.465279, + "nauc_ndcg_at_20_max": 0.330428, + "nauc_ndcg_at_20_std": 0.099997, + "nauc_ndcg_at_20_diff1": 0.470258, + "nauc_ndcg_at_100_max": 0.316971, + "nauc_ndcg_at_100_std": 0.081328, + "nauc_ndcg_at_100_diff1": 0.446386, + "nauc_ndcg_at_1000_max": 0.304158, + "nauc_ndcg_at_1000_std": 0.058117, + "nauc_ndcg_at_1000_diff1": 0.43674, + "nauc_map_at_1_max": 0.339533, + "nauc_map_at_1_std": 0.101734, + "nauc_map_at_1_diff1": 0.54843, + "nauc_map_at_3_max": 0.331801, + "nauc_map_at_3_std": 0.073122, + "nauc_map_at_3_diff1": 0.494409, + "nauc_map_at_5_max": 0.316397, + "nauc_map_at_5_std": 0.045234, + "nauc_map_at_5_diff1": 0.48085, + "nauc_map_at_10_max": 0.333107, + "nauc_map_at_10_std": 0.074198, + "nauc_map_at_10_diff1": 0.482061, + "nauc_map_at_20_max": 0.323868, + "nauc_map_at_20_std": 0.069702, + "nauc_map_at_20_diff1": 0.480211, + "nauc_map_at_100_max": 0.321286, + "nauc_map_at_100_std": 0.067395, + "nauc_map_at_100_diff1": 0.472576, + "nauc_map_at_1000_max": 0.319544, + "nauc_map_at_1000_std": 0.06473, + "nauc_map_at_1000_diff1": 0.47133, + "nauc_recall_at_1_max": 0.339533, + "nauc_recall_at_1_std": 0.101734, + "nauc_recall_at_1_diff1": 0.54843, + "nauc_recall_at_3_max": 0.262111, + "nauc_recall_at_3_std": 0.026159, + "nauc_recall_at_3_diff1": 0.440529, + "nauc_recall_at_5_max": 0.207303, + "nauc_recall_at_5_std": -0.022375, + "nauc_recall_at_5_diff1": 0.401456, + "nauc_recall_at_10_max": 0.349948, + "nauc_recall_at_10_std": 0.205708, + "nauc_recall_at_10_diff1": 0.394692, + "nauc_recall_at_20_max": 0.310751, + "nauc_recall_at_20_std": 0.211799, + "nauc_recall_at_20_diff1": 0.41812, + "nauc_recall_at_100_max": 0.25769, + "nauc_recall_at_100_std": 0.253427, + "nauc_recall_at_100_diff1": 0.3425, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.408059, + "nauc_precision_at_1_std": 0.13832, + "nauc_precision_at_1_diff1": 0.47132, + "nauc_precision_at_3_max": -0.009793, + "nauc_precision_at_3_std": -0.083562, + "nauc_precision_at_3_diff1": -0.020823, + "nauc_precision_at_5_max": -0.089373, + "nauc_precision_at_5_std": -0.136383, + "nauc_precision_at_5_diff1": -0.131093, + "nauc_precision_at_10_max": -0.069705, + "nauc_precision_at_10_std": -0.033622, + "nauc_precision_at_10_diff1": -0.189697, + "nauc_precision_at_20_max": -0.153975, + "nauc_precision_at_20_std": -0.07192, + "nauc_precision_at_20_diff1": -0.253971, + "nauc_precision_at_100_max": -0.234943, + "nauc_precision_at_100_std": -0.151001, + "nauc_precision_at_100_diff1": -0.364687, + "nauc_precision_at_1000_max": -0.303324, + "nauc_precision_at_1000_std": -0.230511, + "nauc_precision_at_1000_diff1": -0.410886, + "nauc_cv_recall_at_1_max": 0.408059, + "nauc_cv_recall_at_1_std": 0.13832, + "nauc_cv_recall_at_1_diff1": 0.47132, + "nauc_cv_recall_at_3_max": 0.181124, + "nauc_cv_recall_at_3_std": -0.096917, + "nauc_cv_recall_at_3_diff1": 0.344514, + "nauc_cv_recall_at_5_max": 0.10003, + "nauc_cv_recall_at_5_std": -0.157274, + "nauc_cv_recall_at_5_diff1": 0.319093, + "nauc_cv_recall_at_10_max": 0.402194, + "nauc_cv_recall_at_10_std": 0.378112, + "nauc_cv_recall_at_10_diff1": 0.321078, + "nauc_cv_recall_at_20_max": 0.410898, + "nauc_cv_recall_at_20_std": 0.410231, + "nauc_cv_recall_at_20_diff1": 0.199213, + "nauc_cv_recall_at_100_max": -1.151727, + "nauc_cv_recall_at_100_std": -0.563492, + "nauc_cv_recall_at_100_diff1": -1.151727, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.408059, + "nauc_mrr_at_1_std": 0.13832, + "nauc_mrr_at_1_diff1": 0.47132, + "nauc_mrr_at_3_max": 0.337097, + "nauc_mrr_at_3_std": 0.067564, + "nauc_mrr_at_3_diff1": 0.438184, + "nauc_mrr_at_5_max": 0.331186, + "nauc_mrr_at_5_std": 0.064474, + "nauc_mrr_at_5_diff1": 0.437115, + "nauc_mrr_at_10_max": 0.356383, + "nauc_mrr_at_10_std": 0.101028, + "nauc_mrr_at_10_diff1": 0.443131, + "nauc_mrr_at_20_max": 0.355692, + "nauc_mrr_at_20_std": 0.098768, + "nauc_mrr_at_20_diff1": 0.442977, + "nauc_mrr_at_100_max": 0.354148, + "nauc_mrr_at_100_std": 0.09655, + "nauc_mrr_at_100_diff1": 0.443181, + "nauc_mrr_at_1000_max": 0.354207, + "nauc_mrr_at_1000_std": 0.096576, + "nauc_mrr_at_1000_diff1": 0.443244, + "main_score": 0.64524, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.6, + "ndcg_at_3": 0.56502, + "ndcg_at_5": 0.59543, + "ndcg_at_10": 0.61926, + "ndcg_at_20": 0.64474, + "ndcg_at_100": 0.68792, + "ndcg_at_1000": 0.70575, + "map_at_1": 0.3694, + "map_at_3": 0.46583, + "map_at_5": 0.50647, + "map_at_10": 0.53248, + "map_at_20": 0.54668, + "map_at_100": 0.55853, + "map_at_1000": 0.56049, + "recall_at_1": 0.3694, + "recall_at_3": 0.5216, + "recall_at_5": 0.62483, + "recall_at_10": 0.69923, + "recall_at_20": 0.76886, + "recall_at_100": 0.92248, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.6, + "cv_recall_at_3": 0.75625, + "cv_recall_at_5": 0.85, + "cv_recall_at_10": 0.89375, + "cv_recall_at_20": 0.91875, + "cv_recall_at_100": 0.9875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.6, + "precision_at_3": 0.33125, + "precision_at_5": 0.25625, + "precision_at_10": 0.16063, + "precision_at_20": 0.09531, + "precision_at_100": 0.02619, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.6, + "mrr_at_3": 0.666667, + "mrr_at_5": 0.688854, + "mrr_at_10": 0.694407, + "mrr_at_20": 0.696432, + "mrr_at_100": 0.698275, + "mrr_at_1000": 0.698306, + "nauc_ndcg_at_1_max": 0.145101, + "nauc_ndcg_at_1_std": 0.064615, + "nauc_ndcg_at_1_diff1": 0.432389, + "nauc_ndcg_at_3_max": 0.219611, + "nauc_ndcg_at_3_std": 0.083791, + "nauc_ndcg_at_3_diff1": 0.385973, + "nauc_ndcg_at_5_max": 0.225812, + "nauc_ndcg_at_5_std": 0.097785, + "nauc_ndcg_at_5_diff1": 0.396821, + "nauc_ndcg_at_10_max": 0.232444, + "nauc_ndcg_at_10_std": 0.100287, + "nauc_ndcg_at_10_diff1": 0.40158, + "nauc_ndcg_at_20_max": 0.24551, + "nauc_ndcg_at_20_std": 0.126672, + "nauc_ndcg_at_20_diff1": 0.402913, + "nauc_ndcg_at_100_max": 0.225488, + "nauc_ndcg_at_100_std": 0.122867, + "nauc_ndcg_at_100_diff1": 0.401374, + "nauc_ndcg_at_1000_max": 0.217087, + "nauc_ndcg_at_1000_std": 0.096083, + "nauc_ndcg_at_1000_diff1": 0.391965, + "nauc_map_at_1_max": 0.200323, + "nauc_map_at_1_std": 0.107561, + "nauc_map_at_1_diff1": 0.512293, + "nauc_map_at_3_max": 0.241917, + "nauc_map_at_3_std": 0.113803, + "nauc_map_at_3_diff1": 0.45461, + "nauc_map_at_5_max": 0.239111, + "nauc_map_at_5_std": 0.103879, + "nauc_map_at_5_diff1": 0.431737, + "nauc_map_at_10_max": 0.225931, + "nauc_map_at_10_std": 0.091649, + "nauc_map_at_10_diff1": 0.414141, + "nauc_map_at_20_max": 0.23144, + "nauc_map_at_20_std": 0.100974, + "nauc_map_at_20_diff1": 0.411765, + "nauc_map_at_100_max": 0.226068, + "nauc_map_at_100_std": 0.098815, + "nauc_map_at_100_diff1": 0.410675, + "nauc_map_at_1000_max": 0.225251, + "nauc_map_at_1000_std": 0.096464, + "nauc_map_at_1000_diff1": 0.40914, + "nauc_recall_at_1_max": 0.200323, + "nauc_recall_at_1_std": 0.107561, + "nauc_recall_at_1_diff1": 0.512293, + "nauc_recall_at_3_max": 0.267089, + "nauc_recall_at_3_std": 0.135634, + "nauc_recall_at_3_diff1": 0.412619, + "nauc_recall_at_5_max": 0.232748, + "nauc_recall_at_5_std": 0.109353, + "nauc_recall_at_5_diff1": 0.339696, + "nauc_recall_at_10_max": 0.244585, + "nauc_recall_at_10_std": 0.09958, + "nauc_recall_at_10_diff1": 0.307536, + "nauc_recall_at_20_max": 0.300525, + "nauc_recall_at_20_std": 0.186379, + "nauc_recall_at_20_diff1": 0.310239, + "nauc_recall_at_100_max": 0.244634, + "nauc_recall_at_100_std": 0.356267, + "nauc_recall_at_100_diff1": 0.324657, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.145101, + "nauc_precision_at_1_std": 0.064615, + "nauc_precision_at_1_diff1": 0.432389, + "nauc_precision_at_3_max": 0.112253, + "nauc_precision_at_3_std": 0.014595, + "nauc_precision_at_3_diff1": 0.019217, + "nauc_precision_at_5_max": 0.032341, + "nauc_precision_at_5_std": -0.036725, + "nauc_precision_at_5_diff1": -0.121874, + "nauc_precision_at_10_max": -0.043616, + "nauc_precision_at_10_std": -0.073182, + "nauc_precision_at_10_diff1": -0.189165, + "nauc_precision_at_20_max": -0.053071, + "nauc_precision_at_20_std": -0.028762, + "nauc_precision_at_20_diff1": -0.215398, + "nauc_precision_at_100_max": -0.1318, + "nauc_precision_at_100_std": -0.081662, + "nauc_precision_at_100_diff1": -0.280437, + "nauc_precision_at_1000_max": -0.174395, + "nauc_precision_at_1000_std": -0.191379, + "nauc_precision_at_1000_diff1": -0.338674, + "nauc_cv_recall_at_1_max": 0.145101, + "nauc_cv_recall_at_1_std": 0.064615, + "nauc_cv_recall_at_1_diff1": 0.432389, + "nauc_cv_recall_at_3_max": 0.26881, + "nauc_cv_recall_at_3_std": 0.133083, + "nauc_cv_recall_at_3_diff1": 0.408783, + "nauc_cv_recall_at_5_max": 0.181378, + "nauc_cv_recall_at_5_std": 0.116844, + "nauc_cv_recall_at_5_diff1": 0.33593, + "nauc_cv_recall_at_10_max": 0.265816, + "nauc_cv_recall_at_10_std": 0.231665, + "nauc_cv_recall_at_10_diff1": 0.328527, + "nauc_cv_recall_at_20_max": 0.204374, + "nauc_cv_recall_at_20_std": 0.239352, + "nauc_cv_recall_at_20_diff1": 0.324319, + "nauc_cv_recall_at_100_max": -0.214753, + "nauc_cv_recall_at_100_std": 0.218254, + "nauc_cv_recall_at_100_diff1": -0.592904, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.145101, + "nauc_mrr_at_1_std": 0.064615, + "nauc_mrr_at_1_diff1": 0.432389, + "nauc_mrr_at_3_max": 0.190058, + "nauc_mrr_at_3_std": 0.087681, + "nauc_mrr_at_3_diff1": 0.421336, + "nauc_mrr_at_5_max": 0.174148, + "nauc_mrr_at_5_std": 0.084868, + "nauc_mrr_at_5_diff1": 0.413163, + "nauc_mrr_at_10_max": 0.176992, + "nauc_mrr_at_10_std": 0.089927, + "nauc_mrr_at_10_diff1": 0.414177, + "nauc_mrr_at_20_max": 0.174936, + "nauc_mrr_at_20_std": 0.088705, + "nauc_mrr_at_20_diff1": 0.414517, + "nauc_mrr_at_100_max": 0.173759, + "nauc_mrr_at_100_std": 0.086873, + "nauc_mrr_at_100_diff1": 0.41381, + "nauc_mrr_at_1000_max": 0.173782, + "nauc_mrr_at_1000_std": 0.086836, + "nauc_mrr_at_1000_diff1": 0.413903, + "main_score": 0.59543, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 1123.2742795944214, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/Vidore2ESGReportsHLRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..581af38b5d --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.73718, + "ndcg_at_3": 0.7518, + "ndcg_at_5": 0.76383, + "ndcg_at_10": 0.80684, + "ndcg_at_20": 0.8168, + "ndcg_at_100": 0.83192, + "ndcg_at_1000": 0.83827, + "map_at_1": 0.52711, + "map_at_3": 0.66486, + "map_at_5": 0.70212, + "map_at_10": 0.73446, + "map_at_20": 0.74025, + "map_at_100": 0.74774, + "map_at_1000": 0.74856, + "recall_at_1": 0.52711, + "recall_at_3": 0.73064, + "recall_at_5": 0.7957, + "recall_at_10": 0.90258, + "recall_at_20": 0.92428, + "recall_at_100": 0.97445, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.75, + "cv_recall_at_3": 0.92308, + "cv_recall_at_5": 0.94231, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.75, + "precision_at_3": 0.41026, + "precision_at_5": 0.29231, + "precision_at_10": 0.17692, + "precision_at_20": 0.09712, + "precision_at_100": 0.02269, + "precision_at_1000": 0.00246, + "mrr_at_1": 0.75, + "mrr_at_3": 0.826923, + "mrr_at_5": 0.830769, + "mrr_at_10": 0.839103, + "mrr_at_20": 0.839103, + "mrr_at_100": 0.839103, + "mrr_at_1000": 0.839103, + "nauc_ndcg_at_1_max": 0.252082, + "nauc_ndcg_at_1_std": 0.17309, + "nauc_ndcg_at_1_diff1": 0.394354, + "nauc_ndcg_at_3_max": 0.344694, + "nauc_ndcg_at_3_std": 0.252656, + "nauc_ndcg_at_3_diff1": 0.372082, + "nauc_ndcg_at_5_max": 0.304117, + "nauc_ndcg_at_5_std": 0.188154, + "nauc_ndcg_at_5_diff1": 0.412562, + "nauc_ndcg_at_10_max": 0.305131, + "nauc_ndcg_at_10_std": 0.189269, + "nauc_ndcg_at_10_diff1": 0.480129, + "nauc_ndcg_at_20_max": 0.318947, + "nauc_ndcg_at_20_std": 0.2136, + "nauc_ndcg_at_20_diff1": 0.451498, + "nauc_ndcg_at_100_max": 0.298628, + "nauc_ndcg_at_100_std": 0.210422, + "nauc_ndcg_at_100_diff1": 0.415073, + "nauc_ndcg_at_1000_max": 0.283283, + "nauc_ndcg_at_1000_std": 0.180704, + "nauc_ndcg_at_1000_diff1": 0.408287, + "nauc_map_at_1_max": 0.206169, + "nauc_map_at_1_std": 0.109305, + "nauc_map_at_1_diff1": 0.499899, + "nauc_map_at_3_max": 0.278977, + "nauc_map_at_3_std": 0.169757, + "nauc_map_at_3_diff1": 0.431403, + "nauc_map_at_5_max": 0.255664, + "nauc_map_at_5_std": 0.151451, + "nauc_map_at_5_diff1": 0.437099, + "nauc_map_at_10_max": 0.264474, + "nauc_map_at_10_std": 0.163568, + "nauc_map_at_10_diff1": 0.464637, + "nauc_map_at_20_max": 0.276985, + "nauc_map_at_20_std": 0.179402, + "nauc_map_at_20_diff1": 0.452192, + "nauc_map_at_100_max": 0.27374, + "nauc_map_at_100_std": 0.18517, + "nauc_map_at_100_diff1": 0.440112, + "nauc_map_at_1000_max": 0.272368, + "nauc_map_at_1000_std": 0.182552, + "nauc_map_at_1000_diff1": 0.439654, + "nauc_recall_at_1_max": 0.206169, + "nauc_recall_at_1_std": 0.109305, + "nauc_recall_at_1_diff1": 0.499899, + "nauc_recall_at_3_max": 0.341219, + "nauc_recall_at_3_std": 0.218297, + "nauc_recall_at_3_diff1": 0.37776, + "nauc_recall_at_5_max": 0.295817, + "nauc_recall_at_5_std": 0.15165, + "nauc_recall_at_5_diff1": 0.427376, + "nauc_recall_at_10_max": 0.394982, + "nauc_recall_at_10_std": 0.26281, + "nauc_recall_at_10_diff1": 0.716352, + "nauc_recall_at_20_max": 0.552892, + "nauc_recall_at_20_std": 0.442871, + "nauc_recall_at_20_diff1": 0.660862, + "nauc_recall_at_100_max": 0.619497, + "nauc_recall_at_100_std": 0.834535, + "nauc_recall_at_100_diff1": 0.541443, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.220742, + "nauc_precision_at_1_std": 0.173307, + "nauc_precision_at_1_diff1": 0.403842, + "nauc_precision_at_3_max": 0.050713, + "nauc_precision_at_3_std": 0.078848, + "nauc_precision_at_3_diff1": -0.2325, + "nauc_precision_at_5_max": -0.050819, + "nauc_precision_at_5_std": -0.003719, + "nauc_precision_at_5_diff1": -0.189682, + "nauc_precision_at_10_max": -0.117011, + "nauc_precision_at_10_std": -0.046556, + "nauc_precision_at_10_diff1": -0.201028, + "nauc_precision_at_20_max": -0.086645, + "nauc_precision_at_20_std": 0.016717, + "nauc_precision_at_20_diff1": -0.307004, + "nauc_precision_at_100_max": -0.150924, + "nauc_precision_at_100_std": -0.022514, + "nauc_precision_at_100_diff1": -0.370226, + "nauc_precision_at_1000_max": -0.184495, + "nauc_precision_at_1000_std": -0.104268, + "nauc_precision_at_1000_diff1": -0.361636, + "nauc_cv_recall_at_1_max": 0.220742, + "nauc_cv_recall_at_1_std": 0.173307, + "nauc_cv_recall_at_1_diff1": 0.403842, + "nauc_cv_recall_at_3_max": 0.25437, + "nauc_cv_recall_at_3_std": 0.196772, + "nauc_cv_recall_at_3_diff1": 0.028705, + "nauc_cv_recall_at_5_max": 0.39573, + "nauc_cv_recall_at_5_std": 0.143396, + "nauc_cv_recall_at_5_diff1": 0.426216, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.220742, + "nauc_mrr_at_1_std": 0.173307, + "nauc_mrr_at_1_diff1": 0.403842, + "nauc_mrr_at_3_max": 0.259184, + "nauc_mrr_at_3_std": 0.20387, + "nauc_mrr_at_3_diff1": 0.339758, + "nauc_mrr_at_5_max": 0.269883, + "nauc_mrr_at_5_std": 0.200052, + "nauc_mrr_at_5_diff1": 0.377265, + "nauc_mrr_at_10_max": 0.257722, + "nauc_mrr_at_10_std": 0.191799, + "nauc_mrr_at_10_diff1": 0.36923, + "nauc_mrr_at_20_max": 0.257722, + "nauc_mrr_at_20_std": 0.191799, + "nauc_mrr_at_20_diff1": 0.36923, + "nauc_mrr_at_100_max": 0.257722, + "nauc_mrr_at_100_std": 0.191799, + "nauc_mrr_at_100_diff1": 0.36923, + "nauc_mrr_at_1000_max": 0.257722, + "nauc_mrr_at_1000_std": 0.191799, + "nauc_mrr_at_1000_diff1": 0.36923, + "main_score": 0.76383, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1303.1721379756927, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/Vidore2ESGReportsRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..788e0baf9e --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.49123, + "ndcg_at_3": 0.50018, + "ndcg_at_5": 0.55865, + "ndcg_at_10": 0.62052, + "ndcg_at_20": 0.64726, + "ndcg_at_100": 0.6832, + "ndcg_at_1000": 0.69039, + "map_at_1": 0.26696, + "map_at_3": 0.37822, + "map_at_5": 0.44701, + "map_at_10": 0.49942, + "map_at_20": 0.51578, + "map_at_100": 0.53558, + "map_at_1000": 0.53623, + "recall_at_1": 0.26696, + "recall_at_3": 0.4519, + "recall_at_5": 0.62416, + "recall_at_10": 0.78939, + "recall_at_20": 0.86992, + "recall_at_100": 0.95468, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.49123, + "cv_recall_at_3": 0.7193, + "cv_recall_at_5": 0.87719, + "cv_recall_at_10": 0.91228, + "cv_recall_at_20": 0.96491, + "cv_recall_at_100": 0.96491, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.49123, + "precision_at_3": 0.34503, + "precision_at_5": 0.29123, + "precision_at_10": 0.20702, + "precision_at_20": 0.12544, + "precision_at_100": 0.03772, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.491228, + "mrr_at_3": 0.593567, + "mrr_at_5": 0.630409, + "mrr_at_10": 0.634357, + "mrr_at_20": 0.637362, + "mrr_at_100": 0.637362, + "mrr_at_1000": 0.637543, + "nauc_ndcg_at_1_max": 0.026838, + "nauc_ndcg_at_1_std": 0.05477, + "nauc_ndcg_at_1_diff1": 0.264017, + "nauc_ndcg_at_3_max": 0.075792, + "nauc_ndcg_at_3_std": 0.138375, + "nauc_ndcg_at_3_diff1": 0.275488, + "nauc_ndcg_at_5_max": 0.143127, + "nauc_ndcg_at_5_std": 0.25224, + "nauc_ndcg_at_5_diff1": 0.333377, + "nauc_ndcg_at_10_max": 0.10075, + "nauc_ndcg_at_10_std": 0.228446, + "nauc_ndcg_at_10_diff1": 0.367267, + "nauc_ndcg_at_20_max": 0.046138, + "nauc_ndcg_at_20_std": 0.182552, + "nauc_ndcg_at_20_diff1": 0.33437, + "nauc_ndcg_at_100_max": -0.057368, + "nauc_ndcg_at_100_std": 0.138598, + "nauc_ndcg_at_100_diff1": 0.334651, + "nauc_ndcg_at_1000_max": -0.042341, + "nauc_ndcg_at_1000_std": 0.127507, + "nauc_ndcg_at_1000_diff1": 0.328075, + "nauc_map_at_1_max": 0.18419, + "nauc_map_at_1_std": 0.101232, + "nauc_map_at_1_diff1": 0.264599, + "nauc_map_at_3_max": 0.17235, + "nauc_map_at_3_std": 0.172513, + "nauc_map_at_3_diff1": 0.303697, + "nauc_map_at_5_max": 0.212574, + "nauc_map_at_5_std": 0.260972, + "nauc_map_at_5_diff1": 0.353395, + "nauc_map_at_10_max": 0.164356, + "nauc_map_at_10_std": 0.227404, + "nauc_map_at_10_diff1": 0.373111, + "nauc_map_at_20_max": 0.134443, + "nauc_map_at_20_std": 0.201061, + "nauc_map_at_20_diff1": 0.358864, + "nauc_map_at_100_max": 0.093364, + "nauc_map_at_100_std": 0.186844, + "nauc_map_at_100_diff1": 0.357869, + "nauc_map_at_1000_max": 0.093054, + "nauc_map_at_1000_std": 0.185733, + "nauc_map_at_1000_diff1": 0.358086, + "nauc_recall_at_1_max": 0.18419, + "nauc_recall_at_1_std": 0.101232, + "nauc_recall_at_1_diff1": 0.264599, + "nauc_recall_at_3_max": 0.130261, + "nauc_recall_at_3_std": 0.177399, + "nauc_recall_at_3_diff1": 0.208661, + "nauc_recall_at_5_max": 0.319293, + "nauc_recall_at_5_std": 0.447148, + "nauc_recall_at_5_diff1": 0.302441, + "nauc_recall_at_10_max": 0.321735, + "nauc_recall_at_10_std": 0.472259, + "nauc_recall_at_10_diff1": 0.41522, + "nauc_recall_at_20_max": 0.146999, + "nauc_recall_at_20_std": 0.406774, + "nauc_recall_at_20_diff1": 0.362477, + "nauc_recall_at_100_max": -0.670622, + "nauc_recall_at_100_std": 0.390855, + "nauc_recall_at_100_diff1": 0.55577, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.026838, + "nauc_precision_at_1_std": 0.05477, + "nauc_precision_at_1_diff1": 0.264017, + "nauc_precision_at_3_max": -0.08459, + "nauc_precision_at_3_std": 0.095749, + "nauc_precision_at_3_diff1": 0.213345, + "nauc_precision_at_5_max": -0.060766, + "nauc_precision_at_5_std": 0.168513, + "nauc_precision_at_5_diff1": 0.243214, + "nauc_precision_at_10_max": -0.2585, + "nauc_precision_at_10_std": -0.040587, + "nauc_precision_at_10_diff1": 0.119842, + "nauc_precision_at_20_max": -0.365273, + "nauc_precision_at_20_std": -0.154937, + "nauc_precision_at_20_diff1": -0.018244, + "nauc_precision_at_100_max": -0.397453, + "nauc_precision_at_100_std": -0.163257, + "nauc_precision_at_100_diff1": -0.078084, + "nauc_precision_at_1000_max": -0.396826, + "nauc_precision_at_1000_std": -0.18089, + "nauc_precision_at_1000_diff1": -0.080357, + "nauc_cv_recall_at_1_max": 0.026838, + "nauc_cv_recall_at_1_std": 0.05477, + "nauc_cv_recall_at_1_diff1": 0.264017, + "nauc_cv_recall_at_3_max": -0.270647, + "nauc_cv_recall_at_3_std": 0.052189, + "nauc_cv_recall_at_3_diff1": 0.155228, + "nauc_cv_recall_at_5_max": -0.319566, + "nauc_cv_recall_at_5_std": 0.325736, + "nauc_cv_recall_at_5_diff1": 0.29474, + "nauc_cv_recall_at_10_max": -0.0933, + "nauc_cv_recall_at_10_std": 0.45513, + "nauc_cv_recall_at_10_diff1": 0.438692, + "nauc_cv_recall_at_20_max": -0.946401, + "nauc_cv_recall_at_20_std": 0.424674, + "nauc_cv_recall_at_20_diff1": 0.638909, + "nauc_cv_recall_at_100_max": -0.946401, + "nauc_cv_recall_at_100_std": 0.424674, + "nauc_cv_recall_at_100_diff1": 0.638909, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.026838, + "nauc_mrr_at_1_std": 0.05477, + "nauc_mrr_at_1_diff1": 0.264017, + "nauc_mrr_at_3_max": -0.077965, + "nauc_mrr_at_3_std": 0.06046, + "nauc_mrr_at_3_diff1": 0.230976, + "nauc_mrr_at_5_max": -0.070148, + "nauc_mrr_at_5_std": 0.081586, + "nauc_mrr_at_5_diff1": 0.246726, + "nauc_mrr_at_10_max": -0.058046, + "nauc_mrr_at_10_std": 0.082727, + "nauc_mrr_at_10_diff1": 0.249525, + "nauc_mrr_at_20_max": -0.063965, + "nauc_mrr_at_20_std": 0.078502, + "nauc_mrr_at_20_diff1": 0.248531, + "nauc_mrr_at_100_max": -0.063965, + "nauc_mrr_at_100_std": 0.078502, + "nauc_mrr_at_100_diff1": 0.248531, + "nauc_mrr_at_1000_max": -0.063317, + "nauc_mrr_at_1000_std": 0.07829, + "nauc_mrr_at_1000_diff1": 0.248263, + "main_score": 0.55865, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.50877, + "ndcg_at_3": 0.51893, + "ndcg_at_5": 0.56667, + "ndcg_at_10": 0.63604, + "ndcg_at_20": 0.66094, + "ndcg_at_100": 0.70163, + "ndcg_at_1000": 0.70365, + "map_at_1": 0.27022, + "map_at_3": 0.3892, + "map_at_5": 0.45606, + "map_at_10": 0.51195, + "map_at_20": 0.53028, + "map_at_100": 0.55192, + "map_at_1000": 0.55224, + "recall_at_1": 0.27022, + "recall_at_3": 0.46729, + "recall_at_5": 0.61539, + "recall_at_10": 0.79754, + "recall_at_20": 0.87153, + "recall_at_100": 0.99123, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.50877, + "cv_recall_at_3": 0.77193, + "cv_recall_at_5": 0.85965, + "cv_recall_at_10": 0.94737, + "cv_recall_at_20": 0.96491, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.50877, + "precision_at_3": 0.35088, + "precision_at_5": 0.29123, + "precision_at_10": 0.21228, + "precision_at_20": 0.12982, + "precision_at_100": 0.03842, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.508772, + "mrr_at_3": 0.628655, + "mrr_at_5": 0.64883, + "mrr_at_10": 0.66077, + "mrr_at_20": 0.661866, + "mrr_at_100": 0.66236, + "mrr_at_1000": 0.66236, + "nauc_ndcg_at_1_max": 0.141839, + "nauc_ndcg_at_1_std": 0.053709, + "nauc_ndcg_at_1_diff1": 0.399524, + "nauc_ndcg_at_3_max": 0.197462, + "nauc_ndcg_at_3_std": 0.195666, + "nauc_ndcg_at_3_diff1": 0.359366, + "nauc_ndcg_at_5_max": 0.137298, + "nauc_ndcg_at_5_std": 0.167858, + "nauc_ndcg_at_5_diff1": 0.375336, + "nauc_ndcg_at_10_max": 0.154466, + "nauc_ndcg_at_10_std": 0.188916, + "nauc_ndcg_at_10_diff1": 0.414372, + "nauc_ndcg_at_20_max": 0.117862, + "nauc_ndcg_at_20_std": 0.163468, + "nauc_ndcg_at_20_diff1": 0.38493, + "nauc_ndcg_at_100_max": 0.056721, + "nauc_ndcg_at_100_std": 0.100083, + "nauc_ndcg_at_100_diff1": 0.363823, + "nauc_ndcg_at_1000_max": 0.056954, + "nauc_ndcg_at_1000_std": 0.100066, + "nauc_ndcg_at_1000_diff1": 0.363131, + "nauc_map_at_1_max": 0.220737, + "nauc_map_at_1_std": 0.055729, + "nauc_map_at_1_diff1": 0.291548, + "nauc_map_at_3_max": 0.257395, + "nauc_map_at_3_std": 0.188213, + "nauc_map_at_3_diff1": 0.329498, + "nauc_map_at_5_max": 0.220014, + "nauc_map_at_5_std": 0.198059, + "nauc_map_at_5_diff1": 0.403287, + "nauc_map_at_10_max": 0.206692, + "nauc_map_at_10_std": 0.199897, + "nauc_map_at_10_diff1": 0.417207, + "nauc_map_at_20_max": 0.182843, + "nauc_map_at_20_std": 0.182228, + "nauc_map_at_20_diff1": 0.403201, + "nauc_map_at_100_max": 0.14907, + "nauc_map_at_100_std": 0.157611, + "nauc_map_at_100_diff1": 0.394867, + "nauc_map_at_1000_max": 0.148741, + "nauc_map_at_1000_std": 0.157193, + "nauc_map_at_1000_diff1": 0.394662, + "nauc_recall_at_1_max": 0.220737, + "nauc_recall_at_1_std": 0.055729, + "nauc_recall_at_1_diff1": 0.291548, + "nauc_recall_at_3_max": 0.278358, + "nauc_recall_at_3_std": 0.273527, + "nauc_recall_at_3_diff1": 0.273694, + "nauc_recall_at_5_max": 0.19221, + "nauc_recall_at_5_std": 0.27068, + "nauc_recall_at_5_diff1": 0.309714, + "nauc_recall_at_10_max": 0.282811, + "nauc_recall_at_10_std": 0.354547, + "nauc_recall_at_10_diff1": 0.42618, + "nauc_recall_at_20_max": 0.191349, + "nauc_recall_at_20_std": 0.392104, + "nauc_recall_at_20_diff1": 0.402265, + "nauc_recall_at_100_max": -0.150834, + "nauc_recall_at_100_std": -0.108922, + "nauc_recall_at_100_diff1": 0.275645, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.141839, + "nauc_precision_at_1_std": 0.053709, + "nauc_precision_at_1_diff1": 0.399524, + "nauc_precision_at_3_max": 0.039176, + "nauc_precision_at_3_std": 0.197921, + "nauc_precision_at_3_diff1": 0.367516, + "nauc_precision_at_5_max": -0.111068, + "nauc_precision_at_5_std": 0.076905, + "nauc_precision_at_5_diff1": 0.305551, + "nauc_precision_at_10_max": -0.236296, + "nauc_precision_at_10_std": -0.047456, + "nauc_precision_at_10_diff1": 0.114714, + "nauc_precision_at_20_max": -0.310933, + "nauc_precision_at_20_std": -0.14832, + "nauc_precision_at_20_diff1": -0.022837, + "nauc_precision_at_100_max": -0.319238, + "nauc_precision_at_100_std": -0.183224, + "nauc_precision_at_100_diff1": -0.106493, + "nauc_precision_at_1000_max": -0.319326, + "nauc_precision_at_1000_std": -0.186118, + "nauc_precision_at_1000_diff1": -0.109616, + "nauc_cv_recall_at_1_max": 0.141839, + "nauc_cv_recall_at_1_std": 0.053709, + "nauc_cv_recall_at_1_diff1": 0.399524, + "nauc_cv_recall_at_3_max": -0.045694, + "nauc_cv_recall_at_3_std": 0.175753, + "nauc_cv_recall_at_3_diff1": 0.264605, + "nauc_cv_recall_at_5_max": -0.226644, + "nauc_cv_recall_at_5_std": 0.131847, + "nauc_cv_recall_at_5_diff1": 0.07124, + "nauc_cv_recall_at_10_max": -0.056422, + "nauc_cv_recall_at_10_std": 0.545223, + "nauc_cv_recall_at_10_diff1": 0.4024, + "nauc_cv_recall_at_20_max": -0.362176, + "nauc_cv_recall_at_20_std": 0.456469, + "nauc_cv_recall_at_20_diff1": 0.424674, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.141839, + "nauc_mrr_at_1_std": 0.053709, + "nauc_mrr_at_1_diff1": 0.399524, + "nauc_mrr_at_3_max": 0.051125, + "nauc_mrr_at_3_std": 0.075349, + "nauc_mrr_at_3_diff1": 0.33309, + "nauc_mrr_at_5_max": 0.035247, + "nauc_mrr_at_5_std": 0.06359, + "nauc_mrr_at_5_diff1": 0.317872, + "nauc_mrr_at_10_max": 0.052575, + "nauc_mrr_at_10_std": 0.073465, + "nauc_mrr_at_10_diff1": 0.336741, + "nauc_mrr_at_20_max": 0.050432, + "nauc_mrr_at_20_std": 0.070695, + "nauc_mrr_at_20_diff1": 0.336651, + "nauc_mrr_at_100_max": 0.051139, + "nauc_mrr_at_100_std": 0.06991, + "nauc_mrr_at_100_diff1": 0.336352, + "nauc_mrr_at_1000_max": 0.051139, + "nauc_mrr_at_1000_std": 0.06991, + "nauc_mrr_at_1000_diff1": 0.336352, + "main_score": 0.56667, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.49123, + "ndcg_at_3": 0.52989, + "ndcg_at_5": 0.60409, + "ndcg_at_10": 0.64603, + "ndcg_at_20": 0.66941, + "ndcg_at_100": 0.70598, + "ndcg_at_1000": 0.70724, + "map_at_1": 0.25004, + "map_at_3": 0.39203, + "map_at_5": 0.46999, + "map_at_10": 0.51658, + "map_at_20": 0.53724, + "map_at_100": 0.55563, + "map_at_1000": 0.55602, + "recall_at_1": 0.25004, + "recall_at_3": 0.49921, + "recall_at_5": 0.68949, + "recall_at_10": 0.82496, + "recall_at_20": 0.89131, + "recall_at_100": 0.99635, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.49123, + "cv_recall_at_3": 0.78947, + "cv_recall_at_5": 0.91228, + "cv_recall_at_10": 0.96491, + "cv_recall_at_20": 0.96491, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.49123, + "precision_at_3": 0.37427, + "precision_at_5": 0.33333, + "precision_at_10": 0.21754, + "precision_at_20": 0.13246, + "precision_at_100": 0.03842, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.491228, + "mrr_at_3": 0.619883, + "mrr_at_5": 0.647076, + "mrr_at_10": 0.653286, + "mrr_at_20": 0.653286, + "mrr_at_100": 0.654346, + "mrr_at_1000": 0.654346, + "nauc_ndcg_at_1_max": -0.047651, + "nauc_ndcg_at_1_std": 0.187409, + "nauc_ndcg_at_1_diff1": 0.47185, + "nauc_ndcg_at_3_max": 0.11041, + "nauc_ndcg_at_3_std": 0.224778, + "nauc_ndcg_at_3_diff1": 0.37071, + "nauc_ndcg_at_5_max": 0.022781, + "nauc_ndcg_at_5_std": 0.181786, + "nauc_ndcg_at_5_diff1": 0.297818, + "nauc_ndcg_at_10_max": 0.055414, + "nauc_ndcg_at_10_std": 0.212432, + "nauc_ndcg_at_10_diff1": 0.371915, + "nauc_ndcg_at_20_max": -0.008327, + "nauc_ndcg_at_20_std": 0.162391, + "nauc_ndcg_at_20_diff1": 0.356372, + "nauc_ndcg_at_100_max": -0.060908, + "nauc_ndcg_at_100_std": 0.144528, + "nauc_ndcg_at_100_diff1": 0.324578, + "nauc_ndcg_at_1000_max": -0.068, + "nauc_ndcg_at_1000_std": 0.1389, + "nauc_ndcg_at_1000_diff1": 0.323693, + "nauc_map_at_1_max": 0.150343, + "nauc_map_at_1_std": 0.25128, + "nauc_map_at_1_diff1": 0.393899, + "nauc_map_at_3_max": 0.192239, + "nauc_map_at_3_std": 0.262156, + "nauc_map_at_3_diff1": 0.397629, + "nauc_map_at_5_max": 0.163949, + "nauc_map_at_5_std": 0.262684, + "nauc_map_at_5_diff1": 0.364362, + "nauc_map_at_10_max": 0.150632, + "nauc_map_at_10_std": 0.265475, + "nauc_map_at_10_diff1": 0.382512, + "nauc_map_at_20_max": 0.107549, + "nauc_map_at_20_std": 0.229723, + "nauc_map_at_20_diff1": 0.373635, + "nauc_map_at_100_max": 0.07332, + "nauc_map_at_100_std": 0.216049, + "nauc_map_at_100_diff1": 0.359765, + "nauc_map_at_1000_max": 0.07178, + "nauc_map_at_1000_std": 0.214829, + "nauc_map_at_1000_diff1": 0.360085, + "nauc_recall_at_1_max": 0.150343, + "nauc_recall_at_1_std": 0.25128, + "nauc_recall_at_1_diff1": 0.393899, + "nauc_recall_at_3_max": 0.228709, + "nauc_recall_at_3_std": 0.264913, + "nauc_recall_at_3_diff1": 0.321936, + "nauc_recall_at_5_max": 0.146784, + "nauc_recall_at_5_std": 0.199046, + "nauc_recall_at_5_diff1": 0.173622, + "nauc_recall_at_10_max": 0.219437, + "nauc_recall_at_10_std": 0.215431, + "nauc_recall_at_10_diff1": 0.287106, + "nauc_recall_at_20_max": -0.006323, + "nauc_recall_at_20_std": 0.055562, + "nauc_recall_at_20_diff1": 0.29244, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.833639, + "nauc_recall_at_100_diff1": 0.376237, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.047651, + "nauc_precision_at_1_std": 0.187409, + "nauc_precision_at_1_diff1": 0.47185, + "nauc_precision_at_3_max": -0.075299, + "nauc_precision_at_3_std": 0.068484, + "nauc_precision_at_3_diff1": 0.196245, + "nauc_precision_at_5_max": -0.236759, + "nauc_precision_at_5_std": -0.079522, + "nauc_precision_at_5_diff1": -0.041368, + "nauc_precision_at_10_max": -0.295302, + "nauc_precision_at_10_std": -0.156302, + "nauc_precision_at_10_diff1": -0.072775, + "nauc_precision_at_20_max": -0.375605, + "nauc_precision_at_20_std": -0.237602, + "nauc_precision_at_20_diff1": -0.133557, + "nauc_precision_at_100_max": -0.376302, + "nauc_precision_at_100_std": -0.217244, + "nauc_precision_at_100_diff1": -0.190663, + "nauc_precision_at_1000_max": -0.382355, + "nauc_precision_at_1000_std": -0.225499, + "nauc_precision_at_1000_diff1": -0.185855, + "nauc_cv_recall_at_1_max": -0.047651, + "nauc_cv_recall_at_1_std": 0.187409, + "nauc_cv_recall_at_1_diff1": 0.47185, + "nauc_cv_recall_at_3_max": -0.099111, + "nauc_cv_recall_at_3_std": 0.206788, + "nauc_cv_recall_at_3_diff1": 0.207054, + "nauc_cv_recall_at_5_max": -0.672471, + "nauc_cv_recall_at_5_std": -0.060428, + "nauc_cv_recall_at_5_diff1": 0.021479, + "nauc_cv_recall_at_10_max": -1.448445, + "nauc_cv_recall_at_10_std": -0.864221, + "nauc_cv_recall_at_10_diff1": -0.084517, + "nauc_cv_recall_at_20_max": -1.448445, + "nauc_cv_recall_at_20_std": -0.864221, + "nauc_cv_recall_at_20_diff1": -0.084517, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.047651, + "nauc_mrr_at_1_std": 0.187409, + "nauc_mrr_at_1_diff1": 0.47185, + "nauc_mrr_at_3_max": -0.066403, + "nauc_mrr_at_3_std": 0.162131, + "nauc_mrr_at_3_diff1": 0.377606, + "nauc_mrr_at_5_max": -0.107467, + "nauc_mrr_at_5_std": 0.13946, + "nauc_mrr_at_5_diff1": 0.374692, + "nauc_mrr_at_10_max": -0.104112, + "nauc_mrr_at_10_std": 0.130285, + "nauc_mrr_at_10_diff1": 0.379213, + "nauc_mrr_at_20_max": -0.104112, + "nauc_mrr_at_20_std": 0.130285, + "nauc_mrr_at_20_diff1": 0.379213, + "nauc_mrr_at_100_max": -0.09919, + "nauc_mrr_at_100_std": 0.133691, + "nauc_mrr_at_100_diff1": 0.378545, + "nauc_mrr_at_1000_max": -0.09919, + "nauc_mrr_at_1000_std": 0.133691, + "nauc_mrr_at_1000_diff1": 0.378545, + "main_score": 0.60409, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.50877, + "ndcg_at_3": 0.49559, + "ndcg_at_5": 0.53583, + "ndcg_at_10": 0.59501, + "ndcg_at_20": 0.63876, + "ndcg_at_100": 0.67904, + "ndcg_at_1000": 0.68395, + "map_at_1": 0.27982, + "map_at_3": 0.38026, + "map_at_5": 0.43771, + "map_at_10": 0.48687, + "map_at_20": 0.50752, + "map_at_100": 0.52865, + "map_at_1000": 0.52929, + "recall_at_1": 0.27982, + "recall_at_3": 0.42318, + "recall_at_5": 0.55242, + "recall_at_10": 0.72746, + "recall_at_20": 0.87529, + "recall_at_100": 0.97222, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.50877, + "cv_recall_at_3": 0.7193, + "cv_recall_at_5": 0.77193, + "cv_recall_at_10": 0.85965, + "cv_recall_at_20": 0.98246, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.50877, + "precision_at_3": 0.33918, + "precision_at_5": 0.2807, + "precision_at_10": 0.19474, + "precision_at_20": 0.12193, + "precision_at_100": 0.03789, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.508772, + "mrr_at_3": 0.599415, + "mrr_at_5": 0.610819, + "mrr_at_10": 0.622563, + "mrr_at_20": 0.631068, + "mrr_at_100": 0.631068, + "mrr_at_1000": 0.631138, + "nauc_ndcg_at_1_max": 0.248228, + "nauc_ndcg_at_1_std": 0.170648, + "nauc_ndcg_at_1_diff1": 0.22638, + "nauc_ndcg_at_3_max": 0.236825, + "nauc_ndcg_at_3_std": 0.257256, + "nauc_ndcg_at_3_diff1": 0.297655, + "nauc_ndcg_at_5_max": 0.213658, + "nauc_ndcg_at_5_std": 0.231156, + "nauc_ndcg_at_5_diff1": 0.213706, + "nauc_ndcg_at_10_max": 0.256703, + "nauc_ndcg_at_10_std": 0.284845, + "nauc_ndcg_at_10_diff1": 0.249256, + "nauc_ndcg_at_20_max": 0.251263, + "nauc_ndcg_at_20_std": 0.264953, + "nauc_ndcg_at_20_diff1": 0.269088, + "nauc_ndcg_at_100_max": 0.19889, + "nauc_ndcg_at_100_std": 0.214018, + "nauc_ndcg_at_100_diff1": 0.209327, + "nauc_ndcg_at_1000_max": 0.195357, + "nauc_ndcg_at_1000_std": 0.211301, + "nauc_ndcg_at_1000_diff1": 0.203665, + "nauc_map_at_1_max": 0.268564, + "nauc_map_at_1_std": 0.163453, + "nauc_map_at_1_diff1": 0.219254, + "nauc_map_at_3_max": 0.249125, + "nauc_map_at_3_std": 0.239332, + "nauc_map_at_3_diff1": 0.30005, + "nauc_map_at_5_max": 0.263352, + "nauc_map_at_5_std": 0.273611, + "nauc_map_at_5_diff1": 0.259631, + "nauc_map_at_10_max": 0.283536, + "nauc_map_at_10_std": 0.303865, + "nauc_map_at_10_diff1": 0.286586, + "nauc_map_at_20_max": 0.277938, + "nauc_map_at_20_std": 0.291019, + "nauc_map_at_20_diff1": 0.287403, + "nauc_map_at_100_max": 0.260078, + "nauc_map_at_100_std": 0.278275, + "nauc_map_at_100_diff1": 0.269701, + "nauc_map_at_1000_max": 0.259063, + "nauc_map_at_1000_std": 0.277156, + "nauc_map_at_1000_diff1": 0.26891, + "nauc_recall_at_1_max": 0.268564, + "nauc_recall_at_1_std": 0.163453, + "nauc_recall_at_1_diff1": 0.219254, + "nauc_recall_at_3_max": 0.210759, + "nauc_recall_at_3_std": 0.247006, + "nauc_recall_at_3_diff1": 0.339129, + "nauc_recall_at_5_max": 0.156984, + "nauc_recall_at_5_std": 0.215464, + "nauc_recall_at_5_diff1": 0.181069, + "nauc_recall_at_10_max": 0.304183, + "nauc_recall_at_10_std": 0.37156, + "nauc_recall_at_10_diff1": 0.251359, + "nauc_recall_at_20_max": 0.353263, + "nauc_recall_at_20_std": 0.396162, + "nauc_recall_at_20_diff1": 0.501929, + "nauc_recall_at_100_max": 0.192757, + "nauc_recall_at_100_std": 0.107167, + "nauc_recall_at_100_diff1": 0.32894, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.248228, + "nauc_precision_at_1_std": 0.170648, + "nauc_precision_at_1_diff1": 0.22638, + "nauc_precision_at_3_max": 0.086168, + "nauc_precision_at_3_std": 0.228042, + "nauc_precision_at_3_diff1": 0.23064, + "nauc_precision_at_5_max": 0.072106, + "nauc_precision_at_5_std": 0.179045, + "nauc_precision_at_5_diff1": 0.063586, + "nauc_precision_at_10_max": 0.03657, + "nauc_precision_at_10_std": 0.12817, + "nauc_precision_at_10_diff1": 0.028442, + "nauc_precision_at_20_max": -0.061364, + "nauc_precision_at_20_std": -0.002054, + "nauc_precision_at_20_diff1": -0.051745, + "nauc_precision_at_100_max": -0.228279, + "nauc_precision_at_100_std": -0.138224, + "nauc_precision_at_100_diff1": -0.161834, + "nauc_precision_at_1000_max": -0.240631, + "nauc_precision_at_1000_std": -0.153374, + "nauc_precision_at_1000_diff1": -0.170956, + "nauc_cv_recall_at_1_max": 0.248228, + "nauc_cv_recall_at_1_std": 0.170648, + "nauc_cv_recall_at_1_diff1": 0.22638, + "nauc_cv_recall_at_3_max": 0.086264, + "nauc_cv_recall_at_3_std": 0.149899, + "nauc_cv_recall_at_3_diff1": 0.172659, + "nauc_cv_recall_at_5_max": -0.060147, + "nauc_cv_recall_at_5_std": -0.015761, + "nauc_cv_recall_at_5_diff1": 0.036561, + "nauc_cv_recall_at_10_max": 0.126751, + "nauc_cv_recall_at_10_std": 0.195771, + "nauc_cv_recall_at_10_diff1": -0.004403, + "nauc_cv_recall_at_20_max": 0.126616, + "nauc_cv_recall_at_20_std": -0.164944, + "nauc_cv_recall_at_20_diff1": 0.126616, + "nauc_cv_recall_at_100_max": 0.126616, + "nauc_cv_recall_at_100_std": -0.164944, + "nauc_cv_recall_at_100_diff1": 0.126616, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.248228, + "nauc_mrr_at_1_std": 0.170648, + "nauc_mrr_at_1_diff1": 0.22638, + "nauc_mrr_at_3_max": 0.187999, + "nauc_mrr_at_3_std": 0.171275, + "nauc_mrr_at_3_diff1": 0.190424, + "nauc_mrr_at_5_max": 0.169526, + "nauc_mrr_at_5_std": 0.146939, + "nauc_mrr_at_5_diff1": 0.170657, + "nauc_mrr_at_10_max": 0.181948, + "nauc_mrr_at_10_std": 0.163225, + "nauc_mrr_at_10_diff1": 0.167485, + "nauc_mrr_at_20_max": 0.183883, + "nauc_mrr_at_20_std": 0.160126, + "nauc_mrr_at_20_diff1": 0.173025, + "nauc_mrr_at_100_max": 0.183883, + "nauc_mrr_at_100_std": 0.160126, + "nauc_mrr_at_100_diff1": 0.173025, + "nauc_mrr_at_1000_max": 0.183897, + "nauc_mrr_at_1000_std": 0.160207, + "nauc_mrr_at_1000_diff1": 0.173037, + "main_score": 0.53583, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 10570.88462805748, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/Vidore2EconomicsReportsRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..a7b9dc5b95 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.56897, + "ndcg_at_3": 0.54406, + "ndcg_at_5": 0.53224, + "ndcg_at_10": 0.5312, + "ndcg_at_20": 0.56636, + "ndcg_at_100": 0.66965, + "ndcg_at_1000": 0.7048, + "map_at_1": 0.05858, + "map_at_3": 0.15753, + "map_at_5": 0.20983, + "map_at_10": 0.2823, + "map_at_20": 0.34178, + "map_at_100": 0.42535, + "map_at_1000": 0.4412, + "recall_at_1": 0.05858, + "recall_at_3": 0.21651, + "recall_at_5": 0.31154, + "recall_at_10": 0.45137, + "recall_at_20": 0.60401, + "recall_at_100": 0.90618, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.56897, + "cv_recall_at_3": 0.86207, + "cv_recall_at_5": 0.94828, + "cv_recall_at_10": 0.96552, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.56897, + "precision_at_3": 0.5, + "precision_at_5": 0.46897, + "precision_at_10": 0.3931, + "precision_at_20": 0.31034, + "precision_at_100": 0.12552, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.568966, + "mrr_at_3": 0.704023, + "mrr_at_5": 0.723851, + "mrr_at_10": 0.726006, + "mrr_at_20": 0.728587, + "mrr_at_100": 0.728587, + "mrr_at_1000": 0.728587, + "nauc_ndcg_at_1_max": 0.231526, + "nauc_ndcg_at_1_std": 0.291562, + "nauc_ndcg_at_1_diff1": 0.259267, + "nauc_ndcg_at_3_max": 0.020335, + "nauc_ndcg_at_3_std": 0.142739, + "nauc_ndcg_at_3_diff1": 0.147969, + "nauc_ndcg_at_5_max": 0.058984, + "nauc_ndcg_at_5_std": 0.18014, + "nauc_ndcg_at_5_diff1": 0.089374, + "nauc_ndcg_at_10_max": -0.008914, + "nauc_ndcg_at_10_std": 0.098533, + "nauc_ndcg_at_10_diff1": 0.140852, + "nauc_ndcg_at_20_max": 0.043266, + "nauc_ndcg_at_20_std": 0.124485, + "nauc_ndcg_at_20_diff1": 0.119546, + "nauc_ndcg_at_100_max": 0.081556, + "nauc_ndcg_at_100_std": 0.21285, + "nauc_ndcg_at_100_diff1": 0.11367, + "nauc_ndcg_at_1000_max": 0.027821, + "nauc_ndcg_at_1000_std": 0.161215, + "nauc_ndcg_at_1000_diff1": 0.100628, + "nauc_map_at_1_max": 0.139754, + "nauc_map_at_1_std": 0.047408, + "nauc_map_at_1_diff1": 0.516385, + "nauc_map_at_3_max": -0.067284, + "nauc_map_at_3_std": -0.157087, + "nauc_map_at_3_diff1": 0.272148, + "nauc_map_at_5_max": 0.023845, + "nauc_map_at_5_std": -0.062771, + "nauc_map_at_5_diff1": 0.248205, + "nauc_map_at_10_max": 0.088422, + "nauc_map_at_10_std": 0.003722, + "nauc_map_at_10_diff1": 0.237386, + "nauc_map_at_20_max": 0.084684, + "nauc_map_at_20_std": 0.059002, + "nauc_map_at_20_diff1": 0.217929, + "nauc_map_at_100_max": 0.04205, + "nauc_map_at_100_std": 0.098047, + "nauc_map_at_100_diff1": 0.135944, + "nauc_map_at_1000_max": -0.003563, + "nauc_map_at_1000_std": 0.070525, + "nauc_map_at_1000_diff1": 0.097998, + "nauc_recall_at_1_max": 0.139754, + "nauc_recall_at_1_std": 0.047408, + "nauc_recall_at_1_diff1": 0.516385, + "nauc_recall_at_3_max": -0.121448, + "nauc_recall_at_3_std": -0.227606, + "nauc_recall_at_3_diff1": 0.179078, + "nauc_recall_at_5_max": -0.032538, + "nauc_recall_at_5_std": -0.109124, + "nauc_recall_at_5_diff1": 0.116716, + "nauc_recall_at_10_max": -0.018542, + "nauc_recall_at_10_std": -0.062396, + "nauc_recall_at_10_diff1": 0.143072, + "nauc_recall_at_20_max": 0.037991, + "nauc_recall_at_20_std": 0.0206, + "nauc_recall_at_20_diff1": 0.129342, + "nauc_recall_at_100_max": 0.083572, + "nauc_recall_at_100_std": 0.23213, + "nauc_recall_at_100_diff1": -0.022331, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.231526, + "nauc_precision_at_1_std": 0.291562, + "nauc_precision_at_1_diff1": 0.259267, + "nauc_precision_at_3_max": 0.025439, + "nauc_precision_at_3_std": 0.208002, + "nauc_precision_at_3_diff1": 0.101147, + "nauc_precision_at_5_max": 0.143593, + "nauc_precision_at_5_std": 0.316271, + "nauc_precision_at_5_diff1": -0.023654, + "nauc_precision_at_10_max": 0.044605, + "nauc_precision_at_10_std": 0.206217, + "nauc_precision_at_10_diff1": -0.049172, + "nauc_precision_at_20_max": -0.046033, + "nauc_precision_at_20_std": 0.149336, + "nauc_precision_at_20_diff1": -0.12874, + "nauc_precision_at_100_max": -0.19253, + "nauc_precision_at_100_std": -0.006248, + "nauc_precision_at_100_diff1": -0.234058, + "nauc_precision_at_1000_max": -0.240838, + "nauc_precision_at_1000_std": -0.066929, + "nauc_precision_at_1000_diff1": -0.231972, + "nauc_cv_recall_at_1_max": 0.231526, + "nauc_cv_recall_at_1_std": 0.291562, + "nauc_cv_recall_at_1_diff1": 0.259267, + "nauc_cv_recall_at_3_max": 0.271892, + "nauc_cv_recall_at_3_std": 0.386494, + "nauc_cv_recall_at_3_diff1": 0.569138, + "nauc_cv_recall_at_5_max": 0.043843, + "nauc_cv_recall_at_5_std": 0.637689, + "nauc_cv_recall_at_5_diff1": 0.472102, + "nauc_cv_recall_at_10_max": -0.211243, + "nauc_cv_recall_at_10_std": 0.456534, + "nauc_cv_recall_at_10_diff1": 0.347932, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.231526, + "nauc_mrr_at_1_std": 0.291562, + "nauc_mrr_at_1_diff1": 0.259267, + "nauc_mrr_at_3_max": 0.222186, + "nauc_mrr_at_3_std": 0.294929, + "nauc_mrr_at_3_diff1": 0.30714, + "nauc_mrr_at_5_max": 0.209781, + "nauc_mrr_at_5_std": 0.305665, + "nauc_mrr_at_5_diff1": 0.281003, + "nauc_mrr_at_10_max": 0.206384, + "nauc_mrr_at_10_std": 0.298813, + "nauc_mrr_at_10_diff1": 0.276667, + "nauc_mrr_at_20_max": 0.208992, + "nauc_mrr_at_20_std": 0.296677, + "nauc_mrr_at_20_diff1": 0.27448, + "nauc_mrr_at_100_max": 0.208992, + "nauc_mrr_at_100_std": 0.296677, + "nauc_mrr_at_100_diff1": 0.27448, + "nauc_mrr_at_1000_max": 0.208992, + "nauc_mrr_at_1000_std": 0.296677, + "nauc_mrr_at_1000_diff1": 0.27448, + "main_score": 0.53224, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.60345, + "ndcg_at_3": 0.55903, + "ndcg_at_5": 0.54772, + "ndcg_at_10": 0.54985, + "ndcg_at_20": 0.57358, + "ndcg_at_100": 0.6853, + "ndcg_at_1000": 0.72595, + "map_at_1": 0.10464, + "map_at_3": 0.19143, + "map_at_5": 0.23873, + "map_at_10": 0.30993, + "map_at_20": 0.36104, + "map_at_100": 0.44812, + "map_at_1000": 0.46619, + "recall_at_1": 0.10464, + "recall_at_3": 0.23552, + "recall_at_5": 0.31483, + "recall_at_10": 0.45156, + "recall_at_20": 0.57867, + "recall_at_100": 0.89796, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.60345, + "cv_recall_at_3": 0.86207, + "cv_recall_at_5": 0.94828, + "cv_recall_at_10": 0.96552, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.60345, + "precision_at_3": 0.50575, + "precision_at_5": 0.46897, + "precision_at_10": 0.39483, + "precision_at_20": 0.29569, + "precision_at_100": 0.12155, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.603448, + "mrr_at_3": 0.727011, + "mrr_at_5": 0.746839, + "mrr_at_10": 0.748994, + "mrr_at_20": 0.749952, + "mrr_at_100": 0.75067, + "mrr_at_1000": 0.75067, + "nauc_ndcg_at_1_max": 0.187507, + "nauc_ndcg_at_1_std": 0.099747, + "nauc_ndcg_at_1_diff1": 0.328966, + "nauc_ndcg_at_3_max": 0.112379, + "nauc_ndcg_at_3_std": 0.030437, + "nauc_ndcg_at_3_diff1": 0.180371, + "nauc_ndcg_at_5_max": 0.090799, + "nauc_ndcg_at_5_std": 0.024668, + "nauc_ndcg_at_5_diff1": 0.094861, + "nauc_ndcg_at_10_max": 0.040737, + "nauc_ndcg_at_10_std": -0.007689, + "nauc_ndcg_at_10_diff1": 0.095568, + "nauc_ndcg_at_20_max": 0.05006, + "nauc_ndcg_at_20_std": 0.001488, + "nauc_ndcg_at_20_diff1": 0.135335, + "nauc_ndcg_at_100_max": 0.092631, + "nauc_ndcg_at_100_std": 0.055502, + "nauc_ndcg_at_100_diff1": 0.042814, + "nauc_ndcg_at_1000_max": 0.081966, + "nauc_ndcg_at_1000_std": 0.021603, + "nauc_ndcg_at_1000_diff1": 0.052303, + "nauc_map_at_1_max": -0.0177, + "nauc_map_at_1_std": -0.193607, + "nauc_map_at_1_diff1": 0.19111, + "nauc_map_at_3_max": -0.030385, + "nauc_map_at_3_std": -0.190304, + "nauc_map_at_3_diff1": 0.135395, + "nauc_map_at_5_max": -0.007399, + "nauc_map_at_5_std": -0.142171, + "nauc_map_at_5_diff1": 0.115911, + "nauc_map_at_10_max": 0.018531, + "nauc_map_at_10_std": -0.091571, + "nauc_map_at_10_diff1": 0.101506, + "nauc_map_at_20_max": 0.038382, + "nauc_map_at_20_std": -0.063357, + "nauc_map_at_20_diff1": 0.11663, + "nauc_map_at_100_max": 0.065717, + "nauc_map_at_100_std": -0.023767, + "nauc_map_at_100_diff1": 0.068343, + "nauc_map_at_1000_max": 0.044793, + "nauc_map_at_1000_std": -0.045252, + "nauc_map_at_1000_diff1": 0.049609, + "nauc_recall_at_1_max": -0.0177, + "nauc_recall_at_1_std": -0.193607, + "nauc_recall_at_1_diff1": 0.19111, + "nauc_recall_at_3_max": -0.084382, + "nauc_recall_at_3_std": -0.225947, + "nauc_recall_at_3_diff1": 0.097776, + "nauc_recall_at_5_max": -0.075285, + "nauc_recall_at_5_std": -0.154524, + "nauc_recall_at_5_diff1": 0.021698, + "nauc_recall_at_10_max": -0.065361, + "nauc_recall_at_10_std": -0.081617, + "nauc_recall_at_10_diff1": -0.038882, + "nauc_recall_at_20_max": -0.037809, + "nauc_recall_at_20_std": -0.045086, + "nauc_recall_at_20_diff1": 0.048166, + "nauc_recall_at_100_max": 0.038107, + "nauc_recall_at_100_std": 0.122766, + "nauc_recall_at_100_diff1": -0.073177, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.187507, + "nauc_precision_at_1_std": 0.099747, + "nauc_precision_at_1_diff1": 0.328966, + "nauc_precision_at_3_max": 0.167842, + "nauc_precision_at_3_std": 0.164375, + "nauc_precision_at_3_diff1": 0.154848, + "nauc_precision_at_5_max": 0.162461, + "nauc_precision_at_5_std": 0.205407, + "nauc_precision_at_5_diff1": -0.019634, + "nauc_precision_at_10_max": 0.109772, + "nauc_precision_at_10_std": 0.173447, + "nauc_precision_at_10_diff1": -0.00992, + "nauc_precision_at_20_max": 0.043112, + "nauc_precision_at_20_std": 0.100483, + "nauc_precision_at_20_diff1": -0.049822, + "nauc_precision_at_100_max": -0.040783, + "nauc_precision_at_100_std": -0.000578, + "nauc_precision_at_100_diff1": -0.154976, + "nauc_precision_at_1000_max": -0.090758, + "nauc_precision_at_1000_std": -0.065768, + "nauc_precision_at_1000_diff1": -0.141809, + "nauc_cv_recall_at_1_max": 0.187507, + "nauc_cv_recall_at_1_std": 0.099747, + "nauc_cv_recall_at_1_diff1": 0.328966, + "nauc_cv_recall_at_3_max": 0.085158, + "nauc_cv_recall_at_3_std": 0.062963, + "nauc_cv_recall_at_3_diff1": 0.40279, + "nauc_cv_recall_at_5_max": 0.3721, + "nauc_cv_recall_at_5_std": 0.572703, + "nauc_cv_recall_at_5_diff1": -0.155051, + "nauc_cv_recall_at_10_max": 0.123761, + "nauc_cv_recall_at_10_std": 0.359055, + "nauc_cv_recall_at_10_diff1": -0.294456, + "nauc_cv_recall_at_20_max": 0.123761, + "nauc_cv_recall_at_20_std": 0.359055, + "nauc_cv_recall_at_20_diff1": -1.142926, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.187507, + "nauc_mrr_at_1_std": 0.099747, + "nauc_mrr_at_1_diff1": 0.328966, + "nauc_mrr_at_3_max": 0.152848, + "nauc_mrr_at_3_std": 0.089923, + "nauc_mrr_at_3_diff1": 0.361282, + "nauc_mrr_at_5_max": 0.181161, + "nauc_mrr_at_5_std": 0.124507, + "nauc_mrr_at_5_diff1": 0.3275, + "nauc_mrr_at_10_max": 0.17401, + "nauc_mrr_at_10_std": 0.115402, + "nauc_mrr_at_10_diff1": 0.329619, + "nauc_mrr_at_20_max": 0.174243, + "nauc_mrr_at_20_std": 0.11427, + "nauc_mrr_at_20_diff1": 0.328577, + "nauc_mrr_at_100_max": 0.174419, + "nauc_mrr_at_100_std": 0.113415, + "nauc_mrr_at_100_diff1": 0.33372, + "nauc_mrr_at_1000_max": 0.174419, + "nauc_mrr_at_1000_std": 0.113415, + "nauc_mrr_at_1000_diff1": 0.33372, + "main_score": 0.54772, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.74138, + "ndcg_at_3": 0.6908, + "ndcg_at_5": 0.64322, + "ndcg_at_10": 0.62756, + "ndcg_at_20": 0.63399, + "ndcg_at_100": 0.74047, + "ndcg_at_1000": 0.76914, + "map_at_1": 0.11186, + "map_at_3": 0.22917, + "map_at_5": 0.28292, + "map_at_10": 0.37047, + "map_at_20": 0.42128, + "map_at_100": 0.5152, + "map_at_1000": 0.53059, + "recall_at_1": 0.11186, + "recall_at_3": 0.2707, + "recall_at_5": 0.34107, + "recall_at_10": 0.48952, + "recall_at_20": 0.61684, + "recall_at_100": 0.92815, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.74138, + "cv_recall_at_3": 0.91379, + "cv_recall_at_5": 0.93103, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.74138, + "precision_at_3": 0.63793, + "precision_at_5": 0.55172, + "precision_at_10": 0.45, + "precision_at_20": 0.31983, + "precision_at_100": 0.13017, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.741379, + "mrr_at_3": 0.818966, + "mrr_at_5": 0.822414, + "mrr_at_10": 0.825287, + "mrr_at_20": 0.8291, + "mrr_at_100": 0.8291, + "mrr_at_1000": 0.8291, + "nauc_ndcg_at_1_max": 0.380565, + "nauc_ndcg_at_1_std": 0.404259, + "nauc_ndcg_at_1_diff1": 0.203933, + "nauc_ndcg_at_3_max": 0.026429, + "nauc_ndcg_at_3_std": 0.103675, + "nauc_ndcg_at_3_diff1": -0.086341, + "nauc_ndcg_at_5_max": -0.01946, + "nauc_ndcg_at_5_std": 0.024029, + "nauc_ndcg_at_5_diff1": -0.11766, + "nauc_ndcg_at_10_max": -0.028951, + "nauc_ndcg_at_10_std": -0.047101, + "nauc_ndcg_at_10_diff1": -0.121016, + "nauc_ndcg_at_20_max": 0.115923, + "nauc_ndcg_at_20_std": 0.055932, + "nauc_ndcg_at_20_diff1": -0.022079, + "nauc_ndcg_at_100_max": 0.079059, + "nauc_ndcg_at_100_std": 0.116274, + "nauc_ndcg_at_100_diff1": -0.190259, + "nauc_ndcg_at_1000_max": 0.074677, + "nauc_ndcg_at_1000_std": 0.072952, + "nauc_ndcg_at_1000_diff1": -0.121777, + "nauc_map_at_1_max": 0.02206, + "nauc_map_at_1_std": -0.153202, + "nauc_map_at_1_diff1": 0.126436, + "nauc_map_at_3_max": -0.034382, + "nauc_map_at_3_std": -0.128998, + "nauc_map_at_3_diff1": 0.078942, + "nauc_map_at_5_max": -0.024308, + "nauc_map_at_5_std": -0.094896, + "nauc_map_at_5_diff1": 0.034637, + "nauc_map_at_10_max": -0.003942, + "nauc_map_at_10_std": -0.03903, + "nauc_map_at_10_diff1": -0.032251, + "nauc_map_at_20_max": 0.054403, + "nauc_map_at_20_std": 0.021584, + "nauc_map_at_20_diff1": -0.017586, + "nauc_map_at_100_max": 0.023069, + "nauc_map_at_100_std": 0.023188, + "nauc_map_at_100_diff1": -0.119157, + "nauc_map_at_1000_max": 0.008203, + "nauc_map_at_1000_std": -0.00377, + "nauc_map_at_1000_diff1": -0.115851, + "nauc_recall_at_1_max": 0.02206, + "nauc_recall_at_1_std": -0.153202, + "nauc_recall_at_1_diff1": 0.126436, + "nauc_recall_at_3_max": -0.103395, + "nauc_recall_at_3_std": -0.206097, + "nauc_recall_at_3_diff1": 0.030628, + "nauc_recall_at_5_max": -0.073583, + "nauc_recall_at_5_std": -0.168297, + "nauc_recall_at_5_diff1": -0.020215, + "nauc_recall_at_10_max": 0.011179, + "nauc_recall_at_10_std": -0.067452, + "nauc_recall_at_10_diff1": -0.086525, + "nauc_recall_at_20_max": 0.149623, + "nauc_recall_at_20_std": 0.106896, + "nauc_recall_at_20_diff1": 0.059795, + "nauc_recall_at_100_max": 0.057835, + "nauc_recall_at_100_std": 0.189766, + "nauc_recall_at_100_diff1": -0.383508, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.380565, + "nauc_precision_at_1_std": 0.404259, + "nauc_precision_at_1_diff1": 0.203933, + "nauc_precision_at_3_max": 0.019543, + "nauc_precision_at_3_std": 0.175002, + "nauc_precision_at_3_diff1": -0.159378, + "nauc_precision_at_5_max": 0.00836, + "nauc_precision_at_5_std": 0.131244, + "nauc_precision_at_5_diff1": -0.174769, + "nauc_precision_at_10_max": 0.006586, + "nauc_precision_at_10_std": 0.106823, + "nauc_precision_at_10_diff1": -0.203657, + "nauc_precision_at_20_max": 0.045012, + "nauc_precision_at_20_std": 0.089611, + "nauc_precision_at_20_diff1": -0.128116, + "nauc_precision_at_100_max": -0.100263, + "nauc_precision_at_100_std": -0.078305, + "nauc_precision_at_100_diff1": -0.172439, + "nauc_precision_at_1000_max": -0.106817, + "nauc_precision_at_1000_std": -0.125258, + "nauc_precision_at_1000_diff1": -0.115942, + "nauc_cv_recall_at_1_max": 0.380565, + "nauc_cv_recall_at_1_std": 0.404259, + "nauc_cv_recall_at_1_diff1": 0.203933, + "nauc_cv_recall_at_3_max": 0.329147, + "nauc_cv_recall_at_3_std": 0.305158, + "nauc_cv_recall_at_3_diff1": -0.14786, + "nauc_cv_recall_at_5_max": 0.272931, + "nauc_cv_recall_at_5_std": 0.164253, + "nauc_cv_recall_at_5_diff1": -0.323328, + "nauc_cv_recall_at_10_max": 0.322654, + "nauc_cv_recall_at_10_std": -0.021143, + "nauc_cv_recall_at_10_diff1": -0.720697, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.380565, + "nauc_mrr_at_1_std": 0.404259, + "nauc_mrr_at_1_diff1": 0.203933, + "nauc_mrr_at_3_max": 0.322865, + "nauc_mrr_at_3_std": 0.341728, + "nauc_mrr_at_3_diff1": 0.130509, + "nauc_mrr_at_5_max": 0.317969, + "nauc_mrr_at_5_std": 0.330565, + "nauc_mrr_at_5_diff1": 0.12154, + "nauc_mrr_at_10_max": 0.321458, + "nauc_mrr_at_10_std": 0.323561, + "nauc_mrr_at_10_diff1": 0.108114, + "nauc_mrr_at_20_max": 0.322411, + "nauc_mrr_at_20_std": 0.335425, + "nauc_mrr_at_20_diff1": 0.130692, + "nauc_mrr_at_100_max": 0.322411, + "nauc_mrr_at_100_std": 0.335425, + "nauc_mrr_at_100_diff1": 0.130692, + "nauc_mrr_at_1000_max": 0.322411, + "nauc_mrr_at_1000_std": 0.335425, + "nauc_mrr_at_1000_diff1": 0.130692, + "main_score": 0.64322, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.62069, + "ndcg_at_3": 0.56554, + "ndcg_at_5": 0.5446, + "ndcg_at_10": 0.53085, + "ndcg_at_20": 0.55319, + "ndcg_at_100": 0.67657, + "ndcg_at_1000": 0.71685, + "map_at_1": 0.08774, + "map_at_3": 0.19134, + "map_at_5": 0.23698, + "map_at_10": 0.30126, + "map_at_20": 0.34888, + "map_at_100": 0.43653, + "map_at_1000": 0.45415, + "recall_at_1": 0.08774, + "recall_at_3": 0.24273, + "recall_at_5": 0.31422, + "recall_at_10": 0.43528, + "recall_at_20": 0.5532, + "recall_at_100": 0.89491, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.62069, + "cv_recall_at_3": 0.87931, + "cv_recall_at_5": 0.93103, + "cv_recall_at_10": 0.96552, + "cv_recall_at_20": 0.96552, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.62069, + "precision_at_3": 0.51149, + "precision_at_5": 0.46207, + "precision_at_10": 0.36897, + "precision_at_20": 0.27931, + "precision_at_100": 0.12397, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.62069, + "mrr_at_3": 0.732759, + "mrr_at_5": 0.744828, + "mrr_at_10": 0.748898, + "mrr_at_20": 0.748898, + "mrr_at_100": 0.750321, + "mrr_at_1000": 0.750321, + "nauc_ndcg_at_1_max": 0.072131, + "nauc_ndcg_at_1_std": 0.167555, + "nauc_ndcg_at_1_diff1": 0.056723, + "nauc_ndcg_at_3_max": -0.099288, + "nauc_ndcg_at_3_std": 0.024389, + "nauc_ndcg_at_3_diff1": -0.009818, + "nauc_ndcg_at_5_max": -0.035542, + "nauc_ndcg_at_5_std": 0.073279, + "nauc_ndcg_at_5_diff1": -0.064717, + "nauc_ndcg_at_10_max": -0.058732, + "nauc_ndcg_at_10_std": 0.081662, + "nauc_ndcg_at_10_diff1": -0.039941, + "nauc_ndcg_at_20_max": -0.111138, + "nauc_ndcg_at_20_std": 0.051276, + "nauc_ndcg_at_20_diff1": -0.057947, + "nauc_ndcg_at_100_max": -0.079779, + "nauc_ndcg_at_100_std": 0.091744, + "nauc_ndcg_at_100_diff1": -0.089938, + "nauc_ndcg_at_1000_max": -0.045288, + "nauc_ndcg_at_1000_std": 0.098066, + "nauc_ndcg_at_1000_diff1": -0.085732, + "nauc_map_at_1_max": -0.002648, + "nauc_map_at_1_std": -0.160457, + "nauc_map_at_1_diff1": -0.073798, + "nauc_map_at_3_max": -0.142967, + "nauc_map_at_3_std": -0.208208, + "nauc_map_at_3_diff1": -0.016939, + "nauc_map_at_5_max": -0.131984, + "nauc_map_at_5_std": -0.137553, + "nauc_map_at_5_diff1": -0.051625, + "nauc_map_at_10_max": -0.116731, + "nauc_map_at_10_std": -0.050127, + "nauc_map_at_10_diff1": -0.048083, + "nauc_map_at_20_max": -0.155358, + "nauc_map_at_20_std": -0.0463, + "nauc_map_at_20_diff1": -0.074692, + "nauc_map_at_100_max": -0.121468, + "nauc_map_at_100_std": 0.009894, + "nauc_map_at_100_diff1": -0.135934, + "nauc_map_at_1000_max": -0.104036, + "nauc_map_at_1000_std": 0.017147, + "nauc_map_at_1000_diff1": -0.146879, + "nauc_recall_at_1_max": -0.002648, + "nauc_recall_at_1_std": -0.160457, + "nauc_recall_at_1_diff1": -0.073798, + "nauc_recall_at_3_max": -0.20206, + "nauc_recall_at_3_std": -0.271523, + "nauc_recall_at_3_diff1": 0.01406, + "nauc_recall_at_5_max": -0.177267, + "nauc_recall_at_5_std": -0.153356, + "nauc_recall_at_5_diff1": -0.033222, + "nauc_recall_at_10_max": -0.172486, + "nauc_recall_at_10_std": -0.033113, + "nauc_recall_at_10_diff1": -0.03393, + "nauc_recall_at_20_max": -0.266468, + "nauc_recall_at_20_std": -0.068669, + "nauc_recall_at_20_diff1": -0.063575, + "nauc_recall_at_100_max": -0.12445, + "nauc_recall_at_100_std": 0.035448, + "nauc_recall_at_100_diff1": -0.064062, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.072131, + "nauc_precision_at_1_std": 0.167555, + "nauc_precision_at_1_diff1": 0.056723, + "nauc_precision_at_3_max": -0.082785, + "nauc_precision_at_3_std": 0.115937, + "nauc_precision_at_3_diff1": 0.009646, + "nauc_precision_at_5_max": 0.049073, + "nauc_precision_at_5_std": 0.241947, + "nauc_precision_at_5_diff1": -0.037055, + "nauc_precision_at_10_max": 0.073114, + "nauc_precision_at_10_std": 0.281856, + "nauc_precision_at_10_diff1": 0.003396, + "nauc_precision_at_20_max": -0.001161, + "nauc_precision_at_20_std": 0.165621, + "nauc_precision_at_20_diff1": -0.10944, + "nauc_precision_at_100_max": 0.124289, + "nauc_precision_at_100_std": 0.138818, + "nauc_precision_at_100_diff1": -0.12044, + "nauc_precision_at_1000_max": 0.127186, + "nauc_precision_at_1000_std": 0.114088, + "nauc_precision_at_1000_diff1": -0.103764, + "nauc_cv_recall_at_1_max": 0.072131, + "nauc_cv_recall_at_1_std": 0.167555, + "nauc_cv_recall_at_1_diff1": 0.056723, + "nauc_cv_recall_at_3_max": 0.086487, + "nauc_cv_recall_at_3_std": 0.303366, + "nauc_cv_recall_at_3_diff1": 0.487779, + "nauc_cv_recall_at_5_max": 0.205979, + "nauc_cv_recall_at_5_std": 0.632704, + "nauc_cv_recall_at_5_diff1": 0.525141, + "nauc_cv_recall_at_10_max": -0.225269, + "nauc_cv_recall_at_10_std": 0.554013, + "nauc_cv_recall_at_10_diff1": 0.338887, + "nauc_cv_recall_at_20_max": -0.225269, + "nauc_cv_recall_at_20_std": 0.554013, + "nauc_cv_recall_at_20_diff1": 0.338887, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.072131, + "nauc_mrr_at_1_std": 0.167555, + "nauc_mrr_at_1_diff1": 0.056723, + "nauc_mrr_at_3_max": 0.015994, + "nauc_mrr_at_3_std": 0.140169, + "nauc_mrr_at_3_diff1": 0.116432, + "nauc_mrr_at_5_max": 0.02372, + "nauc_mrr_at_5_std": 0.160221, + "nauc_mrr_at_5_diff1": 0.099736, + "nauc_mrr_at_10_max": 0.01191, + "nauc_mrr_at_10_std": 0.149705, + "nauc_mrr_at_10_diff1": 0.087688, + "nauc_mrr_at_20_max": 0.01191, + "nauc_mrr_at_20_std": 0.149705, + "nauc_mrr_at_20_diff1": 0.087688, + "nauc_mrr_at_100_max": 0.013286, + "nauc_mrr_at_100_std": 0.146945, + "nauc_mrr_at_100_diff1": 0.086123, + "nauc_mrr_at_1000_max": 0.013286, + "nauc_mrr_at_1000_std": 0.146945, + "nauc_mrr_at_1000_diff1": 0.086123, + "main_score": 0.5446, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 489.8820776939392, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreArxivQARetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..d1cd46ac35 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", + "task_name": "VidoreArxivQARetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.816, + "ndcg_at_3": 0.86359, + "ndcg_at_5": 0.87608, + "ndcg_at_10": 0.88392, + "ndcg_at_20": 0.8879, + "ndcg_at_100": 0.89314, + "ndcg_at_1000": 0.89398, + "map_at_1": 0.816, + "map_at_3": 0.85233, + "map_at_5": 0.85933, + "map_at_10": 0.86262, + "map_at_20": 0.86367, + "map_at_100": 0.86442, + "map_at_1000": 0.86446, + "recall_at_1": 0.816, + "recall_at_3": 0.896, + "recall_at_5": 0.926, + "recall_at_10": 0.95, + "recall_at_20": 0.966, + "recall_at_100": 0.994, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.816, + "cv_recall_at_3": 0.896, + "cv_recall_at_5": 0.926, + "cv_recall_at_10": 0.95, + "cv_recall_at_20": 0.966, + "cv_recall_at_100": 0.994, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.816, + "precision_at_3": 0.29867, + "precision_at_5": 0.1852, + "precision_at_10": 0.095, + "precision_at_20": 0.0483, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "mrr_at_1": 0.816, + "mrr_at_3": 0.852333, + "mrr_at_5": 0.859333, + "mrr_at_10": 0.862619, + "mrr_at_20": 0.863675, + "mrr_at_100": 0.864417, + "mrr_at_1000": 0.864461, + "nauc_ndcg_at_1_max": 0.629301, + "nauc_ndcg_at_1_std": 0.407266, + "nauc_ndcg_at_1_diff1": 0.940854, + "nauc_ndcg_at_3_max": 0.666089, + "nauc_ndcg_at_3_std": 0.446013, + "nauc_ndcg_at_3_diff1": 0.92954, + "nauc_ndcg_at_5_max": 0.652073, + "nauc_ndcg_at_5_std": 0.442711, + "nauc_ndcg_at_5_diff1": 0.92714, + "nauc_ndcg_at_10_max": 0.650821, + "nauc_ndcg_at_10_std": 0.448872, + "nauc_ndcg_at_10_diff1": 0.932212, + "nauc_ndcg_at_20_max": 0.648087, + "nauc_ndcg_at_20_std": 0.44561, + "nauc_ndcg_at_20_diff1": 0.932202, + "nauc_ndcg_at_100_max": 0.646114, + "nauc_ndcg_at_100_std": 0.441938, + "nauc_ndcg_at_100_diff1": 0.932242, + "nauc_ndcg_at_1000_max": 0.647767, + "nauc_ndcg_at_1000_std": 0.439192, + "nauc_ndcg_at_1000_diff1": 0.932429, + "nauc_map_at_1_max": 0.629301, + "nauc_map_at_1_std": 0.407266, + "nauc_map_at_1_diff1": 0.940854, + "nauc_map_at_3_max": 0.65568, + "nauc_map_at_3_std": 0.435854, + "nauc_map_at_3_diff1": 0.932262, + "nauc_map_at_5_max": 0.648343, + "nauc_map_at_5_std": 0.433881, + "nauc_map_at_5_diff1": 0.931293, + "nauc_map_at_10_max": 0.647462, + "nauc_map_at_10_std": 0.435523, + "nauc_map_at_10_diff1": 0.933131, + "nauc_map_at_20_max": 0.646834, + "nauc_map_at_20_std": 0.434582, + "nauc_map_at_20_diff1": 0.933138, + "nauc_map_at_100_max": 0.646512, + "nauc_map_at_100_std": 0.434083, + "nauc_map_at_100_diff1": 0.933152, + "nauc_map_at_1000_max": 0.646623, + "nauc_map_at_1000_std": 0.433984, + "nauc_map_at_1000_diff1": 0.933164, + "nauc_recall_at_1_max": 0.629301, + "nauc_recall_at_1_std": 0.407266, + "nauc_recall_at_1_diff1": 0.940854, + "nauc_recall_at_3_max": 0.706896, + "nauc_recall_at_3_std": 0.485364, + "nauc_recall_at_3_diff1": 0.919151, + "nauc_recall_at_5_max": 0.667537, + "nauc_recall_at_5_std": 0.486815, + "nauc_recall_at_5_diff1": 0.905052, + "nauc_recall_at_10_max": 0.671242, + "nauc_recall_at_10_std": 0.554472, + "nauc_recall_at_10_diff1": 0.930065, + "nauc_recall_at_20_max": 0.646262, + "nauc_recall_at_20_std": 0.566595, + "nauc_recall_at_20_diff1": 0.928874, + "nauc_recall_at_100_max": 0.478836, + "nauc_recall_at_100_std": 0.786026, + "nauc_recall_at_100_diff1": 0.912854, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.629301, + "nauc_precision_at_1_std": 0.407266, + "nauc_precision_at_1_diff1": 0.940854, + "nauc_precision_at_3_max": 0.706896, + "nauc_precision_at_3_std": 0.485364, + "nauc_precision_at_3_diff1": 0.919151, + "nauc_precision_at_5_max": 0.667537, + "nauc_precision_at_5_std": 0.486815, + "nauc_precision_at_5_diff1": 0.905052, + "nauc_precision_at_10_max": 0.671242, + "nauc_precision_at_10_std": 0.554472, + "nauc_precision_at_10_diff1": 0.930065, + "nauc_precision_at_20_max": 0.646262, + "nauc_precision_at_20_std": 0.566595, + "nauc_precision_at_20_diff1": 0.928874, + "nauc_precision_at_100_max": 0.478836, + "nauc_precision_at_100_std": 0.786026, + "nauc_precision_at_100_diff1": 0.912854, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_cv_recall_at_1_max": 0.629301, + "nauc_cv_recall_at_1_std": 0.407266, + "nauc_cv_recall_at_1_diff1": 0.940854, + "nauc_cv_recall_at_3_max": 0.706896, + "nauc_cv_recall_at_3_std": 0.485364, + "nauc_cv_recall_at_3_diff1": 0.919151, + "nauc_cv_recall_at_5_max": 0.667537, + "nauc_cv_recall_at_5_std": 0.486815, + "nauc_cv_recall_at_5_diff1": 0.905052, + "nauc_cv_recall_at_10_max": 0.671242, + "nauc_cv_recall_at_10_std": 0.554472, + "nauc_cv_recall_at_10_diff1": 0.930065, + "nauc_cv_recall_at_20_max": 0.646262, + "nauc_cv_recall_at_20_std": 0.566595, + "nauc_cv_recall_at_20_diff1": 0.928874, + "nauc_cv_recall_at_100_max": 0.478836, + "nauc_cv_recall_at_100_std": 0.786026, + "nauc_cv_recall_at_100_diff1": 0.912854, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.629301, + "nauc_mrr_at_1_std": 0.407266, + "nauc_mrr_at_1_diff1": 0.940854, + "nauc_mrr_at_3_max": 0.65568, + "nauc_mrr_at_3_std": 0.435854, + "nauc_mrr_at_3_diff1": 0.932262, + "nauc_mrr_at_5_max": 0.648343, + "nauc_mrr_at_5_std": 0.433881, + "nauc_mrr_at_5_diff1": 0.931293, + "nauc_mrr_at_10_max": 0.647462, + "nauc_mrr_at_10_std": 0.435523, + "nauc_mrr_at_10_diff1": 0.933131, + "nauc_mrr_at_20_max": 0.646834, + "nauc_mrr_at_20_std": 0.434582, + "nauc_mrr_at_20_diff1": 0.933138, + "nauc_mrr_at_100_max": 0.646512, + "nauc_mrr_at_100_std": 0.434083, + "nauc_mrr_at_100_diff1": 0.933152, + "nauc_mrr_at_1000_max": 0.646623, + "nauc_mrr_at_1000_std": 0.433984, + "nauc_mrr_at_1000_diff1": 0.933164, + "main_score": 0.87608, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 207.87656116485596, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreDocVQARetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..77dd2089ea --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", + "task_name": "VidoreDocVQARetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.55876, + "ndcg_at_3": 0.62654, + "ndcg_at_5": 0.6448, + "ndcg_at_10": 0.66306, + "ndcg_at_20": 0.67604, + "ndcg_at_100": 0.69879, + "ndcg_at_1000": 0.70821, + "map_at_1": 0.55495, + "map_at_3": 0.60775, + "map_at_5": 0.61856, + "map_at_10": 0.62664, + "map_at_20": 0.63055, + "map_at_100": 0.63391, + "map_at_1000": 0.63446, + "recall_at_1": 0.55495, + "recall_at_3": 0.6694, + "recall_at_5": 0.71513, + "recall_at_10": 0.77175, + "recall_at_20": 0.82003, + "recall_at_100": 0.93675, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.55876, + "cv_recall_at_3": 0.67849, + "cv_recall_at_5": 0.72506, + "cv_recall_at_10": 0.78049, + "cv_recall_at_20": 0.82927, + "cv_recall_at_100": 0.94457, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.55876, + "precision_at_3": 0.22912, + "precision_at_5": 0.14767, + "precision_at_10": 0.08004, + "precision_at_20": 0.04302, + "precision_at_100": 0.01004, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.558758, + "mrr_at_3": 0.613082, + "mrr_at_5": 0.623503, + "mrr_at_10": 0.630951, + "mrr_at_20": 0.63469, + "mrr_at_100": 0.637787, + "mrr_at_1000": 0.638143, + "nauc_ndcg_at_1_max": 0.258765, + "nauc_ndcg_at_1_std": 0.350263, + "nauc_ndcg_at_1_diff1": 0.887473, + "nauc_ndcg_at_3_max": 0.219268, + "nauc_ndcg_at_3_std": 0.350245, + "nauc_ndcg_at_3_diff1": 0.859127, + "nauc_ndcg_at_5_max": 0.212543, + "nauc_ndcg_at_5_std": 0.356687, + "nauc_ndcg_at_5_diff1": 0.854457, + "nauc_ndcg_at_10_max": 0.208044, + "nauc_ndcg_at_10_std": 0.360275, + "nauc_ndcg_at_10_diff1": 0.853568, + "nauc_ndcg_at_20_max": 0.203164, + "nauc_ndcg_at_20_std": 0.365773, + "nauc_ndcg_at_20_diff1": 0.84821, + "nauc_ndcg_at_100_max": 0.21172, + "nauc_ndcg_at_100_std": 0.382014, + "nauc_ndcg_at_100_diff1": 0.851359, + "nauc_ndcg_at_1000_max": 0.214273, + "nauc_ndcg_at_1000_std": 0.362385, + "nauc_ndcg_at_1000_diff1": 0.856495, + "nauc_map_at_1_max": 0.267711, + "nauc_map_at_1_std": 0.358846, + "nauc_map_at_1_diff1": 0.891095, + "nauc_map_at_3_max": 0.236668, + "nauc_map_at_3_std": 0.355524, + "nauc_map_at_3_diff1": 0.869088, + "nauc_map_at_5_max": 0.232653, + "nauc_map_at_5_std": 0.357674, + "nauc_map_at_5_diff1": 0.866034, + "nauc_map_at_10_max": 0.230356, + "nauc_map_at_10_std": 0.358362, + "nauc_map_at_10_diff1": 0.865296, + "nauc_map_at_20_max": 0.229442, + "nauc_map_at_20_std": 0.359384, + "nauc_map_at_20_diff1": 0.863764, + "nauc_map_at_100_max": 0.230323, + "nauc_map_at_100_std": 0.361655, + "nauc_map_at_100_diff1": 0.864375, + "nauc_map_at_1000_max": 0.23023, + "nauc_map_at_1000_std": 0.360663, + "nauc_map_at_1000_diff1": 0.864593, + "nauc_recall_at_1_max": 0.267711, + "nauc_recall_at_1_std": 0.358846, + "nauc_recall_at_1_diff1": 0.891095, + "nauc_recall_at_3_max": 0.191601, + "nauc_recall_at_3_std": 0.353905, + "nauc_recall_at_3_diff1": 0.835823, + "nauc_recall_at_5_max": 0.160907, + "nauc_recall_at_5_std": 0.366412, + "nauc_recall_at_5_diff1": 0.816263, + "nauc_recall_at_10_max": 0.128165, + "nauc_recall_at_10_std": 0.37815, + "nauc_recall_at_10_diff1": 0.80824, + "nauc_recall_at_20_max": 0.079571, + "nauc_recall_at_20_std": 0.419438, + "nauc_recall_at_20_diff1": 0.767228, + "nauc_recall_at_100_max": 0.096209, + "nauc_recall_at_100_std": 0.846382, + "nauc_recall_at_100_diff1": 0.69002, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.258765, + "nauc_precision_at_1_std": 0.350263, + "nauc_precision_at_1_diff1": 0.887473, + "nauc_precision_at_3_max": 0.134749, + "nauc_precision_at_3_std": 0.308782, + "nauc_precision_at_3_diff1": 0.759617, + "nauc_precision_at_5_max": 0.086437, + "nauc_precision_at_5_std": 0.299766, + "nauc_precision_at_5_diff1": 0.701629, + "nauc_precision_at_10_max": 0.026159, + "nauc_precision_at_10_std": 0.270492, + "nauc_precision_at_10_diff1": 0.618247, + "nauc_precision_at_20_max": -0.073787, + "nauc_precision_at_20_std": 0.218939, + "nauc_precision_at_20_diff1": 0.462041, + "nauc_precision_at_100_max": -0.275372, + "nauc_precision_at_100_std": 0.074255, + "nauc_precision_at_100_diff1": -0.04142, + "nauc_precision_at_1000_max": -0.405873, + "nauc_precision_at_1000_std": -0.423973, + "nauc_precision_at_1000_diff1": -0.416889, + "nauc_cv_recall_at_1_max": 0.258765, + "nauc_cv_recall_at_1_std": 0.350263, + "nauc_cv_recall_at_1_diff1": 0.887473, + "nauc_cv_recall_at_3_max": 0.170773, + "nauc_cv_recall_at_3_std": 0.349589, + "nauc_cv_recall_at_3_diff1": 0.827543, + "nauc_cv_recall_at_5_max": 0.133206, + "nauc_cv_recall_at_5_std": 0.360912, + "nauc_cv_recall_at_5_diff1": 0.809884, + "nauc_cv_recall_at_10_max": 0.102101, + "nauc_cv_recall_at_10_std": 0.371728, + "nauc_cv_recall_at_10_diff1": 0.804183, + "nauc_cv_recall_at_20_max": 0.043688, + "nauc_cv_recall_at_20_std": 0.414949, + "nauc_cv_recall_at_20_diff1": 0.766199, + "nauc_cv_recall_at_100_max": 0.01522, + "nauc_cv_recall_at_100_std": 0.854147, + "nauc_cv_recall_at_100_diff1": 0.674246, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.258765, + "nauc_mrr_at_1_std": 0.350263, + "nauc_mrr_at_1_diff1": 0.887473, + "nauc_mrr_at_3_max": 0.22552, + "nauc_mrr_at_3_std": 0.34933, + "nauc_mrr_at_3_diff1": 0.864715, + "nauc_mrr_at_5_max": 0.221026, + "nauc_mrr_at_5_std": 0.351173, + "nauc_mrr_at_5_diff1": 0.86202, + "nauc_mrr_at_10_max": 0.220205, + "nauc_mrr_at_10_std": 0.352283, + "nauc_mrr_at_10_diff1": 0.862278, + "nauc_mrr_at_20_max": 0.21972, + "nauc_mrr_at_20_std": 0.353704, + "nauc_mrr_at_20_diff1": 0.861137, + "nauc_mrr_at_100_max": 0.221042, + "nauc_mrr_at_100_std": 0.355339, + "nauc_mrr_at_100_diff1": 0.861629, + "nauc_mrr_at_1000_max": 0.221327, + "nauc_mrr_at_1000_std": 0.354745, + "nauc_mrr_at_1000_diff1": 0.861902, + "main_score": 0.6448, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 301.94915986061096, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreInfoVQARetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..a9a9d92dbb --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", + "task_name": "VidoreInfoVQARetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.91093, + "ndcg_at_3": 0.92978, + "ndcg_at_5": 0.93606, + "ndcg_at_10": 0.94295, + "ndcg_at_20": 0.94527, + "ndcg_at_100": 0.94748, + "ndcg_at_1000": 0.94775, + "map_at_1": 0.91093, + "map_at_3": 0.9251, + "map_at_5": 0.92849, + "map_at_10": 0.93165, + "map_at_20": 0.93226, + "map_at_100": 0.93256, + "map_at_1000": 0.93259, + "recall_at_1": 0.91093, + "recall_at_3": 0.94332, + "recall_at_5": 0.9585, + "recall_at_10": 0.97814, + "recall_at_20": 0.98725, + "recall_at_100": 0.99858, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.91093, + "cv_recall_at_3": 0.94332, + "cv_recall_at_5": 0.95951, + "cv_recall_at_10": 0.97976, + "cv_recall_at_20": 0.98988, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.91093, + "precision_at_3": 0.31444, + "precision_at_5": 0.1919, + "precision_at_10": 0.09818, + "precision_at_20": 0.0496, + "precision_at_100": 0.01008, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.910931, + "mrr_at_3": 0.925101, + "mrr_at_5": 0.928745, + "mrr_at_10": 0.931769, + "mrr_at_20": 0.932432, + "mrr_at_100": 0.932661, + "mrr_at_1000": 0.932661, + "nauc_ndcg_at_1_max": 0.505049, + "nauc_ndcg_at_1_std": 0.246352, + "nauc_ndcg_at_1_diff1": 0.958069, + "nauc_ndcg_at_3_max": 0.49464, + "nauc_ndcg_at_3_std": 0.239627, + "nauc_ndcg_at_3_diff1": 0.960384, + "nauc_ndcg_at_5_max": 0.556874, + "nauc_ndcg_at_5_std": 0.2977, + "nauc_ndcg_at_5_diff1": 0.961471, + "nauc_ndcg_at_10_max": 0.547027, + "nauc_ndcg_at_10_std": 0.286687, + "nauc_ndcg_at_10_diff1": 0.961557, + "nauc_ndcg_at_20_max": 0.539829, + "nauc_ndcg_at_20_std": 0.291239, + "nauc_ndcg_at_20_diff1": 0.960621, + "nauc_ndcg_at_100_max": 0.529954, + "nauc_ndcg_at_100_std": 0.271735, + "nauc_ndcg_at_100_diff1": 0.959924, + "nauc_ndcg_at_1000_max": 0.527973, + "nauc_ndcg_at_1000_std": 0.268916, + "nauc_ndcg_at_1000_diff1": 0.960159, + "nauc_map_at_1_max": 0.505049, + "nauc_map_at_1_std": 0.246352, + "nauc_map_at_1_diff1": 0.958069, + "nauc_map_at_3_max": 0.500404, + "nauc_map_at_3_std": 0.240311, + "nauc_map_at_3_diff1": 0.959773, + "nauc_map_at_5_max": 0.53218, + "nauc_map_at_5_std": 0.269004, + "nauc_map_at_5_diff1": 0.960268, + "nauc_map_at_10_max": 0.526904, + "nauc_map_at_10_std": 0.263422, + "nauc_map_at_10_diff1": 0.960182, + "nauc_map_at_20_max": 0.525282, + "nauc_map_at_20_std": 0.264695, + "nauc_map_at_20_diff1": 0.959933, + "nauc_map_at_100_max": 0.524155, + "nauc_map_at_100_std": 0.262555, + "nauc_map_at_100_diff1": 0.959857, + "nauc_map_at_1000_max": 0.52399, + "nauc_map_at_1000_std": 0.262332, + "nauc_map_at_1000_diff1": 0.959871, + "nauc_recall_at_1_max": 0.505049, + "nauc_recall_at_1_std": 0.246352, + "nauc_recall_at_1_diff1": 0.958069, + "nauc_recall_at_3_max": 0.471142, + "nauc_recall_at_3_std": 0.2377, + "nauc_recall_at_3_diff1": 0.962685, + "nauc_recall_at_5_max": 0.693481, + "nauc_recall_at_5_std": 0.458882, + "nauc_recall_at_5_diff1": 0.968146, + "nauc_recall_at_10_max": 0.75459, + "nauc_recall_at_10_std": 0.52596, + "nauc_recall_at_10_diff1": 0.975815, + "nauc_recall_at_20_max": 0.783053, + "nauc_recall_at_20_std": 0.771021, + "nauc_recall_at_20_diff1": 0.968904, + "nauc_recall_at_100_max": 0.906713, + "nauc_recall_at_100_std": 0.82736, + "nauc_recall_at_100_diff1": 0.906713, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.505049, + "nauc_precision_at_1_std": 0.246352, + "nauc_precision_at_1_diff1": 0.958069, + "nauc_precision_at_3_max": 0.471142, + "nauc_precision_at_3_std": 0.2377, + "nauc_precision_at_3_diff1": 0.962685, + "nauc_precision_at_5_max": 0.692762, + "nauc_precision_at_5_std": 0.456496, + "nauc_precision_at_5_diff1": 0.96735, + "nauc_precision_at_10_max": 0.564332, + "nauc_precision_at_10_std": 0.387296, + "nauc_precision_at_10_diff1": 0.74413, + "nauc_precision_at_20_max": 0.441345, + "nauc_precision_at_20_std": 0.48796, + "nauc_precision_at_20_diff1": 0.573785, + "nauc_precision_at_100_max": -0.533693, + "nauc_precision_at_100_std": -0.390146, + "nauc_precision_at_100_diff1": -0.573514, + "nauc_precision_at_1000_max": -0.534483, + "nauc_precision_at_1000_std": -0.412238, + "nauc_precision_at_1000_diff1": -0.56103, + "nauc_cv_recall_at_1_max": 0.505049, + "nauc_cv_recall_at_1_std": 0.246352, + "nauc_cv_recall_at_1_diff1": 0.958069, + "nauc_cv_recall_at_3_max": 0.471142, + "nauc_cv_recall_at_3_std": 0.2377, + "nauc_cv_recall_at_3_diff1": 0.962685, + "nauc_cv_recall_at_5_max": 0.692762, + "nauc_cv_recall_at_5_std": 0.456496, + "nauc_cv_recall_at_5_diff1": 0.96735, + "nauc_cv_recall_at_10_max": 0.734958, + "nauc_cv_recall_at_10_std": 0.510256, + "nauc_cv_recall_at_10_diff1": 0.97388, + "nauc_cv_recall_at_20_max": 0.739707, + "nauc_cv_recall_at_20_std": 0.768985, + "nauc_cv_recall_at_20_diff1": 0.97388, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.505049, + "nauc_mrr_at_1_std": 0.246352, + "nauc_mrr_at_1_diff1": 0.958069, + "nauc_mrr_at_3_max": 0.500404, + "nauc_mrr_at_3_std": 0.240311, + "nauc_mrr_at_3_diff1": 0.959773, + "nauc_mrr_at_5_max": 0.531505, + "nauc_mrr_at_5_std": 0.26799, + "nauc_mrr_at_5_diff1": 0.960127, + "nauc_mrr_at_10_max": 0.525742, + "nauc_mrr_at_10_std": 0.26242, + "nauc_mrr_at_10_diff1": 0.960113, + "nauc_mrr_at_20_max": 0.523819, + "nauc_mrr_at_20_std": 0.263183, + "nauc_mrr_at_20_diff1": 0.959939, + "nauc_mrr_at_100_max": 0.523233, + "nauc_mrr_at_100_std": 0.261562, + "nauc_mrr_at_100_diff1": 0.959909, + "nauc_mrr_at_1000_max": 0.523233, + "nauc_mrr_at_1000_std": 0.261562, + "nauc_mrr_at_1000_diff1": 0.959909, + "main_score": 0.93606, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 315.8918011188507, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreShiftProjectRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..6973d7181f --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "84a382e05c4473fed9cff2bbae95fe2379416117", + "task_name": "VidoreShiftProjectRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.84, + "ndcg_at_3": 0.9194, + "ndcg_at_5": 0.92327, + "ndcg_at_10": 0.9266, + "ndcg_at_20": 0.9266, + "ndcg_at_100": 0.92821, + "ndcg_at_1000": 0.92821, + "map_at_1": 0.84, + "map_at_3": 0.90167, + "map_at_5": 0.90367, + "map_at_10": 0.9051, + "map_at_20": 0.9051, + "map_at_100": 0.90523, + "map_at_1000": 0.90523, + "recall_at_1": 0.84, + "recall_at_3": 0.97, + "recall_at_5": 0.98, + "recall_at_10": 0.99, + "recall_at_20": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.84, + "cv_recall_at_3": 0.97, + "cv_recall_at_5": 0.98, + "cv_recall_at_10": 0.99, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.84, + "precision_at_3": 0.32333, + "precision_at_5": 0.196, + "precision_at_10": 0.099, + "precision_at_20": 0.0495, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.84, + "mrr_at_3": 0.901667, + "mrr_at_5": 0.903667, + "mrr_at_10": 0.905095, + "mrr_at_20": 0.905095, + "mrr_at_100": 0.905232, + "mrr_at_1000": 0.905232, + "nauc_ndcg_at_1_max": 0.190967, + "nauc_ndcg_at_1_std": -0.301669, + "nauc_ndcg_at_1_diff1": 0.826522, + "nauc_ndcg_at_3_max": 0.08576, + "nauc_ndcg_at_3_std": -0.405153, + "nauc_ndcg_at_3_diff1": 0.84455, + "nauc_ndcg_at_5_max": 0.08382, + "nauc_ndcg_at_5_std": -0.366036, + "nauc_ndcg_at_5_diff1": 0.836406, + "nauc_ndcg_at_10_max": 0.114426, + "nauc_ndcg_at_10_std": -0.328888, + "nauc_ndcg_at_10_diff1": 0.834851, + "nauc_ndcg_at_20_max": 0.114426, + "nauc_ndcg_at_20_std": -0.328888, + "nauc_ndcg_at_20_diff1": 0.834851, + "nauc_ndcg_at_100_max": 0.121106, + "nauc_ndcg_at_100_std": -0.339447, + "nauc_ndcg_at_100_diff1": 0.830991, + "nauc_ndcg_at_1000_max": 0.121106, + "nauc_ndcg_at_1000_std": -0.339447, + "nauc_ndcg_at_1000_diff1": 0.830991, + "nauc_map_at_1_max": 0.190967, + "nauc_map_at_1_std": -0.301669, + "nauc_map_at_1_diff1": 0.826522, + "nauc_map_at_3_max": 0.118792, + "nauc_map_at_3_std": -0.369126, + "nauc_map_at_3_diff1": 0.832897, + "nauc_map_at_5_max": 0.118706, + "nauc_map_at_5_std": -0.352189, + "nauc_map_at_5_diff1": 0.82928, + "nauc_map_at_10_max": 0.129417, + "nauc_map_at_10_std": -0.339635, + "nauc_map_at_10_diff1": 0.828652, + "nauc_map_at_20_max": 0.129417, + "nauc_map_at_20_std": -0.339635, + "nauc_map_at_20_diff1": 0.828652, + "nauc_map_at_100_max": 0.129871, + "nauc_map_at_100_std": -0.340332, + "nauc_map_at_100_diff1": 0.828394, + "nauc_map_at_1000_max": 0.129871, + "nauc_map_at_1000_std": -0.340332, + "nauc_map_at_1000_diff1": 0.828394, + "nauc_recall_at_1_max": 0.190967, + "nauc_recall_at_1_std": -0.301669, + "nauc_recall_at_1_diff1": 0.826522, + "nauc_recall_at_3_max": -0.204015, + "nauc_recall_at_3_std": -0.726891, + "nauc_recall_at_3_diff1": 0.956427, + "nauc_recall_at_5_max": -0.367414, + "nauc_recall_at_5_std": -0.514472, + "nauc_recall_at_5_diff1": 0.934641, + "nauc_recall_at_10_max": -0.171335, + "nauc_recall_at_10_std": 0.122782, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": -0.171335, + "nauc_recall_at_20_std": 0.122782, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.190967, + "nauc_precision_at_1_std": -0.301669, + "nauc_precision_at_1_diff1": 0.826522, + "nauc_precision_at_3_max": -0.204015, + "nauc_precision_at_3_std": -0.726891, + "nauc_precision_at_3_diff1": 0.956427, + "nauc_precision_at_5_max": -0.367414, + "nauc_precision_at_5_std": -0.514472, + "nauc_precision_at_5_diff1": 0.934641, + "nauc_precision_at_10_max": -0.171335, + "nauc_precision_at_10_std": 0.122782, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": -0.171335, + "nauc_precision_at_20_std": 0.122782, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.190967, + "nauc_cv_recall_at_1_std": -0.301669, + "nauc_cv_recall_at_1_diff1": 0.826522, + "nauc_cv_recall_at_3_max": -0.204015, + "nauc_cv_recall_at_3_std": -0.726891, + "nauc_cv_recall_at_3_diff1": 0.956427, + "nauc_cv_recall_at_5_max": -0.367414, + "nauc_cv_recall_at_5_std": -0.514472, + "nauc_cv_recall_at_5_diff1": 0.934641, + "nauc_cv_recall_at_10_max": -0.171335, + "nauc_cv_recall_at_10_std": 0.122782, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": -0.171335, + "nauc_cv_recall_at_20_std": 0.122782, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.190967, + "nauc_mrr_at_1_std": -0.301669, + "nauc_mrr_at_1_diff1": 0.826522, + "nauc_mrr_at_3_max": 0.118792, + "nauc_mrr_at_3_std": -0.369126, + "nauc_mrr_at_3_diff1": 0.832897, + "nauc_mrr_at_5_max": 0.118706, + "nauc_mrr_at_5_std": -0.352189, + "nauc_mrr_at_5_diff1": 0.82928, + "nauc_mrr_at_10_max": 0.129417, + "nauc_mrr_at_10_std": -0.339635, + "nauc_mrr_at_10_diff1": 0.828652, + "nauc_mrr_at_20_max": 0.129417, + "nauc_mrr_at_20_std": -0.339635, + "nauc_mrr_at_20_diff1": 0.828652, + "nauc_mrr_at_100_max": 0.129871, + "nauc_mrr_at_100_std": -0.340332, + "nauc_mrr_at_100_diff1": 0.828394, + "nauc_mrr_at_1000_max": 0.129871, + "nauc_mrr_at_1000_std": -0.340332, + "nauc_mrr_at_1000_diff1": 0.828394, + "main_score": 0.92327, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 556.1751530170441, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreSyntheticDocQAAIRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..9a0ad963a7 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", + "task_name": "VidoreSyntheticDocQAAIRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "map_at_1": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "map_at_10": 1.0, + "map_at_20": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "recall_at_1": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 1.0, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 1.0, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "mrr_at_10": 1.0, + "mrr_at_20": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_1000_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": NaN, + "nauc_cv_recall_at_1_std": NaN, + "nauc_cv_recall_at_1_diff1": NaN, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "main_score": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 446.8306210041046, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreSyntheticDocQAEnergyRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..851dd9b75e --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", + "task_name": "VidoreSyntheticDocQAEnergyRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.96, + "ndcg_at_3": 0.96631, + "ndcg_at_5": 0.96631, + "ndcg_at_10": 0.97235, + "ndcg_at_20": 0.97235, + "ndcg_at_100": 0.97451, + "ndcg_at_1000": 0.97451, + "map_at_1": 0.96, + "map_at_3": 0.965, + "map_at_5": 0.965, + "map_at_10": 0.96725, + "map_at_20": 0.96725, + "map_at_100": 0.96767, + "map_at_1000": 0.96767, + "recall_at_1": 0.96, + "recall_at_3": 0.97, + "recall_at_5": 0.97, + "recall_at_10": 0.99, + "recall_at_20": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.96, + "cv_recall_at_3": 0.97, + "cv_recall_at_5": 0.97, + "cv_recall_at_10": 0.99, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.96, + "precision_at_3": 0.32333, + "precision_at_5": 0.194, + "precision_at_10": 0.099, + "precision_at_20": 0.0495, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.96, + "mrr_at_3": 0.965, + "mrr_at_5": 0.965, + "mrr_at_10": 0.96725, + "mrr_at_20": 0.96725, + "mrr_at_100": 0.967667, + "mrr_at_1000": 0.967667, + "nauc_ndcg_at_1_max": 0.56711, + "nauc_ndcg_at_1_std": -0.50887, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.65032, + "nauc_ndcg_at_3_std": -0.278322, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.65032, + "nauc_ndcg_at_5_std": -0.278322, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.65217, + "nauc_ndcg_at_10_std": -0.239664, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.65217, + "nauc_ndcg_at_20_std": -0.239664, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.63383, + "nauc_ndcg_at_100_std": -0.320917, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.63383, + "nauc_ndcg_at_1000_std": -0.320917, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.56711, + "nauc_map_at_1_std": -0.50887, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.630586, + "nauc_map_at_3_std": -0.333, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.630586, + "nauc_map_at_5_std": -0.333, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.629422, + "nauc_map_at_10_std": -0.330315, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.629422, + "nauc_map_at_20_std": -0.330315, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.626331, + "nauc_map_at_100_std": -0.343878, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.626331, + "nauc_map_at_1000_std": -0.343878, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.56711, + "nauc_recall_at_1_std": -0.50887, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 0.715219, + "nauc_recall_at_3_std": -0.098506, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 0.715219, + "nauc_recall_at_5_std": -0.098506, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 0.869281, + "nauc_recall_at_10_std": 0.722222, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": 0.869281, + "nauc_recall_at_20_std": 0.722222, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.56711, + "nauc_precision_at_1_std": -0.50887, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 0.715219, + "nauc_precision_at_3_std": -0.098506, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 0.715219, + "nauc_precision_at_5_std": -0.098506, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 0.869281, + "nauc_precision_at_10_std": 0.722222, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 0.869281, + "nauc_precision_at_20_std": 0.722222, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.56711, + "nauc_cv_recall_at_1_std": -0.50887, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": 0.715219, + "nauc_cv_recall_at_3_std": -0.098506, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 0.715219, + "nauc_cv_recall_at_5_std": -0.098506, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": 0.869281, + "nauc_cv_recall_at_10_std": 0.722222, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": 0.869281, + "nauc_cv_recall_at_20_std": 0.722222, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.56711, + "nauc_mrr_at_1_std": -0.50887, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.630586, + "nauc_mrr_at_3_std": -0.333, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.630586, + "nauc_mrr_at_5_std": -0.333, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.629422, + "nauc_mrr_at_10_std": -0.330315, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.629422, + "nauc_mrr_at_20_std": -0.330315, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.626331, + "nauc_mrr_at_100_std": -0.343878, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.626331, + "nauc_mrr_at_1000_std": -0.343878, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.96631, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 428.78443908691406, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..878d4b2dfe --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b4909afa930f81282fd20601e860668073ad02aa", + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.93, + "ndcg_at_3": 0.96655, + "ndcg_at_5": 0.96655, + "ndcg_at_10": 0.97011, + "ndcg_at_20": 0.97011, + "ndcg_at_100": 0.97011, + "ndcg_at_1000": 0.97011, + "map_at_1": 0.93, + "map_at_3": 0.95833, + "map_at_5": 0.95833, + "map_at_10": 0.96, + "map_at_20": 0.96, + "map_at_100": 0.96, + "map_at_1000": 0.96, + "recall_at_1": 0.93, + "recall_at_3": 0.99, + "recall_at_5": 0.99, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.93, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 0.99, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.93, + "precision_at_3": 0.33, + "precision_at_5": 0.198, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.93, + "mrr_at_3": 0.958333, + "mrr_at_5": 0.958333, + "mrr_at_10": 0.96, + "mrr_at_20": 0.96, + "mrr_at_100": 0.96, + "mrr_at_1000": 0.96, + "nauc_ndcg_at_1_max": 0.771309, + "nauc_ndcg_at_1_std": 0.254568, + "nauc_ndcg_at_1_diff1": 0.943978, + "nauc_ndcg_at_3_max": 0.82339, + "nauc_ndcg_at_3_std": 0.406881, + "nauc_ndcg_at_3_diff1": 0.95162, + "nauc_ndcg_at_5_max": 0.82339, + "nauc_ndcg_at_5_std": 0.406881, + "nauc_ndcg_at_5_diff1": 0.95162, + "nauc_ndcg_at_10_max": 0.802344, + "nauc_ndcg_at_10_std": 0.336201, + "nauc_ndcg_at_10_diff1": 0.945855, + "nauc_ndcg_at_20_max": 0.802344, + "nauc_ndcg_at_20_std": 0.336201, + "nauc_ndcg_at_20_diff1": 0.945855, + "nauc_ndcg_at_100_max": 0.802344, + "nauc_ndcg_at_100_std": 0.336201, + "nauc_ndcg_at_100_diff1": 0.945855, + "nauc_ndcg_at_1000_max": 0.802344, + "nauc_ndcg_at_1000_std": 0.336201, + "nauc_ndcg_at_1000_diff1": 0.945855, + "nauc_map_at_1_max": 0.771309, + "nauc_map_at_1_std": 0.254568, + "nauc_map_at_1_diff1": 0.943978, + "nauc_map_at_3_max": 0.807899, + "nauc_map_at_3_std": 0.356004, + "nauc_map_at_3_diff1": 0.947712, + "nauc_map_at_5_max": 0.807899, + "nauc_map_at_5_std": 0.356004, + "nauc_map_at_5_diff1": 0.947712, + "nauc_map_at_10_max": 0.799895, + "nauc_map_at_10_std": 0.329171, + "nauc_map_at_10_diff1": 0.945534, + "nauc_map_at_20_max": 0.799895, + "nauc_map_at_20_std": 0.329171, + "nauc_map_at_20_diff1": 0.945534, + "nauc_map_at_100_max": 0.799895, + "nauc_map_at_100_std": 0.329171, + "nauc_map_at_100_diff1": 0.945534, + "nauc_map_at_1000_max": 0.799895, + "nauc_map_at_1000_std": 0.329171, + "nauc_map_at_1000_diff1": 0.945534, + "nauc_recall_at_1_max": 0.771309, + "nauc_recall_at_1_std": 0.254568, + "nauc_recall_at_1_diff1": 0.943978, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": 1.0, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": 1.0, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.771309, + "nauc_precision_at_1_std": 0.254568, + "nauc_precision_at_1_diff1": 0.943978, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.771309, + "nauc_cv_recall_at_1_std": 0.254568, + "nauc_cv_recall_at_1_diff1": 0.943978, + "nauc_cv_recall_at_3_max": 1.0, + "nauc_cv_recall_at_3_std": 1.0, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 1.0, + "nauc_cv_recall_at_5_std": 1.0, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.771309, + "nauc_mrr_at_1_std": 0.254568, + "nauc_mrr_at_1_diff1": 0.943978, + "nauc_mrr_at_3_max": 0.807899, + "nauc_mrr_at_3_std": 0.356004, + "nauc_mrr_at_3_diff1": 0.947712, + "nauc_mrr_at_5_max": 0.807899, + "nauc_mrr_at_5_std": 0.356004, + "nauc_mrr_at_5_diff1": 0.947712, + "nauc_mrr_at_10_max": 0.799895, + "nauc_mrr_at_10_std": 0.329171, + "nauc_mrr_at_10_diff1": 0.945534, + "nauc_mrr_at_20_max": 0.799895, + "nauc_mrr_at_20_std": 0.329171, + "nauc_mrr_at_20_diff1": 0.945534, + "nauc_mrr_at_100_max": 0.799895, + "nauc_mrr_at_100_std": 0.329171, + "nauc_mrr_at_100_diff1": 0.945534, + "nauc_mrr_at_1000_max": 0.799895, + "nauc_mrr_at_1000_std": 0.329171, + "nauc_mrr_at_1000_diff1": 0.945534, + "main_score": 0.96655, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 449.91918754577637, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..a758f88ee5 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.99, + "ndcg_at_3": 0.99631, + "ndcg_at_5": 0.99631, + "ndcg_at_10": 0.99631, + "ndcg_at_20": 0.99631, + "ndcg_at_100": 0.99631, + "ndcg_at_1000": 0.99631, + "map_at_1": 0.99, + "map_at_3": 0.995, + "map_at_5": 0.995, + "map_at_10": 0.995, + "map_at_20": 0.995, + "map_at_100": 0.995, + "map_at_1000": 0.995, + "recall_at_1": 0.99, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.99, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.99, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.99, + "mrr_at_3": 0.995, + "mrr_at_5": 0.995, + "mrr_at_10": 0.995, + "mrr_at_20": 0.995, + "mrr_at_100": 0.995, + "mrr_at_1000": 0.995, + "nauc_ndcg_at_1_max": 0.722222, + "nauc_ndcg_at_1_std": 1.0, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.722222, + "nauc_ndcg_at_3_std": 1.0, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.722222, + "nauc_ndcg_at_5_std": 1.0, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.722222, + "nauc_ndcg_at_10_std": 1.0, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.722222, + "nauc_ndcg_at_20_std": 1.0, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.722222, + "nauc_ndcg_at_100_std": 1.0, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.722222, + "nauc_ndcg_at_1000_std": 1.0, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.722222, + "nauc_map_at_1_std": 1.0, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.722222, + "nauc_map_at_3_std": 1.0, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.722222, + "nauc_map_at_5_std": 1.0, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.722222, + "nauc_map_at_10_std": 1.0, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.722222, + "nauc_map_at_20_std": 1.0, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.722222, + "nauc_map_at_100_std": 1.0, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.722222, + "nauc_map_at_1000_std": 1.0, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.722222, + "nauc_recall_at_1_std": 1.0, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.722222, + "nauc_precision_at_1_std": 1.0, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.722222, + "nauc_cv_recall_at_1_std": 1.0, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.722222, + "nauc_mrr_at_1_std": 1.0, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.722222, + "nauc_mrr_at_3_std": 1.0, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.722222, + "nauc_mrr_at_5_std": 1.0, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.722222, + "nauc_mrr_at_10_std": 1.0, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.722222, + "nauc_mrr_at_20_std": 1.0, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.722222, + "nauc_mrr_at_100_std": 1.0, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.722222, + "nauc_mrr_at_1000_std": 1.0, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.99631, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 440.7572731971741, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreTabfquadRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..f94dc168be --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", + "task_name": "VidoreTabfquadRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.88929, + "ndcg_at_3": 0.93699, + "ndcg_at_5": 0.94298, + "ndcg_at_10": 0.94649, + "ndcg_at_20": 0.94746, + "ndcg_at_100": 0.94808, + "ndcg_at_1000": 0.94808, + "map_at_1": 0.88929, + "map_at_3": 0.92619, + "map_at_5": 0.92958, + "map_at_10": 0.93105, + "map_at_20": 0.93135, + "map_at_100": 0.93142, + "map_at_1000": 0.93142, + "recall_at_1": 0.88929, + "recall_at_3": 0.96786, + "recall_at_5": 0.98214, + "recall_at_10": 0.99286, + "recall_at_20": 0.99643, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.88929, + "cv_recall_at_3": 0.96786, + "cv_recall_at_5": 0.98214, + "cv_recall_at_10": 0.99286, + "cv_recall_at_20": 0.99643, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.88929, + "precision_at_3": 0.32262, + "precision_at_5": 0.19643, + "precision_at_10": 0.09929, + "precision_at_20": 0.04982, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.889286, + "mrr_at_3": 0.92619, + "mrr_at_5": 0.929583, + "mrr_at_10": 0.93105, + "mrr_at_20": 0.931348, + "mrr_at_100": 0.931416, + "mrr_at_1000": 0.931416, + "nauc_ndcg_at_1_max": 0.115356, + "nauc_ndcg_at_1_std": 0.089778, + "nauc_ndcg_at_1_diff1": 0.927092, + "nauc_ndcg_at_3_max": 0.276933, + "nauc_ndcg_at_3_std": 0.240953, + "nauc_ndcg_at_3_diff1": 0.937815, + "nauc_ndcg_at_5_max": 0.235644, + "nauc_ndcg_at_5_std": 0.222536, + "nauc_ndcg_at_5_diff1": 0.937971, + "nauc_ndcg_at_10_max": 0.227084, + "nauc_ndcg_at_10_std": 0.206231, + "nauc_ndcg_at_10_diff1": 0.936802, + "nauc_ndcg_at_20_max": 0.217898, + "nauc_ndcg_at_20_std": 0.193931, + "nauc_ndcg_at_20_diff1": 0.93563, + "nauc_ndcg_at_100_max": 0.208411, + "nauc_ndcg_at_100_std": 0.184153, + "nauc_ndcg_at_100_diff1": 0.934849, + "nauc_ndcg_at_1000_max": 0.208411, + "nauc_ndcg_at_1000_std": 0.184153, + "nauc_ndcg_at_1000_diff1": 0.934849, + "nauc_map_at_1_max": 0.115356, + "nauc_map_at_1_std": 0.089778, + "nauc_map_at_1_diff1": 0.927092, + "nauc_map_at_3_max": 0.224727, + "nauc_map_at_3_std": 0.192095, + "nauc_map_at_3_diff1": 0.934851, + "nauc_map_at_5_max": 0.203673, + "nauc_map_at_5_std": 0.182009, + "nauc_map_at_5_diff1": 0.934694, + "nauc_map_at_10_max": 0.200307, + "nauc_map_at_10_std": 0.175979, + "nauc_map_at_10_diff1": 0.934267, + "nauc_map_at_20_max": 0.198021, + "nauc_map_at_20_std": 0.172942, + "nauc_map_at_20_diff1": 0.933979, + "nauc_map_at_100_max": 0.19721, + "nauc_map_at_100_std": 0.172106, + "nauc_map_at_100_diff1": 0.933912, + "nauc_map_at_1000_max": 0.19721, + "nauc_map_at_1000_std": 0.172106, + "nauc_map_at_1000_diff1": 0.933912, + "nauc_recall_at_1_max": 0.115356, + "nauc_recall_at_1_std": 0.089778, + "nauc_recall_at_1_diff1": 0.927092, + "nauc_recall_at_3_max": 0.613912, + "nauc_recall_at_3_std": 0.556334, + "nauc_recall_at_3_diff1": 0.956427, + "nauc_recall_at_5_max": 0.565453, + "nauc_recall_at_5_std": 0.65789, + "nauc_recall_at_5_diff1": 0.973856, + "nauc_recall_at_10_max": 0.861111, + "nauc_recall_at_10_std": 0.934641, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 1.0, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.115356, + "nauc_precision_at_1_std": 0.089778, + "nauc_precision_at_1_diff1": 0.927092, + "nauc_precision_at_3_max": 0.613912, + "nauc_precision_at_3_std": 0.556334, + "nauc_precision_at_3_diff1": 0.956427, + "nauc_precision_at_5_max": 0.565453, + "nauc_precision_at_5_std": 0.65789, + "nauc_precision_at_5_diff1": 0.973856, + "nauc_precision_at_10_max": 0.861111, + "nauc_precision_at_10_std": 0.934641, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.115356, + "nauc_cv_recall_at_1_std": 0.089778, + "nauc_cv_recall_at_1_diff1": 0.927092, + "nauc_cv_recall_at_3_max": 0.613912, + "nauc_cv_recall_at_3_std": 0.556334, + "nauc_cv_recall_at_3_diff1": 0.956427, + "nauc_cv_recall_at_5_max": 0.565453, + "nauc_cv_recall_at_5_std": 0.65789, + "nauc_cv_recall_at_5_diff1": 0.973856, + "nauc_cv_recall_at_10_max": 0.861111, + "nauc_cv_recall_at_10_std": 0.934641, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": 1.0, + "nauc_cv_recall_at_20_std": 1.0, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.115356, + "nauc_mrr_at_1_std": 0.089778, + "nauc_mrr_at_1_diff1": 0.927092, + "nauc_mrr_at_3_max": 0.224727, + "nauc_mrr_at_3_std": 0.192095, + "nauc_mrr_at_3_diff1": 0.934851, + "nauc_mrr_at_5_max": 0.203673, + "nauc_mrr_at_5_std": 0.182009, + "nauc_mrr_at_5_diff1": 0.934694, + "nauc_mrr_at_10_max": 0.200307, + "nauc_mrr_at_10_std": 0.175979, + "nauc_mrr_at_10_diff1": 0.934267, + "nauc_mrr_at_20_max": 0.198021, + "nauc_mrr_at_20_std": 0.172942, + "nauc_mrr_at_20_diff1": 0.933979, + "nauc_mrr_at_100_max": 0.19721, + "nauc_mrr_at_100_std": 0.172106, + "nauc_mrr_at_100_diff1": 0.933912, + "nauc_mrr_at_1000_max": 0.19721, + "nauc_mrr_at_1000_std": 0.172106, + "nauc_mrr_at_1000_diff1": 0.933912, + "main_score": 0.94298, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 23.781982898712158, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreTatdqaRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..0d0ebfd45e --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", + "task_name": "VidoreTatdqaRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.69259, + "ndcg_at_3": 0.78033, + "ndcg_at_5": 0.79756, + "ndcg_at_10": 0.81302, + "ndcg_at_20": 0.82017, + "ndcg_at_100": 0.8258, + "ndcg_at_1000": 0.8276, + "map_at_1": 0.69137, + "map_at_3": 0.75896, + "map_at_5": 0.76862, + "map_at_10": 0.77514, + "map_at_20": 0.77721, + "map_at_100": 0.77797, + "map_at_1000": 0.77807, + "recall_at_1": 0.69137, + "recall_at_3": 0.84143, + "recall_at_5": 0.88305, + "recall_at_10": 0.92989, + "recall_at_20": 0.95729, + "recall_at_100": 0.98761, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.69988, + "cv_recall_at_3": 0.84265, + "cv_recall_at_5": 0.88335, + "cv_recall_at_10": 0.93013, + "cv_recall_at_20": 0.95808, + "cv_recall_at_100": 0.98846, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.69259, + "precision_at_3": 0.28129, + "precision_at_5": 0.17716, + "precision_at_10": 0.09344, + "precision_at_20": 0.04812, + "precision_at_100": 0.00995, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.699878, + "mrr_at_3": 0.762556, + "mrr_at_5": 0.771851, + "mrr_at_10": 0.778295, + "mrr_at_20": 0.780404, + "mrr_at_100": 0.78115, + "mrr_at_1000": 0.781238, + "nauc_ndcg_at_1_max": 0.238679, + "nauc_ndcg_at_1_std": 0.173904, + "nauc_ndcg_at_1_diff1": 0.857482, + "nauc_ndcg_at_3_max": 0.225838, + "nauc_ndcg_at_3_std": 0.199369, + "nauc_ndcg_at_3_diff1": 0.82004, + "nauc_ndcg_at_5_max": 0.230216, + "nauc_ndcg_at_5_std": 0.200582, + "nauc_ndcg_at_5_diff1": 0.812704, + "nauc_ndcg_at_10_max": 0.235123, + "nauc_ndcg_at_10_std": 0.206398, + "nauc_ndcg_at_10_diff1": 0.815014, + "nauc_ndcg_at_20_max": 0.231446, + "nauc_ndcg_at_20_std": 0.198242, + "nauc_ndcg_at_20_diff1": 0.81661, + "nauc_ndcg_at_100_max": 0.230959, + "nauc_ndcg_at_100_std": 0.196706, + "nauc_ndcg_at_100_diff1": 0.821466, + "nauc_ndcg_at_1000_max": 0.231105, + "nauc_ndcg_at_1000_std": 0.194429, + "nauc_ndcg_at_1000_diff1": 0.822927, + "nauc_map_at_1_max": 0.236212, + "nauc_map_at_1_std": 0.171096, + "nauc_map_at_1_diff1": 0.856401, + "nauc_map_at_3_max": 0.22859, + "nauc_map_at_3_std": 0.190566, + "nauc_map_at_3_diff1": 0.829344, + "nauc_map_at_5_max": 0.230605, + "nauc_map_at_5_std": 0.1906, + "nauc_map_at_5_diff1": 0.825965, + "nauc_map_at_10_max": 0.232115, + "nauc_map_at_10_std": 0.192317, + "nauc_map_at_10_diff1": 0.827006, + "nauc_map_at_20_max": 0.231355, + "nauc_map_at_20_std": 0.190391, + "nauc_map_at_20_diff1": 0.827555, + "nauc_map_at_100_max": 0.231338, + "nauc_map_at_100_std": 0.190311, + "nauc_map_at_100_diff1": 0.828071, + "nauc_map_at_1000_max": 0.231319, + "nauc_map_at_1000_std": 0.190175, + "nauc_map_at_1000_diff1": 0.828131, + "nauc_recall_at_1_max": 0.236212, + "nauc_recall_at_1_std": 0.171096, + "nauc_recall_at_1_diff1": 0.856401, + "nauc_recall_at_3_max": 0.210854, + "nauc_recall_at_3_std": 0.231323, + "nauc_recall_at_3_diff1": 0.781812, + "nauc_recall_at_5_max": 0.223952, + "nauc_recall_at_5_std": 0.246561, + "nauc_recall_at_5_diff1": 0.742533, + "nauc_recall_at_10_max": 0.265031, + "nauc_recall_at_10_std": 0.324632, + "nauc_recall_at_10_diff1": 0.719134, + "nauc_recall_at_20_max": 0.227813, + "nauc_recall_at_20_std": 0.279165, + "nauc_recall_at_20_diff1": 0.679507, + "nauc_recall_at_100_max": 0.200492, + "nauc_recall_at_100_std": 0.371284, + "nauc_recall_at_100_diff1": 0.687553, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.238679, + "nauc_precision_at_1_std": 0.173904, + "nauc_precision_at_1_diff1": 0.857482, + "nauc_precision_at_3_max": 0.208456, + "nauc_precision_at_3_std": 0.230313, + "nauc_precision_at_3_diff1": 0.763377, + "nauc_precision_at_5_max": 0.215845, + "nauc_precision_at_5_std": 0.239949, + "nauc_precision_at_5_diff1": 0.709453, + "nauc_precision_at_10_max": 0.233545, + "nauc_precision_at_10_std": 0.292353, + "nauc_precision_at_10_diff1": 0.636274, + "nauc_precision_at_20_max": 0.167439, + "nauc_precision_at_20_std": 0.217331, + "nauc_precision_at_20_diff1": 0.530415, + "nauc_precision_at_100_max": -0.063834, + "nauc_precision_at_100_std": 0.039082, + "nauc_precision_at_100_diff1": 0.138404, + "nauc_precision_at_1000_max": -0.376303, + "nauc_precision_at_1000_std": -0.361427, + "nauc_precision_at_1000_diff1": -0.444668, + "nauc_cv_recall_at_1_max": 0.236042, + "nauc_cv_recall_at_1_std": 0.173833, + "nauc_cv_recall_at_1_diff1": 0.842722, + "nauc_cv_recall_at_3_max": 0.222258, + "nauc_cv_recall_at_3_std": 0.23474, + "nauc_cv_recall_at_3_diff1": 0.778092, + "nauc_cv_recall_at_5_max": 0.229082, + "nauc_cv_recall_at_5_std": 0.251743, + "nauc_cv_recall_at_5_diff1": 0.74392, + "nauc_cv_recall_at_10_max": 0.237699, + "nauc_cv_recall_at_10_std": 0.290696, + "nauc_cv_recall_at_10_diff1": 0.716766, + "nauc_cv_recall_at_20_max": 0.213265, + "nauc_cv_recall_at_20_std": 0.265584, + "nauc_cv_recall_at_20_diff1": 0.675482, + "nauc_cv_recall_at_100_max": 0.141581, + "nauc_cv_recall_at_100_std": 0.324958, + "nauc_cv_recall_at_100_diff1": 0.675281, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.236042, + "nauc_mrr_at_1_std": 0.173833, + "nauc_mrr_at_1_diff1": 0.842722, + "nauc_mrr_at_3_max": 0.230818, + "nauc_mrr_at_3_std": 0.192819, + "nauc_mrr_at_3_diff1": 0.819837, + "nauc_mrr_at_5_max": 0.231847, + "nauc_mrr_at_5_std": 0.193404, + "nauc_mrr_at_5_diff1": 0.816911, + "nauc_mrr_at_10_max": 0.231675, + "nauc_mrr_at_10_std": 0.193134, + "nauc_mrr_at_10_diff1": 0.817503, + "nauc_mrr_at_20_max": 0.231553, + "nauc_mrr_at_20_std": 0.19204, + "nauc_mrr_at_20_diff1": 0.817998, + "nauc_mrr_at_100_max": 0.231593, + "nauc_mrr_at_100_std": 0.192026, + "nauc_mrr_at_100_diff1": 0.818497, + "nauc_mrr_at_1000_max": 0.23165, + "nauc_mrr_at_1000_std": 0.19197, + "nauc_mrr_at_1000_diff1": 0.818562, + "main_score": 0.79756, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 178.51907181739807, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/model_meta.json b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/model_meta.json new file mode 100644 index 0000000000..4ad62fb7db --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-1b-v1/1f0fdea7f5b19532a750be109b19072d719b8177/model_meta.json @@ -0,0 +1 @@ +{"name": "nvidia/llama-nemoretriever-colembed-1b-v1", "revision": "1f0fdea7f5b19532a750be109b19072d719b8177", "release_date": "2025-06-27", "languages": ["eng-Latn"], "n_parameters": 2418000000, "memory_usage_mb": 9224.0, "max_tokens": 8192.0, "embed_dim": 2048, "license": "https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1/blob/main/LICENSE", "open_weights": true, "public_training_code": "Proprietary Code", "public_training_data": "https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1#training-dataset", "framework": ["PyTorch"], "reference": "https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1", "similarity_fn_name": "max_sim", "use_instructions": true, "training_datasets": {"DocVQA": ["train"], "InfoVQA": ["train"], "TATDQA": ["train"], "arXivQA": ["train"], "hotpotqa": ["train"], "miracl": ["train"], "NQ": ["train"], "stackexchange": ["train"], "SQuAD": ["train"], "WebInstructSub": ["train"], "docmatix-ir": ["train"], "vdr-multilingual-train": ["train"], "colpali_train_set": ["train"], "VisRAG-Ret-Train-Synthetic-data": ["train"], "VisRAG-Ret-Train-In-domain-data": ["train"], "wiki-ss-nq": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "llama_nemoretriever_colembed"} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/Vidore2BioMedicalLecturesRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/Vidore2BioMedicalLecturesRetrieval.json new file mode 100644 index 0000000000..1117dc75b0 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/Vidore2BioMedicalLecturesRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "a29202f0da409034d651614d87cd8938d254e2ea", + "task_name": "Vidore2BioMedicalLecturesRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.63125, + "ndcg_at_3": 0.6221, + "ndcg_at_5": 0.64257, + "ndcg_at_10": 0.67511, + "ndcg_at_20": 0.69448, + "ndcg_at_100": 0.73064, + "ndcg_at_1000": 0.74475, + "map_at_1": 0.37549, + "map_at_3": 0.51346, + "map_at_5": 0.55156, + "map_at_10": 0.58629, + "map_at_20": 0.59803, + "map_at_100": 0.60971, + "map_at_1000": 0.61126, + "recall_at_1": 0.37549, + "recall_at_3": 0.58233, + "recall_at_5": 0.67214, + "recall_at_10": 0.76415, + "recall_at_20": 0.81626, + "recall_at_100": 0.93469, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.63125, + "cv_recall_at_3": 0.80625, + "cv_recall_at_5": 0.89375, + "cv_recall_at_10": 0.93125, + "cv_recall_at_20": 0.95, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.63125, + "precision_at_3": 0.38125, + "precision_at_5": 0.285, + "precision_at_10": 0.18313, + "precision_at_20": 0.105, + "precision_at_100": 0.02769, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.63125, + "mrr_at_3": 0.7125, + "mrr_at_5": 0.731563, + "mrr_at_10": 0.736404, + "mrr_at_20": 0.7377, + "mrr_at_100": 0.73853, + "mrr_at_1000": 0.738602, + "nauc_ndcg_at_1_max": 0.29313, + "nauc_ndcg_at_1_std": 0.099212, + "nauc_ndcg_at_1_diff1": 0.451867, + "nauc_ndcg_at_3_max": 0.374619, + "nauc_ndcg_at_3_std": 0.165837, + "nauc_ndcg_at_3_diff1": 0.36714, + "nauc_ndcg_at_5_max": 0.358316, + "nauc_ndcg_at_5_std": 0.139622, + "nauc_ndcg_at_5_diff1": 0.386517, + "nauc_ndcg_at_10_max": 0.330294, + "nauc_ndcg_at_10_std": 0.108547, + "nauc_ndcg_at_10_diff1": 0.381868, + "nauc_ndcg_at_20_max": 0.325367, + "nauc_ndcg_at_20_std": 0.138847, + "nauc_ndcg_at_20_diff1": 0.362935, + "nauc_ndcg_at_100_max": 0.354748, + "nauc_ndcg_at_100_std": 0.146667, + "nauc_ndcg_at_100_diff1": 0.355974, + "nauc_ndcg_at_1000_max": 0.341867, + "nauc_ndcg_at_1000_std": 0.1319, + "nauc_ndcg_at_1000_diff1": 0.352863, + "nauc_map_at_1_max": 0.257416, + "nauc_map_at_1_std": 0.086183, + "nauc_map_at_1_diff1": 0.529536, + "nauc_map_at_3_max": 0.347285, + "nauc_map_at_3_std": 0.156568, + "nauc_map_at_3_diff1": 0.432036, + "nauc_map_at_5_max": 0.336155, + "nauc_map_at_5_std": 0.125839, + "nauc_map_at_5_diff1": 0.421526, + "nauc_map_at_10_max": 0.314045, + "nauc_map_at_10_std": 0.102835, + "nauc_map_at_10_diff1": 0.405055, + "nauc_map_at_20_max": 0.314121, + "nauc_map_at_20_std": 0.115142, + "nauc_map_at_20_diff1": 0.394717, + "nauc_map_at_100_max": 0.324588, + "nauc_map_at_100_std": 0.119503, + "nauc_map_at_100_diff1": 0.387425, + "nauc_map_at_1000_max": 0.323622, + "nauc_map_at_1000_std": 0.11788, + "nauc_map_at_1000_diff1": 0.38643, + "nauc_recall_at_1_max": 0.257416, + "nauc_recall_at_1_std": 0.086183, + "nauc_recall_at_1_diff1": 0.529536, + "nauc_recall_at_3_max": 0.377925, + "nauc_recall_at_3_std": 0.197888, + "nauc_recall_at_3_diff1": 0.366731, + "nauc_recall_at_5_max": 0.327048, + "nauc_recall_at_5_std": 0.116242, + "nauc_recall_at_5_diff1": 0.324811, + "nauc_recall_at_10_max": 0.254819, + "nauc_recall_at_10_std": 0.051043, + "nauc_recall_at_10_diff1": 0.26899, + "nauc_recall_at_20_max": 0.229003, + "nauc_recall_at_20_std": 0.152117, + "nauc_recall_at_20_diff1": 0.181462, + "nauc_recall_at_100_max": 0.442118, + "nauc_recall_at_100_std": 0.273406, + "nauc_recall_at_100_diff1": 0.187009, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.29313, + "nauc_precision_at_1_std": 0.099212, + "nauc_precision_at_1_diff1": 0.451867, + "nauc_precision_at_3_max": 0.215337, + "nauc_precision_at_3_std": 0.109069, + "nauc_precision_at_3_diff1": -0.047102, + "nauc_precision_at_5_max": 0.104664, + "nauc_precision_at_5_std": 0.015909, + "nauc_precision_at_5_diff1": -0.136448, + "nauc_precision_at_10_max": -0.038014, + "nauc_precision_at_10_std": -0.072152, + "nauc_precision_at_10_diff1": -0.227021, + "nauc_precision_at_20_max": -0.070653, + "nauc_precision_at_20_std": -0.042034, + "nauc_precision_at_20_diff1": -0.291181, + "nauc_precision_at_100_max": -0.081781, + "nauc_precision_at_100_std": -0.076829, + "nauc_precision_at_100_diff1": -0.348791, + "nauc_precision_at_1000_max": -0.169398, + "nauc_precision_at_1000_std": -0.15013, + "nauc_precision_at_1000_diff1": -0.363945, + "nauc_cv_recall_at_1_max": 0.29313, + "nauc_cv_recall_at_1_std": 0.099212, + "nauc_cv_recall_at_1_diff1": 0.451867, + "nauc_cv_recall_at_3_max": 0.443426, + "nauc_cv_recall_at_3_std": 0.209688, + "nauc_cv_recall_at_3_diff1": 0.347207, + "nauc_cv_recall_at_5_max": 0.59855, + "nauc_cv_recall_at_5_std": 0.331129, + "nauc_cv_recall_at_5_diff1": 0.259451, + "nauc_cv_recall_at_10_max": 0.557041, + "nauc_cv_recall_at_10_std": 0.271242, + "nauc_cv_recall_at_10_diff1": 0.066293, + "nauc_cv_recall_at_20_max": 0.612045, + "nauc_cv_recall_at_20_std": 0.5426, + "nauc_cv_recall_at_20_diff1": -0.093954, + "nauc_cv_recall_at_100_max": 0.415188, + "nauc_cv_recall_at_100_std": 0.330221, + "nauc_cv_recall_at_100_diff1": -0.400093, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.29313, + "nauc_mrr_at_1_std": 0.099212, + "nauc_mrr_at_1_diff1": 0.451867, + "nauc_mrr_at_3_max": 0.346123, + "nauc_mrr_at_3_std": 0.141775, + "nauc_mrr_at_3_diff1": 0.417317, + "nauc_mrr_at_5_max": 0.354797, + "nauc_mrr_at_5_std": 0.147938, + "nauc_mrr_at_5_diff1": 0.412382, + "nauc_mrr_at_10_max": 0.34785, + "nauc_mrr_at_10_std": 0.142242, + "nauc_mrr_at_10_diff1": 0.407449, + "nauc_mrr_at_20_max": 0.347657, + "nauc_mrr_at_20_std": 0.146304, + "nauc_mrr_at_20_diff1": 0.406998, + "nauc_mrr_at_100_max": 0.346047, + "nauc_mrr_at_100_std": 0.14451, + "nauc_mrr_at_100_diff1": 0.407369, + "nauc_mrr_at_1000_max": 0.346026, + "nauc_mrr_at_1000_std": 0.144521, + "nauc_mrr_at_1000_diff1": 0.407571, + "main_score": 0.64257, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.59375, + "ndcg_at_3": 0.6116, + "ndcg_at_5": 0.62457, + "ndcg_at_10": 0.65662, + "ndcg_at_20": 0.68346, + "ndcg_at_100": 0.71943, + "ndcg_at_1000": 0.73384, + "map_at_1": 0.36866, + "map_at_3": 0.50528, + "map_at_5": 0.53728, + "map_at_10": 0.5718, + "map_at_20": 0.58725, + "map_at_100": 0.59842, + "map_at_1000": 0.59994, + "recall_at_1": 0.36866, + "recall_at_3": 0.57779, + "recall_at_5": 0.65312, + "recall_at_10": 0.74063, + "recall_at_20": 0.81413, + "recall_at_100": 0.93314, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.59375, + "cv_recall_at_3": 0.80625, + "cv_recall_at_5": 0.8625, + "cv_recall_at_10": 0.9125, + "cv_recall_at_20": 0.95625, + "cv_recall_at_100": 0.98125, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.59375, + "precision_at_3": 0.37708, + "precision_at_5": 0.27375, + "precision_at_10": 0.17625, + "precision_at_20": 0.10438, + "precision_at_100": 0.02762, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.59375, + "mrr_at_3": 0.689583, + "mrr_at_5": 0.702396, + "mrr_at_10": 0.709539, + "mrr_at_20": 0.712512, + "mrr_at_100": 0.713023, + "mrr_at_1000": 0.713126, + "nauc_ndcg_at_1_max": 0.289026, + "nauc_ndcg_at_1_std": 0.041575, + "nauc_ndcg_at_1_diff1": 0.516611, + "nauc_ndcg_at_3_max": 0.273325, + "nauc_ndcg_at_3_std": 0.048842, + "nauc_ndcg_at_3_diff1": 0.382393, + "nauc_ndcg_at_5_max": 0.248466, + "nauc_ndcg_at_5_std": 0.003476, + "nauc_ndcg_at_5_diff1": 0.408347, + "nauc_ndcg_at_10_max": 0.260883, + "nauc_ndcg_at_10_std": 0.007198, + "nauc_ndcg_at_10_diff1": 0.412588, + "nauc_ndcg_at_20_max": 0.252593, + "nauc_ndcg_at_20_std": 0.03131, + "nauc_ndcg_at_20_diff1": 0.394544, + "nauc_ndcg_at_100_max": 0.252471, + "nauc_ndcg_at_100_std": 0.033433, + "nauc_ndcg_at_100_diff1": 0.39361, + "nauc_ndcg_at_1000_max": 0.24661, + "nauc_ndcg_at_1000_std": 0.020398, + "nauc_ndcg_at_1000_diff1": 0.390753, + "nauc_map_at_1_max": 0.24064, + "nauc_map_at_1_std": 0.033601, + "nauc_map_at_1_diff1": 0.525183, + "nauc_map_at_3_max": 0.271057, + "nauc_map_at_3_std": 0.054916, + "nauc_map_at_3_diff1": 0.435341, + "nauc_map_at_5_max": 0.239402, + "nauc_map_at_5_std": 0.00912, + "nauc_map_at_5_diff1": 0.43016, + "nauc_map_at_10_max": 0.240172, + "nauc_map_at_10_std": -0.001372, + "nauc_map_at_10_diff1": 0.42102, + "nauc_map_at_20_max": 0.236102, + "nauc_map_at_20_std": 0.006713, + "nauc_map_at_20_diff1": 0.412217, + "nauc_map_at_100_max": 0.238101, + "nauc_map_at_100_std": 0.009786, + "nauc_map_at_100_diff1": 0.409319, + "nauc_map_at_1000_max": 0.237453, + "nauc_map_at_1000_std": 0.00839, + "nauc_map_at_1000_diff1": 0.408794, + "nauc_recall_at_1_max": 0.24064, + "nauc_recall_at_1_std": 0.033601, + "nauc_recall_at_1_diff1": 0.525183, + "nauc_recall_at_3_max": 0.288535, + "nauc_recall_at_3_std": 0.0877, + "nauc_recall_at_3_diff1": 0.355994, + "nauc_recall_at_5_max": 0.215934, + "nauc_recall_at_5_std": -0.011505, + "nauc_recall_at_5_diff1": 0.337511, + "nauc_recall_at_10_max": 0.229989, + "nauc_recall_at_10_std": -0.013794, + "nauc_recall_at_10_diff1": 0.309533, + "nauc_recall_at_20_max": 0.218444, + "nauc_recall_at_20_std": 0.079199, + "nauc_recall_at_20_diff1": 0.20018, + "nauc_recall_at_100_max": 0.217001, + "nauc_recall_at_100_std": 0.155474, + "nauc_recall_at_100_diff1": 0.205855, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.289026, + "nauc_precision_at_1_std": 0.041575, + "nauc_precision_at_1_diff1": 0.516611, + "nauc_precision_at_3_max": 0.107315, + "nauc_precision_at_3_std": 0.004142, + "nauc_precision_at_3_diff1": -0.040912, + "nauc_precision_at_5_max": -0.020516, + "nauc_precision_at_5_std": -0.117671, + "nauc_precision_at_5_diff1": -0.094469, + "nauc_precision_at_10_max": -0.050782, + "nauc_precision_at_10_std": -0.091738, + "nauc_precision_at_10_diff1": -0.181523, + "nauc_precision_at_20_max": -0.089555, + "nauc_precision_at_20_std": -0.039913, + "nauc_precision_at_20_diff1": -0.248164, + "nauc_precision_at_100_max": -0.124269, + "nauc_precision_at_100_std": -0.061166, + "nauc_precision_at_100_diff1": -0.326229, + "nauc_precision_at_1000_max": -0.180487, + "nauc_precision_at_1000_std": -0.117239, + "nauc_precision_at_1000_diff1": -0.368583, + "nauc_cv_recall_at_1_max": 0.289026, + "nauc_cv_recall_at_1_std": 0.041575, + "nauc_cv_recall_at_1_diff1": 0.516611, + "nauc_cv_recall_at_3_max": 0.346661, + "nauc_cv_recall_at_3_std": 0.024501, + "nauc_cv_recall_at_3_diff1": 0.353895, + "nauc_cv_recall_at_5_max": 0.32803, + "nauc_cv_recall_at_5_std": -0.045947, + "nauc_cv_recall_at_5_diff1": 0.360488, + "nauc_cv_recall_at_10_max": 0.250734, + "nauc_cv_recall_at_10_std": -0.01734, + "nauc_cv_recall_at_10_diff1": 0.227591, + "nauc_cv_recall_at_20_max": 0.294251, + "nauc_cv_recall_at_20_std": 0.509204, + "nauc_cv_recall_at_20_diff1": -0.089502, + "nauc_cv_recall_at_100_max": 0.044818, + "nauc_cv_recall_at_100_std": 0.09057, + "nauc_cv_recall_at_100_diff1": -0.400093, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.289026, + "nauc_mrr_at_1_std": 0.041575, + "nauc_mrr_at_1_diff1": 0.516611, + "nauc_mrr_at_3_max": 0.301905, + "nauc_mrr_at_3_std": 0.037744, + "nauc_mrr_at_3_diff1": 0.470075, + "nauc_mrr_at_5_max": 0.298765, + "nauc_mrr_at_5_std": 0.031153, + "nauc_mrr_at_5_diff1": 0.477913, + "nauc_mrr_at_10_max": 0.294276, + "nauc_mrr_at_10_std": 0.035798, + "nauc_mrr_at_10_diff1": 0.473723, + "nauc_mrr_at_20_max": 0.29445, + "nauc_mrr_at_20_std": 0.042175, + "nauc_mrr_at_20_diff1": 0.472427, + "nauc_mrr_at_100_max": 0.293921, + "nauc_mrr_at_100_std": 0.040596, + "nauc_mrr_at_100_diff1": 0.472531, + "nauc_mrr_at_1000_max": 0.294036, + "nauc_mrr_at_1000_std": 0.040711, + "nauc_mrr_at_1000_diff1": 0.472778, + "main_score": 0.62457, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.61875, + "ndcg_at_3": 0.62632, + "ndcg_at_5": 0.65182, + "ndcg_at_10": 0.68289, + "ndcg_at_20": 0.70812, + "ndcg_at_100": 0.74028, + "ndcg_at_1000": 0.75381, + "map_at_1": 0.38689, + "map_at_3": 0.51858, + "map_at_5": 0.56369, + "map_at_10": 0.59882, + "map_at_20": 0.61315, + "map_at_100": 0.62387, + "map_at_1000": 0.62556, + "recall_at_1": 0.38689, + "recall_at_3": 0.57843, + "recall_at_5": 0.67625, + "recall_at_10": 0.76803, + "recall_at_20": 0.83684, + "recall_at_100": 0.94551, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.61875, + "cv_recall_at_3": 0.80625, + "cv_recall_at_5": 0.875, + "cv_recall_at_10": 0.9125, + "cv_recall_at_20": 0.95625, + "cv_recall_at_100": 0.99375, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.61875, + "precision_at_3": 0.3875, + "precision_at_5": 0.29375, + "precision_at_10": 0.18563, + "precision_at_20": 0.10844, + "precision_at_100": 0.02731, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.61875, + "mrr_at_3": 0.704167, + "mrr_at_5": 0.720104, + "mrr_at_10": 0.725231, + "mrr_at_20": 0.728931, + "mrr_at_100": 0.729937, + "mrr_at_1000": 0.729948, + "nauc_ndcg_at_1_max": 0.413796, + "nauc_ndcg_at_1_std": 0.093912, + "nauc_ndcg_at_1_diff1": 0.5604, + "nauc_ndcg_at_3_max": 0.298015, + "nauc_ndcg_at_3_std": 0.013397, + "nauc_ndcg_at_3_diff1": 0.421515, + "nauc_ndcg_at_5_max": 0.320033, + "nauc_ndcg_at_5_std": 0.020114, + "nauc_ndcg_at_5_diff1": 0.468446, + "nauc_ndcg_at_10_max": 0.334628, + "nauc_ndcg_at_10_std": 0.026379, + "nauc_ndcg_at_10_diff1": 0.464524, + "nauc_ndcg_at_20_max": 0.324634, + "nauc_ndcg_at_20_std": 0.043981, + "nauc_ndcg_at_20_diff1": 0.459949, + "nauc_ndcg_at_100_max": 0.314036, + "nauc_ndcg_at_100_std": 0.032024, + "nauc_ndcg_at_100_diff1": 0.443592, + "nauc_ndcg_at_1000_max": 0.29617, + "nauc_ndcg_at_1000_std": 0.003641, + "nauc_ndcg_at_1000_diff1": 0.442168, + "nauc_map_at_1_max": 0.309777, + "nauc_map_at_1_std": 0.007487, + "nauc_map_at_1_diff1": 0.579225, + "nauc_map_at_3_max": 0.315573, + "nauc_map_at_3_std": 0.005935, + "nauc_map_at_3_diff1": 0.495359, + "nauc_map_at_5_max": 0.313308, + "nauc_map_at_5_std": 0.000544, + "nauc_map_at_5_diff1": 0.502897, + "nauc_map_at_10_max": 0.312619, + "nauc_map_at_10_std": -0.001356, + "nauc_map_at_10_diff1": 0.483501, + "nauc_map_at_20_max": 0.305225, + "nauc_map_at_20_std": -0.000653, + "nauc_map_at_20_diff1": 0.474559, + "nauc_map_at_100_max": 0.301986, + "nauc_map_at_100_std": -0.004088, + "nauc_map_at_100_diff1": 0.467523, + "nauc_map_at_1000_max": 0.299658, + "nauc_map_at_1000_std": -0.00733, + "nauc_map_at_1000_diff1": 0.466938, + "nauc_recall_at_1_max": 0.309777, + "nauc_recall_at_1_std": 0.007487, + "nauc_recall_at_1_diff1": 0.579225, + "nauc_recall_at_3_max": 0.283259, + "nauc_recall_at_3_std": -0.0, + "nauc_recall_at_3_diff1": 0.430771, + "nauc_recall_at_5_max": 0.277748, + "nauc_recall_at_5_std": 0.02967, + "nauc_recall_at_5_diff1": 0.416852, + "nauc_recall_at_10_max": 0.315712, + "nauc_recall_at_10_std": 0.072108, + "nauc_recall_at_10_diff1": 0.365798, + "nauc_recall_at_20_max": 0.30406, + "nauc_recall_at_20_std": 0.162209, + "nauc_recall_at_20_diff1": 0.342882, + "nauc_recall_at_100_max": 0.297749, + "nauc_recall_at_100_std": 0.310968, + "nauc_recall_at_100_diff1": 0.230456, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.413796, + "nauc_precision_at_1_std": 0.093912, + "nauc_precision_at_1_diff1": 0.5604, + "nauc_precision_at_3_max": 0.046767, + "nauc_precision_at_3_std": -0.022482, + "nauc_precision_at_3_diff1": -0.072054, + "nauc_precision_at_5_max": -0.022507, + "nauc_precision_at_5_std": -0.038139, + "nauc_precision_at_5_diff1": -0.151073, + "nauc_precision_at_10_max": -0.076719, + "nauc_precision_at_10_std": -0.044597, + "nauc_precision_at_10_diff1": -0.252457, + "nauc_precision_at_20_max": -0.141029, + "nauc_precision_at_20_std": -0.035443, + "nauc_precision_at_20_diff1": -0.311941, + "nauc_precision_at_100_max": -0.201146, + "nauc_precision_at_100_std": -0.08728, + "nauc_precision_at_100_diff1": -0.393562, + "nauc_precision_at_1000_max": -0.286379, + "nauc_precision_at_1000_std": -0.178283, + "nauc_precision_at_1000_diff1": -0.419032, + "nauc_cv_recall_at_1_max": 0.413796, + "nauc_cv_recall_at_1_std": 0.093912, + "nauc_cv_recall_at_1_diff1": 0.5604, + "nauc_cv_recall_at_3_max": 0.27102, + "nauc_cv_recall_at_3_std": -0.053648, + "nauc_cv_recall_at_3_diff1": 0.394473, + "nauc_cv_recall_at_5_max": 0.395757, + "nauc_cv_recall_at_5_std": 0.099449, + "nauc_cv_recall_at_5_diff1": 0.295614, + "nauc_cv_recall_at_10_max": 0.377218, + "nauc_cv_recall_at_10_std": 0.203381, + "nauc_cv_recall_at_10_diff1": 0.213385, + "nauc_cv_recall_at_20_max": 0.345538, + "nauc_cv_recall_at_20_std": 0.463252, + "nauc_cv_recall_at_20_diff1": 0.118514, + "nauc_cv_recall_at_100_max": -1.739963, + "nauc_cv_recall_at_100_std": -0.171335, + "nauc_cv_recall_at_100_diff1": -0.563492, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.413796, + "nauc_mrr_at_1_std": 0.093912, + "nauc_mrr_at_1_diff1": 0.5604, + "nauc_mrr_at_3_max": 0.371304, + "nauc_mrr_at_3_std": 0.059451, + "nauc_mrr_at_3_diff1": 0.512632, + "nauc_mrr_at_5_max": 0.390326, + "nauc_mrr_at_5_std": 0.085924, + "nauc_mrr_at_5_diff1": 0.507079, + "nauc_mrr_at_10_max": 0.389892, + "nauc_mrr_at_10_std": 0.091633, + "nauc_mrr_at_10_diff1": 0.506475, + "nauc_mrr_at_20_max": 0.389728, + "nauc_mrr_at_20_std": 0.094305, + "nauc_mrr_at_20_diff1": 0.509701, + "nauc_mrr_at_100_max": 0.387773, + "nauc_mrr_at_100_std": 0.091545, + "nauc_mrr_at_100_diff1": 0.511379, + "nauc_mrr_at_1000_max": 0.387877, + "nauc_mrr_at_1000_std": 0.091558, + "nauc_mrr_at_1000_diff1": 0.511431, + "main_score": 0.65182, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.575, + "ndcg_at_3": 0.56842, + "ndcg_at_5": 0.5889, + "ndcg_at_10": 0.63155, + "ndcg_at_20": 0.65464, + "ndcg_at_100": 0.69173, + "ndcg_at_1000": 0.70899, + "map_at_1": 0.35241, + "map_at_3": 0.4662, + "map_at_5": 0.49991, + "map_at_10": 0.53841, + "map_at_20": 0.55192, + "map_at_100": 0.56358, + "map_at_1000": 0.56527, + "recall_at_1": 0.35241, + "recall_at_3": 0.52771, + "recall_at_5": 0.60745, + "recall_at_10": 0.73218, + "recall_at_20": 0.79555, + "recall_at_100": 0.9221, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.575, + "cv_recall_at_3": 0.7625, + "cv_recall_at_5": 0.83125, + "cv_recall_at_10": 0.925, + "cv_recall_at_20": 0.94375, + "cv_recall_at_100": 0.9875, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.575, + "precision_at_3": 0.34375, + "precision_at_5": 0.26, + "precision_at_10": 0.1725, + "precision_at_20": 0.10094, + "precision_at_100": 0.02656, + "precision_at_1000": 0.00322, + "mrr_at_1": 0.575, + "mrr_at_3": 0.660417, + "mrr_at_5": 0.676979, + "mrr_at_10": 0.689407, + "mrr_at_20": 0.690573, + "mrr_at_100": 0.691787, + "mrr_at_1000": 0.691841, + "nauc_ndcg_at_1_max": 0.269283, + "nauc_ndcg_at_1_std": 0.142267, + "nauc_ndcg_at_1_diff1": 0.510832, + "nauc_ndcg_at_3_max": 0.219205, + "nauc_ndcg_at_3_std": 0.045043, + "nauc_ndcg_at_3_diff1": 0.387579, + "nauc_ndcg_at_5_max": 0.241982, + "nauc_ndcg_at_5_std": 0.073854, + "nauc_ndcg_at_5_diff1": 0.410573, + "nauc_ndcg_at_10_max": 0.259578, + "nauc_ndcg_at_10_std": 0.083622, + "nauc_ndcg_at_10_diff1": 0.38566, + "nauc_ndcg_at_20_max": 0.257098, + "nauc_ndcg_at_20_std": 0.097698, + "nauc_ndcg_at_20_diff1": 0.38359, + "nauc_ndcg_at_100_max": 0.254549, + "nauc_ndcg_at_100_std": 0.109362, + "nauc_ndcg_at_100_diff1": 0.390727, + "nauc_ndcg_at_1000_max": 0.248629, + "nauc_ndcg_at_1000_std": 0.087326, + "nauc_ndcg_at_1000_diff1": 0.39361, + "nauc_map_at_1_max": 0.253958, + "nauc_map_at_1_std": 0.133637, + "nauc_map_at_1_diff1": 0.553517, + "nauc_map_at_3_max": 0.240436, + "nauc_map_at_3_std": 0.077996, + "nauc_map_at_3_diff1": 0.44825, + "nauc_map_at_5_max": 0.253417, + "nauc_map_at_5_std": 0.086306, + "nauc_map_at_5_diff1": 0.443737, + "nauc_map_at_10_max": 0.247037, + "nauc_map_at_10_std": 0.07432, + "nauc_map_at_10_diff1": 0.406825, + "nauc_map_at_20_max": 0.246308, + "nauc_map_at_20_std": 0.079993, + "nauc_map_at_20_diff1": 0.404503, + "nauc_map_at_100_max": 0.245749, + "nauc_map_at_100_std": 0.082126, + "nauc_map_at_100_diff1": 0.401693, + "nauc_map_at_1000_max": 0.245605, + "nauc_map_at_1000_std": 0.080566, + "nauc_map_at_1000_diff1": 0.402033, + "nauc_recall_at_1_max": 0.253958, + "nauc_recall_at_1_std": 0.133637, + "nauc_recall_at_1_diff1": 0.553517, + "nauc_recall_at_3_max": 0.207376, + "nauc_recall_at_3_std": 0.033837, + "nauc_recall_at_3_diff1": 0.372797, + "nauc_recall_at_5_max": 0.217963, + "nauc_recall_at_5_std": 0.047773, + "nauc_recall_at_5_diff1": 0.362033, + "nauc_recall_at_10_max": 0.25325, + "nauc_recall_at_10_std": 0.071567, + "nauc_recall_at_10_diff1": 0.221801, + "nauc_recall_at_20_max": 0.256068, + "nauc_recall_at_20_std": 0.110158, + "nauc_recall_at_20_diff1": 0.193474, + "nauc_recall_at_100_max": 0.258146, + "nauc_recall_at_100_std": 0.320709, + "nauc_recall_at_100_diff1": 0.225177, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.269283, + "nauc_precision_at_1_std": 0.142267, + "nauc_precision_at_1_diff1": 0.510832, + "nauc_precision_at_3_max": 0.057566, + "nauc_precision_at_3_std": -0.065677, + "nauc_precision_at_3_diff1": -0.003609, + "nauc_precision_at_5_max": 0.028557, + "nauc_precision_at_5_std": -0.047656, + "nauc_precision_at_5_diff1": -0.104466, + "nauc_precision_at_10_max": -0.061981, + "nauc_precision_at_10_std": -0.100027, + "nauc_precision_at_10_diff1": -0.238226, + "nauc_precision_at_20_max": -0.092173, + "nauc_precision_at_20_std": -0.074617, + "nauc_precision_at_20_diff1": -0.260381, + "nauc_precision_at_100_max": -0.112847, + "nauc_precision_at_100_std": -0.080342, + "nauc_precision_at_100_diff1": -0.274411, + "nauc_precision_at_1000_max": -0.158948, + "nauc_precision_at_1000_std": -0.166164, + "nauc_precision_at_1000_diff1": -0.284278, + "nauc_cv_recall_at_1_max": 0.269283, + "nauc_cv_recall_at_1_std": 0.142267, + "nauc_cv_recall_at_1_diff1": 0.510832, + "nauc_cv_recall_at_3_max": 0.2101, + "nauc_cv_recall_at_3_std": -0.005235, + "nauc_cv_recall_at_3_diff1": 0.362542, + "nauc_cv_recall_at_5_max": 0.146382, + "nauc_cv_recall_at_5_std": -0.021606, + "nauc_cv_recall_at_5_diff1": 0.35276, + "nauc_cv_recall_at_10_max": 0.222884, + "nauc_cv_recall_at_10_std": 0.108154, + "nauc_cv_recall_at_10_diff1": 0.186741, + "nauc_cv_recall_at_20_max": 0.075112, + "nauc_cv_recall_at_20_std": 0.109814, + "nauc_cv_recall_at_20_diff1": 0.074852, + "nauc_cv_recall_at_100_max": -0.214753, + "nauc_cv_recall_at_100_std": 0.275444, + "nauc_cv_recall_at_100_diff1": -0.514472, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.269283, + "nauc_mrr_at_1_std": 0.142267, + "nauc_mrr_at_1_diff1": 0.510832, + "nauc_mrr_at_3_max": 0.253015, + "nauc_mrr_at_3_std": 0.097131, + "nauc_mrr_at_3_diff1": 0.453149, + "nauc_mrr_at_5_max": 0.245565, + "nauc_mrr_at_5_std": 0.100568, + "nauc_mrr_at_5_diff1": 0.455165, + "nauc_mrr_at_10_max": 0.255947, + "nauc_mrr_at_10_std": 0.114553, + "nauc_mrr_at_10_diff1": 0.451965, + "nauc_mrr_at_20_max": 0.254068, + "nauc_mrr_at_20_std": 0.114731, + "nauc_mrr_at_20_diff1": 0.451387, + "nauc_mrr_at_100_max": 0.254635, + "nauc_mrr_at_100_std": 0.114552, + "nauc_mrr_at_100_diff1": 0.452323, + "nauc_mrr_at_1000_max": 0.254619, + "nauc_mrr_at_1000_std": 0.114461, + "nauc_mrr_at_1000_diff1": 0.452451, + "main_score": 0.5889, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 1355.4186460971832, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/Vidore2ESGReportsHLRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/Vidore2ESGReportsHLRetrieval.json new file mode 100644 index 0000000000..34c20111ed --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/Vidore2ESGReportsHLRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "6d467dedb09a75144ede1421747e47cf036857dd", + "task_name": "Vidore2ESGReportsHLRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.63462, + "ndcg_at_3": 0.72337, + "ndcg_at_5": 0.7538, + "ndcg_at_10": 0.78255, + "ndcg_at_20": 0.79455, + "ndcg_at_100": 0.80981, + "ndcg_at_1000": 0.81458, + "map_at_1": 0.45865, + "map_at_3": 0.63365, + "map_at_5": 0.68114, + "map_at_10": 0.71132, + "map_at_20": 0.71831, + "map_at_100": 0.72477, + "map_at_1000": 0.72547, + "recall_at_1": 0.45865, + "recall_at_3": 0.72821, + "recall_at_5": 0.81973, + "recall_at_10": 0.88934, + "recall_at_20": 0.92767, + "recall_at_100": 0.981, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.63462, + "cv_recall_at_3": 0.90385, + "cv_recall_at_5": 0.94231, + "cv_recall_at_10": 0.96154, + "cv_recall_at_20": 0.98077, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.63462, + "precision_at_3": 0.41667, + "precision_at_5": 0.31154, + "precision_at_10": 0.18846, + "precision_at_20": 0.10096, + "precision_at_100": 0.02308, + "precision_at_1000": 0.00246, + "mrr_at_1": 0.634615, + "mrr_at_3": 0.762821, + "mrr_at_5": 0.772436, + "mrr_at_10": 0.775183, + "mrr_at_20": 0.776786, + "mrr_at_100": 0.77766, + "mrr_at_1000": 0.77766, + "nauc_ndcg_at_1_max": 0.240198, + "nauc_ndcg_at_1_std": 0.169206, + "nauc_ndcg_at_1_diff1": 0.544568, + "nauc_ndcg_at_3_max": 0.136656, + "nauc_ndcg_at_3_std": 0.042345, + "nauc_ndcg_at_3_diff1": 0.509297, + "nauc_ndcg_at_5_max": 0.225164, + "nauc_ndcg_at_5_std": 0.14292, + "nauc_ndcg_at_5_diff1": 0.531669, + "nauc_ndcg_at_10_max": 0.215816, + "nauc_ndcg_at_10_std": 0.140183, + "nauc_ndcg_at_10_diff1": 0.546318, + "nauc_ndcg_at_20_max": 0.247005, + "nauc_ndcg_at_20_std": 0.157042, + "nauc_ndcg_at_20_diff1": 0.531535, + "nauc_ndcg_at_100_max": 0.237396, + "nauc_ndcg_at_100_std": 0.15357, + "nauc_ndcg_at_100_diff1": 0.546103, + "nauc_ndcg_at_1000_max": 0.227575, + "nauc_ndcg_at_1000_std": 0.133805, + "nauc_ndcg_at_1000_diff1": 0.550227, + "nauc_map_at_1_max": 0.350383, + "nauc_map_at_1_std": 0.208519, + "nauc_map_at_1_diff1": 0.626876, + "nauc_map_at_3_max": 0.235907, + "nauc_map_at_3_std": 0.091145, + "nauc_map_at_3_diff1": 0.499868, + "nauc_map_at_5_max": 0.256842, + "nauc_map_at_5_std": 0.131902, + "nauc_map_at_5_diff1": 0.514603, + "nauc_map_at_10_max": 0.258342, + "nauc_map_at_10_std": 0.154, + "nauc_map_at_10_diff1": 0.528141, + "nauc_map_at_20_max": 0.274895, + "nauc_map_at_20_std": 0.168431, + "nauc_map_at_20_diff1": 0.523585, + "nauc_map_at_100_max": 0.270769, + "nauc_map_at_100_std": 0.169002, + "nauc_map_at_100_diff1": 0.528239, + "nauc_map_at_1000_max": 0.270198, + "nauc_map_at_1000_std": 0.167205, + "nauc_map_at_1000_diff1": 0.528932, + "nauc_recall_at_1_max": 0.350383, + "nauc_recall_at_1_std": 0.208519, + "nauc_recall_at_1_diff1": 0.626876, + "nauc_recall_at_3_max": 0.12019, + "nauc_recall_at_3_std": -0.018241, + "nauc_recall_at_3_diff1": 0.445098, + "nauc_recall_at_5_max": 0.213837, + "nauc_recall_at_5_std": 0.126844, + "nauc_recall_at_5_diff1": 0.422896, + "nauc_recall_at_10_max": 0.210533, + "nauc_recall_at_10_std": 0.20549, + "nauc_recall_at_10_diff1": 0.523882, + "nauc_recall_at_20_max": 0.312501, + "nauc_recall_at_20_std": 0.239413, + "nauc_recall_at_20_diff1": 0.37205, + "nauc_recall_at_100_max": 0.551911, + "nauc_recall_at_100_std": 0.768546, + "nauc_recall_at_100_diff1": 0.374077, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.240198, + "nauc_precision_at_1_std": 0.169206, + "nauc_precision_at_1_diff1": 0.544568, + "nauc_precision_at_3_max": -0.248708, + "nauc_precision_at_3_std": -0.199027, + "nauc_precision_at_3_diff1": -0.172566, + "nauc_precision_at_5_max": -0.197305, + "nauc_precision_at_5_std": -0.064022, + "nauc_precision_at_5_diff1": -0.20004, + "nauc_precision_at_10_max": -0.155366, + "nauc_precision_at_10_std": -0.005559, + "nauc_precision_at_10_diff1": -0.195495, + "nauc_precision_at_20_max": -0.10805, + "nauc_precision_at_20_std": 0.025153, + "nauc_precision_at_20_diff1": -0.233409, + "nauc_precision_at_100_max": -0.152039, + "nauc_precision_at_100_std": -0.026123, + "nauc_precision_at_100_diff1": -0.215701, + "nauc_precision_at_1000_max": -0.1668, + "nauc_precision_at_1000_std": -0.083083, + "nauc_precision_at_1000_diff1": -0.212148, + "nauc_cv_recall_at_1_max": 0.240198, + "nauc_cv_recall_at_1_std": 0.169206, + "nauc_cv_recall_at_1_diff1": 0.544568, + "nauc_cv_recall_at_3_max": -0.526124, + "nauc_cv_recall_at_3_std": -0.564965, + "nauc_cv_recall_at_3_diff1": 0.509183, + "nauc_cv_recall_at_5_max": 0.102933, + "nauc_cv_recall_at_5_std": 0.24212, + "nauc_cv_recall_at_5_diff1": 0.615262, + "nauc_cv_recall_at_10_max": 0.093596, + "nauc_cv_recall_at_10_std": 0.086058, + "nauc_cv_recall_at_10_diff1": 0.86209, + "nauc_cv_recall_at_20_max": -0.16971, + "nauc_cv_recall_at_20_std": -0.552063, + "nauc_cv_recall_at_20_diff1": 0.724179, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.240198, + "nauc_mrr_at_1_std": 0.169206, + "nauc_mrr_at_1_diff1": 0.544568, + "nauc_mrr_at_3_max": 0.070069, + "nauc_mrr_at_3_std": 0.000337, + "nauc_mrr_at_3_diff1": 0.544094, + "nauc_mrr_at_5_max": 0.147301, + "nauc_mrr_at_5_std": 0.089414, + "nauc_mrr_at_5_diff1": 0.553826, + "nauc_mrr_at_10_max": 0.147675, + "nauc_mrr_at_10_std": 0.082656, + "nauc_mrr_at_10_diff1": 0.56011, + "nauc_mrr_at_20_max": 0.145885, + "nauc_mrr_at_20_std": 0.077169, + "nauc_mrr_at_20_diff1": 0.556347, + "nauc_mrr_at_100_max": 0.147364, + "nauc_mrr_at_100_std": 0.080118, + "nauc_mrr_at_100_diff1": 0.55556, + "nauc_mrr_at_1000_max": 0.147364, + "nauc_mrr_at_1000_std": 0.080118, + "nauc_mrr_at_1000_diff1": 0.55556, + "main_score": 0.7538, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1362.6555981636047, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/Vidore2ESGReportsRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/Vidore2ESGReportsRetrieval.json new file mode 100644 index 0000000000..71bb04765d --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/Vidore2ESGReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3", + "task_name": "Vidore2ESGReportsRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.52632, + "ndcg_at_3": 0.55435, + "ndcg_at_5": 0.58589, + "ndcg_at_10": 0.63405, + "ndcg_at_20": 0.66715, + "ndcg_at_100": 0.70476, + "ndcg_at_1000": 0.71035, + "map_at_1": 0.2632, + "map_at_3": 0.4153, + "map_at_5": 0.47618, + "map_at_10": 0.51795, + "map_at_20": 0.54135, + "map_at_100": 0.5619, + "map_at_1000": 0.56229, + "recall_at_1": 0.2632, + "recall_at_3": 0.51422, + "recall_at_5": 0.61124, + "recall_at_10": 0.77005, + "recall_at_20": 0.86637, + "recall_at_100": 0.96272, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.52632, + "cv_recall_at_3": 0.80702, + "cv_recall_at_5": 0.82456, + "cv_recall_at_10": 0.92982, + "cv_recall_at_20": 0.94737, + "cv_recall_at_100": 0.96491, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.52632, + "precision_at_3": 0.38012, + "precision_at_5": 0.31228, + "precision_at_10": 0.20702, + "precision_at_20": 0.12895, + "precision_at_100": 0.03842, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.526316, + "mrr_at_3": 0.654971, + "mrr_at_5": 0.659357, + "mrr_at_10": 0.674046, + "mrr_at_20": 0.675299, + "mrr_at_100": 0.676135, + "mrr_at_1000": 0.676402, + "nauc_ndcg_at_1_max": -0.034967, + "nauc_ndcg_at_1_std": -0.03875, + "nauc_ndcg_at_1_diff1": 0.301138, + "nauc_ndcg_at_3_max": 0.018637, + "nauc_ndcg_at_3_std": 0.09375, + "nauc_ndcg_at_3_diff1": 0.357249, + "nauc_ndcg_at_5_max": -0.031606, + "nauc_ndcg_at_5_std": 0.073134, + "nauc_ndcg_at_5_diff1": 0.332824, + "nauc_ndcg_at_10_max": -0.026646, + "nauc_ndcg_at_10_std": 0.079983, + "nauc_ndcg_at_10_diff1": 0.378061, + "nauc_ndcg_at_20_max": -0.083137, + "nauc_ndcg_at_20_std": 0.0599, + "nauc_ndcg_at_20_diff1": 0.384598, + "nauc_ndcg_at_100_max": -0.164408, + "nauc_ndcg_at_100_std": -0.00459, + "nauc_ndcg_at_100_diff1": 0.380172, + "nauc_ndcg_at_1000_max": -0.146804, + "nauc_ndcg_at_1000_std": -0.02048, + "nauc_ndcg_at_1000_diff1": 0.366311, + "nauc_map_at_1_max": 0.16918, + "nauc_map_at_1_std": 0.000219, + "nauc_map_at_1_diff1": 0.409346, + "nauc_map_at_3_max": 0.128927, + "nauc_map_at_3_std": 0.114552, + "nauc_map_at_3_diff1": 0.390562, + "nauc_map_at_5_max": 0.095229, + "nauc_map_at_5_std": 0.123881, + "nauc_map_at_5_diff1": 0.398171, + "nauc_map_at_10_max": 0.0738, + "nauc_map_at_10_std": 0.113336, + "nauc_map_at_10_diff1": 0.406776, + "nauc_map_at_20_max": 0.032885, + "nauc_map_at_20_std": 0.084604, + "nauc_map_at_20_diff1": 0.415144, + "nauc_map_at_100_max": -0.00754, + "nauc_map_at_100_std": 0.056426, + "nauc_map_at_100_diff1": 0.414953, + "nauc_map_at_1000_max": -0.007031, + "nauc_map_at_1000_std": 0.055559, + "nauc_map_at_1000_diff1": 0.414176, + "nauc_recall_at_1_max": 0.16918, + "nauc_recall_at_1_std": 0.000219, + "nauc_recall_at_1_diff1": 0.409346, + "nauc_recall_at_3_max": 0.139783, + "nauc_recall_at_3_std": 0.231098, + "nauc_recall_at_3_diff1": 0.318407, + "nauc_recall_at_5_max": 0.059798, + "nauc_recall_at_5_std": 0.211631, + "nauc_recall_at_5_diff1": 0.317136, + "nauc_recall_at_10_max": 0.06297, + "nauc_recall_at_10_std": 0.240775, + "nauc_recall_at_10_diff1": 0.371398, + "nauc_recall_at_20_max": -0.110419, + "nauc_recall_at_20_std": 0.387208, + "nauc_recall_at_20_diff1": 0.441911, + "nauc_recall_at_100_max": -0.848216, + "nauc_recall_at_100_std": 0.510809, + "nauc_recall_at_100_diff1": 0.79991, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.034967, + "nauc_precision_at_1_std": -0.03875, + "nauc_precision_at_1_diff1": 0.301138, + "nauc_precision_at_3_max": -0.173023, + "nauc_precision_at_3_std": 0.044438, + "nauc_precision_at_3_diff1": 0.158195, + "nauc_precision_at_5_max": -0.199967, + "nauc_precision_at_5_std": 0.002999, + "nauc_precision_at_5_diff1": 0.064557, + "nauc_precision_at_10_max": -0.32092, + "nauc_precision_at_10_std": -0.126391, + "nauc_precision_at_10_diff1": 0.010601, + "nauc_precision_at_20_max": -0.393756, + "nauc_precision_at_20_std": -0.214342, + "nauc_precision_at_20_diff1": -0.016402, + "nauc_precision_at_100_max": -0.379194, + "nauc_precision_at_100_std": -0.231639, + "nauc_precision_at_100_diff1": -0.106276, + "nauc_precision_at_1000_max": -0.375086, + "nauc_precision_at_1000_std": -0.240818, + "nauc_precision_at_1000_diff1": -0.11852, + "nauc_cv_recall_at_1_max": -0.034967, + "nauc_cv_recall_at_1_std": -0.03875, + "nauc_cv_recall_at_1_diff1": 0.301138, + "nauc_cv_recall_at_3_max": -0.287929, + "nauc_cv_recall_at_3_std": 0.082917, + "nauc_cv_recall_at_3_diff1": 0.167569, + "nauc_cv_recall_at_5_max": -0.372915, + "nauc_cv_recall_at_5_std": 0.052746, + "nauc_cv_recall_at_5_diff1": 0.145158, + "nauc_cv_recall_at_10_max": -0.905311, + "nauc_cv_recall_at_10_std": -0.151528, + "nauc_cv_recall_at_10_diff1": 0.39673, + "nauc_cv_recall_at_20_max": -1.020612, + "nauc_cv_recall_at_20_std": 0.373915, + "nauc_cv_recall_at_20_diff1": 0.343945, + "nauc_cv_recall_at_100_max": -0.946401, + "nauc_cv_recall_at_100_std": 0.497564, + "nauc_cv_recall_at_100_diff1": 0.795622, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.034967, + "nauc_mrr_at_1_std": -0.03875, + "nauc_mrr_at_1_diff1": 0.301138, + "nauc_mrr_at_3_max": -0.142392, + "nauc_mrr_at_3_std": -0.032854, + "nauc_mrr_at_3_diff1": 0.249168, + "nauc_mrr_at_5_max": -0.153588, + "nauc_mrr_at_5_std": -0.039227, + "nauc_mrr_at_5_diff1": 0.246834, + "nauc_mrr_at_10_max": -0.166137, + "nauc_mrr_at_10_std": -0.056636, + "nauc_mrr_at_10_diff1": 0.262853, + "nauc_mrr_at_20_max": -0.164176, + "nauc_mrr_at_20_std": -0.048304, + "nauc_mrr_at_20_diff1": 0.261396, + "nauc_mrr_at_100_max": -0.160825, + "nauc_mrr_at_100_std": -0.048887, + "nauc_mrr_at_100_diff1": 0.264133, + "nauc_mrr_at_1000_max": -0.160118, + "nauc_mrr_at_1000_std": -0.049535, + "nauc_mrr_at_1000_diff1": 0.263577, + "main_score": 0.58589, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.47368, + "ndcg_at_3": 0.52487, + "ndcg_at_5": 0.56166, + "ndcg_at_10": 0.62558, + "ndcg_at_20": 0.65816, + "ndcg_at_100": 0.68824, + "ndcg_at_1000": 0.69141, + "map_at_1": 0.20618, + "map_at_3": 0.37352, + "map_at_5": 0.43382, + "map_at_10": 0.48901, + "map_at_20": 0.51161, + "map_at_100": 0.53072, + "map_at_1000": 0.531, + "recall_at_1": 0.20618, + "recall_at_3": 0.4779, + "recall_at_5": 0.61138, + "recall_at_10": 0.81245, + "recall_at_20": 0.90495, + "recall_at_100": 0.98026, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.47368, + "cv_recall_at_3": 0.78947, + "cv_recall_at_5": 0.85965, + "cv_recall_at_10": 0.96491, + "cv_recall_at_20": 0.98246, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.47368, + "precision_at_3": 0.38012, + "precision_at_5": 0.30877, + "precision_at_10": 0.21404, + "precision_at_20": 0.13596, + "precision_at_100": 0.0386, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.473684, + "mrr_at_3": 0.622807, + "mrr_at_5": 0.637719, + "mrr_at_10": 0.65197, + "mrr_at_20": 0.653565, + "mrr_at_100": 0.653565, + "mrr_at_1000": 0.653714, + "nauc_ndcg_at_1_max": -0.018131, + "nauc_ndcg_at_1_std": 0.010175, + "nauc_ndcg_at_1_diff1": 0.468514, + "nauc_ndcg_at_3_max": -0.042191, + "nauc_ndcg_at_3_std": -0.027397, + "nauc_ndcg_at_3_diff1": 0.341363, + "nauc_ndcg_at_5_max": 0.005353, + "nauc_ndcg_at_5_std": 0.037262, + "nauc_ndcg_at_5_diff1": 0.415016, + "nauc_ndcg_at_10_max": -0.000966, + "nauc_ndcg_at_10_std": -0.037313, + "nauc_ndcg_at_10_diff1": 0.446053, + "nauc_ndcg_at_20_max": -0.02381, + "nauc_ndcg_at_20_std": -0.02199, + "nauc_ndcg_at_20_diff1": 0.453748, + "nauc_ndcg_at_100_max": -0.101358, + "nauc_ndcg_at_100_std": -0.081192, + "nauc_ndcg_at_100_diff1": 0.42954, + "nauc_ndcg_at_1000_max": -0.1023, + "nauc_ndcg_at_1000_std": -0.086504, + "nauc_ndcg_at_1000_diff1": 0.4212, + "nauc_map_at_1_max": 0.147301, + "nauc_map_at_1_std": 0.017004, + "nauc_map_at_1_diff1": 0.55209, + "nauc_map_at_3_max": 0.087912, + "nauc_map_at_3_std": 0.024809, + "nauc_map_at_3_diff1": 0.417023, + "nauc_map_at_5_max": 0.110218, + "nauc_map_at_5_std": 0.078015, + "nauc_map_at_5_diff1": 0.47388, + "nauc_map_at_10_max": 0.064949, + "nauc_map_at_10_std": 0.014855, + "nauc_map_at_10_diff1": 0.472568, + "nauc_map_at_20_max": 0.03731, + "nauc_map_at_20_std": 0.007321, + "nauc_map_at_20_diff1": 0.474439, + "nauc_map_at_100_max": -0.003683, + "nauc_map_at_100_std": -0.020977, + "nauc_map_at_100_diff1": 0.473114, + "nauc_map_at_1000_max": -0.003884, + "nauc_map_at_1000_std": -0.021458, + "nauc_map_at_1000_diff1": 0.472528, + "nauc_recall_at_1_max": 0.147301, + "nauc_recall_at_1_std": 0.017004, + "nauc_recall_at_1_diff1": 0.55209, + "nauc_recall_at_3_max": 0.024219, + "nauc_recall_at_3_std": 0.005402, + "nauc_recall_at_3_diff1": 0.197624, + "nauc_recall_at_5_max": 0.163613, + "nauc_recall_at_5_std": 0.189792, + "nauc_recall_at_5_diff1": 0.32686, + "nauc_recall_at_10_max": 0.179673, + "nauc_recall_at_10_std": 0.002833, + "nauc_recall_at_10_diff1": 0.38352, + "nauc_recall_at_20_max": 0.318456, + "nauc_recall_at_20_std": 0.293662, + "nauc_recall_at_20_diff1": 0.479619, + "nauc_recall_at_100_max": -0.084941, + "nauc_recall_at_100_std": 0.192851, + "nauc_recall_at_100_diff1": 0.833687, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.018131, + "nauc_precision_at_1_std": 0.010175, + "nauc_precision_at_1_diff1": 0.468514, + "nauc_precision_at_3_max": -0.152949, + "nauc_precision_at_3_std": -0.045786, + "nauc_precision_at_3_diff1": 0.17057, + "nauc_precision_at_5_max": -0.133993, + "nauc_precision_at_5_std": -0.005342, + "nauc_precision_at_5_diff1": 0.185673, + "nauc_precision_at_10_max": -0.261282, + "nauc_precision_at_10_std": -0.173736, + "nauc_precision_at_10_diff1": 0.038699, + "nauc_precision_at_20_max": -0.322206, + "nauc_precision_at_20_std": -0.193009, + "nauc_precision_at_20_diff1": -0.039316, + "nauc_precision_at_100_max": -0.321108, + "nauc_precision_at_100_std": -0.195587, + "nauc_precision_at_100_diff1": -0.105593, + "nauc_precision_at_1000_max": -0.322722, + "nauc_precision_at_1000_std": -0.19951, + "nauc_precision_at_1000_diff1": -0.112348, + "nauc_cv_recall_at_1_max": -0.018131, + "nauc_cv_recall_at_1_std": 0.010175, + "nauc_cv_recall_at_1_diff1": 0.468514, + "nauc_cv_recall_at_3_max": -0.364251, + "nauc_cv_recall_at_3_std": -0.238462, + "nauc_cv_recall_at_3_diff1": 0.040179, + "nauc_cv_recall_at_5_max": -0.254453, + "nauc_cv_recall_at_5_std": -0.074553, + "nauc_cv_recall_at_5_diff1": 0.230437, + "nauc_cv_recall_at_10_max": -0.666989, + "nauc_cv_recall_at_10_std": -0.800621, + "nauc_cv_recall_at_10_diff1": 0.351784, + "nauc_cv_recall_at_20_max": -0.164944, + "nauc_cv_recall_at_20_std": 0.126616, + "nauc_cv_recall_at_20_diff1": 0.868512, + "nauc_cv_recall_at_100_max": -0.164944, + "nauc_cv_recall_at_100_std": 0.126616, + "nauc_cv_recall_at_100_diff1": 0.868512, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.018131, + "nauc_mrr_at_1_std": 0.010175, + "nauc_mrr_at_1_diff1": 0.468514, + "nauc_mrr_at_3_max": -0.131608, + "nauc_mrr_at_3_std": -0.077249, + "nauc_mrr_at_3_diff1": 0.318607, + "nauc_mrr_at_5_max": -0.113866, + "nauc_mrr_at_5_std": -0.057, + "nauc_mrr_at_5_diff1": 0.351429, + "nauc_mrr_at_10_max": -0.114973, + "nauc_mrr_at_10_std": -0.073287, + "nauc_mrr_at_10_diff1": 0.356473, + "nauc_mrr_at_20_max": -0.1083, + "nauc_mrr_at_20_std": -0.062811, + "nauc_mrr_at_20_diff1": 0.359774, + "nauc_mrr_at_100_max": -0.1083, + "nauc_mrr_at_100_std": -0.062811, + "nauc_mrr_at_100_diff1": 0.359774, + "nauc_mrr_at_1000_max": -0.108266, + "nauc_mrr_at_1000_std": -0.062923, + "nauc_mrr_at_1000_diff1": 0.359473, + "main_score": 0.56166, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.49123, + "ndcg_at_3": 0.54471, + "ndcg_at_5": 0.60301, + "ndcg_at_10": 0.65086, + "ndcg_at_20": 0.68309, + "ndcg_at_100": 0.71091, + "ndcg_at_1000": 0.71393, + "map_at_1": 0.24639, + "map_at_3": 0.40347, + "map_at_5": 0.47551, + "map_at_10": 0.52324, + "map_at_20": 0.54822, + "map_at_100": 0.56661, + "map_at_1000": 0.56686, + "recall_at_1": 0.24639, + "recall_at_3": 0.51456, + "recall_at_5": 0.66596, + "recall_at_10": 0.82517, + "recall_at_20": 0.91168, + "recall_at_100": 0.98026, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.49123, + "cv_recall_at_3": 0.78947, + "cv_recall_at_5": 0.87719, + "cv_recall_at_10": 0.96491, + "cv_recall_at_20": 0.98246, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.49123, + "precision_at_3": 0.39181, + "precision_at_5": 0.33684, + "precision_at_10": 0.2193, + "precision_at_20": 0.13947, + "precision_at_100": 0.0386, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.491228, + "mrr_at_3": 0.625731, + "mrr_at_5": 0.646784, + "mrr_at_10": 0.659211, + "mrr_at_20": 0.660307, + "mrr_at_100": 0.660307, + "mrr_at_1000": 0.660418, + "nauc_ndcg_at_1_max": -0.080101, + "nauc_ndcg_at_1_std": 0.023865, + "nauc_ndcg_at_1_diff1": 0.467611, + "nauc_ndcg_at_3_max": -0.048012, + "nauc_ndcg_at_3_std": 0.049315, + "nauc_ndcg_at_3_diff1": 0.30895, + "nauc_ndcg_at_5_max": -0.097845, + "nauc_ndcg_at_5_std": 0.005578, + "nauc_ndcg_at_5_diff1": 0.286391, + "nauc_ndcg_at_10_max": -0.073959, + "nauc_ndcg_at_10_std": 0.003894, + "nauc_ndcg_at_10_diff1": 0.297941, + "nauc_ndcg_at_20_max": -0.133287, + "nauc_ndcg_at_20_std": -0.041212, + "nauc_ndcg_at_20_diff1": 0.300762, + "nauc_ndcg_at_100_max": -0.218173, + "nauc_ndcg_at_100_std": -0.112555, + "nauc_ndcg_at_100_diff1": 0.294014, + "nauc_ndcg_at_1000_max": -0.202071, + "nauc_ndcg_at_1000_std": -0.101818, + "nauc_ndcg_at_1000_diff1": 0.310135, + "nauc_map_at_1_max": 0.095713, + "nauc_map_at_1_std": 0.002219, + "nauc_map_at_1_diff1": 0.337958, + "nauc_map_at_3_max": 0.046994, + "nauc_map_at_3_std": 0.072016, + "nauc_map_at_3_diff1": 0.275936, + "nauc_map_at_5_max": 0.051563, + "nauc_map_at_5_std": 0.098466, + "nauc_map_at_5_diff1": 0.306079, + "nauc_map_at_10_max": 0.017694, + "nauc_map_at_10_std": 0.057304, + "nauc_map_at_10_diff1": 0.313379, + "nauc_map_at_20_max": -0.025786, + "nauc_map_at_20_std": 0.020005, + "nauc_map_at_20_diff1": 0.318986, + "nauc_map_at_100_max": -0.072325, + "nauc_map_at_100_std": -0.017193, + "nauc_map_at_100_diff1": 0.321634, + "nauc_map_at_1000_max": -0.072141, + "nauc_map_at_1000_std": -0.017148, + "nauc_map_at_1000_diff1": 0.321863, + "nauc_recall_at_1_max": 0.095713, + "nauc_recall_at_1_std": 0.002219, + "nauc_recall_at_1_diff1": 0.337958, + "nauc_recall_at_3_max": 0.058324, + "nauc_recall_at_3_std": 0.166311, + "nauc_recall_at_3_diff1": 0.112142, + "nauc_recall_at_5_max": 0.034494, + "nauc_recall_at_5_std": 0.148428, + "nauc_recall_at_5_diff1": 0.111313, + "nauc_recall_at_10_max": 0.137744, + "nauc_recall_at_10_std": 0.16621, + "nauc_recall_at_10_diff1": 0.077682, + "nauc_recall_at_20_max": -0.028193, + "nauc_recall_at_20_std": 0.092378, + "nauc_recall_at_20_diff1": -0.057863, + "nauc_recall_at_100_max": -1.424762, + "nauc_recall_at_100_std": -0.94264, + "nauc_recall_at_100_diff1": -0.92803, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.080101, + "nauc_precision_at_1_std": 0.023865, + "nauc_precision_at_1_diff1": 0.467611, + "nauc_precision_at_3_max": -0.201465, + "nauc_precision_at_3_std": 0.012022, + "nauc_precision_at_3_diff1": 0.210089, + "nauc_precision_at_5_max": -0.233233, + "nauc_precision_at_5_std": -0.042893, + "nauc_precision_at_5_diff1": 0.149157, + "nauc_precision_at_10_max": -0.347763, + "nauc_precision_at_10_std": -0.222073, + "nauc_precision_at_10_diff1": 0.074843, + "nauc_precision_at_20_max": -0.388095, + "nauc_precision_at_20_std": -0.280954, + "nauc_precision_at_20_diff1": 0.010935, + "nauc_precision_at_100_max": -0.387568, + "nauc_precision_at_100_std": -0.287196, + "nauc_precision_at_100_diff1": -0.067055, + "nauc_precision_at_1000_max": -0.383756, + "nauc_precision_at_1000_std": -0.285519, + "nauc_precision_at_1000_diff1": -0.066189, + "nauc_cv_recall_at_1_max": -0.080101, + "nauc_cv_recall_at_1_std": 0.023865, + "nauc_cv_recall_at_1_diff1": 0.467611, + "nauc_cv_recall_at_3_max": -0.314441, + "nauc_cv_recall_at_3_std": -0.014678, + "nauc_cv_recall_at_3_diff1": 0.172982, + "nauc_cv_recall_at_5_max": -0.703516, + "nauc_cv_recall_at_5_std": -0.343974, + "nauc_cv_recall_at_5_diff1": 0.052505, + "nauc_cv_recall_at_10_max": -1.448445, + "nauc_cv_recall_at_10_std": -0.864221, + "nauc_cv_recall_at_10_diff1": -0.521209, + "nauc_cv_recall_at_20_max": -1.727857, + "nauc_cv_recall_at_20_std": -1.169034, + "nauc_cv_recall_at_20_diff1": -1.169034, + "nauc_cv_recall_at_100_max": -1.727857, + "nauc_cv_recall_at_100_std": -1.169034, + "nauc_cv_recall_at_100_diff1": -1.169034, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.080101, + "nauc_mrr_at_1_std": 0.023865, + "nauc_mrr_at_1_diff1": 0.467611, + "nauc_mrr_at_3_max": -0.169838, + "nauc_mrr_at_3_std": -0.021498, + "nauc_mrr_at_3_diff1": 0.373549, + "nauc_mrr_at_5_max": -0.201171, + "nauc_mrr_at_5_std": -0.058192, + "nauc_mrr_at_5_diff1": 0.375105, + "nauc_mrr_at_10_max": -0.189333, + "nauc_mrr_at_10_std": -0.049987, + "nauc_mrr_at_10_diff1": 0.378389, + "nauc_mrr_at_20_max": -0.185063, + "nauc_mrr_at_20_std": -0.047767, + "nauc_mrr_at_20_diff1": 0.379486, + "nauc_mrr_at_100_max": -0.185063, + "nauc_mrr_at_100_std": -0.047767, + "nauc_mrr_at_100_diff1": 0.379486, + "nauc_mrr_at_1000_max": -0.184382, + "nauc_mrr_at_1000_std": -0.047272, + "nauc_mrr_at_1000_diff1": 0.380169, + "main_score": 0.60301, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.49123, + "ndcg_at_3": 0.53151, + "ndcg_at_5": 0.54457, + "ndcg_at_10": 0.59647, + "ndcg_at_20": 0.6405, + "ndcg_at_100": 0.67894, + "ndcg_at_1000": 0.68294, + "map_at_1": 0.24273, + "map_at_3": 0.39271, + "map_at_5": 0.42885, + "map_at_10": 0.47443, + "map_at_20": 0.50068, + "map_at_100": 0.52152, + "map_at_1000": 0.52184, + "recall_at_1": 0.24273, + "recall_at_3": 0.50374, + "recall_at_5": 0.57279, + "recall_at_10": 0.73789, + "recall_at_20": 0.87206, + "recall_at_100": 0.97588, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.49123, + "cv_recall_at_3": 0.78947, + "cv_recall_at_5": 0.82456, + "cv_recall_at_10": 0.91228, + "cv_recall_at_20": 0.96491, + "cv_recall_at_100": 0.98246, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.49123, + "precision_at_3": 0.37427, + "precision_at_5": 0.28421, + "precision_at_10": 0.19649, + "precision_at_20": 0.12807, + "precision_at_100": 0.03842, + "precision_at_1000": 0.00389, + "mrr_at_1": 0.491228, + "mrr_at_3": 0.622807, + "mrr_at_5": 0.629825, + "mrr_at_10": 0.640351, + "mrr_at_20": 0.643901, + "mrr_at_100": 0.644449, + "mrr_at_1000": 0.644592, + "nauc_ndcg_at_1_max": -0.099635, + "nauc_ndcg_at_1_std": -0.03299, + "nauc_ndcg_at_1_diff1": 0.427705, + "nauc_ndcg_at_3_max": -0.078528, + "nauc_ndcg_at_3_std": -0.011237, + "nauc_ndcg_at_3_diff1": 0.339689, + "nauc_ndcg_at_5_max": -0.065322, + "nauc_ndcg_at_5_std": 0.025487, + "nauc_ndcg_at_5_diff1": 0.347093, + "nauc_ndcg_at_10_max": -0.061245, + "nauc_ndcg_at_10_std": 0.061433, + "nauc_ndcg_at_10_diff1": 0.370673, + "nauc_ndcg_at_20_max": -0.061013, + "nauc_ndcg_at_20_std": 0.048317, + "nauc_ndcg_at_20_diff1": 0.361462, + "nauc_ndcg_at_100_max": -0.113875, + "nauc_ndcg_at_100_std": -0.033283, + "nauc_ndcg_at_100_diff1": 0.348904, + "nauc_ndcg_at_1000_max": -0.111669, + "nauc_ndcg_at_1000_std": -0.040204, + "nauc_ndcg_at_1000_diff1": 0.358207, + "nauc_map_at_1_max": 0.04991, + "nauc_map_at_1_std": -0.024519, + "nauc_map_at_1_diff1": 0.399929, + "nauc_map_at_3_max": -0.036116, + "nauc_map_at_3_std": -0.005568, + "nauc_map_at_3_diff1": 0.308487, + "nauc_map_at_5_max": -0.0138, + "nauc_map_at_5_std": 0.034748, + "nauc_map_at_5_diff1": 0.346517, + "nauc_map_at_10_max": -0.007929, + "nauc_map_at_10_std": 0.069903, + "nauc_map_at_10_diff1": 0.36295, + "nauc_map_at_20_max": -0.008776, + "nauc_map_at_20_std": 0.057445, + "nauc_map_at_20_diff1": 0.367338, + "nauc_map_at_100_max": -0.043763, + "nauc_map_at_100_std": 0.021893, + "nauc_map_at_100_diff1": 0.362213, + "nauc_map_at_1000_max": -0.04337, + "nauc_map_at_1000_std": 0.021428, + "nauc_map_at_1000_diff1": 0.362768, + "nauc_recall_at_1_max": 0.04991, + "nauc_recall_at_1_std": -0.024519, + "nauc_recall_at_1_diff1": 0.399929, + "nauc_recall_at_3_max": -0.09652, + "nauc_recall_at_3_std": 0.035699, + "nauc_recall_at_3_diff1": 0.240125, + "nauc_recall_at_5_max": -0.014427, + "nauc_recall_at_5_std": 0.152836, + "nauc_recall_at_5_diff1": 0.270182, + "nauc_recall_at_10_max": -0.034032, + "nauc_recall_at_10_std": 0.216104, + "nauc_recall_at_10_diff1": 0.354347, + "nauc_recall_at_20_max": -0.015959, + "nauc_recall_at_20_std": 0.367866, + "nauc_recall_at_20_diff1": 0.249729, + "nauc_recall_at_100_max": -0.143116, + "nauc_recall_at_100_std": 0.258712, + "nauc_recall_at_100_diff1": -0.171208, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.099635, + "nauc_precision_at_1_std": -0.03299, + "nauc_precision_at_1_diff1": 0.427705, + "nauc_precision_at_3_max": -0.132845, + "nauc_precision_at_3_std": -0.017938, + "nauc_precision_at_3_diff1": 0.141845, + "nauc_precision_at_5_max": -0.099671, + "nauc_precision_at_5_std": 0.002857, + "nauc_precision_at_5_diff1": 0.168056, + "nauc_precision_at_10_max": -0.165596, + "nauc_precision_at_10_std": -0.0464, + "nauc_precision_at_10_diff1": 0.124432, + "nauc_precision_at_20_max": -0.165693, + "nauc_precision_at_20_std": -0.114773, + "nauc_precision_at_20_diff1": 0.012434, + "nauc_precision_at_100_max": -0.259794, + "nauc_precision_at_100_std": -0.224241, + "nauc_precision_at_100_diff1": -0.088435, + "nauc_precision_at_1000_max": -0.255363, + "nauc_precision_at_1000_std": -0.230204, + "nauc_precision_at_1000_diff1": -0.087312, + "nauc_cv_recall_at_1_max": -0.099635, + "nauc_cv_recall_at_1_std": -0.03299, + "nauc_cv_recall_at_1_diff1": 0.427705, + "nauc_cv_recall_at_3_max": -0.205977, + "nauc_cv_recall_at_3_std": -0.0534, + "nauc_cv_recall_at_3_diff1": 0.305921, + "nauc_cv_recall_at_5_max": -0.163746, + "nauc_cv_recall_at_5_std": 0.014458, + "nauc_cv_recall_at_5_diff1": 0.283041, + "nauc_cv_recall_at_10_max": -0.449956, + "nauc_cv_recall_at_10_std": -0.242337, + "nauc_cv_recall_at_10_diff1": 0.408451, + "nauc_cv_recall_at_20_max": -0.521209, + "nauc_cv_recall_at_20_std": -0.019164, + "nauc_cv_recall_at_20_diff1": 0.096454, + "nauc_cv_recall_at_100_max": 0.126616, + "nauc_cv_recall_at_100_std": 0.126616, + "nauc_cv_recall_at_100_diff1": -0.164944, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.099635, + "nauc_mrr_at_1_std": -0.03299, + "nauc_mrr_at_1_diff1": 0.427705, + "nauc_mrr_at_3_max": -0.110296, + "nauc_mrr_at_3_std": -0.039476, + "nauc_mrr_at_3_diff1": 0.370936, + "nauc_mrr_at_5_max": -0.103336, + "nauc_mrr_at_5_std": -0.031096, + "nauc_mrr_at_5_diff1": 0.369508, + "nauc_mrr_at_10_max": -0.110998, + "nauc_mrr_at_10_std": -0.042266, + "nauc_mrr_at_10_diff1": 0.377376, + "nauc_mrr_at_20_max": -0.108126, + "nauc_mrr_at_20_std": -0.039926, + "nauc_mrr_at_20_diff1": 0.373409, + "nauc_mrr_at_100_max": -0.105931, + "nauc_mrr_at_100_std": -0.039667, + "nauc_mrr_at_100_diff1": 0.373442, + "nauc_mrr_at_1000_max": -0.106056, + "nauc_mrr_at_1000_std": -0.039757, + "nauc_mrr_at_1000_diff1": 0.373732, + "main_score": 0.54457, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 11228.700615406036, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/Vidore2EconomicsReportsRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/Vidore2EconomicsReportsRetrieval.json new file mode 100644 index 0000000000..ce3809efd4 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/Vidore2EconomicsReportsRetrieval.json @@ -0,0 +1,711 @@ +{ + "dataset_revision": "b3e3a04b07fbbaffe79be49dabf92f691fbca252", + "task_name": "Vidore2EconomicsReportsRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.67241, + "ndcg_at_3": 0.59373, + "ndcg_at_5": 0.5585, + "ndcg_at_10": 0.54783, + "ndcg_at_20": 0.57628, + "ndcg_at_100": 0.69502, + "ndcg_at_1000": 0.72646, + "map_at_1": 0.11591, + "map_at_3": 0.19284, + "map_at_5": 0.23184, + "map_at_10": 0.29702, + "map_at_20": 0.3563, + "map_at_100": 0.45132, + "map_at_1000": 0.46616, + "recall_at_1": 0.11591, + "recall_at_3": 0.22343, + "recall_at_5": 0.29084, + "recall_at_10": 0.41649, + "recall_at_20": 0.56063, + "recall_at_100": 0.9145, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.67241, + "cv_recall_at_3": 0.86207, + "cv_recall_at_5": 0.94828, + "cv_recall_at_10": 0.94828, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.67241, + "precision_at_3": 0.52299, + "precision_at_5": 0.46207, + "precision_at_10": 0.38621, + "precision_at_20": 0.30603, + "precision_at_100": 0.12862, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.672414, + "mrr_at_3": 0.758621, + "mrr_at_5": 0.776724, + "mrr_at_10": 0.776724, + "mrr_at_20": 0.778831, + "mrr_at_100": 0.779495, + "mrr_at_1000": 0.779495, + "nauc_ndcg_at_1_max": 0.150523, + "nauc_ndcg_at_1_std": 0.186724, + "nauc_ndcg_at_1_diff1": 0.061844, + "nauc_ndcg_at_3_max": 0.036632, + "nauc_ndcg_at_3_std": 0.121787, + "nauc_ndcg_at_3_diff1": -0.012196, + "nauc_ndcg_at_5_max": -0.104752, + "nauc_ndcg_at_5_std": -0.007283, + "nauc_ndcg_at_5_diff1": -0.060748, + "nauc_ndcg_at_10_max": -0.089556, + "nauc_ndcg_at_10_std": -0.003358, + "nauc_ndcg_at_10_diff1": 0.01759, + "nauc_ndcg_at_20_max": -0.017235, + "nauc_ndcg_at_20_std": 0.044041, + "nauc_ndcg_at_20_diff1": 0.021608, + "nauc_ndcg_at_100_max": -0.011702, + "nauc_ndcg_at_100_std": 0.11706, + "nauc_ndcg_at_100_diff1": 0.040137, + "nauc_ndcg_at_1000_max": -0.038081, + "nauc_ndcg_at_1000_std": 0.082774, + "nauc_ndcg_at_1000_diff1": 0.005084, + "nauc_map_at_1_max": -0.078027, + "nauc_map_at_1_std": -0.19215, + "nauc_map_at_1_diff1": 0.023705, + "nauc_map_at_3_max": -0.033581, + "nauc_map_at_3_std": -0.122731, + "nauc_map_at_3_diff1": 0.063352, + "nauc_map_at_5_max": -0.049189, + "nauc_map_at_5_std": -0.127237, + "nauc_map_at_5_diff1": 0.04804, + "nauc_map_at_10_max": -0.013046, + "nauc_map_at_10_std": -0.068365, + "nauc_map_at_10_diff1": 0.081709, + "nauc_map_at_20_max": -0.014974, + "nauc_map_at_20_std": -0.028337, + "nauc_map_at_20_diff1": 0.085287, + "nauc_map_at_100_max": -0.067653, + "nauc_map_at_100_std": 0.004872, + "nauc_map_at_100_diff1": 0.030818, + "nauc_map_at_1000_max": -0.094594, + "nauc_map_at_1000_std": -0.015006, + "nauc_map_at_1000_diff1": 0.008132, + "nauc_recall_at_1_max": -0.078027, + "nauc_recall_at_1_std": -0.19215, + "nauc_recall_at_1_diff1": 0.023705, + "nauc_recall_at_3_max": -0.064653, + "nauc_recall_at_3_std": -0.127423, + "nauc_recall_at_3_diff1": 0.036186, + "nauc_recall_at_5_max": -0.087691, + "nauc_recall_at_5_std": -0.162774, + "nauc_recall_at_5_diff1": 0.053711, + "nauc_recall_at_10_max": -0.004553, + "nauc_recall_at_10_std": -0.044533, + "nauc_recall_at_10_diff1": 0.083625, + "nauc_recall_at_20_max": 0.071443, + "nauc_recall_at_20_std": 0.057209, + "nauc_recall_at_20_diff1": 0.093278, + "nauc_recall_at_100_max": -0.017174, + "nauc_recall_at_100_std": 0.130486, + "nauc_recall_at_100_diff1": 0.211076, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.150523, + "nauc_precision_at_1_std": 0.186724, + "nauc_precision_at_1_diff1": 0.061844, + "nauc_precision_at_3_max": 0.080587, + "nauc_precision_at_3_std": 0.222806, + "nauc_precision_at_3_diff1": -0.015545, + "nauc_precision_at_5_max": -0.082271, + "nauc_precision_at_5_std": 0.086019, + "nauc_precision_at_5_diff1": -0.10673, + "nauc_precision_at_10_max": -0.040774, + "nauc_precision_at_10_std": 0.122913, + "nauc_precision_at_10_diff1": -0.001313, + "nauc_precision_at_20_max": -0.096445, + "nauc_precision_at_20_std": 0.081318, + "nauc_precision_at_20_diff1": -0.055644, + "nauc_precision_at_100_max": -0.201407, + "nauc_precision_at_100_std": -0.035434, + "nauc_precision_at_100_diff1": -0.163663, + "nauc_precision_at_1000_max": -0.225871, + "nauc_precision_at_1000_std": -0.080651, + "nauc_precision_at_1000_diff1": -0.163819, + "nauc_cv_recall_at_1_max": 0.150523, + "nauc_cv_recall_at_1_std": 0.186724, + "nauc_cv_recall_at_1_diff1": 0.061844, + "nauc_cv_recall_at_3_max": 0.099717, + "nauc_cv_recall_at_3_std": 0.263728, + "nauc_cv_recall_at_3_diff1": -0.005121, + "nauc_cv_recall_at_5_max": 0.137029, + "nauc_cv_recall_at_5_std": 0.416626, + "nauc_cv_recall_at_5_diff1": 0.099319, + "nauc_cv_recall_at_10_max": 0.137029, + "nauc_cv_recall_at_10_std": 0.416626, + "nauc_cv_recall_at_10_diff1": 0.099319, + "nauc_cv_recall_at_20_max": 1.0, + "nauc_cv_recall_at_20_std": 0.868778, + "nauc_cv_recall_at_20_diff1": 0.720441, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.150523, + "nauc_mrr_at_1_std": 0.186724, + "nauc_mrr_at_1_diff1": 0.061844, + "nauc_mrr_at_3_max": 0.157812, + "nauc_mrr_at_3_std": 0.217322, + "nauc_mrr_at_3_diff1": 0.038651, + "nauc_mrr_at_5_max": 0.162093, + "nauc_mrr_at_5_std": 0.219201, + "nauc_mrr_at_5_diff1": 0.053106, + "nauc_mrr_at_10_max": 0.162093, + "nauc_mrr_at_10_std": 0.219201, + "nauc_mrr_at_10_diff1": 0.053106, + "nauc_mrr_at_20_max": 0.166252, + "nauc_mrr_at_20_std": 0.219153, + "nauc_mrr_at_20_diff1": 0.056932, + "nauc_mrr_at_100_max": 0.163366, + "nauc_mrr_at_100_std": 0.216905, + "nauc_mrr_at_100_diff1": 0.054635, + "nauc_mrr_at_1000_max": 0.163366, + "nauc_mrr_at_1000_std": 0.216905, + "nauc_mrr_at_1000_diff1": 0.054635, + "main_score": 0.5585, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.65517, + "ndcg_at_3": 0.59669, + "ndcg_at_5": 0.56514, + "ndcg_at_10": 0.56256, + "ndcg_at_20": 0.58917, + "ndcg_at_100": 0.69801, + "ndcg_at_1000": 0.7354, + "map_at_1": 0.1099, + "map_at_3": 0.19764, + "map_at_5": 0.24426, + "map_at_10": 0.3144, + "map_at_20": 0.36858, + "map_at_100": 0.45759, + "map_at_1000": 0.47494, + "recall_at_1": 0.1099, + "recall_at_3": 0.23522, + "recall_at_5": 0.31097, + "recall_at_10": 0.45206, + "recall_at_20": 0.58953, + "recall_at_100": 0.90257, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.65517, + "cv_recall_at_3": 0.89655, + "cv_recall_at_5": 0.93103, + "cv_recall_at_10": 0.96552, + "cv_recall_at_20": 0.98276, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.65517, + "precision_at_3": 0.53448, + "precision_at_5": 0.47241, + "precision_at_10": 0.39655, + "precision_at_20": 0.30172, + "precision_at_100": 0.125, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.655172, + "mrr_at_3": 0.770115, + "mrr_at_5": 0.777874, + "mrr_at_10": 0.78321, + "mrr_at_20": 0.78436, + "mrr_at_100": 0.785023, + "mrr_at_1000": 0.785023, + "nauc_ndcg_at_1_max": -0.266945, + "nauc_ndcg_at_1_std": -0.212233, + "nauc_ndcg_at_1_diff1": 0.22166, + "nauc_ndcg_at_3_max": -0.127748, + "nauc_ndcg_at_3_std": -0.108441, + "nauc_ndcg_at_3_diff1": 0.101229, + "nauc_ndcg_at_5_max": -0.080438, + "nauc_ndcg_at_5_std": -0.083142, + "nauc_ndcg_at_5_diff1": 0.061115, + "nauc_ndcg_at_10_max": -0.062493, + "nauc_ndcg_at_10_std": -0.043933, + "nauc_ndcg_at_10_diff1": 0.001504, + "nauc_ndcg_at_20_max": -0.066353, + "nauc_ndcg_at_20_std": -0.025253, + "nauc_ndcg_at_20_diff1": 0.032034, + "nauc_ndcg_at_100_max": -0.049615, + "nauc_ndcg_at_100_std": -0.004718, + "nauc_ndcg_at_100_diff1": 0.032755, + "nauc_ndcg_at_1000_max": -0.051745, + "nauc_ndcg_at_1000_std": -0.053609, + "nauc_ndcg_at_1000_diff1": 0.071172, + "nauc_map_at_1_max": -0.122641, + "nauc_map_at_1_std": -0.224008, + "nauc_map_at_1_diff1": 0.143742, + "nauc_map_at_3_max": -0.17918, + "nauc_map_at_3_std": -0.228699, + "nauc_map_at_3_diff1": 0.046643, + "nauc_map_at_5_max": -0.118572, + "nauc_map_at_5_std": -0.160681, + "nauc_map_at_5_diff1": 0.025516, + "nauc_map_at_10_max": -0.054385, + "nauc_map_at_10_std": -0.078495, + "nauc_map_at_10_diff1": -0.020173, + "nauc_map_at_20_max": -0.057739, + "nauc_map_at_20_std": -0.053061, + "nauc_map_at_20_diff1": 0.015149, + "nauc_map_at_100_max": -0.036079, + "nauc_map_at_100_std": -0.041433, + "nauc_map_at_100_diff1": -0.026267, + "nauc_map_at_1000_max": -0.049278, + "nauc_map_at_1000_std": -0.062038, + "nauc_map_at_1000_diff1": -0.032527, + "nauc_recall_at_1_max": -0.122641, + "nauc_recall_at_1_std": -0.224008, + "nauc_recall_at_1_diff1": 0.143742, + "nauc_recall_at_3_max": -0.157127, + "nauc_recall_at_3_std": -0.204361, + "nauc_recall_at_3_diff1": 0.000878, + "nauc_recall_at_5_max": -0.091871, + "nauc_recall_at_5_std": -0.090082, + "nauc_recall_at_5_diff1": -0.026898, + "nauc_recall_at_10_max": -0.02271, + "nauc_recall_at_10_std": 0.030096, + "nauc_recall_at_10_diff1": -0.108192, + "nauc_recall_at_20_max": -0.054981, + "nauc_recall_at_20_std": 0.040896, + "nauc_recall_at_20_diff1": -0.038473, + "nauc_recall_at_100_max": -0.025929, + "nauc_recall_at_100_std": 0.176057, + "nauc_recall_at_100_diff1": -0.133576, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.266945, + "nauc_precision_at_1_std": -0.212233, + "nauc_precision_at_1_diff1": 0.22166, + "nauc_precision_at_3_max": -0.000705, + "nauc_precision_at_3_std": 0.060548, + "nauc_precision_at_3_diff1": 0.051672, + "nauc_precision_at_5_max": 0.085279, + "nauc_precision_at_5_std": 0.108458, + "nauc_precision_at_5_diff1": -0.014567, + "nauc_precision_at_10_max": 0.113682, + "nauc_precision_at_10_std": 0.138782, + "nauc_precision_at_10_diff1": -0.040452, + "nauc_precision_at_20_max": 0.039581, + "nauc_precision_at_20_std": 0.072628, + "nauc_precision_at_20_diff1": 0.009364, + "nauc_precision_at_100_max": -0.026203, + "nauc_precision_at_100_std": -0.036086, + "nauc_precision_at_100_diff1": -0.068836, + "nauc_precision_at_1000_max": -0.061122, + "nauc_precision_at_1000_std": -0.085204, + "nauc_precision_at_1000_diff1": -0.062284, + "nauc_cv_recall_at_1_max": -0.266945, + "nauc_cv_recall_at_1_std": -0.212233, + "nauc_cv_recall_at_1_diff1": 0.22166, + "nauc_cv_recall_at_3_max": 0.208826, + "nauc_cv_recall_at_3_std": 0.244334, + "nauc_cv_recall_at_3_diff1": 0.629227, + "nauc_cv_recall_at_5_max": 0.509207, + "nauc_cv_recall_at_5_std": 0.695462, + "nauc_cv_recall_at_5_diff1": 0.604076, + "nauc_cv_recall_at_10_max": 0.241408, + "nauc_cv_recall_at_10_std": 0.456534, + "nauc_cv_recall_at_10_diff1": 0.347932, + "nauc_cv_recall_at_20_max": 0.123761, + "nauc_cv_recall_at_20_std": 0.554013, + "nauc_cv_recall_at_20_diff1": 0.868778, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.266945, + "nauc_mrr_at_1_std": -0.212233, + "nauc_mrr_at_1_diff1": 0.22166, + "nauc_mrr_at_3_max": -0.144941, + "nauc_mrr_at_3_std": -0.091199, + "nauc_mrr_at_3_diff1": 0.336504, + "nauc_mrr_at_5_max": -0.131519, + "nauc_mrr_at_5_std": -0.070367, + "nauc_mrr_at_5_diff1": 0.324017, + "nauc_mrr_at_10_max": -0.158126, + "nauc_mrr_at_10_std": -0.099392, + "nauc_mrr_at_10_diff1": 0.308295, + "nauc_mrr_at_20_max": -0.161347, + "nauc_mrr_at_20_std": -0.102248, + "nauc_mrr_at_20_diff1": 0.311293, + "nauc_mrr_at_100_max": -0.162376, + "nauc_mrr_at_100_std": -0.104615, + "nauc_mrr_at_100_diff1": 0.309282, + "nauc_mrr_at_1000_max": -0.162376, + "nauc_mrr_at_1000_std": -0.104615, + "nauc_mrr_at_1000_diff1": 0.309282, + "main_score": 0.56514, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.77586, + "ndcg_at_3": 0.68662, + "ndcg_at_5": 0.66189, + "ndcg_at_10": 0.62033, + "ndcg_at_20": 0.6386, + "ndcg_at_100": 0.73862, + "ndcg_at_1000": 0.76915, + "map_at_1": 0.1148, + "map_at_3": 0.2227, + "map_at_5": 0.29002, + "map_at_10": 0.35802, + "map_at_20": 0.42168, + "map_at_100": 0.51039, + "map_at_1000": 0.52634, + "recall_at_1": 0.1148, + "recall_at_3": 0.26071, + "recall_at_5": 0.35545, + "recall_at_10": 0.46502, + "recall_at_20": 0.62053, + "recall_at_100": 0.92291, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.77586, + "cv_recall_at_3": 0.93103, + "cv_recall_at_5": 0.94828, + "cv_recall_at_10": 0.96552, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.77586, + "precision_at_3": 0.62069, + "precision_at_5": 0.57241, + "precision_at_10": 0.43966, + "precision_at_20": 0.32241, + "precision_at_100": 0.12931, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.775862, + "mrr_at_3": 0.844828, + "mrr_at_5": 0.848276, + "mrr_at_10": 0.851149, + "mrr_at_20": 0.853395, + "mrr_at_100": 0.853395, + "mrr_at_1000": 0.853395, + "nauc_ndcg_at_1_max": 0.315892, + "nauc_ndcg_at_1_std": 0.470752, + "nauc_ndcg_at_1_diff1": 0.400994, + "nauc_ndcg_at_3_max": 0.008479, + "nauc_ndcg_at_3_std": 0.107436, + "nauc_ndcg_at_3_diff1": 0.080491, + "nauc_ndcg_at_5_max": -0.001192, + "nauc_ndcg_at_5_std": 0.06684, + "nauc_ndcg_at_5_diff1": 0.073212, + "nauc_ndcg_at_10_max": -0.032687, + "nauc_ndcg_at_10_std": -0.010189, + "nauc_ndcg_at_10_diff1": 0.008936, + "nauc_ndcg_at_20_max": 0.128844, + "nauc_ndcg_at_20_std": 0.118011, + "nauc_ndcg_at_20_diff1": 0.036632, + "nauc_ndcg_at_100_max": 0.072256, + "nauc_ndcg_at_100_std": 0.154377, + "nauc_ndcg_at_100_diff1": -0.03077, + "nauc_ndcg_at_1000_max": 0.064598, + "nauc_ndcg_at_1000_std": 0.12088, + "nauc_ndcg_at_1000_diff1": 0.062635, + "nauc_map_at_1_max": -0.059422, + "nauc_map_at_1_std": -0.162921, + "nauc_map_at_1_diff1": 0.009735, + "nauc_map_at_3_max": -0.058085, + "nauc_map_at_3_std": -0.120483, + "nauc_map_at_3_diff1": -0.044202, + "nauc_map_at_5_max": -0.023363, + "nauc_map_at_5_std": -0.06166, + "nauc_map_at_5_diff1": -0.036134, + "nauc_map_at_10_max": -0.019744, + "nauc_map_at_10_std": -0.037903, + "nauc_map_at_10_diff1": -0.063225, + "nauc_map_at_20_max": 0.054687, + "nauc_map_at_20_std": 0.047918, + "nauc_map_at_20_diff1": -0.030479, + "nauc_map_at_100_max": 0.012673, + "nauc_map_at_100_std": 0.061292, + "nauc_map_at_100_diff1": -0.051216, + "nauc_map_at_1000_max": -0.001409, + "nauc_map_at_1000_std": 0.039392, + "nauc_map_at_1000_diff1": -0.035677, + "nauc_recall_at_1_max": -0.059422, + "nauc_recall_at_1_std": -0.162921, + "nauc_recall_at_1_diff1": 0.009735, + "nauc_recall_at_3_max": -0.114339, + "nauc_recall_at_3_std": -0.214265, + "nauc_recall_at_3_diff1": -0.108946, + "nauc_recall_at_5_max": -0.052514, + "nauc_recall_at_5_std": -0.143077, + "nauc_recall_at_5_diff1": -0.103369, + "nauc_recall_at_10_max": -0.012976, + "nauc_recall_at_10_std": -0.111183, + "nauc_recall_at_10_diff1": -0.135264, + "nauc_recall_at_20_max": 0.168203, + "nauc_recall_at_20_std": 0.100133, + "nauc_recall_at_20_diff1": -0.072254, + "nauc_recall_at_100_max": 0.080615, + "nauc_recall_at_100_std": 0.17344, + "nauc_recall_at_100_diff1": -0.36125, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.315892, + "nauc_precision_at_1_std": 0.470752, + "nauc_precision_at_1_diff1": 0.400994, + "nauc_precision_at_3_max": 0.020762, + "nauc_precision_at_3_std": 0.152358, + "nauc_precision_at_3_diff1": 0.065148, + "nauc_precision_at_5_max": 0.053136, + "nauc_precision_at_5_std": 0.158231, + "nauc_precision_at_5_diff1": 0.085841, + "nauc_precision_at_10_max": 0.009187, + "nauc_precision_at_10_std": 0.106726, + "nauc_precision_at_10_diff1": 0.064484, + "nauc_precision_at_20_max": 0.068099, + "nauc_precision_at_20_std": 0.150543, + "nauc_precision_at_20_diff1": 0.095905, + "nauc_precision_at_100_max": -0.074363, + "nauc_precision_at_100_std": -0.01575, + "nauc_precision_at_100_diff1": 0.025589, + "nauc_precision_at_1000_max": -0.086089, + "nauc_precision_at_1000_std": -0.066603, + "nauc_precision_at_1000_diff1": 0.055194, + "nauc_cv_recall_at_1_max": 0.315892, + "nauc_cv_recall_at_1_std": 0.470752, + "nauc_cv_recall_at_1_diff1": 0.400994, + "nauc_cv_recall_at_3_max": 0.257586, + "nauc_cv_recall_at_3_std": 0.110991, + "nauc_cv_recall_at_3_diff1": 0.257586, + "nauc_cv_recall_at_5_max": 0.223763, + "nauc_cv_recall_at_5_std": -0.141605, + "nauc_cv_recall_at_5_diff1": 0.302194, + "nauc_cv_recall_at_10_max": -0.024577, + "nauc_cv_recall_at_10_std": -0.391935, + "nauc_cv_recall_at_10_diff1": 0.093071, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.315892, + "nauc_mrr_at_1_std": 0.470752, + "nauc_mrr_at_1_diff1": 0.400994, + "nauc_mrr_at_3_max": 0.28989, + "nauc_mrr_at_3_std": 0.392364, + "nauc_mrr_at_3_diff1": 0.366066, + "nauc_mrr_at_5_max": 0.288212, + "nauc_mrr_at_5_std": 0.380808, + "nauc_mrr_at_5_diff1": 0.371944, + "nauc_mrr_at_10_max": 0.279294, + "nauc_mrr_at_10_std": 0.381256, + "nauc_mrr_at_10_diff1": 0.364754, + "nauc_mrr_at_20_max": 0.28451, + "nauc_mrr_at_20_std": 0.395119, + "nauc_mrr_at_20_diff1": 0.369628, + "nauc_mrr_at_100_max": 0.28451, + "nauc_mrr_at_100_std": 0.395119, + "nauc_mrr_at_100_diff1": 0.369628, + "nauc_mrr_at_1000_max": 0.28451, + "nauc_mrr_at_1000_std": 0.395119, + "nauc_mrr_at_1000_diff1": 0.369628, + "main_score": 0.66189, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.53448, + "ndcg_at_3": 0.54414, + "ndcg_at_5": 0.52822, + "ndcg_at_10": 0.52437, + "ndcg_at_20": 0.55356, + "ndcg_at_100": 0.66102, + "ndcg_at_1000": 0.69736, + "map_at_1": 0.05618, + "map_at_3": 0.15395, + "map_at_5": 0.20368, + "map_at_10": 0.27376, + "map_at_20": 0.33272, + "map_at_100": 0.41587, + "map_at_1000": 0.4326, + "recall_at_1": 0.05618, + "recall_at_3": 0.22032, + "recall_at_5": 0.30352, + "recall_at_10": 0.43977, + "recall_at_20": 0.58608, + "recall_at_100": 0.9062, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.53448, + "cv_recall_at_3": 0.82759, + "cv_recall_at_5": 0.93103, + "cv_recall_at_10": 0.98276, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.53448, + "precision_at_3": 0.51724, + "precision_at_5": 0.47586, + "precision_at_10": 0.3931, + "precision_at_20": 0.30345, + "precision_at_100": 0.12603, + "precision_at_1000": 0.01564, + "mrr_at_1": 0.534483, + "mrr_at_3": 0.666667, + "mrr_at_5": 0.692529, + "mrr_at_10": 0.700021, + "mrr_at_20": 0.700978, + "mrr_at_100": 0.700978, + "mrr_at_1000": 0.700978, + "nauc_ndcg_at_1_max": 0.102774, + "nauc_ndcg_at_1_std": 0.271963, + "nauc_ndcg_at_1_diff1": 0.042416, + "nauc_ndcg_at_3_max": 0.145554, + "nauc_ndcg_at_3_std": 0.333542, + "nauc_ndcg_at_3_diff1": 0.12461, + "nauc_ndcg_at_5_max": 0.06596, + "nauc_ndcg_at_5_std": 0.212113, + "nauc_ndcg_at_5_diff1": 0.096758, + "nauc_ndcg_at_10_max": -0.020853, + "nauc_ndcg_at_10_std": 0.19314, + "nauc_ndcg_at_10_diff1": 0.127123, + "nauc_ndcg_at_20_max": -0.083236, + "nauc_ndcg_at_20_std": 0.230555, + "nauc_ndcg_at_20_diff1": 0.066723, + "nauc_ndcg_at_100_max": -0.042744, + "nauc_ndcg_at_100_std": 0.304075, + "nauc_ndcg_at_100_diff1": -0.020625, + "nauc_ndcg_at_1000_max": 0.033468, + "nauc_ndcg_at_1000_std": 0.29778, + "nauc_ndcg_at_1000_diff1": 0.084963, + "nauc_map_at_1_max": -0.020802, + "nauc_map_at_1_std": 0.106481, + "nauc_map_at_1_diff1": -0.061961, + "nauc_map_at_3_max": -0.097625, + "nauc_map_at_3_std": 0.110002, + "nauc_map_at_3_diff1": -0.043954, + "nauc_map_at_5_max": -0.139265, + "nauc_map_at_5_std": 0.058539, + "nauc_map_at_5_diff1": -0.083708, + "nauc_map_at_10_max": -0.168382, + "nauc_map_at_10_std": 0.100282, + "nauc_map_at_10_diff1": -0.076132, + "nauc_map_at_20_max": -0.200419, + "nauc_map_at_20_std": 0.14441, + "nauc_map_at_20_diff1": -0.054887, + "nauc_map_at_100_max": -0.112447, + "nauc_map_at_100_std": 0.240161, + "nauc_map_at_100_diff1": -0.014431, + "nauc_map_at_1000_max": -0.07082, + "nauc_map_at_1000_std": 0.237937, + "nauc_map_at_1000_diff1": 0.017264, + "nauc_recall_at_1_max": -0.020802, + "nauc_recall_at_1_std": 0.106481, + "nauc_recall_at_1_diff1": -0.061961, + "nauc_recall_at_3_max": -0.123114, + "nauc_recall_at_3_std": -0.087412, + "nauc_recall_at_3_diff1": -0.053756, + "nauc_recall_at_5_max": -0.187758, + "nauc_recall_at_5_std": -0.144412, + "nauc_recall_at_5_diff1": -0.107506, + "nauc_recall_at_10_max": -0.244669, + "nauc_recall_at_10_std": -0.062721, + "nauc_recall_at_10_diff1": -0.131831, + "nauc_recall_at_20_max": -0.329229, + "nauc_recall_at_20_std": -0.008967, + "nauc_recall_at_20_diff1": -0.137227, + "nauc_recall_at_100_max": -0.299607, + "nauc_recall_at_100_std": 0.102225, + "nauc_recall_at_100_diff1": -0.411083, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.102774, + "nauc_precision_at_1_std": 0.271963, + "nauc_precision_at_1_diff1": 0.042416, + "nauc_precision_at_3_max": 0.209308, + "nauc_precision_at_3_std": 0.408407, + "nauc_precision_at_3_diff1": 0.175998, + "nauc_precision_at_5_max": 0.102165, + "nauc_precision_at_5_std": 0.25691, + "nauc_precision_at_5_diff1": 0.127563, + "nauc_precision_at_10_max": 0.060978, + "nauc_precision_at_10_std": 0.225571, + "nauc_precision_at_10_diff1": 0.190651, + "nauc_precision_at_20_max": 0.02083, + "nauc_precision_at_20_std": 0.17082, + "nauc_precision_at_20_diff1": 0.196377, + "nauc_precision_at_100_max": 0.203301, + "nauc_precision_at_100_std": 0.147642, + "nauc_precision_at_100_diff1": 0.162958, + "nauc_precision_at_1000_max": 0.222646, + "nauc_precision_at_1000_std": 0.114717, + "nauc_precision_at_1000_diff1": 0.187032, + "nauc_cv_recall_at_1_max": 0.102774, + "nauc_cv_recall_at_1_std": 0.271963, + "nauc_cv_recall_at_1_diff1": 0.042416, + "nauc_cv_recall_at_3_max": 0.464692, + "nauc_cv_recall_at_3_std": 0.409671, + "nauc_cv_recall_at_3_diff1": 0.376518, + "nauc_cv_recall_at_5_max": 0.368839, + "nauc_cv_recall_at_5_std": 0.328768, + "nauc_cv_recall_at_5_diff1": 0.250077, + "nauc_cv_recall_at_10_max": 0.123761, + "nauc_cv_recall_at_10_std": 0.868778, + "nauc_cv_recall_at_10_diff1": 0.554013, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.102774, + "nauc_mrr_at_1_std": 0.271963, + "nauc_mrr_at_1_diff1": 0.042416, + "nauc_mrr_at_3_max": 0.200622, + "nauc_mrr_at_3_std": 0.314364, + "nauc_mrr_at_3_diff1": 0.122556, + "nauc_mrr_at_5_max": 0.167499, + "nauc_mrr_at_5_std": 0.299055, + "nauc_mrr_at_5_diff1": 0.088219, + "nauc_mrr_at_10_max": 0.157434, + "nauc_mrr_at_10_std": 0.303883, + "nauc_mrr_at_10_diff1": 0.089315, + "nauc_mrr_at_20_max": 0.157575, + "nauc_mrr_at_20_std": 0.301526, + "nauc_mrr_at_20_diff1": 0.087376, + "nauc_mrr_at_100_max": 0.157575, + "nauc_mrr_at_100_std": 0.301526, + "nauc_mrr_at_100_diff1": 0.087376, + "nauc_mrr_at_1000_max": 0.157575, + "nauc_mrr_at_1000_std": 0.301526, + "nauc_mrr_at_1000_diff1": 0.087376, + "main_score": 0.52822, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ] + } + ] + }, + "evaluation_time": 599.2856538295746, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreArxivQARetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreArxivQARetrieval.json new file mode 100644 index 0000000000..f66e01a8cb --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreArxivQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "7d94d570960eac2408d3baa7a33f9de4822ae3e4", + "task_name": "VidoreArxivQARetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.834, + "ndcg_at_3": 0.87602, + "ndcg_at_5": 0.88351, + "ndcg_at_10": 0.89382, + "ndcg_at_20": 0.89856, + "ndcg_at_100": 0.90271, + "ndcg_at_1000": 0.90327, + "map_at_1": 0.834, + "map_at_3": 0.86567, + "map_at_5": 0.86987, + "map_at_10": 0.87409, + "map_at_20": 0.87549, + "map_at_100": 0.87609, + "map_at_1000": 0.87612, + "recall_at_1": 0.834, + "recall_at_3": 0.906, + "recall_at_5": 0.924, + "recall_at_10": 0.956, + "recall_at_20": 0.974, + "recall_at_100": 0.996, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.834, + "cv_recall_at_3": 0.906, + "cv_recall_at_5": 0.924, + "cv_recall_at_10": 0.956, + "cv_recall_at_20": 0.974, + "cv_recall_at_100": 0.996, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.834, + "precision_at_3": 0.302, + "precision_at_5": 0.1848, + "precision_at_10": 0.0956, + "precision_at_20": 0.0487, + "precision_at_100": 0.00996, + "precision_at_1000": 0.001, + "mrr_at_1": 0.834, + "mrr_at_3": 0.865667, + "mrr_at_5": 0.869867, + "mrr_at_10": 0.87409, + "mrr_at_20": 0.875492, + "mrr_at_100": 0.876088, + "mrr_at_1000": 0.876118, + "nauc_ndcg_at_1_max": 0.670339, + "nauc_ndcg_at_1_std": 0.413276, + "nauc_ndcg_at_1_diff1": 0.952061, + "nauc_ndcg_at_3_max": 0.67569, + "nauc_ndcg_at_3_std": 0.430692, + "nauc_ndcg_at_3_diff1": 0.942806, + "nauc_ndcg_at_5_max": 0.681304, + "nauc_ndcg_at_5_std": 0.434729, + "nauc_ndcg_at_5_diff1": 0.943806, + "nauc_ndcg_at_10_max": 0.678033, + "nauc_ndcg_at_10_std": 0.433585, + "nauc_ndcg_at_10_diff1": 0.946965, + "nauc_ndcg_at_20_max": 0.684001, + "nauc_ndcg_at_20_std": 0.436331, + "nauc_ndcg_at_20_diff1": 0.945865, + "nauc_ndcg_at_100_max": 0.681042, + "nauc_ndcg_at_100_std": 0.435533, + "nauc_ndcg_at_100_diff1": 0.946977, + "nauc_ndcg_at_1000_max": 0.679112, + "nauc_ndcg_at_1000_std": 0.432117, + "nauc_ndcg_at_1000_diff1": 0.946656, + "nauc_map_at_1_max": 0.670339, + "nauc_map_at_1_std": 0.413276, + "nauc_map_at_1_diff1": 0.952061, + "nauc_map_at_3_max": 0.673656, + "nauc_map_at_3_std": 0.425459, + "nauc_map_at_3_diff1": 0.945629, + "nauc_map_at_5_max": 0.676802, + "nauc_map_at_5_std": 0.427729, + "nauc_map_at_5_diff1": 0.946154, + "nauc_map_at_10_max": 0.675974, + "nauc_map_at_10_std": 0.42777, + "nauc_map_at_10_diff1": 0.947476, + "nauc_map_at_20_max": 0.677217, + "nauc_map_at_20_std": 0.428138, + "nauc_map_at_20_diff1": 0.947242, + "nauc_map_at_100_max": 0.676883, + "nauc_map_at_100_std": 0.428186, + "nauc_map_at_100_diff1": 0.947317, + "nauc_map_at_1000_max": 0.676802, + "nauc_map_at_1000_std": 0.428043, + "nauc_map_at_1000_diff1": 0.947304, + "nauc_recall_at_1_max": 0.670339, + "nauc_recall_at_1_std": 0.413276, + "nauc_recall_at_1_diff1": 0.952061, + "nauc_recall_at_3_max": 0.68405, + "nauc_recall_at_3_std": 0.451586, + "nauc_recall_at_3_diff1": 0.931512, + "nauc_recall_at_5_max": 0.703499, + "nauc_recall_at_5_std": 0.46818, + "nauc_recall_at_5_diff1": 0.932921, + "nauc_recall_at_10_max": 0.689097, + "nauc_recall_at_10_std": 0.474196, + "nauc_recall_at_10_diff1": 0.945781, + "nauc_recall_at_20_max": 0.788013, + "nauc_recall_at_20_std": 0.54963, + "nauc_recall_at_20_diff1": 0.928356, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 1.0, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.670339, + "nauc_precision_at_1_std": 0.413276, + "nauc_precision_at_1_diff1": 0.952061, + "nauc_precision_at_3_max": 0.68405, + "nauc_precision_at_3_std": 0.451586, + "nauc_precision_at_3_diff1": 0.931512, + "nauc_precision_at_5_max": 0.703499, + "nauc_precision_at_5_std": 0.46818, + "nauc_precision_at_5_diff1": 0.932921, + "nauc_precision_at_10_max": 0.689097, + "nauc_precision_at_10_std": 0.474196, + "nauc_precision_at_10_diff1": 0.945781, + "nauc_precision_at_20_max": 0.788013, + "nauc_precision_at_20_std": 0.54963, + "nauc_precision_at_20_diff1": 0.928356, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_cv_recall_at_1_max": 0.670339, + "nauc_cv_recall_at_1_std": 0.413276, + "nauc_cv_recall_at_1_diff1": 0.952061, + "nauc_cv_recall_at_3_max": 0.68405, + "nauc_cv_recall_at_3_std": 0.451586, + "nauc_cv_recall_at_3_diff1": 0.931512, + "nauc_cv_recall_at_5_max": 0.703499, + "nauc_cv_recall_at_5_std": 0.46818, + "nauc_cv_recall_at_5_diff1": 0.932921, + "nauc_cv_recall_at_10_max": 0.689097, + "nauc_cv_recall_at_10_std": 0.474196, + "nauc_cv_recall_at_10_diff1": 0.945781, + "nauc_cv_recall_at_20_max": 0.788013, + "nauc_cv_recall_at_20_std": 0.54963, + "nauc_cv_recall_at_20_diff1": 0.928356, + "nauc_cv_recall_at_100_max": 1.0, + "nauc_cv_recall_at_100_std": 1.0, + "nauc_cv_recall_at_100_diff1": 1.0, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.670339, + "nauc_mrr_at_1_std": 0.413276, + "nauc_mrr_at_1_diff1": 0.952061, + "nauc_mrr_at_3_max": 0.673656, + "nauc_mrr_at_3_std": 0.425459, + "nauc_mrr_at_3_diff1": 0.945629, + "nauc_mrr_at_5_max": 0.676802, + "nauc_mrr_at_5_std": 0.427729, + "nauc_mrr_at_5_diff1": 0.946154, + "nauc_mrr_at_10_max": 0.675974, + "nauc_mrr_at_10_std": 0.42777, + "nauc_mrr_at_10_diff1": 0.947476, + "nauc_mrr_at_20_max": 0.677217, + "nauc_mrr_at_20_std": 0.428138, + "nauc_mrr_at_20_diff1": 0.947242, + "nauc_mrr_at_100_max": 0.676883, + "nauc_mrr_at_100_std": 0.428186, + "nauc_mrr_at_100_diff1": 0.947317, + "nauc_mrr_at_1000_max": 0.676802, + "nauc_mrr_at_1000_std": 0.428043, + "nauc_mrr_at_1000_diff1": 0.947304, + "main_score": 0.88351, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 271.81372141838074, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreDocVQARetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreDocVQARetrieval.json new file mode 100644 index 0000000000..f53f7fbc00 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreDocVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "162ba2fc1a8437eda8b6c37b240bc1c0f0deb092", + "task_name": "VidoreDocVQARetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.5898, + "ndcg_at_3": 0.64625, + "ndcg_at_5": 0.66213, + "ndcg_at_10": 0.68123, + "ndcg_at_20": 0.69232, + "ndcg_at_100": 0.7144, + "ndcg_at_1000": 0.72318, + "map_at_1": 0.58444, + "map_at_3": 0.62855, + "map_at_5": 0.63771, + "map_at_10": 0.64618, + "map_at_20": 0.64957, + "map_at_100": 0.6528, + "map_at_1000": 0.65331, + "recall_at_1": 0.58444, + "recall_at_3": 0.68614, + "recall_at_5": 0.7252, + "recall_at_10": 0.78507, + "recall_at_20": 0.82668, + "recall_at_100": 0.94067, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.5898, + "cv_recall_at_3": 0.69623, + "cv_recall_at_5": 0.73614, + "cv_recall_at_10": 0.79157, + "cv_recall_at_20": 0.83592, + "cv_recall_at_100": 0.949, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.5898, + "precision_at_3": 0.23356, + "precision_at_5": 0.15033, + "precision_at_10": 0.0816, + "precision_at_20": 0.04335, + "precision_at_100": 0.01007, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.5898, + "mrr_at_3": 0.635994, + "mrr_at_5": 0.64442, + "mrr_at_10": 0.651916, + "mrr_at_20": 0.655133, + "mrr_at_100": 0.658084, + "mrr_at_1000": 0.658355, + "nauc_ndcg_at_1_max": 0.251285, + "nauc_ndcg_at_1_std": 0.311645, + "nauc_ndcg_at_1_diff1": 0.861045, + "nauc_ndcg_at_3_max": 0.235922, + "nauc_ndcg_at_3_std": 0.280123, + "nauc_ndcg_at_3_diff1": 0.852177, + "nauc_ndcg_at_5_max": 0.226689, + "nauc_ndcg_at_5_std": 0.27192, + "nauc_ndcg_at_5_diff1": 0.848102, + "nauc_ndcg_at_10_max": 0.216868, + "nauc_ndcg_at_10_std": 0.287839, + "nauc_ndcg_at_10_diff1": 0.840902, + "nauc_ndcg_at_20_max": 0.220206, + "nauc_ndcg_at_20_std": 0.295776, + "nauc_ndcg_at_20_diff1": 0.840643, + "nauc_ndcg_at_100_max": 0.218463, + "nauc_ndcg_at_100_std": 0.316034, + "nauc_ndcg_at_100_diff1": 0.843122, + "nauc_ndcg_at_1000_max": 0.220843, + "nauc_ndcg_at_1000_std": 0.294595, + "nauc_ndcg_at_1000_diff1": 0.844299, + "nauc_map_at_1_max": 0.261551, + "nauc_map_at_1_std": 0.315335, + "nauc_map_at_1_diff1": 0.869871, + "nauc_map_at_3_max": 0.246739, + "nauc_map_at_3_std": 0.292379, + "nauc_map_at_3_diff1": 0.859957, + "nauc_map_at_5_max": 0.240336, + "nauc_map_at_5_std": 0.287419, + "nauc_map_at_5_diff1": 0.856521, + "nauc_map_at_10_max": 0.23554, + "nauc_map_at_10_std": 0.293074, + "nauc_map_at_10_diff1": 0.853693, + "nauc_map_at_20_max": 0.236221, + "nauc_map_at_20_std": 0.294821, + "nauc_map_at_20_diff1": 0.853353, + "nauc_map_at_100_max": 0.235653, + "nauc_map_at_100_std": 0.297084, + "nauc_map_at_100_diff1": 0.853364, + "nauc_map_at_1000_max": 0.235391, + "nauc_map_at_1000_std": 0.296056, + "nauc_map_at_1000_diff1": 0.853254, + "nauc_recall_at_1_max": 0.261551, + "nauc_recall_at_1_std": 0.315335, + "nauc_recall_at_1_diff1": 0.869871, + "nauc_recall_at_3_max": 0.231811, + "nauc_recall_at_3_std": 0.256166, + "nauc_recall_at_3_diff1": 0.846036, + "nauc_recall_at_5_max": 0.207776, + "nauc_recall_at_5_std": 0.228719, + "nauc_recall_at_5_diff1": 0.831022, + "nauc_recall_at_10_max": 0.159214, + "nauc_recall_at_10_std": 0.280394, + "nauc_recall_at_10_diff1": 0.793261, + "nauc_recall_at_20_max": 0.16581, + "nauc_recall_at_20_std": 0.327997, + "nauc_recall_at_20_diff1": 0.779896, + "nauc_recall_at_100_max": 0.085792, + "nauc_recall_at_100_std": 0.853987, + "nauc_recall_at_100_diff1": 0.752755, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.251285, + "nauc_precision_at_1_std": 0.311645, + "nauc_precision_at_1_diff1": 0.861045, + "nauc_precision_at_3_max": 0.188908, + "nauc_precision_at_3_std": 0.232513, + "nauc_precision_at_3_diff1": 0.792078, + "nauc_precision_at_5_max": 0.112315, + "nauc_precision_at_5_std": 0.168523, + "nauc_precision_at_5_diff1": 0.683897, + "nauc_precision_at_10_max": 0.028998, + "nauc_precision_at_10_std": 0.177165, + "nauc_precision_at_10_diff1": 0.559158, + "nauc_precision_at_20_max": -0.016693, + "nauc_precision_at_20_std": 0.166931, + "nauc_precision_at_20_diff1": 0.455226, + "nauc_precision_at_100_max": -0.284904, + "nauc_precision_at_100_std": 0.087701, + "nauc_precision_at_100_diff1": -0.061908, + "nauc_precision_at_1000_max": -0.395548, + "nauc_precision_at_1000_std": -0.390383, + "nauc_precision_at_1000_diff1": -0.481253, + "nauc_cv_recall_at_1_max": 0.251285, + "nauc_cv_recall_at_1_std": 0.311645, + "nauc_cv_recall_at_1_diff1": 0.861045, + "nauc_cv_recall_at_3_max": 0.204935, + "nauc_cv_recall_at_3_std": 0.24658, + "nauc_cv_recall_at_3_diff1": 0.829064, + "nauc_cv_recall_at_5_max": 0.174026, + "nauc_cv_recall_at_5_std": 0.215075, + "nauc_cv_recall_at_5_diff1": 0.82237, + "nauc_cv_recall_at_10_max": 0.133502, + "nauc_cv_recall_at_10_std": 0.273964, + "nauc_cv_recall_at_10_diff1": 0.782488, + "nauc_cv_recall_at_20_max": 0.13623, + "nauc_cv_recall_at_20_std": 0.320224, + "nauc_cv_recall_at_20_diff1": 0.774659, + "nauc_cv_recall_at_100_max": -0.009032, + "nauc_cv_recall_at_100_std": 0.87948, + "nauc_cv_recall_at_100_diff1": 0.740336, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.251285, + "nauc_mrr_at_1_std": 0.311645, + "nauc_mrr_at_1_diff1": 0.861045, + "nauc_mrr_at_3_max": 0.229515, + "nauc_mrr_at_3_std": 0.286339, + "nauc_mrr_at_3_diff1": 0.847419, + "nauc_mrr_at_5_max": 0.224013, + "nauc_mrr_at_5_std": 0.281436, + "nauc_mrr_at_5_diff1": 0.846774, + "nauc_mrr_at_10_max": 0.221164, + "nauc_mrr_at_10_std": 0.288209, + "nauc_mrr_at_10_diff1": 0.843812, + "nauc_mrr_at_20_max": 0.222649, + "nauc_mrr_at_20_std": 0.290257, + "nauc_mrr_at_20_diff1": 0.844029, + "nauc_mrr_at_100_max": 0.222451, + "nauc_mrr_at_100_std": 0.292395, + "nauc_mrr_at_100_diff1": 0.844169, + "nauc_mrr_at_1000_max": 0.222687, + "nauc_mrr_at_1000_std": 0.291838, + "nauc_mrr_at_1000_diff1": 0.844253, + "main_score": 0.66213, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 359.27758264541626, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreInfoVQARetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreInfoVQARetrieval.json new file mode 100644 index 0000000000..b1c6915b10 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreInfoVQARetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b802cc5fd6c605df2d673a963667d74881d2c9a4", + "task_name": "VidoreInfoVQARetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.91498, + "ndcg_at_3": 0.94325, + "ndcg_at_5": 0.94923, + "ndcg_at_10": 0.95083, + "ndcg_at_20": 0.95368, + "ndcg_at_100": 0.9549, + "ndcg_at_1000": 0.95508, + "map_at_1": 0.91498, + "map_at_3": 0.93623, + "map_at_5": 0.93963, + "map_at_10": 0.9402, + "map_at_20": 0.94102, + "map_at_100": 0.94122, + "map_at_1000": 0.94123, + "recall_at_1": 0.91498, + "recall_at_3": 0.96356, + "recall_at_5": 0.97773, + "recall_at_10": 0.98279, + "recall_at_20": 0.99332, + "recall_at_100": 0.99919, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.91498, + "cv_recall_at_3": 0.96356, + "cv_recall_at_5": 0.97773, + "cv_recall_at_10": 0.98381, + "cv_recall_at_20": 0.99595, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.91498, + "precision_at_3": 0.32119, + "precision_at_5": 0.19595, + "precision_at_10": 0.09858, + "precision_at_20": 0.0499, + "precision_at_100": 0.01008, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.91498, + "mrr_at_3": 0.936235, + "mrr_at_5": 0.939474, + "mrr_at_10": 0.94019, + "mrr_at_20": 0.941124, + "mrr_at_100": 0.94121, + "mrr_at_1000": 0.94121, + "nauc_ndcg_at_1_max": 0.670382, + "nauc_ndcg_at_1_std": 0.168339, + "nauc_ndcg_at_1_diff1": 0.949853, + "nauc_ndcg_at_3_max": 0.640461, + "nauc_ndcg_at_3_std": 0.137771, + "nauc_ndcg_at_3_diff1": 0.951091, + "nauc_ndcg_at_5_max": 0.676695, + "nauc_ndcg_at_5_std": 0.224688, + "nauc_ndcg_at_5_diff1": 0.954073, + "nauc_ndcg_at_10_max": 0.669425, + "nauc_ndcg_at_10_std": 0.20198, + "nauc_ndcg_at_10_diff1": 0.955224, + "nauc_ndcg_at_20_max": 0.678003, + "nauc_ndcg_at_20_std": 0.21031, + "nauc_ndcg_at_20_diff1": 0.954061, + "nauc_ndcg_at_100_max": 0.670288, + "nauc_ndcg_at_100_std": 0.192332, + "nauc_ndcg_at_100_diff1": 0.953448, + "nauc_ndcg_at_1000_max": 0.668959, + "nauc_ndcg_at_1000_std": 0.190195, + "nauc_ndcg_at_1000_diff1": 0.95326, + "nauc_map_at_1_max": 0.670382, + "nauc_map_at_1_std": 0.168339, + "nauc_map_at_1_diff1": 0.949853, + "nauc_map_at_3_max": 0.65175, + "nauc_map_at_3_std": 0.152024, + "nauc_map_at_3_diff1": 0.951367, + "nauc_map_at_5_max": 0.669864, + "nauc_map_at_5_std": 0.192398, + "nauc_map_at_5_diff1": 0.952795, + "nauc_map_at_10_max": 0.667646, + "nauc_map_at_10_std": 0.185433, + "nauc_map_at_10_diff1": 0.953102, + "nauc_map_at_20_max": 0.669964, + "nauc_map_at_20_std": 0.187995, + "nauc_map_at_20_diff1": 0.952853, + "nauc_map_at_100_max": 0.668949, + "nauc_map_at_100_std": 0.185806, + "nauc_map_at_100_diff1": 0.952783, + "nauc_map_at_1000_max": 0.668842, + "nauc_map_at_1000_std": 0.185633, + "nauc_map_at_1000_diff1": 0.952768, + "nauc_recall_at_1_max": 0.670382, + "nauc_recall_at_1_std": 0.168339, + "nauc_recall_at_1_diff1": 0.949853, + "nauc_recall_at_3_max": 0.582249, + "nauc_recall_at_3_std": 0.062597, + "nauc_recall_at_3_diff1": 0.949211, + "nauc_recall_at_5_max": 0.739556, + "nauc_recall_at_5_std": 0.50789, + "nauc_recall_at_5_diff1": 0.964381, + "nauc_recall_at_10_max": 0.69563, + "nauc_recall_at_10_std": 0.386198, + "nauc_recall_at_10_diff1": 0.976953, + "nauc_recall_at_20_max": 0.960424, + "nauc_recall_at_20_std": 0.873306, + "nauc_recall_at_20_diff1": 0.980212, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.722265, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.670382, + "nauc_precision_at_1_std": 0.168339, + "nauc_precision_at_1_diff1": 0.949853, + "nauc_precision_at_3_max": 0.582249, + "nauc_precision_at_3_std": 0.062597, + "nauc_precision_at_3_diff1": 0.949211, + "nauc_precision_at_5_max": 0.581691, + "nauc_precision_at_5_std": 0.394902, + "nauc_precision_at_5_diff1": 0.75396, + "nauc_precision_at_10_max": 0.4814, + "nauc_precision_at_10_std": 0.235404, + "nauc_precision_at_10_diff1": 0.705014, + "nauc_precision_at_20_max": 0.306422, + "nauc_precision_at_20_std": 0.35129, + "nauc_precision_at_20_diff1": 0.26712, + "nauc_precision_at_100_max": -0.533693, + "nauc_precision_at_100_std": -0.411242, + "nauc_precision_at_100_diff1": -0.554789, + "nauc_precision_at_1000_max": -0.546966, + "nauc_precision_at_1000_std": -0.412238, + "nauc_precision_at_1000_diff1": -0.56103, + "nauc_cv_recall_at_1_max": 0.670382, + "nauc_cv_recall_at_1_std": 0.168339, + "nauc_cv_recall_at_1_diff1": 0.949853, + "nauc_cv_recall_at_3_max": 0.582249, + "nauc_cv_recall_at_3_std": 0.062597, + "nauc_cv_recall_at_3_diff1": 0.949211, + "nauc_cv_recall_at_5_max": 0.739556, + "nauc_cv_recall_at_5_std": 0.50789, + "nauc_cv_recall_at_5_diff1": 0.964381, + "nauc_cv_recall_at_10_max": 0.676607, + "nauc_cv_recall_at_10_std": 0.355998, + "nauc_cv_recall_at_10_diff1": 0.983675, + "nauc_cv_recall_at_20_max": 0.934699, + "nauc_cv_recall_at_20_std": 0.934699, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.670382, + "nauc_mrr_at_1_std": 0.168339, + "nauc_mrr_at_1_diff1": 0.949853, + "nauc_mrr_at_3_max": 0.65175, + "nauc_mrr_at_3_std": 0.152024, + "nauc_mrr_at_3_diff1": 0.951367, + "nauc_mrr_at_5_max": 0.669995, + "nauc_mrr_at_5_std": 0.193306, + "nauc_mrr_at_5_diff1": 0.952914, + "nauc_mrr_at_10_max": 0.666982, + "nauc_mrr_at_10_std": 0.184715, + "nauc_mrr_at_10_diff1": 0.953423, + "nauc_mrr_at_20_max": 0.668683, + "nauc_mrr_at_20_std": 0.186286, + "nauc_mrr_at_20_diff1": 0.953093, + "nauc_mrr_at_100_max": 0.668263, + "nauc_mrr_at_100_std": 0.185157, + "nauc_mrr_at_100_diff1": 0.953024, + "nauc_mrr_at_1000_max": 0.668263, + "nauc_mrr_at_1000_std": 0.185157, + "nauc_mrr_at_1000_diff1": 0.953024, + "main_score": 0.94923, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 376.0076344013214, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreShiftProjectRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreShiftProjectRetrieval.json new file mode 100644 index 0000000000..5b54d1a823 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreShiftProjectRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "84a382e05c4473fed9cff2bbae95fe2379416117", + "task_name": "VidoreShiftProjectRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.81, + "ndcg_at_3": 0.88678, + "ndcg_at_5": 0.907, + "ndcg_at_10": 0.907, + "ndcg_at_20": 0.907, + "ndcg_at_100": 0.90904, + "ndcg_at_1000": 0.90904, + "map_at_1": 0.81, + "map_at_3": 0.86833, + "map_at_5": 0.87933, + "map_at_10": 0.87933, + "map_at_20": 0.87933, + "map_at_100": 0.87968, + "map_at_1000": 0.87968, + "recall_at_1": 0.81, + "recall_at_3": 0.94, + "recall_at_5": 0.99, + "recall_at_10": 0.99, + "recall_at_20": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.81, + "cv_recall_at_3": 0.94, + "cv_recall_at_5": 0.99, + "cv_recall_at_10": 0.99, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.81, + "precision_at_3": 0.31333, + "precision_at_5": 0.198, + "precision_at_10": 0.099, + "precision_at_20": 0.0495, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.81, + "mrr_at_3": 0.868333, + "mrr_at_5": 0.879333, + "mrr_at_10": 0.879333, + "mrr_at_20": 0.879333, + "mrr_at_100": 0.879678, + "mrr_at_1000": 0.879678, + "nauc_ndcg_at_1_max": -0.187307, + "nauc_ndcg_at_1_std": -0.72602, + "nauc_ndcg_at_1_diff1": 0.843365, + "nauc_ndcg_at_3_max": -0.018088, + "nauc_ndcg_at_3_std": -0.58764, + "nauc_ndcg_at_3_diff1": 0.873633, + "nauc_ndcg_at_5_max": -0.11816, + "nauc_ndcg_at_5_std": -0.637082, + "nauc_ndcg_at_5_diff1": 0.864011, + "nauc_ndcg_at_10_max": -0.11816, + "nauc_ndcg_at_10_std": -0.637082, + "nauc_ndcg_at_10_diff1": 0.864011, + "nauc_ndcg_at_20_max": -0.11816, + "nauc_ndcg_at_20_std": -0.637082, + "nauc_ndcg_at_20_diff1": 0.864011, + "nauc_ndcg_at_100_max": -0.123829, + "nauc_ndcg_at_100_std": -0.660496, + "nauc_ndcg_at_100_diff1": 0.860812, + "nauc_ndcg_at_1000_max": -0.123829, + "nauc_ndcg_at_1000_std": -0.660496, + "nauc_ndcg_at_1000_diff1": 0.860812, + "nauc_map_at_1_max": -0.187307, + "nauc_map_at_1_std": -0.72602, + "nauc_map_at_1_diff1": 0.843365, + "nauc_map_at_3_max": -0.079192, + "nauc_map_at_3_std": -0.639616, + "nauc_map_at_3_diff1": 0.86391, + "nauc_map_at_5_max": -0.129196, + "nauc_map_at_5_std": -0.667867, + "nauc_map_at_5_diff1": 0.859196, + "nauc_map_at_10_max": -0.129196, + "nauc_map_at_10_std": -0.667867, + "nauc_map_at_10_diff1": 0.859196, + "nauc_map_at_20_max": -0.129196, + "nauc_map_at_20_std": -0.667867, + "nauc_map_at_20_diff1": 0.859196, + "nauc_map_at_100_max": -0.129955, + "nauc_map_at_100_std": -0.670959, + "nauc_map_at_100_diff1": 0.858772, + "nauc_map_at_1000_max": -0.129955, + "nauc_map_at_1000_std": -0.670959, + "nauc_map_at_1000_diff1": 0.858772, + "nauc_recall_at_1_max": -0.187307, + "nauc_recall_at_1_std": -0.72602, + "nauc_recall_at_1_diff1": 0.843365, + "nauc_recall_at_3_max": 0.35442, + "nauc_recall_at_3_std": -0.269141, + "nauc_recall_at_3_diff1": 0.931917, + "nauc_recall_at_5_max": 0.122782, + "nauc_recall_at_5_std": 0.358077, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 0.122782, + "nauc_recall_at_10_std": 0.358077, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": 0.122782, + "nauc_recall_at_20_std": 0.358077, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.187307, + "nauc_precision_at_1_std": -0.72602, + "nauc_precision_at_1_diff1": 0.843365, + "nauc_precision_at_3_max": 0.35442, + "nauc_precision_at_3_std": -0.269141, + "nauc_precision_at_3_diff1": 0.931917, + "nauc_precision_at_5_max": 0.122782, + "nauc_precision_at_5_std": 0.358077, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 0.122782, + "nauc_precision_at_10_std": 0.358077, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 0.122782, + "nauc_precision_at_20_std": 0.358077, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": -0.187307, + "nauc_cv_recall_at_1_std": -0.72602, + "nauc_cv_recall_at_1_diff1": 0.843365, + "nauc_cv_recall_at_3_max": 0.35442, + "nauc_cv_recall_at_3_std": -0.269141, + "nauc_cv_recall_at_3_diff1": 0.931917, + "nauc_cv_recall_at_5_max": 0.122782, + "nauc_cv_recall_at_5_std": 0.358077, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": 0.122782, + "nauc_cv_recall_at_10_std": 0.358077, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": 0.122782, + "nauc_cv_recall_at_20_std": 0.358077, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.187307, + "nauc_mrr_at_1_std": -0.72602, + "nauc_mrr_at_1_diff1": 0.843365, + "nauc_mrr_at_3_max": -0.079192, + "nauc_mrr_at_3_std": -0.639616, + "nauc_mrr_at_3_diff1": 0.86391, + "nauc_mrr_at_5_max": -0.129196, + "nauc_mrr_at_5_std": -0.667867, + "nauc_mrr_at_5_diff1": 0.859196, + "nauc_mrr_at_10_max": -0.129196, + "nauc_mrr_at_10_std": -0.667867, + "nauc_mrr_at_10_diff1": 0.859196, + "nauc_mrr_at_20_max": -0.129196, + "nauc_mrr_at_20_std": -0.667867, + "nauc_mrr_at_20_diff1": 0.859196, + "nauc_mrr_at_100_max": -0.129955, + "nauc_mrr_at_100_std": -0.670959, + "nauc_mrr_at_100_diff1": 0.858772, + "nauc_mrr_at_1000_max": -0.129955, + "nauc_mrr_at_1000_std": -0.670959, + "nauc_mrr_at_1000_diff1": 0.858772, + "main_score": 0.907, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 604.6610598564148, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreSyntheticDocQAAIRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreSyntheticDocQAAIRetrieval.json new file mode 100644 index 0000000000..28e25e0aba --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreSyntheticDocQAAIRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "2d9ebea5a1c6e9ef4a3b902a612f605dca11261c", + "task_name": "VidoreSyntheticDocQAAIRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.99, + "ndcg_at_3": 0.99631, + "ndcg_at_5": 0.99631, + "ndcg_at_10": 0.99631, + "ndcg_at_20": 0.99631, + "ndcg_at_100": 0.99631, + "ndcg_at_1000": 0.99631, + "map_at_1": 0.99, + "map_at_3": 0.995, + "map_at_5": 0.995, + "map_at_10": 0.995, + "map_at_20": 0.995, + "map_at_100": 0.995, + "map_at_1000": 0.995, + "recall_at_1": 0.99, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.99, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.99, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.99, + "mrr_at_3": 0.995, + "mrr_at_5": 0.995, + "mrr_at_10": 0.995, + "mrr_at_20": 0.995, + "mrr_at_100": 0.995, + "mrr_at_1000": 0.995, + "nauc_ndcg_at_1_max": 0.122782, + "nauc_ndcg_at_1_std": 0.122782, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.122782, + "nauc_ndcg_at_3_std": 0.122782, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.122782, + "nauc_ndcg_at_5_std": 0.122782, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.122782, + "nauc_ndcg_at_10_std": 0.122782, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.122782, + "nauc_ndcg_at_20_std": 0.122782, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.122782, + "nauc_ndcg_at_100_std": 0.122782, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.122782, + "nauc_ndcg_at_1000_std": 0.122782, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.122782, + "nauc_map_at_1_std": 0.122782, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.122782, + "nauc_map_at_3_std": 0.122782, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.122782, + "nauc_map_at_5_std": 0.122782, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.122782, + "nauc_map_at_10_std": 0.122782, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.122782, + "nauc_map_at_20_std": 0.122782, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.122782, + "nauc_map_at_100_std": 0.122782, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.122782, + "nauc_map_at_1000_std": 0.122782, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.122782, + "nauc_recall_at_1_std": 0.122782, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.122782, + "nauc_precision_at_1_std": 0.122782, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.122782, + "nauc_cv_recall_at_1_std": 0.122782, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.122782, + "nauc_mrr_at_1_std": 0.122782, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.122782, + "nauc_mrr_at_3_std": 0.122782, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.122782, + "nauc_mrr_at_5_std": 0.122782, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.122782, + "nauc_mrr_at_10_std": 0.122782, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.122782, + "nauc_mrr_at_20_std": 0.122782, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.122782, + "nauc_mrr_at_100_std": 0.122782, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.122782, + "nauc_mrr_at_1000_std": 0.122782, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.99631, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 504.22863841056824, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreSyntheticDocQAEnergyRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreSyntheticDocQAEnergyRetrieval.json new file mode 100644 index 0000000000..03505b7b5a --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreSyntheticDocQAEnergyRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "9935aadbad5c8deec30910489db1b2c7133ae7a7", + "task_name": "VidoreSyntheticDocQAEnergyRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.96, + "ndcg_at_3": 0.96631, + "ndcg_at_5": 0.96631, + "ndcg_at_10": 0.96946, + "ndcg_at_20": 0.97209, + "ndcg_at_100": 0.97406, + "ndcg_at_1000": 0.97406, + "map_at_1": 0.96, + "map_at_3": 0.965, + "map_at_5": 0.965, + "map_at_10": 0.96625, + "map_at_20": 0.96702, + "map_at_100": 0.96732, + "map_at_1000": 0.96732, + "recall_at_1": 0.96, + "recall_at_3": 0.97, + "recall_at_5": 0.97, + "recall_at_10": 0.98, + "recall_at_20": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.96, + "cv_recall_at_3": 0.97, + "cv_recall_at_5": 0.97, + "cv_recall_at_10": 0.98, + "cv_recall_at_20": 0.99, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.96, + "precision_at_3": 0.32333, + "precision_at_5": 0.194, + "precision_at_10": 0.098, + "precision_at_20": 0.0495, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.96, + "mrr_at_3": 0.965, + "mrr_at_5": 0.965, + "mrr_at_10": 0.96625, + "mrr_at_20": 0.967019, + "mrr_at_100": 0.967322, + "mrr_at_1000": 0.967322, + "nauc_ndcg_at_1_max": 0.71697, + "nauc_ndcg_at_1_std": -0.038632, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.715987, + "nauc_ndcg_at_3_std": -0.208658, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.715987, + "nauc_ndcg_at_5_std": -0.208658, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.732705, + "nauc_ndcg_at_10_std": -0.050461, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.719852, + "nauc_ndcg_at_20_std": -0.088907, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.719673, + "nauc_ndcg_at_100_std": -0.122772, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.719673, + "nauc_ndcg_at_1000_std": -0.122772, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.71697, + "nauc_map_at_1_std": -0.038632, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.71622, + "nauc_map_at_3_std": -0.168334, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.71622, + "nauc_map_at_5_std": -0.168334, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.722222, + "nauc_map_at_10_std": -0.110126, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.718792, + "nauc_map_at_20_std": -0.121046, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.71876, + "nauc_map_at_100_std": -0.125489, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.71876, + "nauc_map_at_1000_std": -0.125489, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.71697, + "nauc_recall_at_1_std": -0.038632, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 0.715219, + "nauc_recall_at_3_std": -0.34127, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 0.715219, + "nauc_recall_at_5_std": -0.34127, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 0.795752, + "nauc_recall_at_10_std": 0.358077, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": 0.722222, + "nauc_recall_at_20_std": 0.358077, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.71697, + "nauc_precision_at_1_std": -0.038632, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 0.715219, + "nauc_precision_at_3_std": -0.34127, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 0.715219, + "nauc_precision_at_5_std": -0.34127, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 0.795752, + "nauc_precision_at_10_std": 0.358077, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 0.722222, + "nauc_precision_at_20_std": 0.358077, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.71697, + "nauc_cv_recall_at_1_std": -0.038632, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": 0.715219, + "nauc_cv_recall_at_3_std": -0.34127, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 0.715219, + "nauc_cv_recall_at_5_std": -0.34127, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": 0.795752, + "nauc_cv_recall_at_10_std": 0.358077, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": 0.722222, + "nauc_cv_recall_at_20_std": 0.358077, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.71697, + "nauc_mrr_at_1_std": -0.038632, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.71622, + "nauc_mrr_at_3_std": -0.168334, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.71622, + "nauc_mrr_at_5_std": -0.168334, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.722222, + "nauc_mrr_at_10_std": -0.110126, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.718792, + "nauc_mrr_at_20_std": -0.121046, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.71876, + "nauc_mrr_at_100_std": -0.125489, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.71876, + "nauc_mrr_at_1000_std": -0.125489, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.96631, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 490.5978660583496, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreSyntheticDocQAGovernmentReportsRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreSyntheticDocQAGovernmentReportsRetrieval.json new file mode 100644 index 0000000000..0c3f07bdda --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreSyntheticDocQAGovernmentReportsRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "b4909afa930f81282fd20601e860668073ad02aa", + "task_name": "VidoreSyntheticDocQAGovernmentReportsRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.95, + "ndcg_at_3": 0.97393, + "ndcg_at_5": 0.97823, + "ndcg_at_10": 0.97823, + "ndcg_at_20": 0.97823, + "ndcg_at_100": 0.97823, + "ndcg_at_1000": 0.97823, + "map_at_1": 0.95, + "map_at_3": 0.96833, + "map_at_5": 0.97083, + "map_at_10": 0.97083, + "map_at_20": 0.97083, + "map_at_100": 0.97083, + "map_at_1000": 0.97083, + "recall_at_1": 0.95, + "recall_at_3": 0.99, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.95, + "cv_recall_at_3": 0.99, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.95, + "precision_at_3": 0.33, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.95, + "mrr_at_3": 0.968333, + "mrr_at_5": 0.970833, + "mrr_at_10": 0.970833, + "mrr_at_20": 0.970833, + "mrr_at_100": 0.970833, + "mrr_at_1000": 0.970833, + "nauc_ndcg_at_1_max": 0.676564, + "nauc_ndcg_at_1_std": 0.555649, + "nauc_ndcg_at_1_diff1": 0.973856, + "nauc_ndcg_at_3_max": 0.712254, + "nauc_ndcg_at_3_std": 0.621624, + "nauc_ndcg_at_3_diff1": 0.949863, + "nauc_ndcg_at_5_max": 0.655316, + "nauc_ndcg_at_5_std": 0.57262, + "nauc_ndcg_at_5_diff1": 0.965807, + "nauc_ndcg_at_10_max": 0.655316, + "nauc_ndcg_at_10_std": 0.57262, + "nauc_ndcg_at_10_diff1": 0.965807, + "nauc_ndcg_at_20_max": 0.655316, + "nauc_ndcg_at_20_std": 0.57262, + "nauc_ndcg_at_20_diff1": 0.965807, + "nauc_ndcg_at_100_max": 0.655316, + "nauc_ndcg_at_100_std": 0.57262, + "nauc_ndcg_at_100_diff1": 0.965807, + "nauc_ndcg_at_1000_max": 0.655316, + "nauc_ndcg_at_1000_std": 0.57262, + "nauc_ndcg_at_1000_diff1": 0.965807, + "nauc_map_at_1_max": 0.676564, + "nauc_map_at_1_std": 0.555649, + "nauc_map_at_1_diff1": 0.973856, + "nauc_map_at_3_max": 0.683007, + "nauc_map_at_3_std": 0.594771, + "nauc_map_at_3_diff1": 0.95872, + "nauc_map_at_5_max": 0.655836, + "nauc_map_at_5_std": 0.571242, + "nauc_map_at_5_diff1": 0.966387, + "nauc_map_at_10_max": 0.655836, + "nauc_map_at_10_std": 0.571242, + "nauc_map_at_10_diff1": 0.966387, + "nauc_map_at_20_max": 0.655836, + "nauc_map_at_20_std": 0.571242, + "nauc_map_at_20_diff1": 0.966387, + "nauc_map_at_100_max": 0.655836, + "nauc_map_at_100_std": 0.571242, + "nauc_map_at_100_diff1": 0.966387, + "nauc_map_at_1000_max": 0.655836, + "nauc_map_at_1000_std": 0.571242, + "nauc_map_at_1000_diff1": 0.966387, + "nauc_recall_at_1_max": 0.676564, + "nauc_recall_at_1_std": 0.555649, + "nauc_recall_at_1_diff1": 0.973856, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": 0.869281, + "nauc_recall_at_3_diff1": 0.869281, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.676564, + "nauc_precision_at_1_std": 0.555649, + "nauc_precision_at_1_diff1": 0.973856, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 0.869281, + "nauc_precision_at_3_diff1": 0.869281, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.676564, + "nauc_cv_recall_at_1_std": 0.555649, + "nauc_cv_recall_at_1_diff1": 0.973856, + "nauc_cv_recall_at_3_max": 1.0, + "nauc_cv_recall_at_3_std": 0.869281, + "nauc_cv_recall_at_3_diff1": 0.869281, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.676564, + "nauc_mrr_at_1_std": 0.555649, + "nauc_mrr_at_1_diff1": 0.973856, + "nauc_mrr_at_3_max": 0.683007, + "nauc_mrr_at_3_std": 0.594771, + "nauc_mrr_at_3_diff1": 0.95872, + "nauc_mrr_at_5_max": 0.655836, + "nauc_mrr_at_5_std": 0.571242, + "nauc_mrr_at_5_diff1": 0.966387, + "nauc_mrr_at_10_max": 0.655836, + "nauc_mrr_at_10_std": 0.571242, + "nauc_mrr_at_10_diff1": 0.966387, + "nauc_mrr_at_20_max": 0.655836, + "nauc_mrr_at_20_std": 0.571242, + "nauc_mrr_at_20_diff1": 0.966387, + "nauc_mrr_at_100_max": 0.655836, + "nauc_mrr_at_100_std": 0.571242, + "nauc_mrr_at_100_diff1": 0.966387, + "nauc_mrr_at_1000_max": 0.655836, + "nauc_mrr_at_1000_std": 0.571242, + "nauc_mrr_at_1000_diff1": 0.966387, + "main_score": 0.97823, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 512.2672967910767, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json new file mode 100644 index 0000000000..2c04ae50fc --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreSyntheticDocQAHealthcareIndustryRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "f9e25d5b6e13e1ad9f5c3cce202565031b3ab164", + "task_name": "VidoreSyntheticDocQAHealthcareIndustryRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.98, + "ndcg_at_3": 0.99262, + "ndcg_at_5": 0.99262, + "ndcg_at_10": 0.99262, + "ndcg_at_20": 0.99262, + "ndcg_at_100": 0.99262, + "ndcg_at_1000": 0.99262, + "map_at_1": 0.98, + "map_at_3": 0.99, + "map_at_5": 0.99, + "map_at_10": 0.99, + "map_at_20": 0.99, + "map_at_100": 0.99, + "map_at_1000": 0.99, + "recall_at_1": 0.98, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.98, + "cv_recall_at_3": 1.0, + "cv_recall_at_5": 1.0, + "cv_recall_at_10": 1.0, + "cv_recall_at_20": 1.0, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.98, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.98, + "mrr_at_3": 0.99, + "mrr_at_5": 0.99, + "mrr_at_10": 0.99, + "mrr_at_20": 0.99, + "mrr_at_100": 0.99, + "mrr_at_1000": 0.99, + "nauc_ndcg_at_1_max": 0.638189, + "nauc_ndcg_at_1_std": -0.141223, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.638189, + "nauc_ndcg_at_3_std": -0.141223, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.638189, + "nauc_ndcg_at_5_std": -0.141223, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.638189, + "nauc_ndcg_at_10_std": -0.141223, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.638189, + "nauc_ndcg_at_20_std": -0.141223, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.638189, + "nauc_ndcg_at_100_std": -0.141223, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.638189, + "nauc_ndcg_at_1000_std": -0.141223, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.638189, + "nauc_map_at_1_std": -0.141223, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.638189, + "nauc_map_at_3_std": -0.141223, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.638189, + "nauc_map_at_5_std": -0.141223, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.638189, + "nauc_map_at_10_std": -0.141223, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.638189, + "nauc_map_at_20_std": -0.141223, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.638189, + "nauc_map_at_100_std": -0.141223, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.638189, + "nauc_map_at_1000_std": -0.141223, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.638189, + "nauc_recall_at_1_std": -0.141223, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.638189, + "nauc_precision_at_1_std": -0.141223, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.638189, + "nauc_cv_recall_at_1_std": -0.141223, + "nauc_cv_recall_at_1_diff1": 1.0, + "nauc_cv_recall_at_3_max": NaN, + "nauc_cv_recall_at_3_std": NaN, + "nauc_cv_recall_at_3_diff1": NaN, + "nauc_cv_recall_at_5_max": NaN, + "nauc_cv_recall_at_5_std": NaN, + "nauc_cv_recall_at_5_diff1": NaN, + "nauc_cv_recall_at_10_max": NaN, + "nauc_cv_recall_at_10_std": NaN, + "nauc_cv_recall_at_10_diff1": NaN, + "nauc_cv_recall_at_20_max": NaN, + "nauc_cv_recall_at_20_std": NaN, + "nauc_cv_recall_at_20_diff1": NaN, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.638189, + "nauc_mrr_at_1_std": -0.141223, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.638189, + "nauc_mrr_at_3_std": -0.141223, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.638189, + "nauc_mrr_at_5_std": -0.141223, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.638189, + "nauc_mrr_at_10_std": -0.141223, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.638189, + "nauc_mrr_at_20_std": -0.141223, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.638189, + "nauc_mrr_at_100_std": -0.141223, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.638189, + "nauc_mrr_at_1000_std": -0.141223, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.99262, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 497.6822040081024, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreTabfquadRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreTabfquadRetrieval.json new file mode 100644 index 0000000000..43c0181c86 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreTabfquadRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "61a2224bcd29b7b261a4892ff4c8bea353527a31", + "task_name": "VidoreTabfquadRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.91786, + "ndcg_at_3": 0.95383, + "ndcg_at_5": 0.95935, + "ndcg_at_10": 0.95935, + "ndcg_at_20": 0.96032, + "ndcg_at_100": 0.96109, + "ndcg_at_1000": 0.96109, + "map_at_1": 0.91786, + "map_at_3": 0.94524, + "map_at_5": 0.9481, + "map_at_10": 0.9481, + "map_at_20": 0.94839, + "map_at_100": 0.94854, + "map_at_1000": 0.94854, + "recall_at_1": 0.91786, + "recall_at_3": 0.97857, + "recall_at_5": 0.99286, + "recall_at_10": 0.99286, + "recall_at_20": 0.99643, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.91786, + "cv_recall_at_3": 0.97857, + "cv_recall_at_5": 0.99286, + "cv_recall_at_10": 0.99286, + "cv_recall_at_20": 0.99643, + "cv_recall_at_100": 1.0, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.91786, + "precision_at_3": 0.32619, + "precision_at_5": 0.19857, + "precision_at_10": 0.09929, + "precision_at_20": 0.04982, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.917857, + "mrr_at_3": 0.945238, + "mrr_at_5": 0.948095, + "mrr_at_10": 0.948095, + "mrr_at_20": 0.948393, + "mrr_at_100": 0.948542, + "mrr_at_1000": 0.948542, + "nauc_ndcg_at_1_max": 0.044879, + "nauc_ndcg_at_1_std": 0.154995, + "nauc_ndcg_at_1_diff1": 0.928754, + "nauc_ndcg_at_3_max": 0.286663, + "nauc_ndcg_at_3_std": 0.383697, + "nauc_ndcg_at_3_diff1": 0.949251, + "nauc_ndcg_at_5_max": 0.218717, + "nauc_ndcg_at_5_std": 0.31379, + "nauc_ndcg_at_5_diff1": 0.942351, + "nauc_ndcg_at_10_max": 0.218717, + "nauc_ndcg_at_10_std": 0.31379, + "nauc_ndcg_at_10_diff1": 0.942351, + "nauc_ndcg_at_20_max": 0.202895, + "nauc_ndcg_at_20_std": 0.30028, + "nauc_ndcg_at_20_diff1": 0.940949, + "nauc_ndcg_at_100_max": 0.187141, + "nauc_ndcg_at_100_std": 0.286451, + "nauc_ndcg_at_100_diff1": 0.939782, + "nauc_ndcg_at_1000_max": 0.187141, + "nauc_ndcg_at_1000_std": 0.286451, + "nauc_ndcg_at_1000_diff1": 0.939782, + "nauc_map_at_1_max": 0.044879, + "nauc_map_at_1_std": 0.154995, + "nauc_map_at_1_diff1": 0.928754, + "nauc_map_at_3_max": 0.210571, + "nauc_map_at_3_std": 0.310894, + "nauc_map_at_3_diff1": 0.942303, + "nauc_map_at_5_max": 0.178873, + "nauc_map_at_5_std": 0.278583, + "nauc_map_at_5_diff1": 0.939127, + "nauc_map_at_10_max": 0.178873, + "nauc_map_at_10_std": 0.278583, + "nauc_map_at_10_diff1": 0.939127, + "nauc_map_at_20_max": 0.174892, + "nauc_map_at_20_std": 0.275176, + "nauc_map_at_20_diff1": 0.938776, + "nauc_map_at_100_max": 0.172506, + "nauc_map_at_100_std": 0.27308, + "nauc_map_at_100_diff1": 0.938599, + "nauc_map_at_1000_max": 0.172506, + "nauc_map_at_1000_std": 0.27308, + "nauc_map_at_1000_diff1": 0.938599, + "nauc_recall_at_1_max": 0.044879, + "nauc_recall_at_1_std": 0.154995, + "nauc_recall_at_1_diff1": 0.928754, + "nauc_recall_at_3_max": 0.835823, + "nauc_recall_at_3_std": 0.910131, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 0.934641, + "nauc_recall_at_5_std": 0.934641, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 0.934641, + "nauc_recall_at_10_std": 0.934641, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 1.0, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.044879, + "nauc_precision_at_1_std": 0.154995, + "nauc_precision_at_1_diff1": 0.928754, + "nauc_precision_at_3_max": 0.835823, + "nauc_precision_at_3_std": 0.910131, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 0.934641, + "nauc_precision_at_5_std": 0.934641, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 0.934641, + "nauc_precision_at_10_std": 0.934641, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_cv_recall_at_1_max": 0.044879, + "nauc_cv_recall_at_1_std": 0.154995, + "nauc_cv_recall_at_1_diff1": 0.928754, + "nauc_cv_recall_at_3_max": 0.835823, + "nauc_cv_recall_at_3_std": 0.910131, + "nauc_cv_recall_at_3_diff1": 1.0, + "nauc_cv_recall_at_5_max": 0.934641, + "nauc_cv_recall_at_5_std": 0.934641, + "nauc_cv_recall_at_5_diff1": 1.0, + "nauc_cv_recall_at_10_max": 0.934641, + "nauc_cv_recall_at_10_std": 0.934641, + "nauc_cv_recall_at_10_diff1": 1.0, + "nauc_cv_recall_at_20_max": 1.0, + "nauc_cv_recall_at_20_std": 1.0, + "nauc_cv_recall_at_20_diff1": 1.0, + "nauc_cv_recall_at_100_max": NaN, + "nauc_cv_recall_at_100_std": NaN, + "nauc_cv_recall_at_100_diff1": NaN, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.044879, + "nauc_mrr_at_1_std": 0.154995, + "nauc_mrr_at_1_diff1": 0.928754, + "nauc_mrr_at_3_max": 0.210571, + "nauc_mrr_at_3_std": 0.310894, + "nauc_mrr_at_3_diff1": 0.942303, + "nauc_mrr_at_5_max": 0.178873, + "nauc_mrr_at_5_std": 0.278583, + "nauc_mrr_at_5_diff1": 0.939127, + "nauc_mrr_at_10_max": 0.178873, + "nauc_mrr_at_10_std": 0.278583, + "nauc_mrr_at_10_diff1": 0.939127, + "nauc_mrr_at_20_max": 0.174892, + "nauc_mrr_at_20_std": 0.275176, + "nauc_mrr_at_20_diff1": 0.938776, + "nauc_mrr_at_100_max": 0.172506, + "nauc_mrr_at_100_std": 0.27308, + "nauc_mrr_at_100_diff1": 0.938599, + "nauc_mrr_at_1000_max": 0.172506, + "nauc_mrr_at_1000_std": 0.27308, + "nauc_mrr_at_1000_diff1": 0.938599, + "main_score": 0.95935, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 30.467877626419067, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreTatdqaRetrieval.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreTatdqaRetrieval.json new file mode 100644 index 0000000000..06293bedf4 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/VidoreTatdqaRetrieval.json @@ -0,0 +1,186 @@ +{ + "dataset_revision": "5feb5630fdff4d8d189ffedb2dba56862fdd45c0", + "task_name": "VidoreTatdqaRetrieval", + "mteb_version": "1.38.32", + "scores": { + "test": [ + { + "ndcg_at_1": 0.70292, + "ndcg_at_3": 0.78513, + "ndcg_at_5": 0.80567, + "ndcg_at_10": 0.82135, + "ndcg_at_20": 0.82604, + "ndcg_at_100": 0.83209, + "ndcg_at_1000": 0.83384, + "map_at_1": 0.7014, + "map_at_3": 0.76552, + "map_at_5": 0.77702, + "map_at_10": 0.78366, + "map_at_20": 0.78506, + "map_at_100": 0.7859, + "map_at_1000": 0.786, + "recall_at_1": 0.7014, + "recall_at_3": 0.84028, + "recall_at_5": 0.88979, + "recall_at_10": 0.93779, + "recall_at_20": 0.95547, + "recall_at_100": 0.98785, + "recall_at_1000": 1.0, + "cv_recall_at_1": 0.70778, + "cv_recall_at_3": 0.84386, + "cv_recall_at_5": 0.89064, + "cv_recall_at_10": 0.93864, + "cv_recall_at_20": 0.95626, + "cv_recall_at_100": 0.98846, + "cv_recall_at_1000": 1.0, + "precision_at_1": 0.70292, + "precision_at_3": 0.28109, + "precision_at_5": 0.17861, + "precision_at_10": 0.09417, + "precision_at_20": 0.04803, + "precision_at_100": 0.00995, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.707776, + "mrr_at_3": 0.768935, + "mrr_at_5": 0.779749, + "mrr_at_10": 0.786369, + "mrr_at_20": 0.78772, + "mrr_at_100": 0.788537, + "mrr_at_1000": 0.78862, + "nauc_ndcg_at_1_max": 0.223511, + "nauc_ndcg_at_1_std": 0.158089, + "nauc_ndcg_at_1_diff1": 0.85601, + "nauc_ndcg_at_3_max": 0.2172, + "nauc_ndcg_at_3_std": 0.184925, + "nauc_ndcg_at_3_diff1": 0.830781, + "nauc_ndcg_at_5_max": 0.232295, + "nauc_ndcg_at_5_std": 0.195936, + "nauc_ndcg_at_5_diff1": 0.82592, + "nauc_ndcg_at_10_max": 0.227716, + "nauc_ndcg_at_10_std": 0.195508, + "nauc_ndcg_at_10_diff1": 0.827113, + "nauc_ndcg_at_20_max": 0.228333, + "nauc_ndcg_at_20_std": 0.194501, + "nauc_ndcg_at_20_diff1": 0.828066, + "nauc_ndcg_at_100_max": 0.224053, + "nauc_ndcg_at_100_std": 0.18792, + "nauc_ndcg_at_100_diff1": 0.831982, + "nauc_ndcg_at_1000_max": 0.225181, + "nauc_ndcg_at_1000_std": 0.186111, + "nauc_ndcg_at_1000_diff1": 0.832643, + "nauc_map_at_1_max": 0.221511, + "nauc_map_at_1_std": 0.155479, + "nauc_map_at_1_diff1": 0.856298, + "nauc_map_at_3_max": 0.219121, + "nauc_map_at_3_std": 0.176691, + "nauc_map_at_3_diff1": 0.837262, + "nauc_map_at_5_max": 0.226842, + "nauc_map_at_5_std": 0.182085, + "nauc_map_at_5_diff1": 0.834999, + "nauc_map_at_10_max": 0.224978, + "nauc_map_at_10_std": 0.181417, + "nauc_map_at_10_diff1": 0.835695, + "nauc_map_at_20_max": 0.225079, + "nauc_map_at_20_std": 0.181157, + "nauc_map_at_20_diff1": 0.83604, + "nauc_map_at_100_max": 0.224568, + "nauc_map_at_100_std": 0.180513, + "nauc_map_at_100_diff1": 0.83644, + "nauc_map_at_1000_max": 0.224592, + "nauc_map_at_1000_std": 0.180404, + "nauc_map_at_1000_diff1": 0.836461, + "nauc_recall_at_1_max": 0.221511, + "nauc_recall_at_1_std": 0.155479, + "nauc_recall_at_1_diff1": 0.856298, + "nauc_recall_at_3_max": 0.208678, + "nauc_recall_at_3_std": 0.216787, + "nauc_recall_at_3_diff1": 0.806494, + "nauc_recall_at_5_max": 0.262269, + "nauc_recall_at_5_std": 0.268811, + "nauc_recall_at_5_diff1": 0.778856, + "nauc_recall_at_10_max": 0.248641, + "nauc_recall_at_10_std": 0.319832, + "nauc_recall_at_10_diff1": 0.752434, + "nauc_recall_at_20_max": 0.268114, + "nauc_recall_at_20_std": 0.355225, + "nauc_recall_at_20_diff1": 0.734036, + "nauc_recall_at_100_max": 0.10636, + "nauc_recall_at_100_std": 0.321928, + "nauc_recall_at_100_diff1": 0.762702, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.223511, + "nauc_precision_at_1_std": 0.158089, + "nauc_precision_at_1_diff1": 0.85601, + "nauc_precision_at_3_max": 0.203712, + "nauc_precision_at_3_std": 0.214682, + "nauc_precision_at_3_diff1": 0.781139, + "nauc_precision_at_5_max": 0.252556, + "nauc_precision_at_5_std": 0.263124, + "nauc_precision_at_5_diff1": 0.743605, + "nauc_precision_at_10_max": 0.228076, + "nauc_precision_at_10_std": 0.301414, + "nauc_precision_at_10_diff1": 0.659825, + "nauc_precision_at_20_max": 0.206433, + "nauc_precision_at_20_std": 0.291261, + "nauc_precision_at_20_diff1": 0.563594, + "nauc_precision_at_100_max": -0.100967, + "nauc_precision_at_100_std": 0.03191, + "nauc_precision_at_100_diff1": 0.139715, + "nauc_precision_at_1000_max": -0.371329, + "nauc_precision_at_1000_std": -0.345792, + "nauc_precision_at_1000_diff1": -0.504959, + "nauc_cv_recall_at_1_max": 0.222014, + "nauc_cv_recall_at_1_std": 0.15416, + "nauc_cv_recall_at_1_diff1": 0.848692, + "nauc_cv_recall_at_3_max": 0.214932, + "nauc_cv_recall_at_3_std": 0.230289, + "nauc_cv_recall_at_3_diff1": 0.800071, + "nauc_cv_recall_at_5_max": 0.256256, + "nauc_cv_recall_at_5_std": 0.262842, + "nauc_cv_recall_at_5_diff1": 0.777752, + "nauc_cv_recall_at_10_max": 0.238874, + "nauc_cv_recall_at_10_std": 0.311051, + "nauc_cv_recall_at_10_diff1": 0.74965, + "nauc_cv_recall_at_20_max": 0.2549, + "nauc_cv_recall_at_20_std": 0.343584, + "nauc_cv_recall_at_20_diff1": 0.731163, + "nauc_cv_recall_at_100_max": 0.059326, + "nauc_cv_recall_at_100_std": 0.28624, + "nauc_cv_recall_at_100_diff1": 0.750213, + "nauc_cv_recall_at_1000_max": NaN, + "nauc_cv_recall_at_1000_std": NaN, + "nauc_cv_recall_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.222014, + "nauc_mrr_at_1_std": 0.15416, + "nauc_mrr_at_1_diff1": 0.848692, + "nauc_mrr_at_3_max": 0.220867, + "nauc_mrr_at_3_std": 0.177899, + "nauc_mrr_at_3_diff1": 0.831192, + "nauc_mrr_at_5_max": 0.226905, + "nauc_mrr_at_5_std": 0.180315, + "nauc_mrr_at_5_diff1": 0.82961, + "nauc_mrr_at_10_max": 0.224762, + "nauc_mrr_at_10_std": 0.179346, + "nauc_mrr_at_10_diff1": 0.830137, + "nauc_mrr_at_20_max": 0.224966, + "nauc_mrr_at_20_std": 0.179172, + "nauc_mrr_at_20_diff1": 0.830495, + "nauc_mrr_at_100_max": 0.22456, + "nauc_mrr_at_100_std": 0.178629, + "nauc_mrr_at_100_diff1": 0.830882, + "nauc_mrr_at_1000_max": 0.22465, + "nauc_mrr_at_1000_std": 0.178588, + "nauc_mrr_at_1000_diff1": 0.830915, + "main_score": 0.80567, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 271.68200302124023, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/model_meta.json b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/model_meta.json new file mode 100644 index 0000000000..92eb9ca931 --- /dev/null +++ b/results/nvidia__llama-nemoretriever-colembed-3b-v1/50c36f4d5271c6851aa08bd26d69f6e7ca8b870c/model_meta.json @@ -0,0 +1 @@ +{"name": "nvidia/llama-nemoretriever-colembed-3b-v1", "revision": "50c36f4d5271c6851aa08bd26d69f6e7ca8b870c", "release_date": "2025-06-27", "languages": ["eng-Latn"], "n_parameters": 4407000000, "memory_usage_mb": 16811.0, "max_tokens": 8192.0, "embed_dim": 3072, "license": "https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1/blob/main/LICENSE", "open_weights": true, "public_training_code": "Proprietary Code", "public_training_data": "https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1#training-dataset", "framework": ["PyTorch"], "reference": "https://huggingface.co/nvidia/llama-nemoretriever-colembed-3b-v1", "similarity_fn_name": "max_sim", "use_instructions": true, "training_datasets": {"DocVQA": ["train"], "InfoVQA": ["train"], "TATDQA": ["train"], "arXivQA": ["train"], "hotpotqa": ["train"], "miracl": ["train"], "NQ": ["train"], "stackexchange": ["train"], "SQuAD": ["train"], "WebInstructSub": ["train"], "docmatix-ir": ["train"], "vdr-multilingual-train": ["train"], "colpali_train_set": ["train"], "VisRAG-Ret-Train-Synthetic-data": ["train"], "VisRAG-Ret-Train-In-domain-data": ["train"], "wiki-ss-nq": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["image", "text"], "loader": "llama_nemoretriever_colembed"} \ No newline at end of file