diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AmazonCounterfactualClassification.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AmazonCounterfactualClassification.json new file mode 100644 index 0000000000..12e51eecd0 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AmazonCounterfactualClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e8379541af4e31359cca9fbcf4b00f2671dba205", + "task_name": "AmazonCounterfactualClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.654478, + "f1": 0.588995, + "f1_weighted": 0.688119, + "ap": 0.275536, + "ap_weighted": 0.275536, + "scores_per_experiment": [ + { + "accuracy": 0.616418, + "f1": 0.549027, + "f1_weighted": 0.655187, + "ap": 0.242238, + "ap_weighted": 0.242238 + }, + { + "accuracy": 0.652239, + "f1": 0.586585, + "f1_weighted": 0.68691, + "ap": 0.271424, + "ap_weighted": 0.271424 + }, + { + "accuracy": 0.613433, + "f1": 0.56813, + "f1_weighted": 0.653307, + "ap": 0.273803, + "ap_weighted": 0.273803 + }, + { + "accuracy": 0.662687, + "f1": 0.5781, + "f1_weighted": 0.693138, + "ap": 0.253635, + "ap_weighted": 0.253635 + }, + { + "accuracy": 0.68209, + "f1": 0.605637, + "f1_weighted": 0.711375, + "ap": 0.279889, + "ap_weighted": 0.279889 + }, + { + "accuracy": 0.676119, + "f1": 0.592849, + "f1_weighted": 0.704976, + "ap": 0.265527, + "ap_weighted": 0.265527 + }, + { + "accuracy": 0.710448, + "f1": 0.632862, + "f1_weighted": 0.735638, + "ap": 0.303321, + "ap_weighted": 0.303321 + }, + { + "accuracy": 0.670149, + "f1": 0.608979, + "f1_weighted": 0.703158, + "ap": 0.294014, + "ap_weighted": 0.294014 + }, + { + "accuracy": 0.620896, + "f1": 0.576009, + "f1_weighted": 0.660017, + "ap": 0.280835, + "ap_weighted": 0.280835 + }, + { + "accuracy": 0.640299, + "f1": 0.591772, + "f1_weighted": 0.677481, + "ap": 0.290673, + "ap_weighted": 0.290673 + } + ], + "main_score": 0.654478, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.934959650039673, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AmazonPolarityClassification.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AmazonPolarityClassification.json new file mode 100644 index 0000000000..ab12cad2ea --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AmazonPolarityClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e2d317d38cd51312af73b3d32a06d1a08b442046", + "task_name": "AmazonPolarityClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.669297, + "f1": 0.666978, + "f1_weighted": 0.666978, + "ap": 0.617461, + "ap_weighted": 0.617461, + "scores_per_experiment": [ + { + "accuracy": 0.67263, + "f1": 0.672483, + "f1_weighted": 0.672483, + "ap": 0.614904, + "ap_weighted": 0.614904 + }, + { + "accuracy": 0.666872, + "f1": 0.66615, + "f1_weighted": 0.66615, + "ap": 0.608912, + "ap_weighted": 0.608912 + }, + { + "accuracy": 0.679342, + "f1": 0.678503, + "f1_weighted": 0.678503, + "ap": 0.618853, + "ap_weighted": 0.618853 + }, + { + "accuracy": 0.677578, + "f1": 0.671302, + "f1_weighted": 0.671302, + "ap": 0.632364, + "ap_weighted": 0.632364 + }, + { + "accuracy": 0.702635, + "f1": 0.70239, + "f1_weighted": 0.70239, + "ap": 0.644877, + "ap_weighted": 0.644877 + }, + { + "accuracy": 0.709087, + "f1": 0.708001, + "f1_weighted": 0.708001, + "ap": 0.654337, + "ap_weighted": 0.654337 + }, + { + "accuracy": 0.659377, + "f1": 0.657118, + "f1_weighted": 0.657118, + "ap": 0.601542, + "ap_weighted": 0.601542 + }, + { + "accuracy": 0.70882, + "f1": 0.704695, + "f1_weighted": 0.704695, + "ap": 0.661513, + "ap_weighted": 0.661513 + }, + { + "accuracy": 0.617692, + "f1": 0.615535, + "f1_weighted": 0.615535, + "ap": 0.575139, + "ap_weighted": 0.575139 + }, + { + "accuracy": 0.598932, + "f1": 0.593598, + "f1_weighted": 0.593598, + "ap": 0.562163, + "ap_weighted": 0.562163 + } + ], + "main_score": 0.669297, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 447.5662610530853, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AmazonReviewsClassification.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AmazonReviewsClassification.json new file mode 100644 index 0000000000..6975fdfd9c --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AmazonReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "1399c76144fd37290681b995c656ef9b2e06e26d", + "task_name": "AmazonReviewsClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.33274, + "f1": 0.330787, + "f1_weighted": 0.330787, + "scores_per_experiment": [ + { + "accuracy": 0.335, + "f1": 0.334234, + "f1_weighted": 0.334234 + }, + { + "accuracy": 0.3362, + "f1": 0.338144, + "f1_weighted": 0.338144 + }, + { + "accuracy": 0.327, + "f1": 0.322692, + "f1_weighted": 0.322692 + }, + { + "accuracy": 0.3332, + "f1": 0.335472, + "f1_weighted": 0.335472 + }, + { + "accuracy": 0.3308, + "f1": 0.329265, + "f1_weighted": 0.329265 + }, + { + "accuracy": 0.3418, + "f1": 0.337586, + "f1_weighted": 0.337586 + }, + { + "accuracy": 0.3196, + "f1": 0.318114, + "f1_weighted": 0.318114 + }, + { + "accuracy": 0.346, + "f1": 0.346881, + "f1_weighted": 0.346881 + }, + { + "accuracy": 0.3448, + "f1": 0.334427, + "f1_weighted": 0.334427 + }, + { + "accuracy": 0.313, + "f1": 0.311053, + "f1_weighted": 0.311053 + } + ], + "main_score": 0.33274, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 12.235694885253906, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AppsRetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AppsRetrieval.json new file mode 100644 index 0000000000..bc35272de3 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AppsRetrieval.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "f22508f96b7a36c2415181ed8bb76f76e04ae2d5", + "task_name": "AppsRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.0842, + "ndcg_at_3": 0.11569, + "ndcg_at_5": 0.12596, + "ndcg_at_10": 0.13962, + "ndcg_at_20": 0.15333, + "ndcg_at_100": 0.18218, + "ndcg_at_1000": 0.2231, + "map_at_1": 0.0842, + "map_at_3": 0.1077, + "map_at_5": 0.11339, + "map_at_10": 0.11898, + "map_at_20": 0.12271, + "map_at_100": 0.12646, + "map_at_1000": 0.1277, + "recall_at_1": 0.0842, + "recall_at_3": 0.13891, + "recall_at_5": 0.16388, + "recall_at_10": 0.20637, + "recall_at_20": 0.26082, + "recall_at_100": 0.41992, + "recall_at_1000": 0.75936, + "precision_at_1": 0.0842, + "precision_at_3": 0.0463, + "precision_at_5": 0.03278, + "precision_at_10": 0.02064, + "precision_at_20": 0.01304, + "precision_at_100": 0.0042, + "precision_at_1000": 0.00076, + "mrr_at_1": 0.083931, + "mrr_at_3": 0.10757, + "mrr_at_5": 0.113254, + "mrr_at_10": 0.118848, + "mrr_at_20": 0.122577, + "mrr_at_100": 0.126328, + "mrr_at_1000": 0.127566, + "nauc_ndcg_at_1_max": 0.303736, + "nauc_ndcg_at_1_std": 0.07742, + "nauc_ndcg_at_1_diff1": 0.397707, + "nauc_ndcg_at_3_max": 0.259152, + "nauc_ndcg_at_3_std": 0.08841, + "nauc_ndcg_at_3_diff1": 0.30148, + "nauc_ndcg_at_5_max": 0.241342, + "nauc_ndcg_at_5_std": 0.09192, + "nauc_ndcg_at_5_diff1": 0.274799, + "nauc_ndcg_at_10_max": 0.227436, + "nauc_ndcg_at_10_std": 0.099086, + "nauc_ndcg_at_10_diff1": 0.251715, + "nauc_ndcg_at_20_max": 0.220382, + "nauc_ndcg_at_20_std": 0.104099, + "nauc_ndcg_at_20_diff1": 0.232518, + "nauc_ndcg_at_100_max": 0.213555, + "nauc_ndcg_at_100_std": 0.116851, + "nauc_ndcg_at_100_diff1": 0.223541, + "nauc_ndcg_at_1000_max": 0.215886, + "nauc_ndcg_at_1000_std": 0.115257, + "nauc_ndcg_at_1000_diff1": 0.230682, + "nauc_map_at_1_max": 0.303736, + "nauc_map_at_1_std": 0.07742, + "nauc_map_at_1_diff1": 0.397707, + "nauc_map_at_3_max": 0.268762, + "nauc_map_at_3_std": 0.085664, + "nauc_map_at_3_diff1": 0.321144, + "nauc_map_at_5_max": 0.257871, + "nauc_map_at_5_std": 0.087847, + "nauc_map_at_5_diff1": 0.304281, + "nauc_map_at_10_max": 0.251025, + "nauc_map_at_10_std": 0.091278, + "nauc_map_at_10_diff1": 0.292635, + "nauc_map_at_20_max": 0.248711, + "nauc_map_at_20_std": 0.09291, + "nauc_map_at_20_diff1": 0.286142, + "nauc_map_at_100_max": 0.247098, + "nauc_map_at_100_std": 0.094891, + "nauc_map_at_100_diff1": 0.284433, + "nauc_map_at_1000_max": 0.247093, + "nauc_map_at_1000_std": 0.0948, + "nauc_map_at_1000_diff1": 0.284737, + "nauc_recall_at_1_max": 0.303736, + "nauc_recall_at_1_std": 0.07742, + "nauc_recall_at_1_diff1": 0.397707, + "nauc_recall_at_3_max": 0.236309, + "nauc_recall_at_3_std": 0.095056, + "nauc_recall_at_3_diff1": 0.25507, + "nauc_recall_at_5_max": 0.203417, + "nauc_recall_at_5_std": 0.101265, + "nauc_recall_at_5_diff1": 0.208332, + "nauc_recall_at_10_max": 0.177232, + "nauc_recall_at_10_std": 0.116276, + "nauc_recall_at_10_diff1": 0.166061, + "nauc_recall_at_20_max": 0.161923, + "nauc_recall_at_20_std": 0.128451, + "nauc_recall_at_20_diff1": 0.121676, + "nauc_recall_at_100_max": 0.147021, + "nauc_recall_at_100_std": 0.171363, + "nauc_recall_at_100_diff1": 0.101242, + "nauc_recall_at_1000_max": 0.129237, + "nauc_recall_at_1000_std": 0.193642, + "nauc_recall_at_1000_diff1": 0.081587, + "nauc_precision_at_1_max": 0.303736, + "nauc_precision_at_1_std": 0.07742, + "nauc_precision_at_1_diff1": 0.397707, + "nauc_precision_at_3_max": 0.236309, + "nauc_precision_at_3_std": 0.095056, + "nauc_precision_at_3_diff1": 0.25507, + "nauc_precision_at_5_max": 0.203417, + "nauc_precision_at_5_std": 0.101265, + "nauc_precision_at_5_diff1": 0.208332, + "nauc_precision_at_10_max": 0.177232, + "nauc_precision_at_10_std": 0.116276, + "nauc_precision_at_10_diff1": 0.166061, + "nauc_precision_at_20_max": 0.161923, + "nauc_precision_at_20_std": 0.128451, + "nauc_precision_at_20_diff1": 0.121676, + "nauc_precision_at_100_max": 0.147021, + "nauc_precision_at_100_std": 0.171363, + "nauc_precision_at_100_diff1": 0.101242, + "nauc_precision_at_1000_max": 0.129237, + "nauc_precision_at_1000_std": 0.193642, + "nauc_precision_at_1000_diff1": 0.081587, + "nauc_mrr_at_1_max": 0.305349, + "nauc_mrr_at_1_std": 0.077353, + "nauc_mrr_at_1_diff1": 0.400785, + "nauc_mrr_at_3_max": 0.269378, + "nauc_mrr_at_3_std": 0.085647, + "nauc_mrr_at_3_diff1": 0.322307, + "nauc_mrr_at_5_max": 0.258452, + "nauc_mrr_at_5_std": 0.087834, + "nauc_mrr_at_5_diff1": 0.305382, + "nauc_mrr_at_10_max": 0.251579, + "nauc_mrr_at_10_std": 0.09127, + "nauc_mrr_at_10_diff1": 0.293686, + "nauc_mrr_at_20_max": 0.249255, + "nauc_mrr_at_20_std": 0.092903, + "nauc_mrr_at_20_diff1": 0.287174, + "nauc_mrr_at_100_max": 0.247638, + "nauc_mrr_at_100_std": 0.094887, + "nauc_mrr_at_100_diff1": 0.285457, + "nauc_mrr_at_1000_max": 0.247634, + "nauc_mrr_at_1000_std": 0.094796, + "nauc_mrr_at_1000_diff1": 0.285763, + "main_score": 0.13962, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 30.23980140686035, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArXivHierarchicalClusteringP2P.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArXivHierarchicalClusteringP2P.json new file mode 100644 index 0000000000..b66e7a7833 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArXivHierarchicalClusteringP2P.json @@ -0,0 +1,46 @@ +{ + "dataset_revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8", + "task_name": "ArXivHierarchicalClusteringP2P", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.570793, + 0.600072, + 0.593033, + 0.599704, + 0.570847, + 0.633009, + 0.560708, + 0.550649, + 0.595455, + 0.554381 + ], + "Level 1": [ + 0.588474, + 0.589332, + 0.600536, + 0.61511, + 0.595803, + 0.642311, + 0.57045, + 0.596294, + 0.594475, + 0.589886 + ] + }, + "v_measure": 0.590566, + "v_measure_std": 0.022853, + "main_score": 0.590566, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 56.0141236782074, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArXivHierarchicalClusteringS2S.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArXivHierarchicalClusteringS2S.json new file mode 100644 index 0000000000..b1b79102af --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArXivHierarchicalClusteringS2S.json @@ -0,0 +1,46 @@ +{ + "dataset_revision": "b73bd54100e5abfa6e3a23dcafb46fe4d2438dc3", + "task_name": "ArXivHierarchicalClusteringS2S", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.546459, + 0.595857, + 0.575522, + 0.540129, + 0.528247, + 0.596374, + 0.562756, + 0.573391, + 0.571596, + 0.565737 + ], + "Level 1": [ + 0.580634, + 0.588903, + 0.578917, + 0.586656, + 0.559496, + 0.599475, + 0.564938, + 0.577674, + 0.585657, + 0.594297 + ] + }, + "v_measure": 0.573636, + "v_measure_std": 0.018883, + "main_score": 0.573636, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 53.68436670303345, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArguAna.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArguAna.json new file mode 100644 index 0000000000..3dcc6864da --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArguAna.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a", + "task_name": "ArguAna", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.34282, + "ndcg_at_3": 0.49123, + "ndcg_at_5": 0.54816, + "ndcg_at_10": 0.59215, + "ndcg_at_20": 0.61219, + "ndcg_at_100": 0.62001, + "ndcg_at_1000": 0.62075, + "map_at_1": 0.34282, + "map_at_3": 0.45413, + "map_at_5": 0.4856, + "map_at_10": 0.50393, + "map_at_20": 0.50956, + "map_at_100": 0.51081, + "map_at_1000": 0.51084, + "recall_at_1": 0.34282, + "recall_at_3": 0.59886, + "recall_at_5": 0.73755, + "recall_at_10": 0.87269, + "recall_at_20": 0.95092, + "recall_at_100": 0.99075, + "recall_at_1000": 0.99644, + "precision_at_1": 0.34282, + "precision_at_3": 0.19962, + "precision_at_5": 0.14751, + "precision_at_10": 0.08727, + "precision_at_20": 0.04755, + "precision_at_100": 0.00991, + "precision_at_1000": 0.001, + "mrr_at_1": 0.349218, + "mrr_at_3": 0.456615, + "mrr_at_5": 0.487909, + "mrr_at_10": 0.506287, + "mrr_at_20": 0.511948, + "mrr_at_100": 0.513231, + "mrr_at_1000": 0.513262, + "nauc_ndcg_at_1_max": -0.105852, + "nauc_ndcg_at_1_std": -0.140017, + "nauc_ndcg_at_1_diff1": 0.20128, + "nauc_ndcg_at_3_max": -0.099355, + "nauc_ndcg_at_3_std": -0.161471, + "nauc_ndcg_at_3_diff1": 0.156493, + "nauc_ndcg_at_5_max": -0.062433, + "nauc_ndcg_at_5_std": -0.123145, + "nauc_ndcg_at_5_diff1": 0.169448, + "nauc_ndcg_at_10_max": -0.063044, + "nauc_ndcg_at_10_std": -0.125284, + "nauc_ndcg_at_10_diff1": 0.170141, + "nauc_ndcg_at_20_max": -0.065791, + "nauc_ndcg_at_20_std": -0.13132, + "nauc_ndcg_at_20_diff1": 0.169599, + "nauc_ndcg_at_100_max": -0.074736, + "nauc_ndcg_at_100_std": -0.131603, + "nauc_ndcg_at_100_diff1": 0.171931, + "nauc_ndcg_at_1000_max": -0.077387, + "nauc_ndcg_at_1000_std": -0.134491, + "nauc_ndcg_at_1000_diff1": 0.171667, + "nauc_map_at_1_max": -0.105852, + "nauc_map_at_1_std": -0.140017, + "nauc_map_at_1_diff1": 0.20128, + "nauc_map_at_3_max": -0.10158, + "nauc_map_at_3_std": -0.155832, + "nauc_map_at_3_diff1": 0.166708, + "nauc_map_at_5_max": -0.083037, + "nauc_map_at_5_std": -0.136208, + "nauc_map_at_5_diff1": 0.173937, + "nauc_map_at_10_max": -0.084304, + "nauc_map_at_10_std": -0.13785, + "nauc_map_at_10_diff1": 0.174187, + "nauc_map_at_20_max": -0.085306, + "nauc_map_at_20_std": -0.139593, + "nauc_map_at_20_diff1": 0.173908, + "nauc_map_at_100_max": -0.086414, + "nauc_map_at_100_std": -0.139869, + "nauc_map_at_100_diff1": 0.174303, + "nauc_map_at_1000_max": -0.086502, + "nauc_map_at_1000_std": -0.139965, + "nauc_map_at_1000_diff1": 0.17429, + "nauc_recall_at_1_max": -0.105852, + "nauc_recall_at_1_std": -0.140017, + "nauc_recall_at_1_diff1": 0.20128, + "nauc_recall_at_3_max": -0.092163, + "nauc_recall_at_3_std": -0.179148, + "nauc_recall_at_3_diff1": 0.12513, + "nauc_recall_at_5_max": 0.028764, + "nauc_recall_at_5_std": -0.062121, + "nauc_recall_at_5_diff1": 0.154966, + "nauc_recall_at_10_max": 0.106021, + "nauc_recall_at_10_std": -0.019045, + "nauc_recall_at_10_diff1": 0.149865, + "nauc_recall_at_20_max": 0.30895, + "nauc_recall_at_20_std": 0.030609, + "nauc_recall_at_20_diff1": 0.118304, + "nauc_recall_at_100_max": 0.710083, + "nauc_recall_at_100_std": 0.776286, + "nauc_recall_at_100_diff1": 0.163677, + "nauc_recall_at_1000_max": 0.614479, + "nauc_recall_at_1000_std": 0.760507, + "nauc_recall_at_1000_diff1": 0.016374, + "nauc_precision_at_1_max": -0.105852, + "nauc_precision_at_1_std": -0.140017, + "nauc_precision_at_1_diff1": 0.20128, + "nauc_precision_at_3_max": -0.092163, + "nauc_precision_at_3_std": -0.179148, + "nauc_precision_at_3_diff1": 0.12513, + "nauc_precision_at_5_max": 0.028764, + "nauc_precision_at_5_std": -0.062121, + "nauc_precision_at_5_diff1": 0.154966, + "nauc_precision_at_10_max": 0.106021, + "nauc_precision_at_10_std": -0.019045, + "nauc_precision_at_10_diff1": 0.149865, + "nauc_precision_at_20_max": 0.30895, + "nauc_precision_at_20_std": 0.030609, + "nauc_precision_at_20_diff1": 0.118304, + "nauc_precision_at_100_max": 0.710083, + "nauc_precision_at_100_std": 0.776286, + "nauc_precision_at_100_diff1": 0.163677, + "nauc_precision_at_1000_max": 0.614479, + "nauc_precision_at_1000_std": 0.760507, + "nauc_precision_at_1000_diff1": 0.016374, + "nauc_mrr_at_1_max": -0.096125, + "nauc_mrr_at_1_std": -0.132844, + "nauc_mrr_at_1_diff1": 0.182574, + "nauc_mrr_at_3_max": -0.104059, + "nauc_mrr_at_3_std": -0.15193, + "nauc_mrr_at_3_diff1": 0.149096, + "nauc_mrr_at_5_max": -0.086561, + "nauc_mrr_at_5_std": -0.134119, + "nauc_mrr_at_5_diff1": 0.154912, + "nauc_mrr_at_10_max": -0.087288, + "nauc_mrr_at_10_std": -0.135906, + "nauc_mrr_at_10_diff1": 0.155103, + "nauc_mrr_at_20_max": -0.087801, + "nauc_mrr_at_20_std": -0.137198, + "nauc_mrr_at_20_diff1": 0.154931, + "nauc_mrr_at_100_max": -0.088805, + "nauc_mrr_at_100_std": -0.137578, + "nauc_mrr_at_100_diff1": 0.155446, + "nauc_mrr_at_1000_max": -0.088892, + "nauc_mrr_at_1000_std": -0.137673, + "nauc_mrr_at_1000_diff1": 0.155432, + "main_score": 0.59215, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 19.515056371688843, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArxivClusteringP2P.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArxivClusteringP2P.json new file mode 100644 index 0000000000..52b4aa6d46 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArxivClusteringP2P.json @@ -0,0 +1,53 @@ +{ + "dataset_revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d", + "task_name": "ArxivClusteringP2P", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.486385, + "v_measure_std": 0.138894, + "v_measures": [ + 0.460394, + 0.485687, + 0.479691, + 0.488717, + 0.498486, + 0.503502, + 0.492285, + 0.486628, + 0.495843, + 0.497073, + 0.566238, + 0.568601, + 0.561994, + 0.569031, + 0.567627, + 0.566343, + 0.570014, + 0.567183, + 0.565689, + 0.565405, + 0.531349, + 0.298333, + 0.485144, + 0.423779, + 0.361888, + 0.28776, + 0.30204, + 0.226195, + 0.338164, + 1.0, + 0.266861 + ], + "main_score": 0.486385, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1528.5315272808075, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArxivClusteringS2S.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArxivClusteringS2S.json new file mode 100644 index 0000000000..6009798f8b --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ArxivClusteringS2S.json @@ -0,0 +1,53 @@ +{ + "dataset_revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53", + "task_name": "ArxivClusteringS2S", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.445902, + "v_measure_std": 0.141809, + "v_measures": [ + 0.456766, + 0.471204, + 0.449493, + 0.455413, + 0.46227, + 0.45745, + 0.453721, + 0.462978, + 0.424835, + 0.447359, + 0.521561, + 0.526169, + 0.520873, + 0.516771, + 0.519548, + 0.516093, + 0.521398, + 0.524308, + 0.513428, + 0.515529, + 0.481943, + 0.244362, + 0.397501, + 0.394395, + 0.316156, + 0.244215, + 0.268882, + 0.207658, + 0.298327, + 1.0, + 0.232358 + ], + "main_score": 0.445902, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 462.79064679145813, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AskUbuntuDupQuestions.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AskUbuntuDupQuestions.json new file mode 100644 index 0000000000..67039501a7 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/AskUbuntuDupQuestions.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2000358ca161889fa9c082cb41daa8dcfb161a54", + "task_name": "AskUbuntuDupQuestions", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "map": 0.664834, + "mrr": 0.786613, + "nAUC_map_max": 0.276478, + "nAUC_map_std": 0.152382, + "nAUC_map_diff1": 0.208995, + "nAUC_mrr_max": 0.404306, + "nAUC_mrr_std": 0.236923, + "nAUC_mrr_diff1": 0.330146, + "main_score": 0.664834, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.1834449768066406, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/BIOSSES.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/BIOSSES.json new file mode 100644 index 0000000000..12084ed3a4 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/BIOSSES.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a", + "task_name": "BIOSSES", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.886673, + "spearman": 0.865923, + "cosine_pearson": 0.886673, + "cosine_spearman": 0.865923, + "manhattan_pearson": 0.867529, + "manhattan_spearman": 0.863484, + "euclidean_pearson": 0.869125, + "euclidean_spearman": 0.863887, + "main_score": 0.865923, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.08469223976135254, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/Banking77Classification.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/Banking77Classification.json new file mode 100644 index 0000000000..8b96b96d83 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/Banking77Classification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "0fd18e25b25c072e09e0d92ab615fda904d66300", + "task_name": "Banking77Classification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.855552, + "f1": 0.855063, + "f1_weighted": 0.855063, + "scores_per_experiment": [ + { + "accuracy": 0.861039, + "f1": 0.860343, + "f1_weighted": 0.860343 + }, + { + "accuracy": 0.861364, + "f1": 0.861485, + "f1_weighted": 0.861485 + }, + { + "accuracy": 0.863961, + "f1": 0.863273, + "f1_weighted": 0.863273 + }, + { + "accuracy": 0.860714, + "f1": 0.860791, + "f1_weighted": 0.860791 + }, + { + "accuracy": 0.851948, + "f1": 0.852175, + "f1_weighted": 0.852175 + }, + { + "accuracy": 0.845779, + "f1": 0.844255, + "f1_weighted": 0.844255 + }, + { + "accuracy": 0.846104, + "f1": 0.845438, + "f1_weighted": 0.845438 + }, + { + "accuracy": 0.85, + "f1": 0.848609, + "f1_weighted": 0.848609 + }, + { + "accuracy": 0.858117, + "f1": 0.857562, + "f1_weighted": 0.857562 + }, + { + "accuracy": 0.856494, + "f1": 0.856704, + "f1_weighted": 0.856704 + } + ], + "main_score": 0.855552, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 16.900850296020508, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/BiorxivClusteringP2P.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/BiorxivClusteringP2P.json new file mode 100644 index 0000000000..b6c5eda483 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/BiorxivClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40", + "task_name": "BiorxivClusteringP2P", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.388883, + "v_measure_std": 0.014242, + "v_measures": [ + 0.404519, + 0.397867, + 0.372887, + 0.376614, + 0.388748, + 0.404754, + 0.362031, + 0.382264, + 0.394505, + 0.404642 + ], + "main_score": 0.388883, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 225.62700986862183, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/BiorxivClusteringP2P.v2.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/BiorxivClusteringP2P.v2.json new file mode 100644 index 0000000000..bdfd6324bb --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/BiorxivClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "f5dbc242e11dd8e24def4c4268607a49e02946dc", + "task_name": "BiorxivClusteringP2P.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.408805, + 0.420227, + 0.409011, + 0.42464, + 0.414895, + 0.418051, + 0.427979, + 0.42261, + 0.422912, + 0.414995 + ] + }, + "v_measure": 0.418412, + "v_measure_std": 0.006135, + "main_score": 0.418412, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 43.978520154953, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/BiorxivClusteringS2S.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/BiorxivClusteringS2S.json new file mode 100644 index 0000000000..50cbc6354b --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/BiorxivClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908", + "task_name": "BiorxivClusteringS2S", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.371811, + "v_measure_std": 0.009831, + "v_measures": [ + 0.382145, + 0.382041, + 0.353861, + 0.371585, + 0.373118, + 0.375321, + 0.359633, + 0.367661, + 0.366087, + 0.386661 + ], + "main_score": 0.371811, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 55.91719889640808, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/COIRCodeSearchNetRetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/COIRCodeSearchNetRetrieval.json new file mode 100644 index 0000000000..3b65a810b5 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/COIRCodeSearchNetRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "4adc7bc41202b5c13543c9c886a25f340634dab3", + "task_name": "COIRCodeSearchNetRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.90776, + "ndcg_at_3": 0.93926, + "ndcg_at_5": 0.94326, + "ndcg_at_10": 0.94646, + "ndcg_at_20": 0.9479, + "ndcg_at_100": 0.94933, + "ndcg_at_1000": 0.94993, + "map_at_1": 0.90776, + "map_at_3": 0.93195, + "map_at_5": 0.93419, + "map_at_10": 0.93552, + "map_at_20": 0.93592, + "map_at_100": 0.93612, + "map_at_1000": 0.93614, + "recall_at_1": 0.90776, + "recall_at_3": 0.96025, + "recall_at_5": 0.96984, + "recall_at_10": 0.97969, + "recall_at_20": 0.98539, + "recall_at_100": 0.99303, + "recall_at_1000": 0.99772, + "precision_at_1": 0.90776, + "precision_at_3": 0.32008, + "precision_at_5": 0.19397, + "precision_at_10": 0.09797, + "precision_at_20": 0.04927, + "precision_at_100": 0.00993, + "precision_at_1000": 0.001, + "mrr_at_1": 0.907762, + "mrr_at_3": 0.93195, + "mrr_at_5": 0.934192, + "mrr_at_10": 0.935524, + "mrr_at_20": 0.935918, + "mrr_at_100": 0.936121, + "mrr_at_1000": 0.936144, + "nauc_ndcg_at_1_max": 0.771056, + "nauc_ndcg_at_1_std": -0.120602, + "nauc_ndcg_at_1_diff1": 0.923368, + "nauc_ndcg_at_3_max": 0.818369, + "nauc_ndcg_at_3_std": -0.108349, + "nauc_ndcg_at_3_diff1": 0.926671, + "nauc_ndcg_at_5_max": 0.816481, + "nauc_ndcg_at_5_std": -0.09896, + "nauc_ndcg_at_5_diff1": 0.927243, + "nauc_ndcg_at_10_max": 0.814738, + "nauc_ndcg_at_10_std": -0.093357, + "nauc_ndcg_at_10_diff1": 0.927102, + "nauc_ndcg_at_20_max": 0.812205, + "nauc_ndcg_at_20_std": -0.087347, + "nauc_ndcg_at_20_diff1": 0.926477, + "nauc_ndcg_at_100_max": 0.810159, + "nauc_ndcg_at_100_std": -0.086597, + "nauc_ndcg_at_100_diff1": 0.925881, + "nauc_ndcg_at_1000_max": 0.808479, + "nauc_ndcg_at_1000_std": -0.092733, + "nauc_ndcg_at_1000_diff1": 0.925895, + "nauc_map_at_1_max": 0.771056, + "nauc_map_at_1_std": -0.120602, + "nauc_map_at_1_diff1": 0.923368, + "nauc_map_at_3_max": 0.805168, + "nauc_map_at_3_std": -0.112349, + "nauc_map_at_3_diff1": 0.92571, + "nauc_map_at_5_max": 0.803801, + "nauc_map_at_5_std": -0.107792, + "nauc_map_at_5_diff1": 0.925934, + "nauc_map_at_10_max": 0.802916, + "nauc_map_at_10_std": -0.106052, + "nauc_map_at_10_diff1": 0.925867, + "nauc_map_at_20_max": 0.802277, + "nauc_map_at_20_std": -0.104808, + "nauc_map_at_20_diff1": 0.925714, + "nauc_map_at_100_max": 0.801994, + "nauc_map_at_100_std": -0.104931, + "nauc_map_at_100_diff1": 0.925652, + "nauc_map_at_1000_max": 0.801937, + "nauc_map_at_1000_std": -0.105145, + "nauc_map_at_1000_diff1": 0.925653, + "nauc_recall_at_1_max": 0.771056, + "nauc_recall_at_1_std": -0.120602, + "nauc_recall_at_1_diff1": 0.923368, + "nauc_recall_at_3_max": 0.881988, + "nauc_recall_at_3_std": -0.088593, + "nauc_recall_at_3_diff1": 0.931337, + "nauc_recall_at_5_max": 0.893705, + "nauc_recall_at_5_std": -0.04125, + "nauc_recall_at_5_diff1": 0.935578, + "nauc_recall_at_10_max": 0.917381, + "nauc_recall_at_10_std": 0.03225, + "nauc_recall_at_10_diff1": 0.93833, + "nauc_recall_at_20_max": 0.921703, + "nauc_recall_at_20_std": 0.166504, + "nauc_recall_at_20_diff1": 0.934093, + "nauc_recall_at_100_max": 0.964918, + "nauc_recall_at_100_std": 0.492313, + "nauc_recall_at_100_diff1": 0.918546, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.661256, + "nauc_recall_at_1000_diff1": 0.903744, + "nauc_precision_at_1_max": 0.771056, + "nauc_precision_at_1_std": -0.120602, + "nauc_precision_at_1_diff1": 0.923368, + "nauc_precision_at_3_max": 0.881988, + "nauc_precision_at_3_std": -0.088593, + "nauc_precision_at_3_diff1": 0.931337, + "nauc_precision_at_5_max": 0.893705, + "nauc_precision_at_5_std": -0.04125, + "nauc_precision_at_5_diff1": 0.935578, + "nauc_precision_at_10_max": 0.917381, + "nauc_precision_at_10_std": 0.03225, + "nauc_precision_at_10_diff1": 0.93833, + "nauc_precision_at_20_max": 0.921703, + "nauc_precision_at_20_std": 0.166504, + "nauc_precision_at_20_diff1": 0.934093, + "nauc_precision_at_100_max": 0.964918, + "nauc_precision_at_100_std": 0.492313, + "nauc_precision_at_100_diff1": 0.918546, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 0.661256, + "nauc_precision_at_1000_diff1": 0.903744, + "nauc_mrr_at_1_max": 0.771056, + "nauc_mrr_at_1_std": -0.120602, + "nauc_mrr_at_1_diff1": 0.923368, + "nauc_mrr_at_3_max": 0.805168, + "nauc_mrr_at_3_std": -0.112349, + "nauc_mrr_at_3_diff1": 0.92571, + "nauc_mrr_at_5_max": 0.803801, + "nauc_mrr_at_5_std": -0.107792, + "nauc_mrr_at_5_diff1": 0.925934, + "nauc_mrr_at_10_max": 0.802916, + "nauc_mrr_at_10_std": -0.106052, + "nauc_mrr_at_10_diff1": 0.925867, + "nauc_mrr_at_20_max": 0.802277, + "nauc_mrr_at_20_std": -0.104808, + "nauc_mrr_at_20_diff1": 0.925714, + "nauc_mrr_at_100_max": 0.801994, + "nauc_mrr_at_100_std": -0.104931, + "nauc_mrr_at_100_diff1": 0.925652, + "nauc_mrr_at_1000_max": 0.801937, + "nauc_mrr_at_1000_std": -0.105145, + "nauc_mrr_at_1000_diff1": 0.925653, + "main_score": 0.94646, + "hf_subset": "python", + "languages": [ + "python-Code" + ] + }, + { + "ndcg_at_1": 0.41264, + "ndcg_at_3": 0.49012, + "ndcg_at_5": 0.50998, + "ndcg_at_10": 0.53161, + "ndcg_at_20": 0.54594, + "ndcg_at_100": 0.56561, + "ndcg_at_1000": 0.57864, + "map_at_1": 0.41264, + "map_at_3": 0.47129, + "map_at_5": 0.48229, + "map_at_10": 0.49122, + "map_at_20": 0.49522, + "map_at_100": 0.49788, + "map_at_1000": 0.49834, + "recall_at_1": 0.41264, + "recall_at_3": 0.54452, + "recall_at_5": 0.59283, + "recall_at_10": 0.65968, + "recall_at_20": 0.71589, + "recall_at_100": 0.82255, + "recall_at_1000": 0.92677, + "precision_at_1": 0.41264, + "precision_at_3": 0.18151, + "precision_at_5": 0.11857, + "precision_at_10": 0.06597, + "precision_at_20": 0.03579, + "precision_at_100": 0.00823, + "precision_at_1000": 0.00093, + "mrr_at_1": 0.412641, + "mrr_at_3": 0.471285, + "mrr_at_5": 0.482285, + "mrr_at_10": 0.491219, + "mrr_at_20": 0.495216, + "mrr_at_100": 0.497879, + "mrr_at_1000": 0.498342, + "nauc_ndcg_at_1_max": 0.492679, + "nauc_ndcg_at_1_std": -0.024898, + "nauc_ndcg_at_1_diff1": 0.660133, + "nauc_ndcg_at_3_max": 0.506463, + "nauc_ndcg_at_3_std": -0.013328, + "nauc_ndcg_at_3_diff1": 0.601742, + "nauc_ndcg_at_5_max": 0.509719, + "nauc_ndcg_at_5_std": -0.003934, + "nauc_ndcg_at_5_diff1": 0.59194, + "nauc_ndcg_at_10_max": 0.511239, + "nauc_ndcg_at_10_std": 0.005406, + "nauc_ndcg_at_10_diff1": 0.588504, + "nauc_ndcg_at_20_max": 0.512178, + "nauc_ndcg_at_20_std": 0.012519, + "nauc_ndcg_at_20_diff1": 0.588595, + "nauc_ndcg_at_100_max": 0.510917, + "nauc_ndcg_at_100_std": 0.018403, + "nauc_ndcg_at_100_diff1": 0.591217, + "nauc_ndcg_at_1000_max": 0.51018, + "nauc_ndcg_at_1000_std": 0.014684, + "nauc_ndcg_at_1000_diff1": 0.593915, + "nauc_map_at_1_max": 0.492679, + "nauc_map_at_1_std": -0.024898, + "nauc_map_at_1_diff1": 0.660133, + "nauc_map_at_3_max": 0.50329, + "nauc_map_at_3_std": -0.01658, + "nauc_map_at_3_diff1": 0.615386, + "nauc_map_at_5_max": 0.505044, + "nauc_map_at_5_std": -0.011655, + "nauc_map_at_5_diff1": 0.610212, + "nauc_map_at_10_max": 0.505998, + "nauc_map_at_10_std": -0.007527, + "nauc_map_at_10_diff1": 0.609048, + "nauc_map_at_20_max": 0.506149, + "nauc_map_at_20_std": -0.005843, + "nauc_map_at_20_diff1": 0.609239, + "nauc_map_at_100_max": 0.506061, + "nauc_map_at_100_std": -0.005166, + "nauc_map_at_100_diff1": 0.60965, + "nauc_map_at_1000_max": 0.506032, + "nauc_map_at_1000_std": -0.005253, + "nauc_map_at_1000_diff1": 0.609761, + "nauc_recall_at_1_max": 0.492679, + "nauc_recall_at_1_std": -0.024898, + "nauc_recall_at_1_diff1": 0.660133, + "nauc_recall_at_3_max": 0.51595, + "nauc_recall_at_3_std": -0.003335, + "nauc_recall_at_3_diff1": 0.560865, + "nauc_recall_at_5_max": 0.524822, + "nauc_recall_at_5_std": 0.022252, + "nauc_recall_at_5_diff1": 0.533193, + "nauc_recall_at_10_max": 0.529513, + "nauc_recall_at_10_std": 0.055765, + "nauc_recall_at_10_diff1": 0.513732, + "nauc_recall_at_20_max": 0.537341, + "nauc_recall_at_20_std": 0.099476, + "nauc_recall_at_20_diff1": 0.503785, + "nauc_recall_at_100_max": 0.533764, + "nauc_recall_at_100_std": 0.197335, + "nauc_recall_at_100_diff1": 0.489369, + "nauc_recall_at_1000_max": 0.539364, + "nauc_recall_at_1000_std": 0.292798, + "nauc_recall_at_1000_diff1": 0.443949, + "nauc_precision_at_1_max": 0.492679, + "nauc_precision_at_1_std": -0.024898, + "nauc_precision_at_1_diff1": 0.660133, + "nauc_precision_at_3_max": 0.51595, + "nauc_precision_at_3_std": -0.003335, + "nauc_precision_at_3_diff1": 0.560865, + "nauc_precision_at_5_max": 0.524822, + "nauc_precision_at_5_std": 0.022252, + "nauc_precision_at_5_diff1": 0.533193, + "nauc_precision_at_10_max": 0.529513, + "nauc_precision_at_10_std": 0.055765, + "nauc_precision_at_10_diff1": 0.513732, + "nauc_precision_at_20_max": 0.537341, + "nauc_precision_at_20_std": 0.099476, + "nauc_precision_at_20_diff1": 0.503785, + "nauc_precision_at_100_max": 0.533764, + "nauc_precision_at_100_std": 0.197335, + "nauc_precision_at_100_diff1": 0.489369, + "nauc_precision_at_1000_max": 0.539364, + "nauc_precision_at_1000_std": 0.292798, + "nauc_precision_at_1000_diff1": 0.443949, + "nauc_mrr_at_1_max": 0.492679, + "nauc_mrr_at_1_std": -0.024898, + "nauc_mrr_at_1_diff1": 0.660133, + "nauc_mrr_at_3_max": 0.50329, + "nauc_mrr_at_3_std": -0.01658, + "nauc_mrr_at_3_diff1": 0.615386, + "nauc_mrr_at_5_max": 0.505044, + "nauc_mrr_at_5_std": -0.011655, + "nauc_mrr_at_5_diff1": 0.610212, + "nauc_mrr_at_10_max": 0.505998, + "nauc_mrr_at_10_std": -0.007527, + "nauc_mrr_at_10_diff1": 0.609048, + "nauc_mrr_at_20_max": 0.506149, + "nauc_mrr_at_20_std": -0.005843, + "nauc_mrr_at_20_diff1": 0.609239, + "nauc_mrr_at_100_max": 0.506061, + "nauc_mrr_at_100_std": -0.005166, + "nauc_mrr_at_100_diff1": 0.60965, + "nauc_mrr_at_1000_max": 0.506032, + "nauc_mrr_at_1000_std": -0.005253, + "nauc_mrr_at_1000_diff1": 0.609761, + "main_score": 0.53161, + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ] + }, + { + "ndcg_at_1": 0.57929, + "ndcg_at_3": 0.67103, + "ndcg_at_5": 0.6927, + "ndcg_at_10": 0.70879, + "ndcg_at_20": 0.71888, + "ndcg_at_100": 0.73124, + "ndcg_at_1000": 0.73645, + "map_at_1": 0.57929, + "map_at_3": 0.64885, + "map_at_5": 0.66092, + "map_at_10": 0.66761, + "map_at_20": 0.67041, + "map_at_100": 0.67217, + "map_at_1000": 0.67237, + "recall_at_1": 0.57929, + "recall_at_3": 0.73504, + "recall_at_5": 0.78749, + "recall_at_10": 0.83699, + "recall_at_20": 0.87663, + "recall_at_100": 0.9425, + "recall_at_1000": 0.98375, + "precision_at_1": 0.57929, + "precision_at_3": 0.24501, + "precision_at_5": 0.1575, + "precision_at_10": 0.0837, + "precision_at_20": 0.04383, + "precision_at_100": 0.00943, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.579291, + "mrr_at_3": 0.648855, + "mrr_at_5": 0.660921, + "mrr_at_10": 0.667608, + "mrr_at_20": 0.670414, + "mrr_at_100": 0.672174, + "mrr_at_1000": 0.672367, + "nauc_ndcg_at_1_max": 0.434671, + "nauc_ndcg_at_1_std": -0.086826, + "nauc_ndcg_at_1_diff1": 0.706722, + "nauc_ndcg_at_3_max": 0.460637, + "nauc_ndcg_at_3_std": -0.09423, + "nauc_ndcg_at_3_diff1": 0.665407, + "nauc_ndcg_at_5_max": 0.462558, + "nauc_ndcg_at_5_std": -0.090871, + "nauc_ndcg_at_5_diff1": 0.662046, + "nauc_ndcg_at_10_max": 0.463942, + "nauc_ndcg_at_10_std": -0.085306, + "nauc_ndcg_at_10_diff1": 0.662955, + "nauc_ndcg_at_20_max": 0.462155, + "nauc_ndcg_at_20_std": -0.080264, + "nauc_ndcg_at_20_diff1": 0.662223, + "nauc_ndcg_at_100_max": 0.463314, + "nauc_ndcg_at_100_std": -0.073975, + "nauc_ndcg_at_100_diff1": 0.66584, + "nauc_ndcg_at_1000_max": 0.461424, + "nauc_ndcg_at_1000_std": -0.07866, + "nauc_ndcg_at_1000_diff1": 0.668815, + "nauc_map_at_1_max": 0.434671, + "nauc_map_at_1_std": -0.086826, + "nauc_map_at_1_diff1": 0.706722, + "nauc_map_at_3_max": 0.454186, + "nauc_map_at_3_std": -0.09241, + "nauc_map_at_3_diff1": 0.675796, + "nauc_map_at_5_max": 0.455136, + "nauc_map_at_5_std": -0.090517, + "nauc_map_at_5_diff1": 0.674475, + "nauc_map_at_10_max": 0.455434, + "nauc_map_at_10_std": -0.088583, + "nauc_map_at_10_diff1": 0.675066, + "nauc_map_at_20_max": 0.454994, + "nauc_map_at_20_std": -0.087372, + "nauc_map_at_20_diff1": 0.674942, + "nauc_map_at_100_max": 0.455052, + "nauc_map_at_100_std": -0.086741, + "nauc_map_at_100_diff1": 0.675398, + "nauc_map_at_1000_max": 0.455001, + "nauc_map_at_1000_std": -0.086861, + "nauc_map_at_1000_diff1": 0.675495, + "nauc_recall_at_1_max": 0.434671, + "nauc_recall_at_1_std": -0.086826, + "nauc_recall_at_1_diff1": 0.706722, + "nauc_recall_at_3_max": 0.482876, + "nauc_recall_at_3_std": -0.100494, + "nauc_recall_at_3_diff1": 0.629522, + "nauc_recall_at_5_max": 0.492293, + "nauc_recall_at_5_std": -0.091697, + "nauc_recall_at_5_diff1": 0.611497, + "nauc_recall_at_10_max": 0.507454, + "nauc_recall_at_10_std": -0.063785, + "nauc_recall_at_10_diff1": 0.602415, + "nauc_recall_at_20_max": 0.505054, + "nauc_recall_at_20_std": -0.01953, + "nauc_recall_at_20_diff1": 0.580455, + "nauc_recall_at_100_max": 0.577777, + "nauc_recall_at_100_std": 0.179626, + "nauc_recall_at_100_diff1": 0.568694, + "nauc_recall_at_1000_max": 0.660767, + "nauc_recall_at_1000_std": 0.305437, + "nauc_recall_at_1000_diff1": 0.633023, + "nauc_precision_at_1_max": 0.434671, + "nauc_precision_at_1_std": -0.086826, + "nauc_precision_at_1_diff1": 0.706722, + "nauc_precision_at_3_max": 0.482876, + "nauc_precision_at_3_std": -0.100494, + "nauc_precision_at_3_diff1": 0.629522, + "nauc_precision_at_5_max": 0.492293, + "nauc_precision_at_5_std": -0.091697, + "nauc_precision_at_5_diff1": 0.611497, + "nauc_precision_at_10_max": 0.507454, + "nauc_precision_at_10_std": -0.063785, + "nauc_precision_at_10_diff1": 0.602415, + "nauc_precision_at_20_max": 0.505054, + "nauc_precision_at_20_std": -0.01953, + "nauc_precision_at_20_diff1": 0.580455, + "nauc_precision_at_100_max": 0.577777, + "nauc_precision_at_100_std": 0.179626, + "nauc_precision_at_100_diff1": 0.568694, + "nauc_precision_at_1000_max": 0.660767, + "nauc_precision_at_1000_std": 0.305437, + "nauc_precision_at_1000_diff1": 0.633023, + "nauc_mrr_at_1_max": 0.434671, + "nauc_mrr_at_1_std": -0.086826, + "nauc_mrr_at_1_diff1": 0.706722, + "nauc_mrr_at_3_max": 0.454186, + "nauc_mrr_at_3_std": -0.09241, + "nauc_mrr_at_3_diff1": 0.675796, + "nauc_mrr_at_5_max": 0.455136, + "nauc_mrr_at_5_std": -0.090517, + "nauc_mrr_at_5_diff1": 0.674475, + "nauc_mrr_at_10_max": 0.455434, + "nauc_mrr_at_10_std": -0.088583, + "nauc_mrr_at_10_diff1": 0.675066, + "nauc_mrr_at_20_max": 0.454994, + "nauc_mrr_at_20_std": -0.087372, + "nauc_mrr_at_20_diff1": 0.674942, + "nauc_mrr_at_100_max": 0.455052, + "nauc_mrr_at_100_std": -0.086741, + "nauc_mrr_at_100_diff1": 0.675398, + "nauc_mrr_at_1000_max": 0.455001, + "nauc_mrr_at_1000_std": -0.086861, + "nauc_mrr_at_1000_diff1": 0.675495, + "main_score": 0.70879, + "hf_subset": "go", + "languages": [ + "go-Code" + ] + }, + { + "ndcg_at_1": 0.43299, + "ndcg_at_3": 0.52448, + "ndcg_at_5": 0.5439, + "ndcg_at_10": 0.5644, + "ndcg_at_20": 0.57697, + "ndcg_at_100": 0.59956, + "ndcg_at_1000": 0.61062, + "map_at_1": 0.43299, + "map_at_3": 0.50185, + "map_at_5": 0.51271, + "map_at_10": 0.52132, + "map_at_20": 0.52474, + "map_at_100": 0.52802, + "map_at_1000": 0.52843, + "recall_at_1": 0.43299, + "recall_at_3": 0.59001, + "recall_at_5": 0.6368, + "recall_at_10": 0.69944, + "recall_at_20": 0.74941, + "recall_at_100": 0.86915, + "recall_at_1000": 0.95638, + "precision_at_1": 0.43299, + "precision_at_3": 0.19667, + "precision_at_5": 0.12736, + "precision_at_10": 0.06994, + "precision_at_20": 0.03747, + "precision_at_100": 0.00869, + "precision_at_1000": 0.00096, + "mrr_at_1": 0.43299, + "mrr_at_3": 0.50185, + "mrr_at_5": 0.512715, + "mrr_at_10": 0.521317, + "mrr_at_20": 0.524737, + "mrr_at_100": 0.528017, + "mrr_at_1000": 0.528425, + "nauc_ndcg_at_1_max": 0.475367, + "nauc_ndcg_at_1_std": -0.068581, + "nauc_ndcg_at_1_diff1": 0.606441, + "nauc_ndcg_at_3_max": 0.473039, + "nauc_ndcg_at_3_std": -0.071457, + "nauc_ndcg_at_3_diff1": 0.54625, + "nauc_ndcg_at_5_max": 0.474664, + "nauc_ndcg_at_5_std": -0.062658, + "nauc_ndcg_at_5_diff1": 0.545846, + "nauc_ndcg_at_10_max": 0.465762, + "nauc_ndcg_at_10_std": -0.063581, + "nauc_ndcg_at_10_diff1": 0.537222, + "nauc_ndcg_at_20_max": 0.462656, + "nauc_ndcg_at_20_std": -0.064879, + "nauc_ndcg_at_20_diff1": 0.537404, + "nauc_ndcg_at_100_max": 0.462585, + "nauc_ndcg_at_100_std": -0.05661, + "nauc_ndcg_at_100_diff1": 0.541201, + "nauc_ndcg_at_1000_max": 0.46849, + "nauc_ndcg_at_1000_std": -0.054894, + "nauc_ndcg_at_1000_diff1": 0.546628, + "nauc_map_at_1_max": 0.475367, + "nauc_map_at_1_std": -0.068581, + "nauc_map_at_1_diff1": 0.606441, + "nauc_map_at_3_max": 0.474256, + "nauc_map_at_3_std": -0.069619, + "nauc_map_at_3_diff1": 0.560673, + "nauc_map_at_5_max": 0.475098, + "nauc_map_at_5_std": -0.064728, + "nauc_map_at_5_diff1": 0.560703, + "nauc_map_at_10_max": 0.471568, + "nauc_map_at_10_std": -0.065293, + "nauc_map_at_10_diff1": 0.557701, + "nauc_map_at_20_max": 0.470913, + "nauc_map_at_20_std": -0.065553, + "nauc_map_at_20_diff1": 0.557951, + "nauc_map_at_100_max": 0.47103, + "nauc_map_at_100_std": -0.063988, + "nauc_map_at_100_diff1": 0.558438, + "nauc_map_at_1000_max": 0.471268, + "nauc_map_at_1000_std": -0.06383, + "nauc_map_at_1000_diff1": 0.558601, + "nauc_recall_at_1_max": 0.475367, + "nauc_recall_at_1_std": -0.068581, + "nauc_recall_at_1_diff1": 0.606441, + "nauc_recall_at_3_max": 0.468987, + "nauc_recall_at_3_std": -0.077647, + "nauc_recall_at_3_diff1": 0.501887, + "nauc_recall_at_5_max": 0.473273, + "nauc_recall_at_5_std": -0.055102, + "nauc_recall_at_5_diff1": 0.497147, + "nauc_recall_at_10_max": 0.440004, + "nauc_recall_at_10_std": -0.056392, + "nauc_recall_at_10_diff1": 0.457159, + "nauc_recall_at_20_max": 0.41807, + "nauc_recall_at_20_std": -0.063051, + "nauc_recall_at_20_diff1": 0.444638, + "nauc_recall_at_100_max": 0.381128, + "nauc_recall_at_100_std": 0.019635, + "nauc_recall_at_100_diff1": 0.425543, + "nauc_recall_at_1000_max": 0.498716, + "nauc_recall_at_1000_std": 0.201166, + "nauc_recall_at_1000_diff1": 0.485163, + "nauc_precision_at_1_max": 0.475367, + "nauc_precision_at_1_std": -0.068581, + "nauc_precision_at_1_diff1": 0.606441, + "nauc_precision_at_3_max": 0.468987, + "nauc_precision_at_3_std": -0.077647, + "nauc_precision_at_3_diff1": 0.501887, + "nauc_precision_at_5_max": 0.473273, + "nauc_precision_at_5_std": -0.055102, + "nauc_precision_at_5_diff1": 0.497147, + "nauc_precision_at_10_max": 0.440004, + "nauc_precision_at_10_std": -0.056392, + "nauc_precision_at_10_diff1": 0.457159, + "nauc_precision_at_20_max": 0.41807, + "nauc_precision_at_20_std": -0.063051, + "nauc_precision_at_20_diff1": 0.444638, + "nauc_precision_at_100_max": 0.381128, + "nauc_precision_at_100_std": 0.019635, + "nauc_precision_at_100_diff1": 0.425543, + "nauc_precision_at_1000_max": 0.498716, + "nauc_precision_at_1000_std": 0.201166, + "nauc_precision_at_1000_diff1": 0.485163, + "nauc_mrr_at_1_max": 0.475367, + "nauc_mrr_at_1_std": -0.068581, + "nauc_mrr_at_1_diff1": 0.606441, + "nauc_mrr_at_3_max": 0.474256, + "nauc_mrr_at_3_std": -0.069619, + "nauc_mrr_at_3_diff1": 0.560673, + "nauc_mrr_at_5_max": 0.475098, + "nauc_mrr_at_5_std": -0.064728, + "nauc_mrr_at_5_diff1": 0.560703, + "nauc_mrr_at_10_max": 0.471568, + "nauc_mrr_at_10_std": -0.065293, + "nauc_mrr_at_10_diff1": 0.557701, + "nauc_mrr_at_20_max": 0.470913, + "nauc_mrr_at_20_std": -0.065553, + "nauc_mrr_at_20_diff1": 0.557951, + "nauc_mrr_at_100_max": 0.47103, + "nauc_mrr_at_100_std": -0.063988, + "nauc_mrr_at_100_diff1": 0.558438, + "nauc_mrr_at_1000_max": 0.471268, + "nauc_mrr_at_1000_std": -0.06383, + "nauc_mrr_at_1000_diff1": 0.558601, + "main_score": 0.5644, + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ] + }, + { + "ndcg_at_1": 0.48115, + "ndcg_at_3": 0.57045, + "ndcg_at_5": 0.59176, + "ndcg_at_10": 0.61135, + "ndcg_at_20": 0.62309, + "ndcg_at_100": 0.63928, + "ndcg_at_1000": 0.64795, + "map_at_1": 0.48115, + "map_at_3": 0.54911, + "map_at_5": 0.56098, + "map_at_10": 0.56916, + "map_at_20": 0.57239, + "map_at_100": 0.57466, + "map_at_1000": 0.57497, + "recall_at_1": 0.48115, + "recall_at_3": 0.63195, + "recall_at_5": 0.68352, + "recall_at_10": 0.74359, + "recall_at_20": 0.78996, + "recall_at_100": 0.87677, + "recall_at_1000": 0.94605, + "precision_at_1": 0.48115, + "precision_at_3": 0.21065, + "precision_at_5": 0.1367, + "precision_at_10": 0.07436, + "precision_at_20": 0.0395, + "precision_at_100": 0.00877, + "precision_at_1000": 0.00095, + "mrr_at_1": 0.48115, + "mrr_at_3": 0.54911, + "mrr_at_5": 0.560977, + "mrr_at_10": 0.569161, + "mrr_at_20": 0.572389, + "mrr_at_100": 0.574662, + "mrr_at_1000": 0.574968, + "nauc_ndcg_at_1_max": 0.348482, + "nauc_ndcg_at_1_std": -0.108202, + "nauc_ndcg_at_1_diff1": 0.664691, + "nauc_ndcg_at_3_max": 0.367179, + "nauc_ndcg_at_3_std": -0.104012, + "nauc_ndcg_at_3_diff1": 0.61965, + "nauc_ndcg_at_5_max": 0.371127, + "nauc_ndcg_at_5_std": -0.090815, + "nauc_ndcg_at_5_diff1": 0.611136, + "nauc_ndcg_at_10_max": 0.372846, + "nauc_ndcg_at_10_std": -0.080004, + "nauc_ndcg_at_10_diff1": 0.609571, + "nauc_ndcg_at_20_max": 0.374279, + "nauc_ndcg_at_20_std": -0.071836, + "nauc_ndcg_at_20_diff1": 0.607986, + "nauc_ndcg_at_100_max": 0.377849, + "nauc_ndcg_at_100_std": -0.066679, + "nauc_ndcg_at_100_diff1": 0.609674, + "nauc_ndcg_at_1000_max": 0.376997, + "nauc_ndcg_at_1000_std": -0.069917, + "nauc_ndcg_at_1000_diff1": 0.612154, + "nauc_map_at_1_max": 0.348482, + "nauc_map_at_1_std": -0.108202, + "nauc_map_at_1_diff1": 0.664691, + "nauc_map_at_3_max": 0.362753, + "nauc_map_at_3_std": -0.105475, + "nauc_map_at_3_diff1": 0.63056, + "nauc_map_at_5_max": 0.364686, + "nauc_map_at_5_std": -0.098592, + "nauc_map_at_5_diff1": 0.626318, + "nauc_map_at_10_max": 0.365177, + "nauc_map_at_10_std": -0.094438, + "nauc_map_at_10_diff1": 0.62594, + "nauc_map_at_20_max": 0.365468, + "nauc_map_at_20_std": -0.092509, + "nauc_map_at_20_diff1": 0.625606, + "nauc_map_at_100_max": 0.365886, + "nauc_map_at_100_std": -0.091872, + "nauc_map_at_100_diff1": 0.625861, + "nauc_map_at_1000_max": 0.365871, + "nauc_map_at_1000_std": -0.091941, + "nauc_map_at_1000_diff1": 0.625939, + "nauc_recall_at_1_max": 0.348482, + "nauc_recall_at_1_std": -0.108202, + "nauc_recall_at_1_diff1": 0.664691, + "nauc_recall_at_3_max": 0.381121, + "nauc_recall_at_3_std": -0.099171, + "nauc_recall_at_3_diff1": 0.585154, + "nauc_recall_at_5_max": 0.394012, + "nauc_recall_at_5_std": -0.061265, + "nauc_recall_at_5_diff1": 0.557387, + "nauc_recall_at_10_max": 0.405423, + "nauc_recall_at_10_std": -0.013536, + "nauc_recall_at_10_diff1": 0.541776, + "nauc_recall_at_20_max": 0.419555, + "nauc_recall_at_20_std": 0.04581, + "nauc_recall_at_20_diff1": 0.521501, + "nauc_recall_at_100_max": 0.489373, + "nauc_recall_at_100_std": 0.175866, + "nauc_recall_at_100_diff1": 0.491031, + "nauc_recall_at_1000_max": 0.58553, + "nauc_recall_at_1000_std": 0.329355, + "nauc_recall_at_1000_diff1": 0.446313, + "nauc_precision_at_1_max": 0.348482, + "nauc_precision_at_1_std": -0.108202, + "nauc_precision_at_1_diff1": 0.664691, + "nauc_precision_at_3_max": 0.381121, + "nauc_precision_at_3_std": -0.099171, + "nauc_precision_at_3_diff1": 0.585154, + "nauc_precision_at_5_max": 0.394012, + "nauc_precision_at_5_std": -0.061265, + "nauc_precision_at_5_diff1": 0.557387, + "nauc_precision_at_10_max": 0.405423, + "nauc_precision_at_10_std": -0.013536, + "nauc_precision_at_10_diff1": 0.541776, + "nauc_precision_at_20_max": 0.419555, + "nauc_precision_at_20_std": 0.04581, + "nauc_precision_at_20_diff1": 0.521501, + "nauc_precision_at_100_max": 0.489373, + "nauc_precision_at_100_std": 0.175866, + "nauc_precision_at_100_diff1": 0.491031, + "nauc_precision_at_1000_max": 0.58553, + "nauc_precision_at_1000_std": 0.329355, + "nauc_precision_at_1000_diff1": 0.446313, + "nauc_mrr_at_1_max": 0.348482, + "nauc_mrr_at_1_std": -0.108202, + "nauc_mrr_at_1_diff1": 0.664691, + "nauc_mrr_at_3_max": 0.362753, + "nauc_mrr_at_3_std": -0.105475, + "nauc_mrr_at_3_diff1": 0.63056, + "nauc_mrr_at_5_max": 0.364686, + "nauc_mrr_at_5_std": -0.098592, + "nauc_mrr_at_5_diff1": 0.626318, + "nauc_mrr_at_10_max": 0.365177, + "nauc_mrr_at_10_std": -0.094438, + "nauc_mrr_at_10_diff1": 0.62594, + "nauc_mrr_at_20_max": 0.365468, + "nauc_mrr_at_20_std": -0.092509, + "nauc_mrr_at_20_diff1": 0.625606, + "nauc_mrr_at_100_max": 0.365886, + "nauc_mrr_at_100_std": -0.091872, + "nauc_mrr_at_100_diff1": 0.625861, + "nauc_mrr_at_1000_max": 0.365871, + "nauc_mrr_at_1000_std": -0.091941, + "nauc_mrr_at_1000_diff1": 0.625939, + "main_score": 0.61135, + "hf_subset": "java", + "languages": [ + "java-Code" + ] + }, + { + "ndcg_at_1": 0.37762, + "ndcg_at_3": 0.46787, + "ndcg_at_5": 0.49239, + "ndcg_at_10": 0.51661, + "ndcg_at_20": 0.53352, + "ndcg_at_100": 0.55575, + "ndcg_at_1000": 0.56876, + "map_at_1": 0.37762, + "map_at_3": 0.44578, + "map_at_5": 0.45939, + "map_at_10": 0.46941, + "map_at_20": 0.47407, + "map_at_100": 0.47716, + "map_at_1000": 0.47763, + "recall_at_1": 0.37762, + "recall_at_3": 0.53175, + "recall_at_5": 0.59127, + "recall_at_10": 0.66605, + "recall_at_20": 0.73277, + "recall_at_100": 0.85236, + "recall_at_1000": 0.95583, + "precision_at_1": 0.37762, + "precision_at_3": 0.17725, + "precision_at_5": 0.11825, + "precision_at_10": 0.0666, + "precision_at_20": 0.03664, + "precision_at_100": 0.00852, + "precision_at_1000": 0.00096, + "mrr_at_1": 0.377622, + "mrr_at_3": 0.44578, + "mrr_at_5": 0.459392, + "mrr_at_10": 0.469408, + "mrr_at_20": 0.474073, + "mrr_at_100": 0.47716, + "mrr_at_1000": 0.477631, + "nauc_ndcg_at_1_max": 0.317688, + "nauc_ndcg_at_1_std": -0.078856, + "nauc_ndcg_at_1_diff1": 0.59775, + "nauc_ndcg_at_3_max": 0.327194, + "nauc_ndcg_at_3_std": -0.069744, + "nauc_ndcg_at_3_diff1": 0.532416, + "nauc_ndcg_at_5_max": 0.326666, + "nauc_ndcg_at_5_std": -0.060723, + "nauc_ndcg_at_5_diff1": 0.520852, + "nauc_ndcg_at_10_max": 0.328004, + "nauc_ndcg_at_10_std": -0.052736, + "nauc_ndcg_at_10_diff1": 0.515692, + "nauc_ndcg_at_20_max": 0.33228, + "nauc_ndcg_at_20_std": -0.042192, + "nauc_ndcg_at_20_diff1": 0.51516, + "nauc_ndcg_at_100_max": 0.335211, + "nauc_ndcg_at_100_std": -0.036361, + "nauc_ndcg_at_100_diff1": 0.519711, + "nauc_ndcg_at_1000_max": 0.334801, + "nauc_ndcg_at_1000_std": -0.041022, + "nauc_ndcg_at_1000_diff1": 0.524798, + "nauc_map_at_1_max": 0.317688, + "nauc_map_at_1_std": -0.078856, + "nauc_map_at_1_diff1": 0.59775, + "nauc_map_at_3_max": 0.325343, + "nauc_map_at_3_std": -0.07211, + "nauc_map_at_3_diff1": 0.548099, + "nauc_map_at_5_max": 0.325134, + "nauc_map_at_5_std": -0.067211, + "nauc_map_at_5_diff1": 0.541944, + "nauc_map_at_10_max": 0.325691, + "nauc_map_at_10_std": -0.064202, + "nauc_map_at_10_diff1": 0.540148, + "nauc_map_at_20_max": 0.326774, + "nauc_map_at_20_std": -0.061482, + "nauc_map_at_20_diff1": 0.540133, + "nauc_map_at_100_max": 0.327038, + "nauc_map_at_100_std": -0.060874, + "nauc_map_at_100_diff1": 0.540833, + "nauc_map_at_1000_max": 0.327039, + "nauc_map_at_1000_std": -0.060995, + "nauc_map_at_1000_diff1": 0.540994, + "nauc_recall_at_1_max": 0.317688, + "nauc_recall_at_1_std": -0.078856, + "nauc_recall_at_1_diff1": 0.59775, + "nauc_recall_at_3_max": 0.332516, + "nauc_recall_at_3_std": -0.062619, + "nauc_recall_at_3_diff1": 0.485699, + "nauc_recall_at_5_max": 0.331037, + "nauc_recall_at_5_std": -0.03887, + "nauc_recall_at_5_diff1": 0.452977, + "nauc_recall_at_10_max": 0.336178, + "nauc_recall_at_10_std": -0.006422, + "nauc_recall_at_10_diff1": 0.425047, + "nauc_recall_at_20_max": 0.360272, + "nauc_recall_at_20_std": 0.056604, + "nauc_recall_at_20_diff1": 0.406814, + "nauc_recall_at_100_max": 0.414473, + "nauc_recall_at_100_std": 0.187333, + "nauc_recall_at_100_diff1": 0.38776, + "nauc_recall_at_1000_max": 0.559654, + "nauc_recall_at_1000_std": 0.415667, + "nauc_recall_at_1000_diff1": 0.369639, + "nauc_precision_at_1_max": 0.317688, + "nauc_precision_at_1_std": -0.078856, + "nauc_precision_at_1_diff1": 0.59775, + "nauc_precision_at_3_max": 0.332516, + "nauc_precision_at_3_std": -0.062619, + "nauc_precision_at_3_diff1": 0.485699, + "nauc_precision_at_5_max": 0.331037, + "nauc_precision_at_5_std": -0.03887, + "nauc_precision_at_5_diff1": 0.452977, + "nauc_precision_at_10_max": 0.336178, + "nauc_precision_at_10_std": -0.006422, + "nauc_precision_at_10_diff1": 0.425047, + "nauc_precision_at_20_max": 0.360272, + "nauc_precision_at_20_std": 0.056604, + "nauc_precision_at_20_diff1": 0.406814, + "nauc_precision_at_100_max": 0.414473, + "nauc_precision_at_100_std": 0.187333, + "nauc_precision_at_100_diff1": 0.38776, + "nauc_precision_at_1000_max": 0.559654, + "nauc_precision_at_1000_std": 0.415667, + "nauc_precision_at_1000_diff1": 0.369639, + "nauc_mrr_at_1_max": 0.317688, + "nauc_mrr_at_1_std": -0.078856, + "nauc_mrr_at_1_diff1": 0.59775, + "nauc_mrr_at_3_max": 0.325343, + "nauc_mrr_at_3_std": -0.07211, + "nauc_mrr_at_3_diff1": 0.548099, + "nauc_mrr_at_5_max": 0.325134, + "nauc_mrr_at_5_std": -0.067211, + "nauc_mrr_at_5_diff1": 0.541944, + "nauc_mrr_at_10_max": 0.325691, + "nauc_mrr_at_10_std": -0.064202, + "nauc_mrr_at_10_diff1": 0.540148, + "nauc_mrr_at_20_max": 0.326774, + "nauc_mrr_at_20_std": -0.061482, + "nauc_mrr_at_20_diff1": 0.540133, + "nauc_mrr_at_100_max": 0.327038, + "nauc_mrr_at_100_std": -0.060874, + "nauc_mrr_at_100_diff1": 0.540833, + "nauc_mrr_at_1000_max": 0.327039, + "nauc_mrr_at_1000_std": -0.060995, + "nauc_mrr_at_1000_diff1": 0.540994, + "main_score": 0.51661, + "hf_subset": "php", + "languages": [ + "php-Code" + ] + } + ] + }, + "evaluation_time": 556.1160366535187, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CQADupstackGamingRetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CQADupstackGamingRetrieval.json new file mode 100644 index 0000000000..39096363e6 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CQADupstackGamingRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4885aa143210c98657558c04aaf3dc47cfb54340", + "task_name": "CQADupstackGamingRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.52915, + "ndcg_at_3": 0.59843, + "ndcg_at_5": 0.6229, + "ndcg_at_10": 0.65045, + "ndcg_at_20": 0.66638, + "ndcg_at_100": 0.6844, + "ndcg_at_1000": 0.69215, + "map_at_1": 0.46044, + "map_at_3": 0.55998, + "map_at_5": 0.57809, + "map_at_10": 0.5928, + "map_at_20": 0.59923, + "map_at_100": 0.60278, + "map_at_1000": 0.60318, + "recall_at_1": 0.46044, + "recall_at_3": 0.64493, + "recall_at_5": 0.70585, + "recall_at_10": 0.78465, + "recall_at_20": 0.84247, + "recall_at_100": 0.92754, + "recall_at_1000": 0.98158, + "precision_at_1": 0.52915, + "precision_at_3": 0.26499, + "precision_at_5": 0.17944, + "precision_at_10": 0.10307, + "precision_at_20": 0.0568, + "precision_at_100": 0.01288, + "precision_at_1000": 0.00139, + "mrr_at_1": 0.529154, + "mrr_at_3": 0.604284, + "mrr_at_5": 0.617607, + "mrr_at_10": 0.627241, + "mrr_at_20": 0.630613, + "mrr_at_100": 0.632465, + "mrr_at_1000": 0.632677, + "nauc_ndcg_at_1_max": 0.414943, + "nauc_ndcg_at_1_std": -0.058725, + "nauc_ndcg_at_1_diff1": 0.603937, + "nauc_ndcg_at_3_max": 0.439731, + "nauc_ndcg_at_3_std": -0.067107, + "nauc_ndcg_at_3_diff1": 0.575945, + "nauc_ndcg_at_5_max": 0.440894, + "nauc_ndcg_at_5_std": -0.064389, + "nauc_ndcg_at_5_diff1": 0.5702, + "nauc_ndcg_at_10_max": 0.452444, + "nauc_ndcg_at_10_std": -0.048817, + "nauc_ndcg_at_10_diff1": 0.567978, + "nauc_ndcg_at_20_max": 0.457359, + "nauc_ndcg_at_20_std": -0.033964, + "nauc_ndcg_at_20_diff1": 0.563873, + "nauc_ndcg_at_100_max": 0.459135, + "nauc_ndcg_at_100_std": -0.022831, + "nauc_ndcg_at_100_diff1": 0.564374, + "nauc_ndcg_at_1000_max": 0.456774, + "nauc_ndcg_at_1000_std": -0.028143, + "nauc_ndcg_at_1000_diff1": 0.569449, + "nauc_map_at_1_max": 0.334504, + "nauc_map_at_1_std": -0.089622, + "nauc_map_at_1_diff1": 0.593579, + "nauc_map_at_3_max": 0.412577, + "nauc_map_at_3_std": -0.094307, + "nauc_map_at_3_diff1": 0.581859, + "nauc_map_at_5_max": 0.417511, + "nauc_map_at_5_std": -0.087121, + "nauc_map_at_5_diff1": 0.57727, + "nauc_map_at_10_max": 0.427169, + "nauc_map_at_10_std": -0.075729, + "nauc_map_at_10_diff1": 0.576812, + "nauc_map_at_20_max": 0.432389, + "nauc_map_at_20_std": -0.066385, + "nauc_map_at_20_diff1": 0.575306, + "nauc_map_at_100_max": 0.434224, + "nauc_map_at_100_std": -0.062049, + "nauc_map_at_100_diff1": 0.575548, + "nauc_map_at_1000_max": 0.434324, + "nauc_map_at_1000_std": -0.062024, + "nauc_map_at_1000_diff1": 0.575817, + "nauc_recall_at_1_max": 0.334504, + "nauc_recall_at_1_std": -0.089622, + "nauc_recall_at_1_diff1": 0.593579, + "nauc_recall_at_3_max": 0.42984, + "nauc_recall_at_3_std": -0.0918, + "nauc_recall_at_3_diff1": 0.543869, + "nauc_recall_at_5_max": 0.431946, + "nauc_recall_at_5_std": -0.080123, + "nauc_recall_at_5_diff1": 0.519222, + "nauc_recall_at_10_max": 0.479297, + "nauc_recall_at_10_std": -0.018662, + "nauc_recall_at_10_diff1": 0.501219, + "nauc_recall_at_20_max": 0.526926, + "nauc_recall_at_20_std": 0.088537, + "nauc_recall_at_20_diff1": 0.468008, + "nauc_recall_at_100_max": 0.598699, + "nauc_recall_at_100_std": 0.316383, + "nauc_recall_at_100_diff1": 0.392384, + "nauc_recall_at_1000_max": 0.798452, + "nauc_recall_at_1000_std": 0.689309, + "nauc_recall_at_1000_diff1": 0.459165, + "nauc_precision_at_1_max": 0.414943, + "nauc_precision_at_1_std": -0.058725, + "nauc_precision_at_1_diff1": 0.603937, + "nauc_precision_at_3_max": 0.429614, + "nauc_precision_at_3_std": 0.022028, + "nauc_precision_at_3_diff1": 0.320071, + "nauc_precision_at_5_max": 0.387045, + "nauc_precision_at_5_std": 0.084413, + "nauc_precision_at_5_diff1": 0.211619, + "nauc_precision_at_10_max": 0.351488, + "nauc_precision_at_10_std": 0.177778, + "nauc_precision_at_10_diff1": 0.098478, + "nauc_precision_at_20_max": 0.318427, + "nauc_precision_at_20_std": 0.270326, + "nauc_precision_at_20_diff1": 0.004477, + "nauc_precision_at_100_max": 0.247758, + "nauc_precision_at_100_std": 0.347545, + "nauc_precision_at_100_diff1": -0.091547, + "nauc_precision_at_1000_max": 0.191532, + "nauc_precision_at_1000_std": 0.335751, + "nauc_precision_at_1000_diff1": -0.124359, + "nauc_mrr_at_1_max": 0.414943, + "nauc_mrr_at_1_std": -0.058725, + "nauc_mrr_at_1_diff1": 0.603937, + "nauc_mrr_at_3_max": 0.448264, + "nauc_mrr_at_3_std": -0.045233, + "nauc_mrr_at_3_diff1": 0.5831, + "nauc_mrr_at_5_max": 0.447594, + "nauc_mrr_at_5_std": -0.042329, + "nauc_mrr_at_5_diff1": 0.580065, + "nauc_mrr_at_10_max": 0.449105, + "nauc_mrr_at_10_std": -0.038261, + "nauc_mrr_at_10_diff1": 0.579813, + "nauc_mrr_at_20_max": 0.449275, + "nauc_mrr_at_20_std": -0.035927, + "nauc_mrr_at_20_diff1": 0.579458, + "nauc_mrr_at_100_max": 0.448894, + "nauc_mrr_at_100_std": -0.03591, + "nauc_mrr_at_100_diff1": 0.579555, + "nauc_mrr_at_1000_max": 0.448799, + "nauc_mrr_at_1000_std": -0.036103, + "nauc_mrr_at_1000_diff1": 0.579715, + "main_score": 0.65045, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 35.04284930229187, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CQADupstackRetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CQADupstackRetrieval.json new file mode 100644 index 0000000000..5507e6ad71 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CQADupstackRetrieval.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "1", + "task_name": "CQADupstackRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_10": 0.500012, + "main_score": 0.500012, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 694.6106235980988, + "kg_co2_emissions": NaN +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CQADupstackUnixRetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CQADupstackUnixRetrieval.json new file mode 100644 index 0000000000..9ca9156202 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CQADupstackUnixRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53", + "task_name": "CQADupstackUnixRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.40951, + "ndcg_at_3": 0.46739, + "ndcg_at_5": 0.49992, + "ndcg_at_10": 0.52852, + "ndcg_at_20": 0.55196, + "ndcg_at_100": 0.58053, + "ndcg_at_1000": 0.59453, + "map_at_1": 0.34482, + "map_at_3": 0.42822, + "map_at_5": 0.45196, + "map_at_10": 0.46666, + "map_at_20": 0.47471, + "map_at_100": 0.47953, + "map_at_1000": 0.48033, + "recall_at_1": 0.34482, + "recall_at_3": 0.50712, + "recall_at_5": 0.58965, + "recall_at_10": 0.67029, + "recall_at_20": 0.75367, + "recall_at_100": 0.88998, + "recall_at_1000": 0.98101, + "precision_at_1": 0.40951, + "precision_at_3": 0.21393, + "precision_at_5": 0.15336, + "precision_at_10": 0.09123, + "precision_at_20": 0.05243, + "precision_at_100": 0.01305, + "precision_at_1000": 0.00152, + "mrr_at_1": 0.409515, + "mrr_at_3": 0.4801, + "mrr_at_5": 0.497637, + "mrr_at_10": 0.508867, + "mrr_at_20": 0.513858, + "mrr_at_100": 0.516696, + "mrr_at_1000": 0.517038, + "nauc_ndcg_at_1_max": 0.436955, + "nauc_ndcg_at_1_std": -0.063223, + "nauc_ndcg_at_1_diff1": 0.502977, + "nauc_ndcg_at_3_max": 0.405959, + "nauc_ndcg_at_3_std": -0.051439, + "nauc_ndcg_at_3_diff1": 0.471281, + "nauc_ndcg_at_5_max": 0.424364, + "nauc_ndcg_at_5_std": -0.030203, + "nauc_ndcg_at_5_diff1": 0.465126, + "nauc_ndcg_at_10_max": 0.428817, + "nauc_ndcg_at_10_std": -0.022247, + "nauc_ndcg_at_10_diff1": 0.461371, + "nauc_ndcg_at_20_max": 0.431363, + "nauc_ndcg_at_20_std": -0.019328, + "nauc_ndcg_at_20_diff1": 0.462879, + "nauc_ndcg_at_100_max": 0.429453, + "nauc_ndcg_at_100_std": -0.013703, + "nauc_ndcg_at_100_diff1": 0.461154, + "nauc_ndcg_at_1000_max": 0.428455, + "nauc_ndcg_at_1000_std": -0.022606, + "nauc_ndcg_at_1000_diff1": 0.465215, + "nauc_map_at_1_max": 0.390734, + "nauc_map_at_1_std": -0.073628, + "nauc_map_at_1_diff1": 0.525266, + "nauc_map_at_3_max": 0.401832, + "nauc_map_at_3_std": -0.062734, + "nauc_map_at_3_diff1": 0.494795, + "nauc_map_at_5_max": 0.416642, + "nauc_map_at_5_std": -0.048197, + "nauc_map_at_5_diff1": 0.485121, + "nauc_map_at_10_max": 0.420429, + "nauc_map_at_10_std": -0.04463, + "nauc_map_at_10_diff1": 0.481697, + "nauc_map_at_20_max": 0.421897, + "nauc_map_at_20_std": -0.044169, + "nauc_map_at_20_diff1": 0.481384, + "nauc_map_at_100_max": 0.421507, + "nauc_map_at_100_std": -0.043098, + "nauc_map_at_100_diff1": 0.480624, + "nauc_map_at_1000_max": 0.421425, + "nauc_map_at_1000_std": -0.043365, + "nauc_map_at_1000_diff1": 0.480767, + "nauc_recall_at_1_max": 0.390734, + "nauc_recall_at_1_std": -0.073628, + "nauc_recall_at_1_diff1": 0.525266, + "nauc_recall_at_3_max": 0.367714, + "nauc_recall_at_3_std": -0.040441, + "nauc_recall_at_3_diff1": 0.440824, + "nauc_recall_at_5_max": 0.407371, + "nauc_recall_at_5_std": 0.013209, + "nauc_recall_at_5_diff1": 0.40903, + "nauc_recall_at_10_max": 0.414391, + "nauc_recall_at_10_std": 0.046398, + "nauc_recall_at_10_diff1": 0.390279, + "nauc_recall_at_20_max": 0.426186, + "nauc_recall_at_20_std": 0.075967, + "nauc_recall_at_20_diff1": 0.395191, + "nauc_recall_at_100_max": 0.427787, + "nauc_recall_at_100_std": 0.229402, + "nauc_recall_at_100_diff1": 0.35792, + "nauc_recall_at_1000_max": 0.511867, + "nauc_recall_at_1000_std": 0.484588, + "nauc_recall_at_1000_diff1": 0.445413, + "nauc_precision_at_1_max": 0.436955, + "nauc_precision_at_1_std": -0.063223, + "nauc_precision_at_1_diff1": 0.502977, + "nauc_precision_at_3_max": 0.36338, + "nauc_precision_at_3_std": -0.011344, + "nauc_precision_at_3_diff1": 0.2875, + "nauc_precision_at_5_max": 0.352435, + "nauc_precision_at_5_std": 0.042995, + "nauc_precision_at_5_diff1": 0.189108, + "nauc_precision_at_10_max": 0.285426, + "nauc_precision_at_10_std": 0.062337, + "nauc_precision_at_10_diff1": 0.08479, + "nauc_precision_at_20_max": 0.224387, + "nauc_precision_at_20_std": 0.068915, + "nauc_precision_at_20_diff1": 0.011131, + "nauc_precision_at_100_max": 0.058134, + "nauc_precision_at_100_std": 0.08663, + "nauc_precision_at_100_diff1": -0.151896, + "nauc_precision_at_1000_max": -0.062514, + "nauc_precision_at_1000_std": 0.02125, + "nauc_precision_at_1000_diff1": -0.227137, + "nauc_mrr_at_1_max": 0.436955, + "nauc_mrr_at_1_std": -0.063223, + "nauc_mrr_at_1_diff1": 0.502977, + "nauc_mrr_at_3_max": 0.425101, + "nauc_mrr_at_3_std": -0.04279, + "nauc_mrr_at_3_diff1": 0.468043, + "nauc_mrr_at_5_max": 0.43491, + "nauc_mrr_at_5_std": -0.03438, + "nauc_mrr_at_5_diff1": 0.467007, + "nauc_mrr_at_10_max": 0.433958, + "nauc_mrr_at_10_std": -0.032291, + "nauc_mrr_at_10_diff1": 0.464335, + "nauc_mrr_at_20_max": 0.434202, + "nauc_mrr_at_20_std": -0.031385, + "nauc_mrr_at_20_diff1": 0.465901, + "nauc_mrr_at_100_max": 0.43413, + "nauc_mrr_at_100_std": -0.03166, + "nauc_mrr_at_100_diff1": 0.466534, + "nauc_mrr_at_1000_max": 0.4341, + "nauc_mrr_at_1000_std": -0.031923, + "nauc_mrr_at_1000_diff1": 0.466752, + "main_score": 0.52852, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 77.7349898815155, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ClimateFEVER.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ClimateFEVER.json new file mode 100644 index 0000000000..f75fc4a318 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ClimateFEVER.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380", + "task_name": "ClimateFEVER", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.32964, + "ndcg_at_3": 0.29781, + "ndcg_at_5": 0.31709, + "ndcg_at_10": 0.35824, + "ndcg_at_20": 0.38614, + "ndcg_at_100": 0.42517, + "ndcg_at_1000": 0.4554, + "map_at_1": 0.14792, + "map_at_3": 0.21891, + "map_at_5": 0.24034, + "map_at_10": 0.26193, + "map_at_20": 0.27284, + "map_at_100": 0.28085, + "map_at_1000": 0.28262, + "recall_at_1": 0.14792, + "recall_at_3": 0.27529, + "recall_at_5": 0.33483, + "recall_at_10": 0.42366, + "recall_at_20": 0.50228, + "recall_at_100": 0.65002, + "recall_at_1000": 0.81784, + "precision_at_1": 0.32964, + "precision_at_3": 0.22345, + "precision_at_5": 0.16769, + "precision_at_10": 0.11199, + "precision_at_20": 0.06805, + "precision_at_100": 0.0184, + "precision_at_1000": 0.00241, + "mrr_at_1": 0.329642, + "mrr_at_3": 0.433768, + "mrr_at_5": 0.452237, + "mrr_at_10": 0.465978, + "mrr_at_20": 0.469988, + "mrr_at_100": 0.472559, + "mrr_at_1000": 0.472871, + "nauc_ndcg_at_1_max": 0.332512, + "nauc_ndcg_at_1_std": 0.144187, + "nauc_ndcg_at_1_diff1": 0.318847, + "nauc_ndcg_at_3_max": 0.382535, + "nauc_ndcg_at_3_std": 0.177212, + "nauc_ndcg_at_3_diff1": 0.259802, + "nauc_ndcg_at_5_max": 0.392893, + "nauc_ndcg_at_5_std": 0.191711, + "nauc_ndcg_at_5_diff1": 0.245262, + "nauc_ndcg_at_10_max": 0.396292, + "nauc_ndcg_at_10_std": 0.221432, + "nauc_ndcg_at_10_diff1": 0.233805, + "nauc_ndcg_at_20_max": 0.39813, + "nauc_ndcg_at_20_std": 0.234779, + "nauc_ndcg_at_20_diff1": 0.228817, + "nauc_ndcg_at_100_max": 0.395158, + "nauc_ndcg_at_100_std": 0.24193, + "nauc_ndcg_at_100_diff1": 0.235801, + "nauc_ndcg_at_1000_max": 0.398025, + "nauc_ndcg_at_1000_std": 0.251414, + "nauc_ndcg_at_1000_diff1": 0.236025, + "nauc_map_at_1_max": 0.385185, + "nauc_map_at_1_std": 0.118127, + "nauc_map_at_1_diff1": 0.330093, + "nauc_map_at_3_max": 0.38911, + "nauc_map_at_3_std": 0.151554, + "nauc_map_at_3_diff1": 0.269008, + "nauc_map_at_5_max": 0.390208, + "nauc_map_at_5_std": 0.164933, + "nauc_map_at_5_diff1": 0.257221, + "nauc_map_at_10_max": 0.394928, + "nauc_map_at_10_std": 0.186614, + "nauc_map_at_10_diff1": 0.251139, + "nauc_map_at_20_max": 0.395647, + "nauc_map_at_20_std": 0.193656, + "nauc_map_at_20_diff1": 0.248607, + "nauc_map_at_100_max": 0.395527, + "nauc_map_at_100_std": 0.195971, + "nauc_map_at_100_diff1": 0.249788, + "nauc_map_at_1000_max": 0.395714, + "nauc_map_at_1000_std": 0.196714, + "nauc_map_at_1000_diff1": 0.249787, + "nauc_recall_at_1_max": 0.385185, + "nauc_recall_at_1_std": 0.118127, + "nauc_recall_at_1_diff1": 0.330093, + "nauc_recall_at_3_max": 0.378687, + "nauc_recall_at_3_std": 0.176703, + "nauc_recall_at_3_diff1": 0.21526, + "nauc_recall_at_5_max": 0.368755, + "nauc_recall_at_5_std": 0.191003, + "nauc_recall_at_5_diff1": 0.181673, + "nauc_recall_at_10_max": 0.359744, + "nauc_recall_at_10_std": 0.239363, + "nauc_recall_at_10_diff1": 0.149768, + "nauc_recall_at_20_max": 0.346768, + "nauc_recall_at_20_std": 0.262812, + "nauc_recall_at_20_diff1": 0.128325, + "nauc_recall_at_100_max": 0.316376, + "nauc_recall_at_100_std": 0.286145, + "nauc_recall_at_100_diff1": 0.136873, + "nauc_recall_at_1000_max": 0.322225, + "nauc_recall_at_1000_std": 0.401095, + "nauc_recall_at_1000_diff1": 0.096327, + "nauc_precision_at_1_max": 0.332512, + "nauc_precision_at_1_std": 0.144187, + "nauc_precision_at_1_diff1": 0.318847, + "nauc_precision_at_3_max": 0.32737, + "nauc_precision_at_3_std": 0.211732, + "nauc_precision_at_3_diff1": 0.181368, + "nauc_precision_at_5_max": 0.299012, + "nauc_precision_at_5_std": 0.231379, + "nauc_precision_at_5_diff1": 0.12591, + "nauc_precision_at_10_max": 0.251901, + "nauc_precision_at_10_std": 0.269281, + "nauc_precision_at_10_diff1": 0.07328, + "nauc_precision_at_20_max": 0.214506, + "nauc_precision_at_20_std": 0.268769, + "nauc_precision_at_20_diff1": 0.040889, + "nauc_precision_at_100_max": 0.120298, + "nauc_precision_at_100_std": 0.231278, + "nauc_precision_at_100_diff1": 0.028055, + "nauc_precision_at_1000_max": 0.013245, + "nauc_precision_at_1000_std": 0.19842, + "nauc_precision_at_1000_diff1": -0.043171, + "nauc_mrr_at_1_max": 0.332512, + "nauc_mrr_at_1_std": 0.144187, + "nauc_mrr_at_1_diff1": 0.318847, + "nauc_mrr_at_3_max": 0.366384, + "nauc_mrr_at_3_std": 0.193122, + "nauc_mrr_at_3_diff1": 0.279885, + "nauc_mrr_at_5_max": 0.371221, + "nauc_mrr_at_5_std": 0.198772, + "nauc_mrr_at_5_diff1": 0.275418, + "nauc_mrr_at_10_max": 0.369117, + "nauc_mrr_at_10_std": 0.200814, + "nauc_mrr_at_10_diff1": 0.274185, + "nauc_mrr_at_20_max": 0.369461, + "nauc_mrr_at_20_std": 0.201578, + "nauc_mrr_at_20_diff1": 0.274411, + "nauc_mrr_at_100_max": 0.368786, + "nauc_mrr_at_100_std": 0.200825, + "nauc_mrr_at_100_diff1": 0.274799, + "nauc_mrr_at_1000_max": 0.368716, + "nauc_mrr_at_1000_std": 0.200652, + "nauc_mrr_at_1000_diff1": 0.274922, + "main_score": 0.35824, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 4722.929804801941, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ClimateFEVERHardNegatives.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ClimateFEVERHardNegatives.json new file mode 100644 index 0000000000..58b920b28a --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ClimateFEVERHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "3a309e201f3c2c4b13bd4a367a8f37eee2ec1d21", + "task_name": "ClimateFEVERHardNegatives", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.315, + "ndcg_at_3": 0.29186, + "ndcg_at_5": 0.31501, + "ndcg_at_10": 0.35965, + "ndcg_at_20": 0.38787, + "ndcg_at_100": 0.43024, + "ndcg_at_1000": 0.46366, + "map_at_1": 0.1464, + "map_at_3": 0.21684, + "map_at_5": 0.24005, + "map_at_10": 0.26298, + "map_at_20": 0.27427, + "map_at_100": 0.28285, + "map_at_1000": 0.28486, + "recall_at_1": 0.1464, + "recall_at_3": 0.27333, + "recall_at_5": 0.33693, + "recall_at_10": 0.43337, + "recall_at_20": 0.51357, + "recall_at_100": 0.67243, + "recall_at_1000": 0.85697, + "precision_at_1": 0.315, + "precision_at_3": 0.218, + "precision_at_5": 0.1674, + "precision_at_10": 0.1143, + "precision_at_20": 0.0692, + "precision_at_100": 0.01911, + "precision_at_1000": 0.00254, + "mrr_at_1": 0.315, + "mrr_at_3": 0.4195, + "mrr_at_5": 0.43865, + "mrr_at_10": 0.45397, + "mrr_at_20": 0.458021, + "mrr_at_100": 0.46099, + "mrr_at_1000": 0.461307, + "nauc_ndcg_at_1_max": 0.335247, + "nauc_ndcg_at_1_std": 0.110561, + "nauc_ndcg_at_1_diff1": 0.283952, + "nauc_ndcg_at_3_max": 0.375279, + "nauc_ndcg_at_3_std": 0.125388, + "nauc_ndcg_at_3_diff1": 0.226296, + "nauc_ndcg_at_5_max": 0.378361, + "nauc_ndcg_at_5_std": 0.142427, + "nauc_ndcg_at_5_diff1": 0.208539, + "nauc_ndcg_at_10_max": 0.381085, + "nauc_ndcg_at_10_std": 0.159907, + "nauc_ndcg_at_10_diff1": 0.197218, + "nauc_ndcg_at_20_max": 0.381816, + "nauc_ndcg_at_20_std": 0.164231, + "nauc_ndcg_at_20_diff1": 0.194694, + "nauc_ndcg_at_100_max": 0.386306, + "nauc_ndcg_at_100_std": 0.165641, + "nauc_ndcg_at_100_diff1": 0.203539, + "nauc_ndcg_at_1000_max": 0.389576, + "nauc_ndcg_at_1000_std": 0.173646, + "nauc_ndcg_at_1000_diff1": 0.203372, + "nauc_map_at_1_max": 0.386827, + "nauc_map_at_1_std": 0.099142, + "nauc_map_at_1_diff1": 0.304198, + "nauc_map_at_3_max": 0.378638, + "nauc_map_at_3_std": 0.114207, + "nauc_map_at_3_diff1": 0.236005, + "nauc_map_at_5_max": 0.375106, + "nauc_map_at_5_std": 0.127374, + "nauc_map_at_5_diff1": 0.219623, + "nauc_map_at_10_max": 0.37902, + "nauc_map_at_10_std": 0.141918, + "nauc_map_at_10_diff1": 0.211741, + "nauc_map_at_20_max": 0.379448, + "nauc_map_at_20_std": 0.144659, + "nauc_map_at_20_diff1": 0.210645, + "nauc_map_at_100_max": 0.381477, + "nauc_map_at_100_std": 0.14484, + "nauc_map_at_100_diff1": 0.213058, + "nauc_map_at_1000_max": 0.381472, + "nauc_map_at_1000_std": 0.145537, + "nauc_map_at_1000_diff1": 0.212715, + "nauc_recall_at_1_max": 0.386827, + "nauc_recall_at_1_std": 0.099142, + "nauc_recall_at_1_diff1": 0.304198, + "nauc_recall_at_3_max": 0.36195, + "nauc_recall_at_3_std": 0.123158, + "nauc_recall_at_3_diff1": 0.189432, + "nauc_recall_at_5_max": 0.348954, + "nauc_recall_at_5_std": 0.146915, + "nauc_recall_at_5_diff1": 0.150979, + "nauc_recall_at_10_max": 0.337463, + "nauc_recall_at_10_std": 0.171421, + "nauc_recall_at_10_diff1": 0.118648, + "nauc_recall_at_20_max": 0.324533, + "nauc_recall_at_20_std": 0.176374, + "nauc_recall_at_20_diff1": 0.106812, + "nauc_recall_at_100_max": 0.316934, + "nauc_recall_at_100_std": 0.18232, + "nauc_recall_at_100_diff1": 0.117756, + "nauc_recall_at_1000_max": 0.337481, + "nauc_recall_at_1000_std": 0.283961, + "nauc_recall_at_1000_diff1": 0.075277, + "nauc_precision_at_1_max": 0.335247, + "nauc_precision_at_1_std": 0.110561, + "nauc_precision_at_1_diff1": 0.283952, + "nauc_precision_at_3_max": 0.322297, + "nauc_precision_at_3_std": 0.133728, + "nauc_precision_at_3_diff1": 0.141714, + "nauc_precision_at_5_max": 0.279082, + "nauc_precision_at_5_std": 0.156234, + "nauc_precision_at_5_diff1": 0.081042, + "nauc_precision_at_10_max": 0.235039, + "nauc_precision_at_10_std": 0.179248, + "nauc_precision_at_10_diff1": 0.031521, + "nauc_precision_at_20_max": 0.200877, + "nauc_precision_at_20_std": 0.166913, + "nauc_precision_at_20_diff1": 0.011898, + "nauc_precision_at_100_max": 0.11996, + "nauc_precision_at_100_std": 0.123527, + "nauc_precision_at_100_diff1": 0.003087, + "nauc_precision_at_1000_max": -0.000539, + "nauc_precision_at_1000_std": 0.101828, + "nauc_precision_at_1000_diff1": -0.070142, + "nauc_mrr_at_1_max": 0.335247, + "nauc_mrr_at_1_std": 0.110561, + "nauc_mrr_at_1_diff1": 0.283952, + "nauc_mrr_at_3_max": 0.364547, + "nauc_mrr_at_3_std": 0.130996, + "nauc_mrr_at_3_diff1": 0.248371, + "nauc_mrr_at_5_max": 0.370262, + "nauc_mrr_at_5_std": 0.138374, + "nauc_mrr_at_5_diff1": 0.244044, + "nauc_mrr_at_10_max": 0.369342, + "nauc_mrr_at_10_std": 0.139722, + "nauc_mrr_at_10_diff1": 0.244747, + "nauc_mrr_at_20_max": 0.368684, + "nauc_mrr_at_20_std": 0.139642, + "nauc_mrr_at_20_diff1": 0.244132, + "nauc_mrr_at_100_max": 0.368295, + "nauc_mrr_at_100_std": 0.139564, + "nauc_mrr_at_100_diff1": 0.244491, + "nauc_mrr_at_1000_max": 0.368221, + "nauc_mrr_at_1000_std": 0.139405, + "nauc_mrr_at_1000_diff1": 0.24472, + "main_score": 0.35965, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 71.25993585586548, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeFeedbackMT.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeFeedbackMT.json new file mode 100644 index 0000000000..c7f77bb2ac --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeFeedbackMT.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b0f12fa0c0dd67f59c95a5c33d02aeeb4c398c5f", + "task_name": "CodeFeedbackMT", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.39294, + "ndcg_at_3": 0.48009, + "ndcg_at_5": 0.50293, + "ndcg_at_10": 0.52543, + "ndcg_at_20": 0.54044, + "ndcg_at_100": 0.56215, + "ndcg_at_1000": 0.57578, + "map_at_1": 0.39294, + "map_at_3": 0.45869, + "map_at_5": 0.47139, + "map_at_10": 0.4807, + "map_at_20": 0.48484, + "map_at_100": 0.48786, + "map_at_1000": 0.48834, + "recall_at_1": 0.39294, + "recall_at_3": 0.54199, + "recall_at_5": 0.59735, + "recall_at_10": 0.66679, + "recall_at_20": 0.72599, + "recall_at_100": 0.84289, + "recall_at_1000": 0.95187, + "precision_at_1": 0.39294, + "precision_at_3": 0.18066, + "precision_at_5": 0.11947, + "precision_at_10": 0.06668, + "precision_at_20": 0.0363, + "precision_at_100": 0.00843, + "precision_at_1000": 0.00095, + "mrr_at_1": 0.392935, + "mrr_at_3": 0.458688, + "mrr_at_5": 0.47139, + "mrr_at_10": 0.480697, + "mrr_at_20": 0.484844, + "mrr_at_100": 0.487856, + "mrr_at_1000": 0.488338, + "nauc_ndcg_at_1_max": 0.152177, + "nauc_ndcg_at_1_std": -0.233054, + "nauc_ndcg_at_1_diff1": 0.609481, + "nauc_ndcg_at_3_max": 0.169447, + "nauc_ndcg_at_3_std": -0.24746, + "nauc_ndcg_at_3_diff1": 0.569266, + "nauc_ndcg_at_5_max": 0.17587, + "nauc_ndcg_at_5_std": -0.241671, + "nauc_ndcg_at_5_diff1": 0.562268, + "nauc_ndcg_at_10_max": 0.180132, + "nauc_ndcg_at_10_std": -0.236924, + "nauc_ndcg_at_10_diff1": 0.558113, + "nauc_ndcg_at_20_max": 0.180248, + "nauc_ndcg_at_20_std": -0.231642, + "nauc_ndcg_at_20_diff1": 0.554808, + "nauc_ndcg_at_100_max": 0.18118, + "nauc_ndcg_at_100_std": -0.221861, + "nauc_ndcg_at_100_diff1": 0.555037, + "nauc_ndcg_at_1000_max": 0.180096, + "nauc_ndcg_at_1000_std": -0.22332, + "nauc_ndcg_at_1000_diff1": 0.560694, + "nauc_map_at_1_max": 0.152177, + "nauc_map_at_1_std": -0.233054, + "nauc_map_at_1_diff1": 0.609481, + "nauc_map_at_3_max": 0.165411, + "nauc_map_at_3_std": -0.243484, + "nauc_map_at_3_diff1": 0.578973, + "nauc_map_at_5_max": 0.168846, + "nauc_map_at_5_std": -0.240273, + "nauc_map_at_5_diff1": 0.575259, + "nauc_map_at_10_max": 0.170545, + "nauc_map_at_10_std": -0.23834, + "nauc_map_at_10_diff1": 0.573909, + "nauc_map_at_20_max": 0.170494, + "nauc_map_at_20_std": -0.236962, + "nauc_map_at_20_diff1": 0.573095, + "nauc_map_at_100_max": 0.170624, + "nauc_map_at_100_std": -0.235719, + "nauc_map_at_100_diff1": 0.573248, + "nauc_map_at_1000_max": 0.170615, + "nauc_map_at_1000_std": -0.235714, + "nauc_map_at_1000_diff1": 0.573434, + "nauc_recall_at_1_max": 0.152177, + "nauc_recall_at_1_std": -0.233054, + "nauc_recall_at_1_diff1": 0.609481, + "nauc_recall_at_3_max": 0.181534, + "nauc_recall_at_3_std": -0.259647, + "nauc_recall_at_3_diff1": 0.540054, + "nauc_recall_at_5_max": 0.199035, + "nauc_recall_at_5_std": -0.245392, + "nauc_recall_at_5_diff1": 0.520166, + "nauc_recall_at_10_max": 0.216936, + "nauc_recall_at_10_std": -0.228859, + "nauc_recall_at_10_diff1": 0.498635, + "nauc_recall_at_20_max": 0.223317, + "nauc_recall_at_20_std": -0.200369, + "nauc_recall_at_20_diff1": 0.473054, + "nauc_recall_at_100_max": 0.256757, + "nauc_recall_at_100_std": -0.078794, + "nauc_recall_at_100_diff1": 0.426147, + "nauc_recall_at_1000_max": 0.350367, + "nauc_recall_at_1000_std": 0.12729, + "nauc_recall_at_1000_diff1": 0.428092, + "nauc_precision_at_1_max": 0.152177, + "nauc_precision_at_1_std": -0.233054, + "nauc_precision_at_1_diff1": 0.609481, + "nauc_precision_at_3_max": 0.181534, + "nauc_precision_at_3_std": -0.259647, + "nauc_precision_at_3_diff1": 0.540054, + "nauc_precision_at_5_max": 0.199035, + "nauc_precision_at_5_std": -0.245392, + "nauc_precision_at_5_diff1": 0.520166, + "nauc_precision_at_10_max": 0.216936, + "nauc_precision_at_10_std": -0.228859, + "nauc_precision_at_10_diff1": 0.498635, + "nauc_precision_at_20_max": 0.223317, + "nauc_precision_at_20_std": -0.200369, + "nauc_precision_at_20_diff1": 0.473054, + "nauc_precision_at_100_max": 0.256757, + "nauc_precision_at_100_std": -0.078794, + "nauc_precision_at_100_diff1": 0.426147, + "nauc_precision_at_1000_max": 0.350367, + "nauc_precision_at_1000_std": 0.12729, + "nauc_precision_at_1000_diff1": 0.428092, + "nauc_mrr_at_1_max": 0.152177, + "nauc_mrr_at_1_std": -0.233054, + "nauc_mrr_at_1_diff1": 0.609481, + "nauc_mrr_at_3_max": 0.165411, + "nauc_mrr_at_3_std": -0.243484, + "nauc_mrr_at_3_diff1": 0.578973, + "nauc_mrr_at_5_max": 0.168846, + "nauc_mrr_at_5_std": -0.240273, + "nauc_mrr_at_5_diff1": 0.575259, + "nauc_mrr_at_10_max": 0.170545, + "nauc_mrr_at_10_std": -0.23834, + "nauc_mrr_at_10_diff1": 0.573909, + "nauc_mrr_at_20_max": 0.170494, + "nauc_mrr_at_20_std": -0.236962, + "nauc_mrr_at_20_diff1": 0.573095, + "nauc_mrr_at_100_max": 0.170624, + "nauc_mrr_at_100_std": -0.235719, + "nauc_mrr_at_100_diff1": 0.573248, + "nauc_mrr_at_1000_max": 0.170615, + "nauc_mrr_at_1000_std": -0.235714, + "nauc_mrr_at_1000_diff1": 0.573434, + "main_score": 0.52543, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 248.71036863327026, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeFeedbackST.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeFeedbackST.json new file mode 100644 index 0000000000..e473a6d373 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeFeedbackST.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "d213819e87aab9010628da8b73ab4eb337c89340", + "task_name": "CodeFeedbackST", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.61512, + "ndcg_at_3": 0.73246, + "ndcg_at_5": 0.7568, + "ndcg_at_10": 0.77181, + "ndcg_at_20": 0.77882, + "ndcg_at_100": 0.78552, + "ndcg_at_1000": 0.78745, + "map_at_1": 0.61512, + "map_at_3": 0.70422, + "map_at_5": 0.71779, + "map_at_10": 0.72412, + "map_at_20": 0.72608, + "map_at_100": 0.72704, + "map_at_1000": 0.72712, + "recall_at_1": 0.61512, + "recall_at_3": 0.81393, + "recall_at_5": 0.87277, + "recall_at_10": 0.91848, + "recall_at_20": 0.94595, + "recall_at_100": 0.98151, + "recall_at_1000": 0.99639, + "precision_at_1": 0.61512, + "precision_at_3": 0.27131, + "precision_at_5": 0.17455, + "precision_at_10": 0.09185, + "precision_at_20": 0.0473, + "precision_at_100": 0.00982, + "precision_at_1000": 0.001, + "mrr_at_1": 0.613844, + "mrr_at_3": 0.703853, + "mrr_at_5": 0.717234, + "mrr_at_10": 0.72362, + "mrr_at_20": 0.725584, + "mrr_at_100": 0.726542, + "mrr_at_1000": 0.726621, + "nauc_ndcg_at_1_max": 0.022901, + "nauc_ndcg_at_1_std": -0.305979, + "nauc_ndcg_at_1_diff1": 0.754009, + "nauc_ndcg_at_3_max": 0.074759, + "nauc_ndcg_at_3_std": -0.357657, + "nauc_ndcg_at_3_diff1": 0.706416, + "nauc_ndcg_at_5_max": 0.077382, + "nauc_ndcg_at_5_std": -0.359503, + "nauc_ndcg_at_5_diff1": 0.703346, + "nauc_ndcg_at_10_max": 0.07163, + "nauc_ndcg_at_10_std": -0.350826, + "nauc_ndcg_at_10_diff1": 0.706743, + "nauc_ndcg_at_20_max": 0.069564, + "nauc_ndcg_at_20_std": -0.345682, + "nauc_ndcg_at_20_diff1": 0.711152, + "nauc_ndcg_at_100_max": 0.064848, + "nauc_ndcg_at_100_std": -0.340201, + "nauc_ndcg_at_100_diff1": 0.714547, + "nauc_ndcg_at_1000_max": 0.063639, + "nauc_ndcg_at_1000_std": -0.340001, + "nauc_ndcg_at_1000_diff1": 0.715708, + "nauc_map_at_1_max": 0.022901, + "nauc_map_at_1_std": -0.305979, + "nauc_map_at_1_diff1": 0.754009, + "nauc_map_at_3_max": 0.059607, + "nauc_map_at_3_std": -0.342802, + "nauc_map_at_3_diff1": 0.719492, + "nauc_map_at_5_max": 0.060196, + "nauc_map_at_5_std": -0.343016, + "nauc_map_at_5_diff1": 0.718632, + "nauc_map_at_10_max": 0.057773, + "nauc_map_at_10_std": -0.33968, + "nauc_map_at_10_diff1": 0.720094, + "nauc_map_at_20_max": 0.057169, + "nauc_map_at_20_std": -0.338423, + "nauc_map_at_20_diff1": 0.72121, + "nauc_map_at_100_max": 0.056585, + "nauc_map_at_100_std": -0.337746, + "nauc_map_at_100_diff1": 0.721636, + "nauc_map_at_1000_max": 0.056563, + "nauc_map_at_1000_std": -0.337705, + "nauc_map_at_1000_diff1": 0.721673, + "nauc_recall_at_1_max": 0.022901, + "nauc_recall_at_1_std": -0.305979, + "nauc_recall_at_1_diff1": 0.754009, + "nauc_recall_at_3_max": 0.136702, + "nauc_recall_at_3_std": -0.418239, + "nauc_recall_at_3_diff1": 0.653453, + "nauc_recall_at_5_max": 0.17255, + "nauc_recall_at_5_std": -0.450422, + "nauc_recall_at_5_diff1": 0.619095, + "nauc_recall_at_10_max": 0.179319, + "nauc_recall_at_10_std": -0.43198, + "nauc_recall_at_10_diff1": 0.602132, + "nauc_recall_at_20_max": 0.206302, + "nauc_recall_at_20_std": -0.402791, + "nauc_recall_at_20_diff1": 0.609094, + "nauc_recall_at_100_max": 0.223274, + "nauc_recall_at_100_std": -0.23129, + "nauc_recall_at_100_diff1": 0.589586, + "nauc_recall_at_1000_max": 0.34741, + "nauc_recall_at_1000_std": 0.178023, + "nauc_recall_at_1000_diff1": 0.523168, + "nauc_precision_at_1_max": 0.022901, + "nauc_precision_at_1_std": -0.305979, + "nauc_precision_at_1_diff1": 0.754009, + "nauc_precision_at_3_max": 0.136702, + "nauc_precision_at_3_std": -0.418239, + "nauc_precision_at_3_diff1": 0.653453, + "nauc_precision_at_5_max": 0.17255, + "nauc_precision_at_5_std": -0.450422, + "nauc_precision_at_5_diff1": 0.619095, + "nauc_precision_at_10_max": 0.179319, + "nauc_precision_at_10_std": -0.43198, + "nauc_precision_at_10_diff1": 0.602132, + "nauc_precision_at_20_max": 0.206302, + "nauc_precision_at_20_std": -0.402791, + "nauc_precision_at_20_diff1": 0.609094, + "nauc_precision_at_100_max": 0.223274, + "nauc_precision_at_100_std": -0.23129, + "nauc_precision_at_100_diff1": 0.589586, + "nauc_precision_at_1000_max": 0.34741, + "nauc_precision_at_1000_std": 0.178023, + "nauc_precision_at_1000_diff1": 0.523168, + "nauc_mrr_at_1_max": 0.015169, + "nauc_mrr_at_1_std": -0.304571, + "nauc_mrr_at_1_diff1": 0.756201, + "nauc_mrr_at_3_max": 0.054451, + "nauc_mrr_at_3_std": -0.341686, + "nauc_mrr_at_3_diff1": 0.720847, + "nauc_mrr_at_5_max": 0.055096, + "nauc_mrr_at_5_std": -0.341937, + "nauc_mrr_at_5_diff1": 0.720259, + "nauc_mrr_at_10_max": 0.052553, + "nauc_mrr_at_10_std": -0.338466, + "nauc_mrr_at_10_diff1": 0.721691, + "nauc_mrr_at_20_max": 0.051869, + "nauc_mrr_at_20_std": -0.337264, + "nauc_mrr_at_20_diff1": 0.722833, + "nauc_mrr_at_100_max": 0.051254, + "nauc_mrr_at_100_std": -0.336573, + "nauc_mrr_at_100_diff1": 0.723262, + "nauc_mrr_at_1000_max": 0.051231, + "nauc_mrr_at_1000_std": -0.336533, + "nauc_mrr_at_1000_diff1": 0.723301, + "main_score": 0.77181, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 474.1298174858093, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeSearchNetCCRetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeSearchNetCCRetrieval.json new file mode 100644 index 0000000000..89e48efda0 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeSearchNetCCRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "6e1effa2c03723c5fde48ee912b5ee08d4f211e8", + "task_name": "CodeSearchNetCCRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.37076, + "ndcg_at_3": 0.46325, + "ndcg_at_5": 0.48886, + "ndcg_at_10": 0.51202, + "ndcg_at_20": 0.52731, + "ndcg_at_100": 0.54879, + "ndcg_at_1000": 0.56187, + "map_at_1": 0.37076, + "map_at_3": 0.4406, + "map_at_5": 0.45479, + "map_at_10": 0.46449, + "map_at_20": 0.4687, + "map_at_100": 0.47163, + "map_at_1000": 0.47209, + "recall_at_1": 0.37076, + "recall_at_3": 0.52876, + "recall_at_5": 0.59103, + "recall_at_10": 0.66188, + "recall_at_20": 0.72228, + "recall_at_100": 0.83852, + "recall_at_1000": 0.94336, + "precision_at_1": 0.37076, + "precision_at_3": 0.17625, + "precision_at_5": 0.11821, + "precision_at_10": 0.06619, + "precision_at_20": 0.03611, + "precision_at_100": 0.00839, + "precision_at_1000": 0.00094, + "mrr_at_1": 0.37076, + "mrr_at_3": 0.440609, + "mrr_at_5": 0.4548, + "mrr_at_10": 0.464506, + "mrr_at_20": 0.46871, + "mrr_at_100": 0.471648, + "mrr_at_1000": 0.472109, + "nauc_ndcg_at_1_max": 0.285558, + "nauc_ndcg_at_1_std": -0.071786, + "nauc_ndcg_at_1_diff1": 0.540111, + "nauc_ndcg_at_3_max": 0.295146, + "nauc_ndcg_at_3_std": -0.065911, + "nauc_ndcg_at_3_diff1": 0.498294, + "nauc_ndcg_at_5_max": 0.293276, + "nauc_ndcg_at_5_std": -0.062188, + "nauc_ndcg_at_5_diff1": 0.492752, + "nauc_ndcg_at_10_max": 0.293073, + "nauc_ndcg_at_10_std": -0.054568, + "nauc_ndcg_at_10_diff1": 0.490641, + "nauc_ndcg_at_20_max": 0.293201, + "nauc_ndcg_at_20_std": -0.047894, + "nauc_ndcg_at_20_diff1": 0.488089, + "nauc_ndcg_at_100_max": 0.291595, + "nauc_ndcg_at_100_std": -0.04451, + "nauc_ndcg_at_100_diff1": 0.489606, + "nauc_ndcg_at_1000_max": 0.292695, + "nauc_ndcg_at_1000_std": -0.045733, + "nauc_ndcg_at_1000_diff1": 0.493621, + "nauc_map_at_1_max": 0.285558, + "nauc_map_at_1_std": -0.071786, + "nauc_map_at_1_diff1": 0.540111, + "nauc_map_at_3_max": 0.293025, + "nauc_map_at_3_std": -0.067903, + "nauc_map_at_3_diff1": 0.50861, + "nauc_map_at_5_max": 0.291945, + "nauc_map_at_5_std": -0.066048, + "nauc_map_at_5_diff1": 0.505719, + "nauc_map_at_10_max": 0.291894, + "nauc_map_at_10_std": -0.062969, + "nauc_map_at_10_diff1": 0.505059, + "nauc_map_at_20_max": 0.291901, + "nauc_map_at_20_std": -0.061262, + "nauc_map_at_20_diff1": 0.504439, + "nauc_map_at_100_max": 0.291599, + "nauc_map_at_100_std": -0.060926, + "nauc_map_at_100_diff1": 0.504729, + "nauc_map_at_1000_max": 0.291624, + "nauc_map_at_1000_std": -0.060948, + "nauc_map_at_1000_diff1": 0.504872, + "nauc_recall_at_1_max": 0.285558, + "nauc_recall_at_1_std": -0.071786, + "nauc_recall_at_1_diff1": 0.540111, + "nauc_recall_at_3_max": 0.301367, + "nauc_recall_at_3_std": -0.059742, + "nauc_recall_at_3_diff1": 0.467538, + "nauc_recall_at_5_max": 0.297099, + "nauc_recall_at_5_std": -0.049029, + "nauc_recall_at_5_diff1": 0.451147, + "nauc_recall_at_10_max": 0.296581, + "nauc_recall_at_10_std": -0.020579, + "nauc_recall_at_10_diff1": 0.438074, + "nauc_recall_at_20_max": 0.297947, + "nauc_recall_at_20_std": 0.018551, + "nauc_recall_at_20_diff1": 0.417633, + "nauc_recall_at_100_max": 0.286865, + "nauc_recall_at_100_std": 0.092923, + "nauc_recall_at_100_diff1": 0.392874, + "nauc_recall_at_1000_max": 0.32537, + "nauc_recall_at_1000_std": 0.264071, + "nauc_recall_at_1000_diff1": 0.391202, + "nauc_precision_at_1_max": 0.285558, + "nauc_precision_at_1_std": -0.071786, + "nauc_precision_at_1_diff1": 0.540111, + "nauc_precision_at_3_max": 0.301367, + "nauc_precision_at_3_std": -0.059742, + "nauc_precision_at_3_diff1": 0.467538, + "nauc_precision_at_5_max": 0.297099, + "nauc_precision_at_5_std": -0.049029, + "nauc_precision_at_5_diff1": 0.451147, + "nauc_precision_at_10_max": 0.296581, + "nauc_precision_at_10_std": -0.020579, + "nauc_precision_at_10_diff1": 0.438074, + "nauc_precision_at_20_max": 0.297947, + "nauc_precision_at_20_std": 0.018551, + "nauc_precision_at_20_diff1": 0.417633, + "nauc_precision_at_100_max": 0.286865, + "nauc_precision_at_100_std": 0.092923, + "nauc_precision_at_100_diff1": 0.392874, + "nauc_precision_at_1000_max": 0.32537, + "nauc_precision_at_1000_std": 0.264071, + "nauc_precision_at_1000_diff1": 0.391202, + "nauc_mrr_at_1_max": 0.285462, + "nauc_mrr_at_1_std": -0.071786, + "nauc_mrr_at_1_diff1": 0.540111, + "nauc_mrr_at_3_max": 0.292823, + "nauc_mrr_at_3_std": -0.067972, + "nauc_mrr_at_3_diff1": 0.508534, + "nauc_mrr_at_5_max": 0.291835, + "nauc_mrr_at_5_std": -0.066071, + "nauc_mrr_at_5_diff1": 0.505686, + "nauc_mrr_at_10_max": 0.291774, + "nauc_mrr_at_10_std": -0.063007, + "nauc_mrr_at_10_diff1": 0.505008, + "nauc_mrr_at_20_max": 0.291796, + "nauc_mrr_at_20_std": -0.061283, + "nauc_mrr_at_20_diff1": 0.504393, + "nauc_mrr_at_100_max": 0.291488, + "nauc_mrr_at_100_std": -0.060948, + "nauc_mrr_at_100_diff1": 0.504692, + "nauc_mrr_at_1000_max": 0.291513, + "nauc_mrr_at_1000_std": -0.06097, + "nauc_mrr_at_1000_diff1": 0.504835, + "main_score": 0.51202, + "hf_subset": "python", + "languages": [ + "python-Code" + ] + }, + { + "ndcg_at_1": 0.37891, + "ndcg_at_3": 0.474, + "ndcg_at_5": 0.49778, + "ndcg_at_10": 0.52021, + "ndcg_at_20": 0.53429, + "ndcg_at_100": 0.5547, + "ndcg_at_1000": 0.56803, + "map_at_1": 0.37891, + "map_at_3": 0.45123, + "map_at_5": 0.46449, + "map_at_10": 0.47383, + "map_at_20": 0.47766, + "map_at_100": 0.48041, + "map_at_1000": 0.48087, + "recall_at_1": 0.37891, + "recall_at_3": 0.53965, + "recall_at_5": 0.59708, + "recall_at_10": 0.66606, + "recall_at_20": 0.72197, + "recall_at_100": 0.83318, + "recall_at_1000": 0.94044, + "precision_at_1": 0.37891, + "precision_at_3": 0.17988, + "precision_at_5": 0.11942, + "precision_at_10": 0.06661, + "precision_at_20": 0.0361, + "precision_at_100": 0.00833, + "precision_at_1000": 0.00094, + "mrr_at_1": 0.378001, + "mrr_at_3": 0.450674, + "mrr_at_5": 0.464013, + "mrr_at_10": 0.473349, + "mrr_at_20": 0.477179, + "mrr_at_100": 0.479929, + "mrr_at_1000": 0.480387, + "nauc_ndcg_at_1_max": 0.27369, + "nauc_ndcg_at_1_std": -0.083093, + "nauc_ndcg_at_1_diff1": 0.547282, + "nauc_ndcg_at_3_max": 0.303905, + "nauc_ndcg_at_3_std": -0.073156, + "nauc_ndcg_at_3_diff1": 0.50112, + "nauc_ndcg_at_5_max": 0.306877, + "nauc_ndcg_at_5_std": -0.066117, + "nauc_ndcg_at_5_diff1": 0.493687, + "nauc_ndcg_at_10_max": 0.305179, + "nauc_ndcg_at_10_std": -0.061239, + "nauc_ndcg_at_10_diff1": 0.493583, + "nauc_ndcg_at_20_max": 0.301456, + "nauc_ndcg_at_20_std": -0.055329, + "nauc_ndcg_at_20_diff1": 0.491342, + "nauc_ndcg_at_100_max": 0.299185, + "nauc_ndcg_at_100_std": -0.052962, + "nauc_ndcg_at_100_diff1": 0.492905, + "nauc_ndcg_at_1000_max": 0.297932, + "nauc_ndcg_at_1000_std": -0.056763, + "nauc_ndcg_at_1000_diff1": 0.49728, + "nauc_map_at_1_max": 0.27369, + "nauc_map_at_1_std": -0.083093, + "nauc_map_at_1_diff1": 0.547282, + "nauc_map_at_3_max": 0.297279, + "nauc_map_at_3_std": -0.075593, + "nauc_map_at_3_diff1": 0.511991, + "nauc_map_at_5_max": 0.298988, + "nauc_map_at_5_std": -0.071785, + "nauc_map_at_5_diff1": 0.50823, + "nauc_map_at_10_max": 0.298196, + "nauc_map_at_10_std": -0.069947, + "nauc_map_at_10_diff1": 0.508619, + "nauc_map_at_20_max": 0.297124, + "nauc_map_at_20_std": -0.068355, + "nauc_map_at_20_diff1": 0.508012, + "nauc_map_at_100_max": 0.29679, + "nauc_map_at_100_std": -0.068086, + "nauc_map_at_100_diff1": 0.508153, + "nauc_map_at_1000_max": 0.29674, + "nauc_map_at_1000_std": -0.068184, + "nauc_map_at_1000_diff1": 0.508311, + "nauc_recall_at_1_max": 0.27369, + "nauc_recall_at_1_std": -0.083093, + "nauc_recall_at_1_diff1": 0.547282, + "nauc_recall_at_3_max": 0.323458, + "nauc_recall_at_3_std": -0.06584, + "nauc_recall_at_3_diff1": 0.468676, + "nauc_recall_at_5_max": 0.331622, + "nauc_recall_at_5_std": -0.047137, + "nauc_recall_at_5_diff1": 0.44665, + "nauc_recall_at_10_max": 0.328853, + "nauc_recall_at_10_std": -0.026858, + "nauc_recall_at_10_diff1": 0.438523, + "nauc_recall_at_20_max": 0.313746, + "nauc_recall_at_20_std": 0.007202, + "nauc_recall_at_20_diff1": 0.42058, + "nauc_recall_at_100_max": 0.297375, + "nauc_recall_at_100_std": 0.063197, + "nauc_recall_at_100_diff1": 0.403039, + "nauc_recall_at_1000_max": 0.250451, + "nauc_recall_at_1000_std": 0.099517, + "nauc_recall_at_1000_diff1": 0.423315, + "nauc_precision_at_1_max": 0.27369, + "nauc_precision_at_1_std": -0.083093, + "nauc_precision_at_1_diff1": 0.547282, + "nauc_precision_at_3_max": 0.323458, + "nauc_precision_at_3_std": -0.06584, + "nauc_precision_at_3_diff1": 0.468676, + "nauc_precision_at_5_max": 0.331622, + "nauc_precision_at_5_std": -0.047137, + "nauc_precision_at_5_diff1": 0.44665, + "nauc_precision_at_10_max": 0.328853, + "nauc_precision_at_10_std": -0.026858, + "nauc_precision_at_10_diff1": 0.438523, + "nauc_precision_at_20_max": 0.313746, + "nauc_precision_at_20_std": 0.007202, + "nauc_precision_at_20_diff1": 0.42058, + "nauc_precision_at_100_max": 0.297375, + "nauc_precision_at_100_std": 0.063197, + "nauc_precision_at_100_diff1": 0.403039, + "nauc_precision_at_1000_max": 0.250451, + "nauc_precision_at_1000_std": 0.099517, + "nauc_precision_at_1000_diff1": 0.423315, + "nauc_mrr_at_1_max": 0.274246, + "nauc_mrr_at_1_std": -0.084196, + "nauc_mrr_at_1_diff1": 0.55006, + "nauc_mrr_at_3_max": 0.297206, + "nauc_mrr_at_3_std": -0.076519, + "nauc_mrr_at_3_diff1": 0.513588, + "nauc_mrr_at_5_max": 0.299191, + "nauc_mrr_at_5_std": -0.072346, + "nauc_mrr_at_5_diff1": 0.509767, + "nauc_mrr_at_10_max": 0.298402, + "nauc_mrr_at_10_std": -0.070516, + "nauc_mrr_at_10_diff1": 0.510179, + "nauc_mrr_at_20_max": 0.297331, + "nauc_mrr_at_20_std": -0.068928, + "nauc_mrr_at_20_diff1": 0.509583, + "nauc_mrr_at_100_max": 0.296998, + "nauc_mrr_at_100_std": -0.068662, + "nauc_mrr_at_100_diff1": 0.509733, + "nauc_mrr_at_1000_max": 0.296948, + "nauc_mrr_at_1000_std": -0.068761, + "nauc_mrr_at_1000_diff1": 0.509893, + "main_score": 0.52021, + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ] + }, + { + "ndcg_at_1": 0.28983, + "ndcg_at_3": 0.37003, + "ndcg_at_5": 0.39226, + "ndcg_at_10": 0.41414, + "ndcg_at_20": 0.43021, + "ndcg_at_100": 0.45439, + "ndcg_at_1000": 0.47171, + "map_at_1": 0.28983, + "map_at_3": 0.35039, + "map_at_5": 0.36273, + "map_at_10": 0.37184, + "map_at_20": 0.3763, + "map_at_100": 0.37956, + "map_at_1000": 0.38014, + "recall_at_1": 0.28983, + "recall_at_3": 0.42687, + "recall_at_5": 0.48079, + "recall_at_10": 0.54802, + "recall_at_20": 0.61118, + "recall_at_100": 0.7428, + "recall_at_1000": 0.88316, + "precision_at_1": 0.28983, + "precision_at_3": 0.14229, + "precision_at_5": 0.09616, + "precision_at_10": 0.0548, + "precision_at_20": 0.03056, + "precision_at_100": 0.00743, + "precision_at_1000": 0.00088, + "mrr_at_1": 0.289953, + "mrr_at_3": 0.350612, + "mrr_at_5": 0.36293, + "mrr_at_10": 0.37203, + "mrr_at_20": 0.376464, + "mrr_at_100": 0.379728, + "mrr_at_1000": 0.380311, + "nauc_ndcg_at_1_max": 0.262391, + "nauc_ndcg_at_1_std": -0.004428, + "nauc_ndcg_at_1_diff1": 0.494316, + "nauc_ndcg_at_3_max": 0.248334, + "nauc_ndcg_at_3_std": -0.014121, + "nauc_ndcg_at_3_diff1": 0.440258, + "nauc_ndcg_at_5_max": 0.244737, + "nauc_ndcg_at_5_std": -0.011028, + "nauc_ndcg_at_5_diff1": 0.430201, + "nauc_ndcg_at_10_max": 0.240683, + "nauc_ndcg_at_10_std": -0.003271, + "nauc_ndcg_at_10_diff1": 0.425497, + "nauc_ndcg_at_20_max": 0.237104, + "nauc_ndcg_at_20_std": -0.002829, + "nauc_ndcg_at_20_diff1": 0.422819, + "nauc_ndcg_at_100_max": 0.236406, + "nauc_ndcg_at_100_std": 0.004846, + "nauc_ndcg_at_100_diff1": 0.423077, + "nauc_ndcg_at_1000_max": 0.238197, + "nauc_ndcg_at_1000_std": 0.006103, + "nauc_ndcg_at_1000_diff1": 0.427462, + "nauc_map_at_1_max": 0.262391, + "nauc_map_at_1_std": -0.004428, + "nauc_map_at_1_diff1": 0.494316, + "nauc_map_at_3_max": 0.251741, + "nauc_map_at_3_std": -0.01221, + "nauc_map_at_3_diff1": 0.452482, + "nauc_map_at_5_max": 0.249627, + "nauc_map_at_5_std": -0.010616, + "nauc_map_at_5_diff1": 0.446962, + "nauc_map_at_10_max": 0.247961, + "nauc_map_at_10_std": -0.00757, + "nauc_map_at_10_diff1": 0.445177, + "nauc_map_at_20_max": 0.247038, + "nauc_map_at_20_std": -0.007495, + "nauc_map_at_20_diff1": 0.444564, + "nauc_map_at_100_max": 0.24693, + "nauc_map_at_100_std": -0.006436, + "nauc_map_at_100_diff1": 0.444646, + "nauc_map_at_1000_max": 0.246982, + "nauc_map_at_1000_std": -0.006349, + "nauc_map_at_1000_diff1": 0.444808, + "nauc_recall_at_1_max": 0.262391, + "nauc_recall_at_1_std": -0.004428, + "nauc_recall_at_1_diff1": 0.494316, + "nauc_recall_at_3_max": 0.238724, + "nauc_recall_at_3_std": -0.019336, + "nauc_recall_at_3_diff1": 0.406128, + "nauc_recall_at_5_max": 0.230409, + "nauc_recall_at_5_std": -0.011438, + "nauc_recall_at_5_diff1": 0.381183, + "nauc_recall_at_10_max": 0.21687, + "nauc_recall_at_10_std": 0.013981, + "nauc_recall_at_10_diff1": 0.363209, + "nauc_recall_at_20_max": 0.199495, + "nauc_recall_at_20_std": 0.017509, + "nauc_recall_at_20_diff1": 0.346551, + "nauc_recall_at_100_max": 0.182568, + "nauc_recall_at_100_std": 0.081249, + "nauc_recall_at_100_diff1": 0.32378, + "nauc_recall_at_1000_max": 0.170625, + "nauc_recall_at_1000_std": 0.172181, + "nauc_recall_at_1000_diff1": 0.320204, + "nauc_precision_at_1_max": 0.262391, + "nauc_precision_at_1_std": -0.004428, + "nauc_precision_at_1_diff1": 0.494316, + "nauc_precision_at_3_max": 0.238724, + "nauc_precision_at_3_std": -0.019336, + "nauc_precision_at_3_diff1": 0.406128, + "nauc_precision_at_5_max": 0.230409, + "nauc_precision_at_5_std": -0.011438, + "nauc_precision_at_5_diff1": 0.381183, + "nauc_precision_at_10_max": 0.21687, + "nauc_precision_at_10_std": 0.013981, + "nauc_precision_at_10_diff1": 0.363209, + "nauc_precision_at_20_max": 0.199495, + "nauc_precision_at_20_std": 0.017509, + "nauc_precision_at_20_diff1": 0.346551, + "nauc_precision_at_100_max": 0.182568, + "nauc_precision_at_100_std": 0.081249, + "nauc_precision_at_100_diff1": 0.32378, + "nauc_precision_at_1000_max": 0.170625, + "nauc_precision_at_1000_std": 0.172181, + "nauc_precision_at_1000_diff1": 0.320204, + "nauc_mrr_at_1_max": 0.262673, + "nauc_mrr_at_1_std": -0.003462, + "nauc_mrr_at_1_diff1": 0.493853, + "nauc_mrr_at_3_max": 0.251671, + "nauc_mrr_at_3_std": -0.011609, + "nauc_mrr_at_3_diff1": 0.45212, + "nauc_mrr_at_5_max": 0.249578, + "nauc_mrr_at_5_std": -0.010022, + "nauc_mrr_at_5_diff1": 0.446432, + "nauc_mrr_at_10_max": 0.248023, + "nauc_mrr_at_10_std": -0.007157, + "nauc_mrr_at_10_diff1": 0.444701, + "nauc_mrr_at_20_max": 0.246978, + "nauc_mrr_at_20_std": -0.007054, + "nauc_mrr_at_20_diff1": 0.444126, + "nauc_mrr_at_100_max": 0.246928, + "nauc_mrr_at_100_std": -0.005938, + "nauc_mrr_at_100_diff1": 0.444211, + "nauc_mrr_at_1000_max": 0.246978, + "nauc_mrr_at_1000_std": -0.005854, + "nauc_mrr_at_1000_diff1": 0.444371, + "main_score": 0.41414, + "hf_subset": "go", + "languages": [ + "go-Code" + ] + }, + { + "ndcg_at_1": 0.38699, + "ndcg_at_3": 0.46779, + "ndcg_at_5": 0.48826, + "ndcg_at_10": 0.50928, + "ndcg_at_20": 0.52243, + "ndcg_at_100": 0.54319, + "ndcg_at_1000": 0.55689, + "map_at_1": 0.38699, + "map_at_3": 0.44753, + "map_at_5": 0.45883, + "map_at_10": 0.46735, + "map_at_20": 0.47092, + "map_at_100": 0.47386, + "map_at_1000": 0.47432, + "recall_at_1": 0.38699, + "recall_at_3": 0.52657, + "recall_at_5": 0.57653, + "recall_at_10": 0.64235, + "recall_at_20": 0.69469, + "recall_at_100": 0.80571, + "recall_at_1000": 0.91673, + "precision_at_1": 0.38699, + "precision_at_3": 0.17552, + "precision_at_5": 0.11531, + "precision_at_10": 0.06423, + "precision_at_20": 0.03473, + "precision_at_100": 0.00806, + "precision_at_1000": 0.00092, + "mrr_at_1": 0.387787, + "mrr_at_3": 0.447925, + "mrr_at_5": 0.459225, + "mrr_at_10": 0.467757, + "mrr_at_20": 0.471324, + "mrr_at_100": 0.474265, + "mrr_at_1000": 0.474728, + "nauc_ndcg_at_1_max": 0.322463, + "nauc_ndcg_at_1_std": -0.138304, + "nauc_ndcg_at_1_diff1": 0.586379, + "nauc_ndcg_at_3_max": 0.331357, + "nauc_ndcg_at_3_std": -0.131646, + "nauc_ndcg_at_3_diff1": 0.534648, + "nauc_ndcg_at_5_max": 0.324152, + "nauc_ndcg_at_5_std": -0.14435, + "nauc_ndcg_at_5_diff1": 0.528121, + "nauc_ndcg_at_10_max": 0.329021, + "nauc_ndcg_at_10_std": -0.138195, + "nauc_ndcg_at_10_diff1": 0.527157, + "nauc_ndcg_at_20_max": 0.33024, + "nauc_ndcg_at_20_std": -0.138513, + "nauc_ndcg_at_20_diff1": 0.529737, + "nauc_ndcg_at_100_max": 0.330983, + "nauc_ndcg_at_100_std": -0.131028, + "nauc_ndcg_at_100_diff1": 0.528145, + "nauc_ndcg_at_1000_max": 0.328878, + "nauc_ndcg_at_1000_std": -0.130592, + "nauc_ndcg_at_1000_diff1": 0.530934, + "nauc_map_at_1_max": 0.322463, + "nauc_map_at_1_std": -0.138304, + "nauc_map_at_1_diff1": 0.586379, + "nauc_map_at_3_max": 0.329696, + "nauc_map_at_3_std": -0.133009, + "nauc_map_at_3_diff1": 0.547286, + "nauc_map_at_5_max": 0.32568, + "nauc_map_at_5_std": -0.139981, + "nauc_map_at_5_diff1": 0.543792, + "nauc_map_at_10_max": 0.327721, + "nauc_map_at_10_std": -0.137826, + "nauc_map_at_10_diff1": 0.543646, + "nauc_map_at_20_max": 0.327905, + "nauc_map_at_20_std": -0.13807, + "nauc_map_at_20_diff1": 0.544297, + "nauc_map_at_100_max": 0.327938, + "nauc_map_at_100_std": -0.137034, + "nauc_map_at_100_diff1": 0.544031, + "nauc_map_at_1000_max": 0.327933, + "nauc_map_at_1000_std": -0.137009, + "nauc_map_at_1000_diff1": 0.544128, + "nauc_recall_at_1_max": 0.322463, + "nauc_recall_at_1_std": -0.138304, + "nauc_recall_at_1_diff1": 0.586379, + "nauc_recall_at_3_max": 0.336092, + "nauc_recall_at_3_std": -0.12769, + "nauc_recall_at_3_diff1": 0.496945, + "nauc_recall_at_5_max": 0.318126, + "nauc_recall_at_5_std": -0.160029, + "nauc_recall_at_5_diff1": 0.478423, + "nauc_recall_at_10_max": 0.334166, + "nauc_recall_at_10_std": -0.137969, + "nauc_recall_at_10_diff1": 0.469248, + "nauc_recall_at_20_max": 0.342034, + "nauc_recall_at_20_std": -0.137812, + "nauc_recall_at_20_diff1": 0.476686, + "nauc_recall_at_100_max": 0.354405, + "nauc_recall_at_100_std": -0.07195, + "nauc_recall_at_100_diff1": 0.442486, + "nauc_recall_at_1000_max": 0.310304, + "nauc_recall_at_1000_std": 0.00683, + "nauc_recall_at_1000_diff1": 0.425138, + "nauc_precision_at_1_max": 0.322463, + "nauc_precision_at_1_std": -0.138304, + "nauc_precision_at_1_diff1": 0.586379, + "nauc_precision_at_3_max": 0.336092, + "nauc_precision_at_3_std": -0.12769, + "nauc_precision_at_3_diff1": 0.496945, + "nauc_precision_at_5_max": 0.318126, + "nauc_precision_at_5_std": -0.160029, + "nauc_precision_at_5_diff1": 0.478423, + "nauc_precision_at_10_max": 0.334166, + "nauc_precision_at_10_std": -0.137969, + "nauc_precision_at_10_diff1": 0.469248, + "nauc_precision_at_20_max": 0.342034, + "nauc_precision_at_20_std": -0.137812, + "nauc_precision_at_20_diff1": 0.476686, + "nauc_precision_at_100_max": 0.354405, + "nauc_precision_at_100_std": -0.07195, + "nauc_precision_at_100_diff1": 0.442486, + "nauc_precision_at_1000_max": 0.310304, + "nauc_precision_at_1000_std": 0.00683, + "nauc_precision_at_1000_diff1": 0.425138, + "nauc_mrr_at_1_max": 0.322718, + "nauc_mrr_at_1_std": -0.139041, + "nauc_mrr_at_1_diff1": 0.583958, + "nauc_mrr_at_3_max": 0.329825, + "nauc_mrr_at_3_std": -0.133386, + "nauc_mrr_at_3_diff1": 0.546056, + "nauc_mrr_at_5_max": 0.325811, + "nauc_mrr_at_5_std": -0.140361, + "nauc_mrr_at_5_diff1": 0.54255, + "nauc_mrr_at_10_max": 0.32782, + "nauc_mrr_at_10_std": -0.138237, + "nauc_mrr_at_10_diff1": 0.542379, + "nauc_mrr_at_20_max": 0.328014, + "nauc_mrr_at_20_std": -0.138472, + "nauc_mrr_at_20_diff1": 0.543014, + "nauc_mrr_at_100_max": 0.328047, + "nauc_mrr_at_100_std": -0.137439, + "nauc_mrr_at_100_diff1": 0.542739, + "nauc_mrr_at_1000_max": 0.328042, + "nauc_mrr_at_1000_std": -0.137414, + "nauc_mrr_at_1000_diff1": 0.542836, + "main_score": 0.50928, + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ] + }, + { + "ndcg_at_1": 0.39133, + "ndcg_at_3": 0.48002, + "ndcg_at_5": 0.50222, + "ndcg_at_10": 0.52123, + "ndcg_at_20": 0.53605, + "ndcg_at_100": 0.55569, + "ndcg_at_1000": 0.56875, + "map_at_1": 0.39133, + "map_at_3": 0.45865, + "map_at_5": 0.47103, + "map_at_10": 0.47884, + "map_at_20": 0.48291, + "map_at_100": 0.48557, + "map_at_1000": 0.48602, + "recall_at_1": 0.39133, + "recall_at_3": 0.54167, + "recall_at_5": 0.59534, + "recall_at_10": 0.65431, + "recall_at_20": 0.71292, + "recall_at_100": 0.81953, + "recall_at_1000": 0.92451, + "precision_at_1": 0.39133, + "precision_at_3": 0.18056, + "precision_at_5": 0.11907, + "precision_at_10": 0.06543, + "precision_at_20": 0.03565, + "precision_at_100": 0.0082, + "precision_at_1000": 0.00092, + "mrr_at_1": 0.391785, + "mrr_at_3": 0.458801, + "mrr_at_5": 0.471257, + "mrr_at_10": 0.479086, + "mrr_at_20": 0.483151, + "mrr_at_100": 0.485808, + "mrr_at_1000": 0.486262, + "nauc_ndcg_at_1_max": 0.299198, + "nauc_ndcg_at_1_std": -0.064879, + "nauc_ndcg_at_1_diff1": 0.567995, + "nauc_ndcg_at_3_max": 0.321162, + "nauc_ndcg_at_3_std": -0.054695, + "nauc_ndcg_at_3_diff1": 0.536166, + "nauc_ndcg_at_5_max": 0.319497, + "nauc_ndcg_at_5_std": -0.049442, + "nauc_ndcg_at_5_diff1": 0.528725, + "nauc_ndcg_at_10_max": 0.317157, + "nauc_ndcg_at_10_std": -0.044336, + "nauc_ndcg_at_10_diff1": 0.524483, + "nauc_ndcg_at_20_max": 0.315631, + "nauc_ndcg_at_20_std": -0.038817, + "nauc_ndcg_at_20_diff1": 0.522349, + "nauc_ndcg_at_100_max": 0.315789, + "nauc_ndcg_at_100_std": -0.034435, + "nauc_ndcg_at_100_diff1": 0.52464, + "nauc_ndcg_at_1000_max": 0.316379, + "nauc_ndcg_at_1000_std": -0.036319, + "nauc_ndcg_at_1000_diff1": 0.527722, + "nauc_map_at_1_max": 0.299198, + "nauc_map_at_1_std": -0.064879, + "nauc_map_at_1_diff1": 0.567995, + "nauc_map_at_3_max": 0.315838, + "nauc_map_at_3_std": -0.057926, + "nauc_map_at_3_diff1": 0.543558, + "nauc_map_at_5_max": 0.314815, + "nauc_map_at_5_std": -0.055134, + "nauc_map_at_5_diff1": 0.53953, + "nauc_map_at_10_max": 0.313776, + "nauc_map_at_10_std": -0.053176, + "nauc_map_at_10_diff1": 0.537943, + "nauc_map_at_20_max": 0.313424, + "nauc_map_at_20_std": -0.05174, + "nauc_map_at_20_diff1": 0.53745, + "nauc_map_at_100_max": 0.313423, + "nauc_map_at_100_std": -0.05128, + "nauc_map_at_100_diff1": 0.537838, + "nauc_map_at_1000_max": 0.313436, + "nauc_map_at_1000_std": -0.051315, + "nauc_map_at_1000_diff1": 0.537948, + "nauc_recall_at_1_max": 0.299198, + "nauc_recall_at_1_std": -0.064879, + "nauc_recall_at_1_diff1": 0.567995, + "nauc_recall_at_3_max": 0.337178, + "nauc_recall_at_3_std": -0.044638, + "nauc_recall_at_3_diff1": 0.51407, + "nauc_recall_at_5_max": 0.33405, + "nauc_recall_at_5_std": -0.030271, + "nauc_recall_at_5_diff1": 0.493731, + "nauc_recall_at_10_max": 0.327645, + "nauc_recall_at_10_std": -0.009884, + "nauc_recall_at_10_diff1": 0.474888, + "nauc_recall_at_20_max": 0.320761, + "nauc_recall_at_20_std": 0.022172, + "nauc_recall_at_20_diff1": 0.457593, + "nauc_recall_at_100_max": 0.324818, + "nauc_recall_at_100_std": 0.093184, + "nauc_recall_at_100_diff1": 0.449406, + "nauc_recall_at_1000_max": 0.354483, + "nauc_recall_at_1000_std": 0.182467, + "nauc_recall_at_1000_diff1": 0.452334, + "nauc_precision_at_1_max": 0.299198, + "nauc_precision_at_1_std": -0.064879, + "nauc_precision_at_1_diff1": 0.567995, + "nauc_precision_at_3_max": 0.337178, + "nauc_precision_at_3_std": -0.044638, + "nauc_precision_at_3_diff1": 0.51407, + "nauc_precision_at_5_max": 0.33405, + "nauc_precision_at_5_std": -0.030271, + "nauc_precision_at_5_diff1": 0.493731, + "nauc_precision_at_10_max": 0.327645, + "nauc_precision_at_10_std": -0.009884, + "nauc_precision_at_10_diff1": 0.474888, + "nauc_precision_at_20_max": 0.320761, + "nauc_precision_at_20_std": 0.022172, + "nauc_precision_at_20_diff1": 0.457593, + "nauc_precision_at_100_max": 0.324818, + "nauc_precision_at_100_std": 0.093184, + "nauc_precision_at_100_diff1": 0.449406, + "nauc_precision_at_1000_max": 0.354483, + "nauc_precision_at_1000_std": 0.182467, + "nauc_precision_at_1000_diff1": 0.452334, + "nauc_mrr_at_1_max": 0.300457, + "nauc_mrr_at_1_std": -0.06435, + "nauc_mrr_at_1_diff1": 0.566609, + "nauc_mrr_at_3_max": 0.31662, + "nauc_mrr_at_3_std": -0.057707, + "nauc_mrr_at_3_diff1": 0.542763, + "nauc_mrr_at_5_max": 0.315531, + "nauc_mrr_at_5_std": -0.054898, + "nauc_mrr_at_5_diff1": 0.538685, + "nauc_mrr_at_10_max": 0.314514, + "nauc_mrr_at_10_std": -0.052829, + "nauc_mrr_at_10_diff1": 0.537162, + "nauc_mrr_at_20_max": 0.314185, + "nauc_mrr_at_20_std": -0.051417, + "nauc_mrr_at_20_diff1": 0.536612, + "nauc_mrr_at_100_max": 0.31418, + "nauc_mrr_at_100_std": -0.050975, + "nauc_mrr_at_100_diff1": 0.537003, + "nauc_mrr_at_1000_max": 0.314194, + "nauc_mrr_at_1000_std": -0.051009, + "nauc_mrr_at_1000_diff1": 0.537113, + "main_score": 0.52123, + "hf_subset": "java", + "languages": [ + "java-Code" + ] + }, + { + "ndcg_at_1": 0.26488, + "ndcg_at_3": 0.34044, + "ndcg_at_5": 0.36155, + "ndcg_at_10": 0.38314, + "ndcg_at_20": 0.39909, + "ndcg_at_100": 0.42569, + "ndcg_at_1000": 0.44569, + "map_at_1": 0.26488, + "map_at_3": 0.32198, + "map_at_5": 0.33368, + "map_at_10": 0.34258, + "map_at_20": 0.34693, + "map_at_100": 0.35057, + "map_at_1000": 0.35124, + "recall_at_1": 0.26488, + "recall_at_3": 0.39382, + "recall_at_5": 0.44513, + "recall_at_10": 0.51192, + "recall_at_20": 0.57521, + "recall_at_100": 0.71935, + "recall_at_1000": 0.88126, + "precision_at_1": 0.26488, + "precision_at_3": 0.13127, + "precision_at_5": 0.08903, + "precision_at_10": 0.05119, + "precision_at_20": 0.02876, + "precision_at_100": 0.00719, + "precision_at_1000": 0.00088, + "mrr_at_1": 0.264807, + "mrr_at_3": 0.32194, + "mrr_at_5": 0.333628, + "mrr_at_10": 0.342556, + "mrr_at_20": 0.346902, + "mrr_at_100": 0.350541, + "mrr_at_1000": 0.351214, + "nauc_ndcg_at_1_max": 0.213064, + "nauc_ndcg_at_1_std": -0.07808, + "nauc_ndcg_at_1_diff1": 0.501073, + "nauc_ndcg_at_3_max": 0.207269, + "nauc_ndcg_at_3_std": -0.071304, + "nauc_ndcg_at_3_diff1": 0.441807, + "nauc_ndcg_at_5_max": 0.201632, + "nauc_ndcg_at_5_std": -0.068598, + "nauc_ndcg_at_5_diff1": 0.436082, + "nauc_ndcg_at_10_max": 0.197227, + "nauc_ndcg_at_10_std": -0.062462, + "nauc_ndcg_at_10_diff1": 0.429945, + "nauc_ndcg_at_20_max": 0.194988, + "nauc_ndcg_at_20_std": -0.057325, + "nauc_ndcg_at_20_diff1": 0.426757, + "nauc_ndcg_at_100_max": 0.19181, + "nauc_ndcg_at_100_std": -0.050173, + "nauc_ndcg_at_100_diff1": 0.423981, + "nauc_ndcg_at_1000_max": 0.195651, + "nauc_ndcg_at_1000_std": -0.049785, + "nauc_ndcg_at_1000_diff1": 0.427297, + "nauc_map_at_1_max": 0.213064, + "nauc_map_at_1_std": -0.07808, + "nauc_map_at_1_diff1": 0.501073, + "nauc_map_at_3_max": 0.208648, + "nauc_map_at_3_std": -0.073155, + "nauc_map_at_3_diff1": 0.455232, + "nauc_map_at_5_max": 0.205561, + "nauc_map_at_5_std": -0.071672, + "nauc_map_at_5_diff1": 0.452006, + "nauc_map_at_10_max": 0.203769, + "nauc_map_at_10_std": -0.069217, + "nauc_map_at_10_diff1": 0.44956, + "nauc_map_at_20_max": 0.203181, + "nauc_map_at_20_std": -0.067874, + "nauc_map_at_20_diff1": 0.448792, + "nauc_map_at_100_max": 0.202666, + "nauc_map_at_100_std": -0.067002, + "nauc_map_at_100_diff1": 0.448438, + "nauc_map_at_1000_max": 0.202784, + "nauc_map_at_1000_std": -0.066967, + "nauc_map_at_1000_diff1": 0.448563, + "nauc_recall_at_1_max": 0.213064, + "nauc_recall_at_1_std": -0.07808, + "nauc_recall_at_1_diff1": 0.501073, + "nauc_recall_at_3_max": 0.20348, + "nauc_recall_at_3_std": -0.066129, + "nauc_recall_at_3_diff1": 0.40519, + "nauc_recall_at_5_max": 0.189894, + "nauc_recall_at_5_std": -0.059583, + "nauc_recall_at_5_diff1": 0.391478, + "nauc_recall_at_10_max": 0.175923, + "nauc_recall_at_10_std": -0.040134, + "nauc_recall_at_10_diff1": 0.371092, + "nauc_recall_at_20_max": 0.165288, + "nauc_recall_at_20_std": -0.01724, + "nauc_recall_at_20_diff1": 0.353762, + "nauc_recall_at_100_max": 0.135867, + "nauc_recall_at_100_std": 0.046534, + "nauc_recall_at_100_diff1": 0.312587, + "nauc_recall_at_1000_max": 0.158811, + "nauc_recall_at_1000_std": 0.154613, + "nauc_recall_at_1000_diff1": 0.262628, + "nauc_precision_at_1_max": 0.213064, + "nauc_precision_at_1_std": -0.07808, + "nauc_precision_at_1_diff1": 0.501073, + "nauc_precision_at_3_max": 0.20348, + "nauc_precision_at_3_std": -0.066129, + "nauc_precision_at_3_diff1": 0.40519, + "nauc_precision_at_5_max": 0.189894, + "nauc_precision_at_5_std": -0.059583, + "nauc_precision_at_5_diff1": 0.391478, + "nauc_precision_at_10_max": 0.175923, + "nauc_precision_at_10_std": -0.040134, + "nauc_precision_at_10_diff1": 0.371092, + "nauc_precision_at_20_max": 0.165288, + "nauc_precision_at_20_std": -0.01724, + "nauc_precision_at_20_diff1": 0.353762, + "nauc_precision_at_100_max": 0.135867, + "nauc_precision_at_100_std": 0.046534, + "nauc_precision_at_100_diff1": 0.312587, + "nauc_precision_at_1000_max": 0.158811, + "nauc_precision_at_1000_std": 0.154613, + "nauc_precision_at_1000_diff1": 0.262628, + "nauc_mrr_at_1_max": 0.214082, + "nauc_mrr_at_1_std": -0.076799, + "nauc_mrr_at_1_diff1": 0.501354, + "nauc_mrr_at_3_max": 0.209173, + "nauc_mrr_at_3_std": -0.072461, + "nauc_mrr_at_3_diff1": 0.455444, + "nauc_mrr_at_5_max": 0.206022, + "nauc_mrr_at_5_std": -0.070992, + "nauc_mrr_at_5_diff1": 0.452157, + "nauc_mrr_at_10_max": 0.204224, + "nauc_mrr_at_10_std": -0.068585, + "nauc_mrr_at_10_diff1": 0.449734, + "nauc_mrr_at_20_max": 0.203647, + "nauc_mrr_at_20_std": -0.067233, + "nauc_mrr_at_20_diff1": 0.448972, + "nauc_mrr_at_100_max": 0.203126, + "nauc_mrr_at_100_std": -0.06636, + "nauc_mrr_at_100_diff1": 0.448615, + "nauc_mrr_at_1000_max": 0.203246, + "nauc_mrr_at_1000_std": -0.066323, + "nauc_mrr_at_1000_diff1": 0.44874, + "main_score": 0.38314, + "hf_subset": "php", + "languages": [ + "php-Code" + ] + } + ] + }, + "evaluation_time": 713.5910089015961, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeTransOceanContest.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeTransOceanContest.json new file mode 100644 index 0000000000..e443de31bd --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeTransOceanContest.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "20da4eb20a4b17300c0986ee148c90867a7f2a4d", + "task_name": "CodeTransOceanContest", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.66063, + "ndcg_at_3": 0.73535, + "ndcg_at_5": 0.75619, + "ndcg_at_10": 0.77071, + "ndcg_at_20": 0.78116, + "ndcg_at_100": 0.79065, + "ndcg_at_1000": 0.79463, + "map_at_1": 0.66063, + "map_at_3": 0.7187, + "map_at_5": 0.73047, + "map_at_10": 0.73641, + "map_at_20": 0.73935, + "map_at_100": 0.74075, + "map_at_1000": 0.74089, + "recall_at_1": 0.66063, + "recall_at_3": 0.78281, + "recall_at_5": 0.83258, + "recall_at_10": 0.87783, + "recall_at_20": 0.91855, + "recall_at_100": 0.96833, + "recall_at_1000": 1.0, + "precision_at_1": 0.66063, + "precision_at_3": 0.26094, + "precision_at_5": 0.16652, + "precision_at_10": 0.08778, + "precision_at_20": 0.04593, + "precision_at_100": 0.00968, + "precision_at_1000": 0.001, + "mrr_at_1": 0.660633, + "mrr_at_3": 0.718703, + "mrr_at_5": 0.730468, + "mrr_at_10": 0.736406, + "mrr_at_20": 0.739347, + "mrr_at_100": 0.74075, + "mrr_at_1000": 0.740889, + "nauc_ndcg_at_1_max": 0.765004, + "nauc_ndcg_at_1_std": 0.111067, + "nauc_ndcg_at_1_diff1": 0.907238, + "nauc_ndcg_at_3_max": 0.752574, + "nauc_ndcg_at_3_std": 0.056022, + "nauc_ndcg_at_3_diff1": 0.863673, + "nauc_ndcg_at_5_max": 0.746373, + "nauc_ndcg_at_5_std": 0.078156, + "nauc_ndcg_at_5_diff1": 0.854296, + "nauc_ndcg_at_10_max": 0.745624, + "nauc_ndcg_at_10_std": 0.074677, + "nauc_ndcg_at_10_diff1": 0.861517, + "nauc_ndcg_at_20_max": 0.740206, + "nauc_ndcg_at_20_std": 0.069016, + "nauc_ndcg_at_20_diff1": 0.858016, + "nauc_ndcg_at_100_max": 0.751921, + "nauc_ndcg_at_100_std": 0.095766, + "nauc_ndcg_at_100_diff1": 0.864109, + "nauc_ndcg_at_1000_max": 0.750218, + "nauc_ndcg_at_1000_std": 0.084614, + "nauc_ndcg_at_1000_diff1": 0.867148, + "nauc_map_at_1_max": 0.765004, + "nauc_map_at_1_std": 0.111067, + "nauc_map_at_1_diff1": 0.907238, + "nauc_map_at_3_max": 0.756527, + "nauc_map_at_3_std": 0.072773, + "nauc_map_at_3_diff1": 0.874745, + "nauc_map_at_5_max": 0.753528, + "nauc_map_at_5_std": 0.084565, + "nauc_map_at_5_diff1": 0.870141, + "nauc_map_at_10_max": 0.753386, + "nauc_map_at_10_std": 0.081951, + "nauc_map_at_10_diff1": 0.873556, + "nauc_map_at_20_max": 0.751989, + "nauc_map_at_20_std": 0.080565, + "nauc_map_at_20_diff1": 0.872646, + "nauc_map_at_100_max": 0.75329, + "nauc_map_at_100_std": 0.084397, + "nauc_map_at_100_diff1": 0.873601, + "nauc_map_at_1000_max": 0.753269, + "nauc_map_at_1000_std": 0.084086, + "nauc_map_at_1000_diff1": 0.873683, + "nauc_recall_at_1_max": 0.765004, + "nauc_recall_at_1_std": 0.111067, + "nauc_recall_at_1_diff1": 0.907238, + "nauc_recall_at_3_max": 0.738466, + "nauc_recall_at_3_std": -0.003414, + "nauc_recall_at_3_diff1": 0.825395, + "nauc_recall_at_5_max": 0.71451, + "nauc_recall_at_5_std": 0.055089, + "nauc_recall_at_5_diff1": 0.786517, + "nauc_recall_at_10_max": 0.700386, + "nauc_recall_at_10_std": 0.044121, + "nauc_recall_at_10_diff1": 0.797635, + "nauc_recall_at_20_max": 0.633552, + "nauc_recall_at_20_std": -0.019745, + "nauc_recall_at_20_diff1": 0.740297, + "nauc_recall_at_100_max": 0.838424, + "nauc_recall_at_100_std": 0.593956, + "nauc_recall_at_100_diff1": 0.72822, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.765004, + "nauc_precision_at_1_std": 0.111067, + "nauc_precision_at_1_diff1": 0.907238, + "nauc_precision_at_3_max": 0.738466, + "nauc_precision_at_3_std": -0.003414, + "nauc_precision_at_3_diff1": 0.825395, + "nauc_precision_at_5_max": 0.71451, + "nauc_precision_at_5_std": 0.055089, + "nauc_precision_at_5_diff1": 0.786517, + "nauc_precision_at_10_max": 0.700386, + "nauc_precision_at_10_std": 0.044121, + "nauc_precision_at_10_diff1": 0.797635, + "nauc_precision_at_20_max": 0.633552, + "nauc_precision_at_20_std": -0.019745, + "nauc_precision_at_20_diff1": 0.740297, + "nauc_precision_at_100_max": 0.838424, + "nauc_precision_at_100_std": 0.593956, + "nauc_precision_at_100_diff1": 0.72822, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.765004, + "nauc_mrr_at_1_std": 0.111067, + "nauc_mrr_at_1_diff1": 0.907238, + "nauc_mrr_at_3_max": 0.756527, + "nauc_mrr_at_3_std": 0.072773, + "nauc_mrr_at_3_diff1": 0.874745, + "nauc_mrr_at_5_max": 0.753528, + "nauc_mrr_at_5_std": 0.084565, + "nauc_mrr_at_5_diff1": 0.870141, + "nauc_mrr_at_10_max": 0.753386, + "nauc_mrr_at_10_std": 0.081951, + "nauc_mrr_at_10_diff1": 0.873556, + "nauc_mrr_at_20_max": 0.751989, + "nauc_mrr_at_20_std": 0.080565, + "nauc_mrr_at_20_diff1": 0.872646, + "nauc_mrr_at_100_max": 0.75329, + "nauc_mrr_at_100_std": 0.084397, + "nauc_mrr_at_100_diff1": 0.873601, + "nauc_mrr_at_1000_max": 0.753269, + "nauc_mrr_at_1000_std": 0.084086, + "nauc_mrr_at_1000_diff1": 0.873683, + "main_score": 0.77071, + "hf_subset": "default", + "languages": [ + "python-Code", + "c++-Code" + ] + } + ] + }, + "evaluation_time": 4.555964469909668, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeTransOceanDL.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeTransOceanDL.json new file mode 100644 index 0000000000..998fcbfea6 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CodeTransOceanDL.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "281562cb8a1265ab5c0824bfa6ddcd9b0a15618f", + "task_name": "CodeTransOceanDL", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.08889, + "ndcg_at_3": 0.14458, + "ndcg_at_5": 0.21104, + "ndcg_at_10": 0.35028, + "ndcg_at_20": 0.40219, + "ndcg_at_100": 0.40585, + "ndcg_at_1000": 0.40585, + "map_at_1": 0.08889, + "map_at_3": 0.12778, + "map_at_5": 0.16472, + "map_at_10": 0.22164, + "map_at_20": 0.23814, + "map_at_100": 0.23848, + "map_at_1000": 0.23848, + "recall_at_1": 0.08889, + "recall_at_3": 0.19444, + "recall_at_5": 0.35556, + "recall_at_10": 0.78889, + "recall_at_20": 0.97778, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.08889, + "precision_at_3": 0.06481, + "precision_at_5": 0.07111, + "precision_at_10": 0.07889, + "precision_at_20": 0.04889, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.077778, + "mrr_at_3": 0.12963, + "mrr_at_5": 0.15963, + "mrr_at_10": 0.223616, + "mrr_at_20": 0.237168, + "mrr_at_100": 0.237509, + "mrr_at_1000": 0.237509, + "nauc_ndcg_at_1_max": -0.413486, + "nauc_ndcg_at_1_std": -0.160199, + "nauc_ndcg_at_1_diff1": 0.82392, + "nauc_ndcg_at_3_max": -0.448573, + "nauc_ndcg_at_3_std": -0.182932, + "nauc_ndcg_at_3_diff1": 0.522438, + "nauc_ndcg_at_5_max": -0.394629, + "nauc_ndcg_at_5_std": -0.146782, + "nauc_ndcg_at_5_diff1": 0.531075, + "nauc_ndcg_at_10_max": -0.354619, + "nauc_ndcg_at_10_std": -0.15486, + "nauc_ndcg_at_10_diff1": 0.603375, + "nauc_ndcg_at_20_max": -0.421293, + "nauc_ndcg_at_20_std": -0.125269, + "nauc_ndcg_at_20_diff1": 0.576587, + "nauc_ndcg_at_100_max": -0.403879, + "nauc_ndcg_at_100_std": -0.143333, + "nauc_ndcg_at_100_diff1": 0.585523, + "nauc_ndcg_at_1000_max": -0.403879, + "nauc_ndcg_at_1000_std": -0.143333, + "nauc_ndcg_at_1000_diff1": 0.585523, + "nauc_map_at_1_max": -0.413486, + "nauc_map_at_1_std": -0.160199, + "nauc_map_at_1_diff1": 0.82392, + "nauc_map_at_3_max": -0.441535, + "nauc_map_at_3_std": -0.180469, + "nauc_map_at_3_diff1": 0.5862, + "nauc_map_at_5_max": -0.402712, + "nauc_map_at_5_std": -0.156832, + "nauc_map_at_5_diff1": 0.582988, + "nauc_map_at_10_max": -0.384229, + "nauc_map_at_10_std": -0.15827, + "nauc_map_at_10_diff1": 0.610083, + "nauc_map_at_20_max": -0.404565, + "nauc_map_at_20_std": -0.150586, + "nauc_map_at_20_diff1": 0.603938, + "nauc_map_at_100_max": -0.403191, + "nauc_map_at_100_std": -0.151992, + "nauc_map_at_100_diff1": 0.604752, + "nauc_map_at_1000_max": -0.403191, + "nauc_map_at_1000_std": -0.151992, + "nauc_map_at_1000_diff1": 0.604752, + "nauc_recall_at_1_max": -0.413486, + "nauc_recall_at_1_std": -0.160199, + "nauc_recall_at_1_diff1": 0.82392, + "nauc_recall_at_3_max": -0.463307, + "nauc_recall_at_3_std": -0.187517, + "nauc_recall_at_3_diff1": 0.388067, + "nauc_recall_at_5_max": -0.379434, + "nauc_recall_at_5_std": -0.125825, + "nauc_recall_at_5_diff1": 0.439944, + "nauc_recall_at_10_max": -0.250672, + "nauc_recall_at_10_std": -0.150959, + "nauc_recall_at_10_diff1": 0.640582, + "nauc_recall_at_20_max": -1.151727, + "nauc_recall_at_20_std": 0.638189, + "nauc_recall_at_20_diff1": 0.214169, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.413486, + "nauc_precision_at_1_std": -0.160199, + "nauc_precision_at_1_diff1": 0.82392, + "nauc_precision_at_3_max": -0.463307, + "nauc_precision_at_3_std": -0.187517, + "nauc_precision_at_3_diff1": 0.388067, + "nauc_precision_at_5_max": -0.379434, + "nauc_precision_at_5_std": -0.125825, + "nauc_precision_at_5_diff1": 0.439944, + "nauc_precision_at_10_max": -0.250672, + "nauc_precision_at_10_std": -0.150959, + "nauc_precision_at_10_diff1": 0.640582, + "nauc_precision_at_20_max": -1.151727, + "nauc_precision_at_20_std": 0.638189, + "nauc_precision_at_20_diff1": 0.214169, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": -0.390452, + "nauc_mrr_at_1_std": -0.114397, + "nauc_mrr_at_1_diff1": 0.25022, + "nauc_mrr_at_3_max": -0.405702, + "nauc_mrr_at_3_std": -0.137603, + "nauc_mrr_at_3_diff1": 0.224334, + "nauc_mrr_at_5_max": -0.374574, + "nauc_mrr_at_5_std": -0.148711, + "nauc_mrr_at_5_diff1": 0.211526, + "nauc_mrr_at_10_max": -0.352883, + "nauc_mrr_at_10_std": -0.119456, + "nauc_mrr_at_10_diff1": 0.226377, + "nauc_mrr_at_20_max": -0.384933, + "nauc_mrr_at_20_std": -0.120095, + "nauc_mrr_at_20_diff1": 0.213119, + "nauc_mrr_at_100_max": -0.383497, + "nauc_mrr_at_100_std": -0.12147, + "nauc_mrr_at_100_diff1": 0.213215, + "nauc_mrr_at_1000_max": -0.383497, + "nauc_mrr_at_1000_std": -0.12147, + "nauc_mrr_at_1000_diff1": 0.213215, + "main_score": 0.35028, + "hf_subset": "default", + "languages": [ + "python-Code" + ] + } + ] + }, + "evaluation_time": 4.34340238571167, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CosQA.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CosQA.json new file mode 100644 index 0000000000..3f8c9a7739 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/CosQA.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "bc5efb7e9d437246ce393ed19d772e08e4a79535", + "task_name": "CosQA", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.17, + "ndcg_at_3": 0.26047, + "ndcg_at_5": 0.3177, + "ndcg_at_10": 0.37014, + "ndcg_at_20": 0.40433, + "ndcg_at_100": 0.43635, + "ndcg_at_1000": 0.44447, + "map_at_1": 0.17, + "map_at_3": 0.23667, + "map_at_5": 0.26817, + "map_at_10": 0.29022, + "map_at_20": 0.29977, + "map_at_100": 0.30453, + "map_at_1000": 0.30486, + "recall_at_1": 0.17, + "recall_at_3": 0.33, + "recall_at_5": 0.47, + "recall_at_10": 0.63, + "recall_at_20": 0.764, + "recall_at_100": 0.932, + "recall_at_1000": 0.994, + "precision_at_1": 0.17, + "precision_at_3": 0.11, + "precision_at_5": 0.094, + "precision_at_10": 0.063, + "precision_at_20": 0.0382, + "precision_at_100": 0.00932, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.154, + "mrr_at_3": 0.213333, + "mrr_at_5": 0.237733, + "mrr_at_10": 0.263047, + "mrr_at_20": 0.273963, + "mrr_at_100": 0.279104, + "mrr_at_1000": 0.279432, + "nauc_ndcg_at_1_max": 0.067827, + "nauc_ndcg_at_1_std": -0.090016, + "nauc_ndcg_at_1_diff1": 0.361374, + "nauc_ndcg_at_3_max": 0.086512, + "nauc_ndcg_at_3_std": -0.08639, + "nauc_ndcg_at_3_diff1": 0.248546, + "nauc_ndcg_at_5_max": 0.108969, + "nauc_ndcg_at_5_std": -0.096553, + "nauc_ndcg_at_5_diff1": 0.206776, + "nauc_ndcg_at_10_max": 0.111226, + "nauc_ndcg_at_10_std": -0.086477, + "nauc_ndcg_at_10_diff1": 0.223139, + "nauc_ndcg_at_20_max": 0.111889, + "nauc_ndcg_at_20_std": -0.089642, + "nauc_ndcg_at_20_diff1": 0.211282, + "nauc_ndcg_at_100_max": 0.115512, + "nauc_ndcg_at_100_std": -0.061768, + "nauc_ndcg_at_100_diff1": 0.222334, + "nauc_ndcg_at_1000_max": 0.10804, + "nauc_ndcg_at_1000_std": -0.078251, + "nauc_ndcg_at_1000_diff1": 0.235227, + "nauc_map_at_1_max": 0.067827, + "nauc_map_at_1_std": -0.090016, + "nauc_map_at_1_diff1": 0.361374, + "nauc_map_at_3_max": 0.082266, + "nauc_map_at_3_std": -0.089738, + "nauc_map_at_3_diff1": 0.275176, + "nauc_map_at_5_max": 0.094866, + "nauc_map_at_5_std": -0.095972, + "nauc_map_at_5_diff1": 0.250307, + "nauc_map_at_10_max": 0.096099, + "nauc_map_at_10_std": -0.092303, + "nauc_map_at_10_diff1": 0.258111, + "nauc_map_at_20_max": 0.095315, + "nauc_map_at_20_std": -0.093699, + "nauc_map_at_20_diff1": 0.255376, + "nauc_map_at_100_max": 0.095965, + "nauc_map_at_100_std": -0.089533, + "nauc_map_at_100_diff1": 0.257619, + "nauc_map_at_1000_max": 0.095727, + "nauc_map_at_1000_std": -0.090085, + "nauc_map_at_1000_diff1": 0.258078, + "nauc_recall_at_1_max": 0.067827, + "nauc_recall_at_1_std": -0.090016, + "nauc_recall_at_1_diff1": 0.361374, + "nauc_recall_at_3_max": 0.097027, + "nauc_recall_at_3_std": -0.077296, + "nauc_recall_at_3_diff1": 0.18228, + "nauc_recall_at_5_max": 0.146745, + "nauc_recall_at_5_std": -0.097994, + "nauc_recall_at_5_diff1": 0.09497, + "nauc_recall_at_10_max": 0.157665, + "nauc_recall_at_10_std": -0.064398, + "nauc_recall_at_10_diff1": 0.124417, + "nauc_recall_at_20_max": 0.185813, + "nauc_recall_at_20_std": -0.06728, + "nauc_recall_at_20_diff1": 0.028227, + "nauc_recall_at_100_max": 0.386596, + "nauc_recall_at_100_std": 0.475463, + "nauc_recall_at_100_diff1": -0.151714, + "nauc_recall_at_1000_max": 0.814815, + "nauc_recall_at_1000_std": 0.912854, + "nauc_recall_at_1000_diff1": -0.073296, + "nauc_precision_at_1_max": 0.067827, + "nauc_precision_at_1_std": -0.090016, + "nauc_precision_at_1_diff1": 0.361374, + "nauc_precision_at_3_max": 0.097027, + "nauc_precision_at_3_std": -0.077296, + "nauc_precision_at_3_diff1": 0.18228, + "nauc_precision_at_5_max": 0.146745, + "nauc_precision_at_5_std": -0.097994, + "nauc_precision_at_5_diff1": 0.09497, + "nauc_precision_at_10_max": 0.157665, + "nauc_precision_at_10_std": -0.064398, + "nauc_precision_at_10_diff1": 0.124417, + "nauc_precision_at_20_max": 0.185813, + "nauc_precision_at_20_std": -0.06728, + "nauc_precision_at_20_diff1": 0.028227, + "nauc_precision_at_100_max": 0.386596, + "nauc_precision_at_100_std": 0.475463, + "nauc_precision_at_100_diff1": -0.151714, + "nauc_precision_at_1000_max": 0.814815, + "nauc_precision_at_1000_std": 0.912854, + "nauc_precision_at_1000_diff1": -0.073296, + "nauc_mrr_at_1_max": 0.016149, + "nauc_mrr_at_1_std": -0.253527, + "nauc_mrr_at_1_diff1": 0.428338, + "nauc_mrr_at_3_max": 0.049457, + "nauc_mrr_at_3_std": -0.212489, + "nauc_mrr_at_3_diff1": 0.318832, + "nauc_mrr_at_5_max": 0.067019, + "nauc_mrr_at_5_std": -0.200678, + "nauc_mrr_at_5_diff1": 0.303222, + "nauc_mrr_at_10_max": 0.0645, + "nauc_mrr_at_10_std": -0.201378, + "nauc_mrr_at_10_diff1": 0.300744, + "nauc_mrr_at_20_max": 0.062538, + "nauc_mrr_at_20_std": -0.201168, + "nauc_mrr_at_20_diff1": 0.298222, + "nauc_mrr_at_100_max": 0.064959, + "nauc_mrr_at_100_std": -0.19861, + "nauc_mrr_at_100_diff1": 0.301106, + "nauc_mrr_at_1000_max": 0.064678, + "nauc_mrr_at_1000_std": -0.19931, + "nauc_mrr_at_1000_diff1": 0.30163, + "main_score": 0.37014, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 11.473597049713135, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/DBPedia.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/DBPedia.json new file mode 100644 index 0000000000..65c3de9154 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/DBPedia.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659", + "task_name": "DBPedia", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.53, + "ndcg_at_3": 0.43483, + "ndcg_at_5": 0.41593, + "ndcg_at_10": 0.39597, + "ndcg_at_20": 0.3948, + "ndcg_at_100": 0.44926, + "ndcg_at_1000": 0.52802, + "map_at_1": 0.07842, + "map_at_3": 0.13432, + "map_at_5": 0.15959, + "map_at_10": 0.18937, + "map_at_20": 0.22068, + "map_at_100": 0.2673, + "map_at_1000": 0.28416, + "recall_at_1": 0.07842, + "recall_at_3": 0.15328, + "recall_at_5": 0.19504, + "recall_at_10": 0.25564, + "recall_at_20": 0.33294, + "recall_at_100": 0.53048, + "recall_at_1000": 0.7714, + "precision_at_1": 0.6475, + "precision_at_3": 0.47417, + "precision_at_5": 0.408, + "precision_at_10": 0.31175, + "precision_at_20": 0.2385, + "precision_at_100": 0.09943, + "precision_at_1000": 0.02145, + "mrr_at_1": 0.6475, + "mrr_at_3": 0.715417, + "mrr_at_5": 0.728167, + "mrr_at_10": 0.734486, + "mrr_at_20": 0.737165, + "mrr_at_100": 0.738311, + "mrr_at_1000": 0.738319, + "nauc_ndcg_at_1_max": 0.421765, + "nauc_ndcg_at_1_std": 0.179738, + "nauc_ndcg_at_1_diff1": 0.427344, + "nauc_ndcg_at_3_max": 0.354186, + "nauc_ndcg_at_3_std": 0.189434, + "nauc_ndcg_at_3_diff1": 0.243705, + "nauc_ndcg_at_5_max": 0.317556, + "nauc_ndcg_at_5_std": 0.179121, + "nauc_ndcg_at_5_diff1": 0.204825, + "nauc_ndcg_at_10_max": 0.296274, + "nauc_ndcg_at_10_std": 0.173441, + "nauc_ndcg_at_10_diff1": 0.207754, + "nauc_ndcg_at_20_max": 0.268978, + "nauc_ndcg_at_20_std": 0.149254, + "nauc_ndcg_at_20_diff1": 0.21143, + "nauc_ndcg_at_100_max": 0.257339, + "nauc_ndcg_at_100_std": 0.208981, + "nauc_ndcg_at_100_diff1": 0.182753, + "nauc_ndcg_at_1000_max": 0.332263, + "nauc_ndcg_at_1000_std": 0.295607, + "nauc_ndcg_at_1000_diff1": 0.167623, + "nauc_map_at_1_max": 0.089357, + "nauc_map_at_1_std": -0.13262, + "nauc_map_at_1_diff1": 0.397184, + "nauc_map_at_3_max": 0.049604, + "nauc_map_at_3_std": -0.126429, + "nauc_map_at_3_diff1": 0.252795, + "nauc_map_at_5_max": 0.05316, + "nauc_map_at_5_std": -0.104452, + "nauc_map_at_5_diff1": 0.218992, + "nauc_map_at_10_max": 0.083823, + "nauc_map_at_10_std": -0.045407, + "nauc_map_at_10_diff1": 0.196169, + "nauc_map_at_20_max": 0.13973, + "nauc_map_at_20_std": 0.035846, + "nauc_map_at_20_diff1": 0.187792, + "nauc_map_at_100_max": 0.20358, + "nauc_map_at_100_std": 0.166509, + "nauc_map_at_100_diff1": 0.165323, + "nauc_map_at_1000_max": 0.230671, + "nauc_map_at_1000_std": 0.197281, + "nauc_map_at_1000_diff1": 0.160657, + "nauc_recall_at_1_max": 0.089357, + "nauc_recall_at_1_std": -0.13262, + "nauc_recall_at_1_diff1": 0.397184, + "nauc_recall_at_3_max": -0.025491, + "nauc_recall_at_3_std": -0.157256, + "nauc_recall_at_3_diff1": 0.181157, + "nauc_recall_at_5_max": -0.041138, + "nauc_recall_at_5_std": -0.157303, + "nauc_recall_at_5_diff1": 0.134946, + "nauc_recall_at_10_max": -0.040014, + "nauc_recall_at_10_std": -0.108202, + "nauc_recall_at_10_diff1": 0.09328, + "nauc_recall_at_20_max": 0.008901, + "nauc_recall_at_20_std": -0.043042, + "nauc_recall_at_20_diff1": 0.080776, + "nauc_recall_at_100_max": 0.073528, + "nauc_recall_at_100_std": 0.153796, + "nauc_recall_at_100_diff1": 0.045211, + "nauc_recall_at_1000_max": 0.171257, + "nauc_recall_at_1000_std": 0.347604, + "nauc_recall_at_1000_diff1": -0.014305, + "nauc_precision_at_1_max": 0.51602, + "nauc_precision_at_1_std": 0.244608, + "nauc_precision_at_1_diff1": 0.496674, + "nauc_precision_at_3_max": 0.376748, + "nauc_precision_at_3_std": 0.297508, + "nauc_precision_at_3_diff1": 0.095757, + "nauc_precision_at_5_max": 0.334144, + "nauc_precision_at_5_std": 0.329315, + "nauc_precision_at_5_diff1": 0.009508, + "nauc_precision_at_10_max": 0.347027, + "nauc_precision_at_10_std": 0.392431, + "nauc_precision_at_10_diff1": -0.004966, + "nauc_precision_at_20_max": 0.354442, + "nauc_precision_at_20_std": 0.429686, + "nauc_precision_at_20_diff1": 0.001377, + "nauc_precision_at_100_max": 0.310648, + "nauc_precision_at_100_std": 0.442508, + "nauc_precision_at_100_diff1": -0.04522, + "nauc_precision_at_1000_max": 0.294048, + "nauc_precision_at_1000_std": 0.199334, + "nauc_precision_at_1000_diff1": -0.063379, + "nauc_mrr_at_1_max": 0.51602, + "nauc_mrr_at_1_std": 0.244608, + "nauc_mrr_at_1_diff1": 0.496674, + "nauc_mrr_at_3_max": 0.52211, + "nauc_mrr_at_3_std": 0.282533, + "nauc_mrr_at_3_diff1": 0.476117, + "nauc_mrr_at_5_max": 0.524878, + "nauc_mrr_at_5_std": 0.281795, + "nauc_mrr_at_5_diff1": 0.471371, + "nauc_mrr_at_10_max": 0.524403, + "nauc_mrr_at_10_std": 0.27862, + "nauc_mrr_at_10_diff1": 0.47171, + "nauc_mrr_at_20_max": 0.523353, + "nauc_mrr_at_20_std": 0.276762, + "nauc_mrr_at_20_diff1": 0.471354, + "nauc_mrr_at_100_max": 0.524333, + "nauc_mrr_at_100_std": 0.277215, + "nauc_mrr_at_100_diff1": 0.473397, + "nauc_mrr_at_1000_max": 0.524317, + "nauc_mrr_at_1000_std": 0.277188, + "nauc_mrr_at_1000_diff1": 0.473386, + "main_score": 0.39597, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2768.8670661449432, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/EmotionClassification.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/EmotionClassification.json new file mode 100644 index 0000000000..81616f6dcb --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/EmotionClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37", + "task_name": "EmotionClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.4131, + "f1": 0.375272, + "f1_weighted": 0.435148, + "scores_per_experiment": [ + { + "accuracy": 0.448, + "f1": 0.401822, + "f1_weighted": 0.469849 + }, + { + "accuracy": 0.3865, + "f1": 0.359004, + "f1_weighted": 0.400752 + }, + { + "accuracy": 0.399, + "f1": 0.357066, + "f1_weighted": 0.423293 + }, + { + "accuracy": 0.3915, + "f1": 0.36337, + "f1_weighted": 0.411695 + }, + { + "accuracy": 0.436, + "f1": 0.392395, + "f1_weighted": 0.460459 + }, + { + "accuracy": 0.421, + "f1": 0.376977, + "f1_weighted": 0.442089 + }, + { + "accuracy": 0.396, + "f1": 0.361051, + "f1_weighted": 0.41868 + }, + { + "accuracy": 0.3835, + "f1": 0.352505, + "f1_weighted": 0.401426 + }, + { + "accuracy": 0.4305, + "f1": 0.391196, + "f1_weighted": 0.458549 + }, + { + "accuracy": 0.439, + "f1": 0.39733, + "f1_weighted": 0.464691 + } + ], + "main_score": 0.4131, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 21.86512565612793, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/FEVER.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/FEVER.json new file mode 100644 index 0000000000..345614bc39 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/FEVER.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12", + "task_name": "FEVER", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.83528, + "ndcg_at_3": 0.86843, + "ndcg_at_5": 0.8761, + "ndcg_at_10": 0.88042, + "ndcg_at_20": 0.88296, + "ndcg_at_100": 0.88602, + "ndcg_at_1000": 0.88844, + "map_at_1": 0.77411, + "map_at_3": 0.83731, + "map_at_5": 0.84309, + "map_at_10": 0.84541, + "map_at_20": 0.8464, + "map_at_100": 0.84702, + "map_at_1000": 0.84715, + "recall_at_1": 0.77411, + "recall_at_3": 0.90176, + "recall_at_5": 0.92228, + "recall_at_10": 0.93571, + "recall_at_20": 0.94422, + "recall_at_100": 0.95782, + "recall_at_1000": 0.97307, + "precision_at_1": 0.83528, + "precision_at_3": 0.32678, + "precision_at_5": 0.20123, + "precision_at_10": 0.10255, + "precision_at_20": 0.05207, + "precision_at_100": 0.0107, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.835284, + "mrr_at_3": 0.894789, + "mrr_at_5": 0.898825, + "mrr_at_10": 0.900198, + "mrr_at_20": 0.900465, + "mrr_at_100": 0.900594, + "mrr_at_1000": 0.900603, + "nauc_ndcg_at_1_max": 0.363067, + "nauc_ndcg_at_1_std": -0.139701, + "nauc_ndcg_at_1_diff1": 0.734498, + "nauc_ndcg_at_3_max": 0.280159, + "nauc_ndcg_at_3_std": -0.061132, + "nauc_ndcg_at_3_diff1": 0.515685, + "nauc_ndcg_at_5_max": 0.277992, + "nauc_ndcg_at_5_std": -0.040704, + "nauc_ndcg_at_5_diff1": 0.507479, + "nauc_ndcg_at_10_max": 0.274489, + "nauc_ndcg_at_10_std": -0.029196, + "nauc_ndcg_at_10_diff1": 0.503535, + "nauc_ndcg_at_20_max": 0.278532, + "nauc_ndcg_at_20_std": -0.024132, + "nauc_ndcg_at_20_diff1": 0.504829, + "nauc_ndcg_at_100_max": 0.280299, + "nauc_ndcg_at_100_std": -0.026325, + "nauc_ndcg_at_100_diff1": 0.506778, + "nauc_ndcg_at_1000_max": 0.280741, + "nauc_ndcg_at_1000_std": -0.030858, + "nauc_ndcg_at_1000_diff1": 0.512745, + "nauc_map_at_1_max": 0.287307, + "nauc_map_at_1_std": -0.064379, + "nauc_map_at_1_diff1": 0.567158, + "nauc_map_at_3_max": 0.263726, + "nauc_map_at_3_std": -0.050782, + "nauc_map_at_3_diff1": 0.503872, + "nauc_map_at_5_max": 0.26729, + "nauc_map_at_5_std": -0.042324, + "nauc_map_at_5_diff1": 0.503345, + "nauc_map_at_10_max": 0.267807, + "nauc_map_at_10_std": -0.038839, + "nauc_map_at_10_diff1": 0.503142, + "nauc_map_at_20_max": 0.269287, + "nauc_map_at_20_std": -0.037698, + "nauc_map_at_20_diff1": 0.503734, + "nauc_map_at_100_max": 0.269807, + "nauc_map_at_100_std": -0.037926, + "nauc_map_at_100_diff1": 0.503986, + "nauc_map_at_1000_max": 0.269825, + "nauc_map_at_1000_std": -0.038075, + "nauc_map_at_1000_diff1": 0.504191, + "nauc_recall_at_1_max": 0.287307, + "nauc_recall_at_1_std": -0.064379, + "nauc_recall_at_1_diff1": 0.567158, + "nauc_recall_at_3_max": 0.191474, + "nauc_recall_at_3_std": 0.010089, + "nauc_recall_at_3_diff1": 0.312126, + "nauc_recall_at_5_max": 0.189222, + "nauc_recall_at_5_std": 0.092293, + "nauc_recall_at_5_diff1": 0.246244, + "nauc_recall_at_10_max": 0.161568, + "nauc_recall_at_10_std": 0.175959, + "nauc_recall_at_10_diff1": 0.179113, + "nauc_recall_at_20_max": 0.166564, + "nauc_recall_at_20_std": 0.239147, + "nauc_recall_at_20_diff1": 0.143313, + "nauc_recall_at_100_max": 0.146326, + "nauc_recall_at_100_std": 0.292692, + "nauc_recall_at_100_diff1": 0.065309, + "nauc_recall_at_1000_max": 0.08059, + "nauc_recall_at_1000_std": 0.346538, + "nauc_recall_at_1000_diff1": -0.000647, + "nauc_precision_at_1_max": 0.363067, + "nauc_precision_at_1_std": -0.139701, + "nauc_precision_at_1_diff1": 0.734498, + "nauc_precision_at_3_max": 0.206436, + "nauc_precision_at_3_std": -0.110437, + "nauc_precision_at_3_diff1": 0.30033, + "nauc_precision_at_5_max": 0.148259, + "nauc_precision_at_5_std": -0.017164, + "nauc_precision_at_5_diff1": 0.123436, + "nauc_precision_at_10_max": 0.088327, + "nauc_precision_at_10_std": 0.053426, + "nauc_precision_at_10_diff1": -0.011637, + "nauc_precision_at_20_max": 0.081554, + "nauc_precision_at_20_std": 0.086269, + "nauc_precision_at_20_diff1": -0.073291, + "nauc_precision_at_100_max": 0.049003, + "nauc_precision_at_100_std": 0.067657, + "nauc_precision_at_100_diff1": -0.123495, + "nauc_precision_at_1000_max": 0.015282, + "nauc_precision_at_1000_std": 0.02069, + "nauc_precision_at_1000_diff1": -0.111825, + "nauc_mrr_at_1_max": 0.363067, + "nauc_mrr_at_1_std": -0.139701, + "nauc_mrr_at_1_diff1": 0.734498, + "nauc_mrr_at_3_max": 0.366487, + "nauc_mrr_at_3_std": -0.158566, + "nauc_mrr_at_3_diff1": 0.714864, + "nauc_mrr_at_5_max": 0.372706, + "nauc_mrr_at_5_std": -0.149412, + "nauc_mrr_at_5_diff1": 0.719427, + "nauc_mrr_at_10_max": 0.37138, + "nauc_mrr_at_10_std": -0.147304, + "nauc_mrr_at_10_diff1": 0.720441, + "nauc_mrr_at_20_max": 0.371129, + "nauc_mrr_at_20_std": -0.14659, + "nauc_mrr_at_20_diff1": 0.720482, + "nauc_mrr_at_100_max": 0.370578, + "nauc_mrr_at_100_std": -0.14741, + "nauc_mrr_at_100_diff1": 0.720418, + "nauc_mrr_at_1000_max": 0.370521, + "nauc_mrr_at_1000_std": -0.147506, + "nauc_mrr_at_1000_diff1": 0.720407, + "main_score": 0.88042, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5132.671888113022, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/FEVERHardNegatives.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/FEVERHardNegatives.json new file mode 100644 index 0000000000..fa21676e35 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/FEVERHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "080c9ed6267b65029207906e815d44a9240bafca", + "task_name": "FEVERHardNegatives", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.842, + "ndcg_at_3": 0.87662, + "ndcg_at_5": 0.88447, + "ndcg_at_10": 0.88922, + "ndcg_at_20": 0.89155, + "ndcg_at_100": 0.89537, + "ndcg_at_1000": 0.89819, + "map_at_1": 0.78955, + "map_at_3": 0.84733, + "map_at_5": 0.85309, + "map_at_10": 0.85547, + "map_at_20": 0.85643, + "map_at_100": 0.85723, + "map_at_1000": 0.8574, + "recall_at_1": 0.78955, + "recall_at_3": 0.91108, + "recall_at_5": 0.93275, + "recall_at_10": 0.94766, + "recall_at_20": 0.95508, + "recall_at_100": 0.97178, + "recall_at_1000": 0.98846, + "precision_at_1": 0.842, + "precision_at_3": 0.32467, + "precision_at_5": 0.1998, + "precision_at_10": 0.102, + "precision_at_20": 0.0519, + "precision_at_100": 0.01075, + "precision_at_1000": 0.00112, + "mrr_at_1": 0.842, + "mrr_at_3": 0.8975, + "mrr_at_5": 0.9023, + "mrr_at_10": 0.903802, + "mrr_at_20": 0.90394, + "mrr_at_100": 0.904058, + "mrr_at_1000": 0.904058, + "nauc_ndcg_at_1_max": 0.379086, + "nauc_ndcg_at_1_std": -0.206939, + "nauc_ndcg_at_1_diff1": 0.769895, + "nauc_ndcg_at_3_max": 0.277529, + "nauc_ndcg_at_3_std": -0.076273, + "nauc_ndcg_at_3_diff1": 0.567534, + "nauc_ndcg_at_5_max": 0.27356, + "nauc_ndcg_at_5_std": -0.082178, + "nauc_ndcg_at_5_diff1": 0.563305, + "nauc_ndcg_at_10_max": 0.267798, + "nauc_ndcg_at_10_std": -0.07262, + "nauc_ndcg_at_10_diff1": 0.564388, + "nauc_ndcg_at_20_max": 0.277457, + "nauc_ndcg_at_20_std": -0.074222, + "nauc_ndcg_at_20_diff1": 0.570143, + "nauc_ndcg_at_100_max": 0.27627, + "nauc_ndcg_at_100_std": -0.072343, + "nauc_ndcg_at_100_diff1": 0.569364, + "nauc_ndcg_at_1000_max": 0.281798, + "nauc_ndcg_at_1000_std": -0.079716, + "nauc_ndcg_at_1000_diff1": 0.583232, + "nauc_map_at_1_max": 0.297747, + "nauc_map_at_1_std": -0.122853, + "nauc_map_at_1_diff1": 0.634237, + "nauc_map_at_3_max": 0.262, + "nauc_map_at_3_std": -0.076607, + "nauc_map_at_3_diff1": 0.563795, + "nauc_map_at_5_max": 0.26647, + "nauc_map_at_5_std": -0.081864, + "nauc_map_at_5_diff1": 0.565622, + "nauc_map_at_10_max": 0.264646, + "nauc_map_at_10_std": -0.078913, + "nauc_map_at_10_diff1": 0.567145, + "nauc_map_at_20_max": 0.268116, + "nauc_map_at_20_std": -0.079705, + "nauc_map_at_20_diff1": 0.569405, + "nauc_map_at_100_max": 0.267864, + "nauc_map_at_100_std": -0.079748, + "nauc_map_at_100_diff1": 0.569085, + "nauc_map_at_1000_max": 0.268043, + "nauc_map_at_1000_std": -0.079897, + "nauc_map_at_1000_diff1": 0.569589, + "nauc_recall_at_1_max": 0.297747, + "nauc_recall_at_1_std": -0.122853, + "nauc_recall_at_1_diff1": 0.634237, + "nauc_recall_at_3_max": 0.18003, + "nauc_recall_at_3_std": 0.049733, + "nauc_recall_at_3_diff1": 0.361584, + "nauc_recall_at_5_max": 0.167003, + "nauc_recall_at_5_std": 0.057739, + "nauc_recall_at_5_diff1": 0.290552, + "nauc_recall_at_10_max": 0.100074, + "nauc_recall_at_10_std": 0.144849, + "nauc_recall_at_10_diff1": 0.223686, + "nauc_recall_at_20_max": 0.134758, + "nauc_recall_at_20_std": 0.166421, + "nauc_recall_at_20_diff1": 0.21943, + "nauc_recall_at_100_max": 0.043737, + "nauc_recall_at_100_std": 0.337055, + "nauc_recall_at_100_diff1": 0.063535, + "nauc_recall_at_1000_max": 0.017, + "nauc_recall_at_1000_std": 0.514394, + "nauc_recall_at_1000_diff1": 0.092937, + "nauc_precision_at_1_max": 0.379086, + "nauc_precision_at_1_std": -0.206939, + "nauc_precision_at_1_diff1": 0.769895, + "nauc_precision_at_3_max": 0.232389, + "nauc_precision_at_3_std": -0.032174, + "nauc_precision_at_3_diff1": 0.334509, + "nauc_precision_at_5_max": 0.184911, + "nauc_precision_at_5_std": -0.072111, + "nauc_precision_at_5_diff1": 0.134665, + "nauc_precision_at_10_max": 0.012254, + "nauc_precision_at_10_std": 0.060577, + "nauc_precision_at_10_diff1": -0.075343, + "nauc_precision_at_20_max": 0.070249, + "nauc_precision_at_20_std": 0.038032, + "nauc_precision_at_20_diff1": -0.110429, + "nauc_precision_at_100_max": 0.002426, + "nauc_precision_at_100_std": 0.06701, + "nauc_precision_at_100_diff1": -0.209441, + "nauc_precision_at_1000_max": -0.008365, + "nauc_precision_at_1000_std": -0.001762, + "nauc_precision_at_1000_diff1": -0.139123, + "nauc_mrr_at_1_max": 0.379086, + "nauc_mrr_at_1_std": -0.206939, + "nauc_mrr_at_1_diff1": 0.769895, + "nauc_mrr_at_3_max": 0.371442, + "nauc_mrr_at_3_std": -0.187016, + "nauc_mrr_at_3_diff1": 0.73374, + "nauc_mrr_at_5_max": 0.377715, + "nauc_mrr_at_5_std": -0.197169, + "nauc_mrr_at_5_diff1": 0.740104, + "nauc_mrr_at_10_max": 0.375667, + "nauc_mrr_at_10_std": -0.197244, + "nauc_mrr_at_10_diff1": 0.740976, + "nauc_mrr_at_20_max": 0.375581, + "nauc_mrr_at_20_std": -0.196271, + "nauc_mrr_at_20_diff1": 0.740804, + "nauc_mrr_at_100_max": 0.374982, + "nauc_mrr_at_100_std": -0.196542, + "nauc_mrr_at_100_diff1": 0.741347, + "nauc_mrr_at_1000_max": 0.374982, + "nauc_mrr_at_1000_std": -0.196542, + "nauc_mrr_at_1000_diff1": 0.741347, + "main_score": 0.88922, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 150.56389570236206, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/FiQA2018.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/FiQA2018.json new file mode 100644 index 0000000000..f6035fe995 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/FiQA2018.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "27a168819829fe9bcd655c2df245fb19452e8e06", + "task_name": "FiQA2018", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.44907, + "ndcg_at_3": 0.42173, + "ndcg_at_5": 0.43869, + "ndcg_at_10": 0.46324, + "ndcg_at_20": 0.48453, + "ndcg_at_100": 0.52701, + "ndcg_at_1000": 0.55505, + "map_at_1": 0.23395, + "map_at_3": 0.32997, + "map_at_5": 0.35933, + "map_at_10": 0.37955, + "map_at_20": 0.389, + "map_at_100": 0.39765, + "map_at_1000": 0.39945, + "recall_at_1": 0.23395, + "recall_at_3": 0.38439, + "recall_at_5": 0.45707, + "recall_at_10": 0.5374, + "recall_at_20": 0.59969, + "recall_at_100": 0.77631, + "recall_at_1000": 0.94385, + "precision_at_1": 0.44907, + "precision_at_3": 0.27984, + "precision_at_5": 0.20988, + "precision_at_10": 0.1284, + "precision_at_20": 0.07423, + "precision_at_100": 0.01957, + "precision_at_1000": 0.00245, + "mrr_at_1": 0.449074, + "mrr_at_3": 0.519547, + "mrr_at_5": 0.534208, + "mrr_at_10": 0.542444, + "mrr_at_20": 0.546029, + "mrr_at_100": 0.549152, + "mrr_at_1000": 0.549474, + "nauc_ndcg_at_1_max": 0.360524, + "nauc_ndcg_at_1_std": -0.046111, + "nauc_ndcg_at_1_diff1": 0.568359, + "nauc_ndcg_at_3_max": 0.33295, + "nauc_ndcg_at_3_std": -0.019471, + "nauc_ndcg_at_3_diff1": 0.430148, + "nauc_ndcg_at_5_max": 0.314372, + "nauc_ndcg_at_5_std": -0.036986, + "nauc_ndcg_at_5_diff1": 0.415123, + "nauc_ndcg_at_10_max": 0.31122, + "nauc_ndcg_at_10_std": -0.015457, + "nauc_ndcg_at_10_diff1": 0.417993, + "nauc_ndcg_at_20_max": 0.329545, + "nauc_ndcg_at_20_std": 0.012263, + "nauc_ndcg_at_20_diff1": 0.419643, + "nauc_ndcg_at_100_max": 0.344954, + "nauc_ndcg_at_100_std": 0.030137, + "nauc_ndcg_at_100_diff1": 0.422843, + "nauc_ndcg_at_1000_max": 0.353327, + "nauc_ndcg_at_1000_std": 0.02189, + "nauc_ndcg_at_1000_diff1": 0.435875, + "nauc_map_at_1_max": 0.168448, + "nauc_map_at_1_std": -0.071616, + "nauc_map_at_1_diff1": 0.452394, + "nauc_map_at_3_max": 0.247171, + "nauc_map_at_3_std": -0.045547, + "nauc_map_at_3_diff1": 0.413155, + "nauc_map_at_5_max": 0.262809, + "nauc_map_at_5_std": -0.051528, + "nauc_map_at_5_diff1": 0.401168, + "nauc_map_at_10_max": 0.277924, + "nauc_map_at_10_std": -0.035395, + "nauc_map_at_10_diff1": 0.401639, + "nauc_map_at_20_max": 0.289856, + "nauc_map_at_20_std": -0.023736, + "nauc_map_at_20_diff1": 0.403927, + "nauc_map_at_100_max": 0.29488, + "nauc_map_at_100_std": -0.01844, + "nauc_map_at_100_diff1": 0.404263, + "nauc_map_at_1000_max": 0.296143, + "nauc_map_at_1000_std": -0.018544, + "nauc_map_at_1000_diff1": 0.405337, + "nauc_recall_at_1_max": 0.168448, + "nauc_recall_at_1_std": -0.071616, + "nauc_recall_at_1_diff1": 0.452394, + "nauc_recall_at_3_max": 0.224809, + "nauc_recall_at_3_std": -0.01712, + "nauc_recall_at_3_diff1": 0.340133, + "nauc_recall_at_5_max": 0.223863, + "nauc_recall_at_5_std": -0.04782, + "nauc_recall_at_5_diff1": 0.300714, + "nauc_recall_at_10_max": 0.223178, + "nauc_recall_at_10_std": 0.008488, + "nauc_recall_at_10_diff1": 0.290201, + "nauc_recall_at_20_max": 0.266708, + "nauc_recall_at_20_std": 0.096209, + "nauc_recall_at_20_diff1": 0.287199, + "nauc_recall_at_100_max": 0.27465, + "nauc_recall_at_100_std": 0.217751, + "nauc_recall_at_100_diff1": 0.232729, + "nauc_recall_at_1000_max": 0.311056, + "nauc_recall_at_1000_std": 0.373453, + "nauc_recall_at_1000_diff1": 0.282073, + "nauc_precision_at_1_max": 0.360524, + "nauc_precision_at_1_std": -0.046111, + "nauc_precision_at_1_diff1": 0.568359, + "nauc_precision_at_3_max": 0.404955, + "nauc_precision_at_3_std": 0.040956, + "nauc_precision_at_3_diff1": 0.300488, + "nauc_precision_at_5_max": 0.394855, + "nauc_precision_at_5_std": 0.036942, + "nauc_precision_at_5_diff1": 0.226795, + "nauc_precision_at_10_max": 0.391291, + "nauc_precision_at_10_std": 0.092583, + "nauc_precision_at_10_diff1": 0.190554, + "nauc_precision_at_20_max": 0.403865, + "nauc_precision_at_20_std": 0.145473, + "nauc_precision_at_20_diff1": 0.161442, + "nauc_precision_at_100_max": 0.376669, + "nauc_precision_at_100_std": 0.182774, + "nauc_precision_at_100_diff1": 0.094687, + "nauc_precision_at_1000_max": 0.318456, + "nauc_precision_at_1000_std": 0.135928, + "nauc_precision_at_1000_diff1": 0.052251, + "nauc_mrr_at_1_max": 0.360524, + "nauc_mrr_at_1_std": -0.046111, + "nauc_mrr_at_1_diff1": 0.568359, + "nauc_mrr_at_3_max": 0.382248, + "nauc_mrr_at_3_std": -0.02169, + "nauc_mrr_at_3_diff1": 0.528316, + "nauc_mrr_at_5_max": 0.386911, + "nauc_mrr_at_5_std": -0.023003, + "nauc_mrr_at_5_diff1": 0.528862, + "nauc_mrr_at_10_max": 0.384084, + "nauc_mrr_at_10_std": -0.018704, + "nauc_mrr_at_10_diff1": 0.529995, + "nauc_mrr_at_20_max": 0.384787, + "nauc_mrr_at_20_std": -0.015758, + "nauc_mrr_at_20_diff1": 0.528917, + "nauc_mrr_at_100_max": 0.384946, + "nauc_mrr_at_100_std": -0.016346, + "nauc_mrr_at_100_diff1": 0.52891, + "nauc_mrr_at_1000_max": 0.384924, + "nauc_mrr_at_1000_std": -0.016476, + "nauc_mrr_at_1000_diff1": 0.529191, + "main_score": 0.46324, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 58.39927649497986, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/HotpotQA.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/HotpotQA.json new file mode 100644 index 0000000000..244c7366e0 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/HotpotQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ab518f4d6fcca38d87c25209f94beba119d02014", + "task_name": "HotpotQA", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.75814, + "ndcg_at_3": 0.62568, + "ndcg_at_5": 0.65356, + "ndcg_at_10": 0.67361, + "ndcg_at_20": 0.68631, + "ndcg_at_100": 0.70399, + "ndcg_at_1000": 0.7163, + "map_at_1": 0.37907, + "map_at_3": 0.55146, + "map_at_5": 0.57382, + "map_at_10": 0.58577, + "map_at_20": 0.59085, + "map_at_100": 0.5944, + "map_at_1000": 0.59502, + "recall_at_1": 0.37907, + "recall_at_3": 0.59932, + "recall_at_5": 0.65436, + "recall_at_10": 0.70452, + "recall_at_20": 0.74537, + "recall_at_100": 0.82363, + "recall_at_1000": 0.9056, + "precision_at_1": 0.75814, + "precision_at_3": 0.39955, + "precision_at_5": 0.26174, + "precision_at_10": 0.1409, + "precision_at_20": 0.07454, + "precision_at_100": 0.01647, + "precision_at_1000": 0.00181, + "mrr_at_1": 0.758136, + "mrr_at_3": 0.814067, + "mrr_at_5": 0.819968, + "mrr_at_10": 0.82337, + "mrr_at_20": 0.82465, + "mrr_at_100": 0.825365, + "mrr_at_1000": 0.825438, + "nauc_ndcg_at_1_max": 0.453966, + "nauc_ndcg_at_1_std": -0.064472, + "nauc_ndcg_at_1_diff1": 0.659559, + "nauc_ndcg_at_3_max": 0.292726, + "nauc_ndcg_at_3_std": 0.032977, + "nauc_ndcg_at_3_diff1": 0.226815, + "nauc_ndcg_at_5_max": 0.286174, + "nauc_ndcg_at_5_std": 0.058399, + "nauc_ndcg_at_5_diff1": 0.204984, + "nauc_ndcg_at_10_max": 0.286375, + "nauc_ndcg_at_10_std": 0.081299, + "nauc_ndcg_at_10_diff1": 0.195832, + "nauc_ndcg_at_20_max": 0.287371, + "nauc_ndcg_at_20_std": 0.094413, + "nauc_ndcg_at_20_diff1": 0.194201, + "nauc_ndcg_at_100_max": 0.292663, + "nauc_ndcg_at_100_std": 0.106587, + "nauc_ndcg_at_100_diff1": 0.199449, + "nauc_ndcg_at_1000_max": 0.293428, + "nauc_ndcg_at_1000_std": 0.102244, + "nauc_ndcg_at_1000_diff1": 0.207124, + "nauc_map_at_1_max": 0.453966, + "nauc_map_at_1_std": -0.064472, + "nauc_map_at_1_diff1": 0.659559, + "nauc_map_at_3_max": 0.258246, + "nauc_map_at_3_std": 0.038571, + "nauc_map_at_3_diff1": 0.170456, + "nauc_map_at_5_max": 0.254909, + "nauc_map_at_5_std": 0.056537, + "nauc_map_at_5_diff1": 0.156886, + "nauc_map_at_10_max": 0.256017, + "nauc_map_at_10_std": 0.068258, + "nauc_map_at_10_diff1": 0.153393, + "nauc_map_at_20_max": 0.257001, + "nauc_map_at_20_std": 0.073005, + "nauc_map_at_20_diff1": 0.153555, + "nauc_map_at_100_max": 0.258144, + "nauc_map_at_100_std": 0.074992, + "nauc_map_at_100_diff1": 0.154656, + "nauc_map_at_1000_max": 0.25815, + "nauc_map_at_1000_std": 0.074932, + "nauc_map_at_1000_diff1": 0.154861, + "nauc_recall_at_1_max": 0.453966, + "nauc_recall_at_1_std": -0.064472, + "nauc_recall_at_1_diff1": 0.659559, + "nauc_recall_at_3_max": 0.236479, + "nauc_recall_at_3_std": 0.064596, + "nauc_recall_at_3_diff1": 0.089441, + "nauc_recall_at_5_max": 0.209557, + "nauc_recall_at_5_std": 0.111594, + "nauc_recall_at_5_diff1": 0.035031, + "nauc_recall_at_10_max": 0.198843, + "nauc_recall_at_10_std": 0.171704, + "nauc_recall_at_10_diff1": -0.004837, + "nauc_recall_at_20_max": 0.189894, + "nauc_recall_at_20_std": 0.22219, + "nauc_recall_at_20_diff1": -0.031434, + "nauc_recall_at_100_max": 0.188287, + "nauc_recall_at_100_std": 0.327311, + "nauc_recall_at_100_diff1": -0.068645, + "nauc_recall_at_1000_max": 0.142252, + "nauc_recall_at_1000_std": 0.4094, + "nauc_recall_at_1000_diff1": -0.129241, + "nauc_precision_at_1_max": 0.453966, + "nauc_precision_at_1_std": -0.064472, + "nauc_precision_at_1_diff1": 0.659559, + "nauc_precision_at_3_max": 0.236479, + "nauc_precision_at_3_std": 0.064596, + "nauc_precision_at_3_diff1": 0.089441, + "nauc_precision_at_5_max": 0.209557, + "nauc_precision_at_5_std": 0.111594, + "nauc_precision_at_5_diff1": 0.035031, + "nauc_precision_at_10_max": 0.198843, + "nauc_precision_at_10_std": 0.171704, + "nauc_precision_at_10_diff1": -0.004837, + "nauc_precision_at_20_max": 0.189894, + "nauc_precision_at_20_std": 0.22219, + "nauc_precision_at_20_diff1": -0.031434, + "nauc_precision_at_100_max": 0.188287, + "nauc_precision_at_100_std": 0.327311, + "nauc_precision_at_100_diff1": -0.068645, + "nauc_precision_at_1000_max": 0.142252, + "nauc_precision_at_1000_std": 0.4094, + "nauc_precision_at_1000_diff1": -0.129241, + "nauc_mrr_at_1_max": 0.453966, + "nauc_mrr_at_1_std": -0.064472, + "nauc_mrr_at_1_diff1": 0.659559, + "nauc_mrr_at_3_max": 0.486094, + "nauc_mrr_at_3_std": -0.044009, + "nauc_mrr_at_3_diff1": 0.640478, + "nauc_mrr_at_5_max": 0.485936, + "nauc_mrr_at_5_std": -0.038806, + "nauc_mrr_at_5_diff1": 0.639911, + "nauc_mrr_at_10_max": 0.485693, + "nauc_mrr_at_10_std": -0.035688, + "nauc_mrr_at_10_diff1": 0.640822, + "nauc_mrr_at_20_max": 0.485218, + "nauc_mrr_at_20_std": -0.035575, + "nauc_mrr_at_20_diff1": 0.641107, + "nauc_mrr_at_100_max": 0.484779, + "nauc_mrr_at_100_std": -0.036061, + "nauc_mrr_at_100_diff1": 0.641399, + "nauc_mrr_at_1000_max": 0.484707, + "nauc_mrr_at_1000_std": -0.036163, + "nauc_mrr_at_1000_diff1": 0.641423, + "main_score": 0.67361, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3599.2914655208588, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/HotpotQAHardNegatives.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/HotpotQAHardNegatives.json new file mode 100644 index 0000000000..b4a63e3c31 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/HotpotQAHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "617612fa63afcb60e3b134bed8b7216a99707c37", + "task_name": "HotpotQAHardNegatives", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.737, + "ndcg_at_3": 0.61895, + "ndcg_at_5": 0.65004, + "ndcg_at_10": 0.67078, + "ndcg_at_20": 0.68931, + "ndcg_at_100": 0.71457, + "ndcg_at_1000": 0.7278, + "map_at_1": 0.3685, + "map_at_3": 0.54592, + "map_at_5": 0.56999, + "map_at_10": 0.58239, + "map_at_20": 0.58973, + "map_at_100": 0.59476, + "map_at_1000": 0.5955, + "recall_at_1": 0.3685, + "recall_at_3": 0.5975, + "recall_at_5": 0.659, + "recall_at_10": 0.711, + "recall_at_20": 0.7705, + "recall_at_100": 0.883, + "recall_at_1000": 0.9695, + "precision_at_1": 0.737, + "precision_at_3": 0.39833, + "precision_at_5": 0.2636, + "precision_at_10": 0.1422, + "precision_at_20": 0.07705, + "precision_at_100": 0.01766, + "precision_at_1000": 0.00194, + "mrr_at_1": 0.737, + "mrr_at_3": 0.798167, + "mrr_at_5": 0.806317, + "mrr_at_10": 0.809686, + "mrr_at_20": 0.811737, + "mrr_at_100": 0.812737, + "mrr_at_1000": 0.812772, + "nauc_ndcg_at_1_max": 0.413816, + "nauc_ndcg_at_1_std": -0.096483, + "nauc_ndcg_at_1_diff1": 0.608055, + "nauc_ndcg_at_3_max": 0.250245, + "nauc_ndcg_at_3_std": -0.065438, + "nauc_ndcg_at_3_diff1": 0.190163, + "nauc_ndcg_at_5_max": 0.246522, + "nauc_ndcg_at_5_std": -0.060158, + "nauc_ndcg_at_5_diff1": 0.168783, + "nauc_ndcg_at_10_max": 0.247629, + "nauc_ndcg_at_10_std": -0.041746, + "nauc_ndcg_at_10_diff1": 0.161666, + "nauc_ndcg_at_20_max": 0.25414, + "nauc_ndcg_at_20_std": -0.038811, + "nauc_ndcg_at_20_diff1": 0.172646, + "nauc_ndcg_at_100_max": 0.262637, + "nauc_ndcg_at_100_std": -0.021925, + "nauc_ndcg_at_100_diff1": 0.183758, + "nauc_ndcg_at_1000_max": 0.262706, + "nauc_ndcg_at_1000_std": -0.02612, + "nauc_ndcg_at_1000_diff1": 0.193876, + "nauc_map_at_1_max": 0.413816, + "nauc_map_at_1_std": -0.096483, + "nauc_map_at_1_diff1": 0.608055, + "nauc_map_at_3_max": 0.21128, + "nauc_map_at_3_std": -0.060731, + "nauc_map_at_3_diff1": 0.135723, + "nauc_map_at_5_max": 0.212174, + "nauc_map_at_5_std": -0.057734, + "nauc_map_at_5_diff1": 0.126018, + "nauc_map_at_10_max": 0.214071, + "nauc_map_at_10_std": -0.04528, + "nauc_map_at_10_diff1": 0.122006, + "nauc_map_at_20_max": 0.2174, + "nauc_map_at_20_std": -0.044993, + "nauc_map_at_20_diff1": 0.126957, + "nauc_map_at_100_max": 0.218648, + "nauc_map_at_100_std": -0.043239, + "nauc_map_at_100_diff1": 0.129174, + "nauc_map_at_1000_max": 0.218533, + "nauc_map_at_1000_std": -0.043109, + "nauc_map_at_1000_diff1": 0.129412, + "nauc_recall_at_1_max": 0.413816, + "nauc_recall_at_1_std": -0.096483, + "nauc_recall_at_1_diff1": 0.608055, + "nauc_recall_at_3_max": 0.188462, + "nauc_recall_at_3_std": -0.053238, + "nauc_recall_at_3_diff1": 0.050122, + "nauc_recall_at_5_max": 0.168895, + "nauc_recall_at_5_std": -0.037725, + "nauc_recall_at_5_diff1": -0.006463, + "nauc_recall_at_10_max": 0.161721, + "nauc_recall_at_10_std": 0.010954, + "nauc_recall_at_10_diff1": -0.040513, + "nauc_recall_at_20_max": 0.164824, + "nauc_recall_at_20_std": 0.032471, + "nauc_recall_at_20_diff1": -0.03934, + "nauc_recall_at_100_max": 0.172361, + "nauc_recall_at_100_std": 0.222015, + "nauc_recall_at_100_diff1": -0.096228, + "nauc_recall_at_1000_max": 0.074367, + "nauc_recall_at_1000_std": 0.587715, + "nauc_recall_at_1000_diff1": -0.314873, + "nauc_precision_at_1_max": 0.413816, + "nauc_precision_at_1_std": -0.096483, + "nauc_precision_at_1_diff1": 0.608055, + "nauc_precision_at_3_max": 0.188462, + "nauc_precision_at_3_std": -0.053238, + "nauc_precision_at_3_diff1": 0.050122, + "nauc_precision_at_5_max": 0.168895, + "nauc_precision_at_5_std": -0.037725, + "nauc_precision_at_5_diff1": -0.006463, + "nauc_precision_at_10_max": 0.161721, + "nauc_precision_at_10_std": 0.010954, + "nauc_precision_at_10_diff1": -0.040513, + "nauc_precision_at_20_max": 0.164824, + "nauc_precision_at_20_std": 0.032471, + "nauc_precision_at_20_diff1": -0.03934, + "nauc_precision_at_100_max": 0.172361, + "nauc_precision_at_100_std": 0.222015, + "nauc_precision_at_100_diff1": -0.096228, + "nauc_precision_at_1000_max": 0.074367, + "nauc_precision_at_1000_std": 0.587715, + "nauc_precision_at_1000_diff1": -0.314873, + "nauc_mrr_at_1_max": 0.413816, + "nauc_mrr_at_1_std": -0.096483, + "nauc_mrr_at_1_diff1": 0.608055, + "nauc_mrr_at_3_max": 0.454603, + "nauc_mrr_at_3_std": -0.096612, + "nauc_mrr_at_3_diff1": 0.582596, + "nauc_mrr_at_5_max": 0.450769, + "nauc_mrr_at_5_std": -0.097232, + "nauc_mrr_at_5_diff1": 0.577414, + "nauc_mrr_at_10_max": 0.449449, + "nauc_mrr_at_10_std": -0.101609, + "nauc_mrr_at_10_diff1": 0.581881, + "nauc_mrr_at_20_max": 0.449033, + "nauc_mrr_at_20_std": -0.100989, + "nauc_mrr_at_20_diff1": 0.584164, + "nauc_mrr_at_100_max": 0.449335, + "nauc_mrr_at_100_std": -0.099887, + "nauc_mrr_at_100_diff1": 0.583909, + "nauc_mrr_at_1000_max": 0.449237, + "nauc_mrr_at_1000_std": -0.099957, + "nauc_mrr_at_1000_diff1": 0.583906, + "main_score": 0.67078, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 122.98756408691406, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ImdbClassification.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ImdbClassification.json new file mode 100644 index 0000000000..a21f031e56 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ImdbClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7", + "task_name": "ImdbClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.619128, + "f1": 0.616877, + "f1_weighted": 0.616877, + "ap": 0.575814, + "ap_weighted": 0.575814, + "scores_per_experiment": [ + { + "accuracy": 0.65152, + "f1": 0.647214, + "f1_weighted": 0.647214, + "ap": 0.60523, + "ap_weighted": 0.60523 + }, + { + "accuracy": 0.64384, + "f1": 0.640289, + "f1_weighted": 0.640289, + "ap": 0.597741, + "ap_weighted": 0.597741 + }, + { + "accuracy": 0.54436, + "f1": 0.543176, + "f1_weighted": 0.543176, + "ap": 0.523966, + "ap_weighted": 0.523966 + }, + { + "accuracy": 0.65984, + "f1": 0.658898, + "f1_weighted": 0.658898, + "ap": 0.603039, + "ap_weighted": 0.603039 + }, + { + "accuracy": 0.62124, + "f1": 0.619268, + "f1_weighted": 0.619268, + "ap": 0.57779, + "ap_weighted": 0.57779 + }, + { + "accuracy": 0.63368, + "f1": 0.633549, + "f1_weighted": 0.633549, + "ap": 0.585412, + "ap_weighted": 0.585412 + }, + { + "accuracy": 0.57668, + "f1": 0.576014, + "f1_weighted": 0.576014, + "ap": 0.544726, + "ap_weighted": 0.544726 + }, + { + "accuracy": 0.631, + "f1": 0.630971, + "f1_weighted": 0.630971, + "ap": 0.582973, + "ap_weighted": 0.582973 + }, + { + "accuracy": 0.58276, + "f1": 0.579595, + "f1_weighted": 0.579595, + "ap": 0.547216, + "ap_weighted": 0.547216 + }, + { + "accuracy": 0.64636, + "f1": 0.639795, + "f1_weighted": 0.639795, + "ap": 0.590047, + "ap_weighted": 0.590047 + } + ], + "main_score": 0.619128, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 52.78667402267456, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBNarrativeQARetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBNarrativeQARetrieval.json new file mode 100644 index 0000000000..a4a8439682 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBNarrativeQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBNarrativeQARetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.37286, + "ndcg_at_3": 0.44008, + "ndcg_at_5": 0.45854, + "ndcg_at_10": 0.47847, + "ndcg_at_20": 0.49533, + "ndcg_at_100": 0.52875, + "ndcg_at_1000": 0.54946, + "map_at_1": 0.37286, + "map_at_3": 0.42372, + "map_at_5": 0.43398, + "map_at_10": 0.44224, + "map_at_20": 0.44686, + "map_at_100": 0.45118, + "map_at_1000": 0.45209, + "recall_at_1": 0.37286, + "recall_at_3": 0.48732, + "recall_at_5": 0.53211, + "recall_at_10": 0.59355, + "recall_at_20": 0.66025, + "recall_at_100": 0.84487, + "recall_at_1000": 1.0, + "precision_at_1": 0.37286, + "precision_at_3": 0.16244, + "precision_at_5": 0.10642, + "precision_at_10": 0.05935, + "precision_at_20": 0.03301, + "precision_at_100": 0.00845, + "precision_at_1000": 0.001, + "mrr_at_1": 0.372859, + "mrr_at_3": 0.423725, + "mrr_at_5": 0.433984, + "mrr_at_10": 0.442241, + "mrr_at_20": 0.446862, + "mrr_at_100": 0.451183, + "mrr_at_1000": 0.452085, + "nauc_ndcg_at_1_max": 0.55995, + "nauc_ndcg_at_1_std": 0.082915, + "nauc_ndcg_at_1_diff1": 0.685478, + "nauc_ndcg_at_3_max": 0.543495, + "nauc_ndcg_at_3_std": 0.112641, + "nauc_ndcg_at_3_diff1": 0.633544, + "nauc_ndcg_at_5_max": 0.537914, + "nauc_ndcg_at_5_std": 0.120278, + "nauc_ndcg_at_5_diff1": 0.624743, + "nauc_ndcg_at_10_max": 0.527193, + "nauc_ndcg_at_10_std": 0.122371, + "nauc_ndcg_at_10_diff1": 0.614383, + "nauc_ndcg_at_20_max": 0.520894, + "nauc_ndcg_at_20_std": 0.123761, + "nauc_ndcg_at_20_diff1": 0.609564, + "nauc_ndcg_at_100_max": 0.519923, + "nauc_ndcg_at_100_std": 0.129473, + "nauc_ndcg_at_100_diff1": 0.611927, + "nauc_ndcg_at_1000_max": 0.530172, + "nauc_ndcg_at_1000_std": 0.119537, + "nauc_ndcg_at_1000_diff1": 0.623312, + "nauc_map_at_1_max": 0.55995, + "nauc_map_at_1_std": 0.082915, + "nauc_map_at_1_diff1": 0.685478, + "nauc_map_at_3_max": 0.548406, + "nauc_map_at_3_std": 0.105528, + "nauc_map_at_3_diff1": 0.646183, + "nauc_map_at_5_max": 0.545441, + "nauc_map_at_5_std": 0.10964, + "nauc_map_at_5_diff1": 0.641409, + "nauc_map_at_10_max": 0.541249, + "nauc_map_at_10_std": 0.110458, + "nauc_map_at_10_diff1": 0.637376, + "nauc_map_at_20_max": 0.539661, + "nauc_map_at_20_std": 0.110723, + "nauc_map_at_20_diff1": 0.636296, + "nauc_map_at_100_max": 0.539465, + "nauc_map_at_100_std": 0.111325, + "nauc_map_at_100_diff1": 0.636539, + "nauc_map_at_1000_max": 0.539844, + "nauc_map_at_1000_std": 0.111004, + "nauc_map_at_1000_diff1": 0.636989, + "nauc_recall_at_1_max": 0.55995, + "nauc_recall_at_1_std": 0.082915, + "nauc_recall_at_1_diff1": 0.685478, + "nauc_recall_at_3_max": 0.528787, + "nauc_recall_at_3_std": 0.133347, + "nauc_recall_at_3_diff1": 0.59666, + "nauc_recall_at_5_max": 0.513983, + "nauc_recall_at_5_std": 0.15338, + "nauc_recall_at_5_diff1": 0.573388, + "nauc_recall_at_10_max": 0.476083, + "nauc_recall_at_10_std": 0.162327, + "nauc_recall_at_10_diff1": 0.535172, + "nauc_recall_at_20_max": 0.440323, + "nauc_recall_at_20_std": 0.174099, + "nauc_recall_at_20_diff1": 0.502277, + "nauc_recall_at_100_max": 0.365042, + "nauc_recall_at_100_std": 0.284301, + "nauc_recall_at_100_diff1": 0.442113, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.55995, + "nauc_precision_at_1_std": 0.082915, + "nauc_precision_at_1_diff1": 0.685478, + "nauc_precision_at_3_max": 0.528787, + "nauc_precision_at_3_std": 0.133347, + "nauc_precision_at_3_diff1": 0.59666, + "nauc_precision_at_5_max": 0.513983, + "nauc_precision_at_5_std": 0.15338, + "nauc_precision_at_5_diff1": 0.573388, + "nauc_precision_at_10_max": 0.476083, + "nauc_precision_at_10_std": 0.162327, + "nauc_precision_at_10_diff1": 0.535172, + "nauc_precision_at_20_max": 0.440323, + "nauc_precision_at_20_std": 0.174099, + "nauc_precision_at_20_diff1": 0.502277, + "nauc_precision_at_100_max": 0.365042, + "nauc_precision_at_100_std": 0.284301, + "nauc_precision_at_100_diff1": 0.442113, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.55995, + "nauc_mrr_at_1_std": 0.082915, + "nauc_mrr_at_1_diff1": 0.685478, + "nauc_mrr_at_3_max": 0.548406, + "nauc_mrr_at_3_std": 0.105528, + "nauc_mrr_at_3_diff1": 0.646183, + "nauc_mrr_at_5_max": 0.545441, + "nauc_mrr_at_5_std": 0.10964, + "nauc_mrr_at_5_diff1": 0.641409, + "nauc_mrr_at_10_max": 0.541249, + "nauc_mrr_at_10_std": 0.110458, + "nauc_mrr_at_10_diff1": 0.637376, + "nauc_mrr_at_20_max": 0.539661, + "nauc_mrr_at_20_std": 0.110723, + "nauc_mrr_at_20_diff1": 0.636296, + "nauc_mrr_at_100_max": 0.539465, + "nauc_mrr_at_100_std": 0.111325, + "nauc_mrr_at_100_diff1": 0.636539, + "nauc_mrr_at_1000_max": 0.539844, + "nauc_mrr_at_1000_std": 0.111004, + "nauc_mrr_at_1000_diff1": 0.636989, + "main_score": 0.47847, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 81.34865379333496, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBNeedleRetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBNeedleRetrieval.json new file mode 100644 index 0000000000..c709e34660 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBNeedleRetrieval.json @@ -0,0 +1,1201 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBNeedleRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test_256": [ + { + "ndcg_at_1": 0.84, + "ndcg_at_3": 0.93833, + "ndcg_at_5": 0.93833, + "ndcg_at_10": 0.93833, + "ndcg_at_20": 0.93833, + "ndcg_at_100": 0.93833, + "ndcg_at_1000": 0.93833, + "map_at_1": 0.84, + "map_at_3": 0.91667, + "map_at_5": 0.91667, + "map_at_10": 0.91667, + "map_at_20": 0.91667, + "map_at_100": 0.91667, + "map_at_1000": 0.91667, + "recall_at_1": 0.84, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.84, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.84, + "mrr_at_3": 0.916667, + "mrr_at_5": 0.916667, + "mrr_at_10": 0.916667, + "mrr_at_20": 0.916667, + "mrr_at_100": 0.916667, + "mrr_at_1000": 0.916667, + "nauc_ndcg_at_1_max": 0.619354, + "nauc_ndcg_at_1_std": -0.37457, + "nauc_ndcg_at_1_diff1": 0.913476, + "nauc_ndcg_at_3_max": 0.636313, + "nauc_ndcg_at_3_std": -0.325705, + "nauc_ndcg_at_3_diff1": 0.917331, + "nauc_ndcg_at_5_max": 0.636313, + "nauc_ndcg_at_5_std": -0.325705, + "nauc_ndcg_at_5_diff1": 0.917331, + "nauc_ndcg_at_10_max": 0.636313, + "nauc_ndcg_at_10_std": -0.325705, + "nauc_ndcg_at_10_diff1": 0.917331, + "nauc_ndcg_at_20_max": 0.636313, + "nauc_ndcg_at_20_std": -0.325705, + "nauc_ndcg_at_20_diff1": 0.917331, + "nauc_ndcg_at_100_max": 0.636313, + "nauc_ndcg_at_100_std": -0.325705, + "nauc_ndcg_at_100_diff1": 0.917331, + "nauc_ndcg_at_1000_max": 0.636313, + "nauc_ndcg_at_1000_std": -0.325705, + "nauc_ndcg_at_1000_diff1": 0.917331, + "nauc_map_at_1_max": 0.619354, + "nauc_map_at_1_std": -0.37457, + "nauc_map_at_1_diff1": 0.913476, + "nauc_map_at_3_max": 0.635332, + "nauc_map_at_3_std": -0.328532, + "nauc_map_at_3_diff1": 0.917108, + "nauc_map_at_5_max": 0.635332, + "nauc_map_at_5_std": -0.328532, + "nauc_map_at_5_diff1": 0.917108, + "nauc_map_at_10_max": 0.635332, + "nauc_map_at_10_std": -0.328532, + "nauc_map_at_10_diff1": 0.917108, + "nauc_map_at_20_max": 0.635332, + "nauc_map_at_20_std": -0.328532, + "nauc_map_at_20_diff1": 0.917108, + "nauc_map_at_100_max": 0.635332, + "nauc_map_at_100_std": -0.328532, + "nauc_map_at_100_diff1": 0.917108, + "nauc_map_at_1000_max": 0.635332, + "nauc_map_at_1000_std": -0.328532, + "nauc_map_at_1000_diff1": 0.917108, + "nauc_recall_at_1_max": 0.619354, + "nauc_recall_at_1_std": -0.37457, + "nauc_recall_at_1_diff1": 0.913476, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.619354, + "nauc_precision_at_1_std": -0.37457, + "nauc_precision_at_1_diff1": 0.913476, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.619354, + "nauc_mrr_at_1_std": -0.37457, + "nauc_mrr_at_1_diff1": 0.913476, + "nauc_mrr_at_3_max": 0.635332, + "nauc_mrr_at_3_std": -0.328532, + "nauc_mrr_at_3_diff1": 0.917108, + "nauc_mrr_at_5_max": 0.635332, + "nauc_mrr_at_5_std": -0.328532, + "nauc_mrr_at_5_diff1": 0.917108, + "nauc_mrr_at_10_max": 0.635332, + "nauc_mrr_at_10_std": -0.328532, + "nauc_mrr_at_10_diff1": 0.917108, + "nauc_mrr_at_20_max": 0.635332, + "nauc_mrr_at_20_std": -0.328532, + "nauc_mrr_at_20_diff1": 0.917108, + "nauc_mrr_at_100_max": 0.635332, + "nauc_mrr_at_100_std": -0.328532, + "nauc_mrr_at_100_diff1": 0.917108, + "nauc_mrr_at_1000_max": 0.635332, + "nauc_mrr_at_1000_std": -0.328532, + "nauc_mrr_at_1000_diff1": 0.917108, + "main_score": 0.84, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_512": [ + { + "ndcg_at_1": 0.74, + "ndcg_at_3": 0.82309, + "ndcg_at_5": 0.85579, + "ndcg_at_10": 0.86292, + "ndcg_at_20": 0.86792, + "ndcg_at_100": 0.86792, + "ndcg_at_1000": 0.86792, + "map_at_1": 0.74, + "map_at_3": 0.80333, + "map_at_5": 0.82133, + "map_at_10": 0.82467, + "map_at_20": 0.826, + "map_at_100": 0.826, + "map_at_1000": 0.826, + "recall_at_1": 0.74, + "recall_at_3": 0.88, + "recall_at_5": 0.96, + "recall_at_10": 0.98, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.74, + "precision_at_3": 0.29333, + "precision_at_5": 0.192, + "precision_at_10": 0.098, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.74, + "mrr_at_3": 0.803333, + "mrr_at_5": 0.821333, + "mrr_at_10": 0.824667, + "mrr_at_20": 0.826, + "mrr_at_100": 0.826, + "mrr_at_1000": 0.826, + "nauc_ndcg_at_1_max": 0.241474, + "nauc_ndcg_at_1_std": -0.606788, + "nauc_ndcg_at_1_diff1": 0.715827, + "nauc_ndcg_at_3_max": 0.330219, + "nauc_ndcg_at_3_std": -0.715517, + "nauc_ndcg_at_3_diff1": 0.737061, + "nauc_ndcg_at_5_max": 0.242673, + "nauc_ndcg_at_5_std": -0.719724, + "nauc_ndcg_at_5_diff1": 0.709553, + "nauc_ndcg_at_10_max": 0.266288, + "nauc_ndcg_at_10_std": -0.66153, + "nauc_ndcg_at_10_diff1": 0.718416, + "nauc_ndcg_at_20_max": 0.26246, + "nauc_ndcg_at_20_std": -0.665619, + "nauc_ndcg_at_20_diff1": 0.718258, + "nauc_ndcg_at_100_max": 0.26246, + "nauc_ndcg_at_100_std": -0.665619, + "nauc_ndcg_at_100_diff1": 0.718258, + "nauc_ndcg_at_1000_max": 0.26246, + "nauc_ndcg_at_1000_std": -0.665619, + "nauc_ndcg_at_1000_diff1": 0.718258, + "nauc_map_at_1_max": 0.241474, + "nauc_map_at_1_std": -0.606788, + "nauc_map_at_1_diff1": 0.715827, + "nauc_map_at_3_max": 0.294283, + "nauc_map_at_3_std": -0.683061, + "nauc_map_at_3_diff1": 0.727325, + "nauc_map_at_5_max": 0.251915, + "nauc_map_at_5_std": -0.68244, + "nauc_map_at_5_diff1": 0.714489, + "nauc_map_at_10_max": 0.260791, + "nauc_map_at_10_std": -0.660261, + "nauc_map_at_10_diff1": 0.717852, + "nauc_map_at_20_max": 0.259968, + "nauc_map_at_20_std": -0.661079, + "nauc_map_at_20_diff1": 0.717815, + "nauc_map_at_100_max": 0.259968, + "nauc_map_at_100_std": -0.661079, + "nauc_map_at_100_diff1": 0.717815, + "nauc_map_at_1000_max": 0.259968, + "nauc_map_at_1000_std": -0.661079, + "nauc_map_at_1000_diff1": 0.717815, + "nauc_recall_at_1_max": 0.241474, + "nauc_recall_at_1_std": -0.606788, + "nauc_recall_at_1_diff1": 0.715827, + "nauc_recall_at_3_max": 0.494273, + "nauc_recall_at_3_std": -0.858097, + "nauc_recall_at_3_diff1": 0.782055, + "nauc_recall_at_5_max": 0.093371, + "nauc_recall_at_5_std": -1.151727, + "nauc_recall_at_5_diff1": 0.638189, + "nauc_recall_at_10_max": 0.358077, + "nauc_recall_at_10_std": -0.563492, + "nauc_recall_at_10_diff1": 0.722222, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.241474, + "nauc_precision_at_1_std": -0.606788, + "nauc_precision_at_1_diff1": 0.715827, + "nauc_precision_at_3_max": 0.494273, + "nauc_precision_at_3_std": -0.858097, + "nauc_precision_at_3_diff1": 0.782055, + "nauc_precision_at_5_max": 0.093371, + "nauc_precision_at_5_std": -1.151727, + "nauc_precision_at_5_diff1": 0.638189, + "nauc_precision_at_10_max": 0.358077, + "nauc_precision_at_10_std": -0.563492, + "nauc_precision_at_10_diff1": 0.722222, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.241474, + "nauc_mrr_at_1_std": -0.606788, + "nauc_mrr_at_1_diff1": 0.715827, + "nauc_mrr_at_3_max": 0.294283, + "nauc_mrr_at_3_std": -0.683061, + "nauc_mrr_at_3_diff1": 0.727325, + "nauc_mrr_at_5_max": 0.251915, + "nauc_mrr_at_5_std": -0.68244, + "nauc_mrr_at_5_diff1": 0.714489, + "nauc_mrr_at_10_max": 0.260791, + "nauc_mrr_at_10_std": -0.660261, + "nauc_mrr_at_10_diff1": 0.717852, + "nauc_mrr_at_20_max": 0.259968, + "nauc_mrr_at_20_std": -0.661079, + "nauc_mrr_at_20_diff1": 0.717815, + "nauc_mrr_at_100_max": 0.259968, + "nauc_mrr_at_100_std": -0.661079, + "nauc_mrr_at_100_diff1": 0.717815, + "nauc_mrr_at_1000_max": 0.259968, + "nauc_mrr_at_1000_std": -0.661079, + "nauc_mrr_at_1000_diff1": 0.717815, + "main_score": 0.74, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_1024": [ + { + "ndcg_at_1": 0.54, + "ndcg_at_3": 0.64309, + "ndcg_at_5": 0.66806, + "ndcg_at_10": 0.69394, + "ndcg_at_20": 0.72061, + "ndcg_at_100": 0.72881, + "ndcg_at_1000": 0.72881, + "map_at_1": 0.54, + "map_at_3": 0.61667, + "map_at_5": 0.63067, + "map_at_10": 0.64136, + "map_at_20": 0.64944, + "map_at_100": 0.65088, + "map_at_1000": 0.65088, + "recall_at_1": 0.54, + "recall_at_3": 0.72, + "recall_at_5": 0.78, + "recall_at_10": 0.86, + "recall_at_20": 0.96, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.54, + "precision_at_3": 0.24, + "precision_at_5": 0.156, + "precision_at_10": 0.086, + "precision_at_20": 0.048, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.54, + "mrr_at_3": 0.616667, + "mrr_at_5": 0.630667, + "mrr_at_10": 0.641357, + "mrr_at_20": 0.649438, + "mrr_at_100": 0.650878, + "mrr_at_1000": 0.650878, + "nauc_ndcg_at_1_max": 0.593287, + "nauc_ndcg_at_1_std": 0.323442, + "nauc_ndcg_at_1_diff1": 0.565808, + "nauc_ndcg_at_3_max": 0.514765, + "nauc_ndcg_at_3_std": 0.306704, + "nauc_ndcg_at_3_diff1": 0.547065, + "nauc_ndcg_at_5_max": 0.530984, + "nauc_ndcg_at_5_std": 0.265947, + "nauc_ndcg_at_5_diff1": 0.551799, + "nauc_ndcg_at_10_max": 0.560313, + "nauc_ndcg_at_10_std": 0.263631, + "nauc_ndcg_at_10_diff1": 0.527542, + "nauc_ndcg_at_20_max": 0.54096, + "nauc_ndcg_at_20_std": 0.268965, + "nauc_ndcg_at_20_diff1": 0.552141, + "nauc_ndcg_at_100_max": 0.550294, + "nauc_ndcg_at_100_std": 0.294057, + "nauc_ndcg_at_100_diff1": 0.549119, + "nauc_ndcg_at_1000_max": 0.550294, + "nauc_ndcg_at_1000_std": 0.294057, + "nauc_ndcg_at_1000_diff1": 0.549119, + "nauc_map_at_1_max": 0.593287, + "nauc_map_at_1_std": 0.323442, + "nauc_map_at_1_diff1": 0.565808, + "nauc_map_at_3_max": 0.535442, + "nauc_map_at_3_std": 0.315652, + "nauc_map_at_3_diff1": 0.551123, + "nauc_map_at_5_max": 0.545726, + "nauc_map_at_5_std": 0.295183, + "nauc_map_at_5_diff1": 0.553744, + "nauc_map_at_10_max": 0.556467, + "nauc_map_at_10_std": 0.293154, + "nauc_map_at_10_diff1": 0.545328, + "nauc_map_at_20_max": 0.5513, + "nauc_map_at_20_std": 0.296253, + "nauc_map_at_20_diff1": 0.551298, + "nauc_map_at_100_max": 0.553271, + "nauc_map_at_100_std": 0.300889, + "nauc_map_at_100_diff1": 0.55098, + "nauc_map_at_1000_max": 0.553271, + "nauc_map_at_1000_std": 0.300889, + "nauc_map_at_1000_diff1": 0.55098, + "nauc_recall_at_1_max": 0.593287, + "nauc_recall_at_1_std": 0.323442, + "nauc_recall_at_1_diff1": 0.565808, + "nauc_recall_at_3_max": 0.442346, + "nauc_recall_at_3_std": 0.273008, + "nauc_recall_at_3_diff1": 0.53331, + "nauc_recall_at_5_max": 0.4716, + "nauc_recall_at_5_std": 0.133908, + "nauc_recall_at_5_diff1": 0.545278, + "nauc_recall_at_10_max": 0.611404, + "nauc_recall_at_10_std": 0.078495, + "nauc_recall_at_10_diff1": 0.401968, + "nauc_recall_at_20_max": 0.348973, + "nauc_recall_at_20_std": -0.298786, + "nauc_recall_at_20_diff1": 0.638189, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.593287, + "nauc_precision_at_1_std": 0.323442, + "nauc_precision_at_1_diff1": 0.565808, + "nauc_precision_at_3_max": 0.442346, + "nauc_precision_at_3_std": 0.273008, + "nauc_precision_at_3_diff1": 0.53331, + "nauc_precision_at_5_max": 0.4716, + "nauc_precision_at_5_std": 0.133908, + "nauc_precision_at_5_diff1": 0.545278, + "nauc_precision_at_10_max": 0.611404, + "nauc_precision_at_10_std": 0.078495, + "nauc_precision_at_10_diff1": 0.401968, + "nauc_precision_at_20_max": 0.348973, + "nauc_precision_at_20_std": -0.298786, + "nauc_precision_at_20_diff1": 0.638189, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.593287, + "nauc_mrr_at_1_std": 0.323442, + "nauc_mrr_at_1_diff1": 0.565808, + "nauc_mrr_at_3_max": 0.535442, + "nauc_mrr_at_3_std": 0.315652, + "nauc_mrr_at_3_diff1": 0.551123, + "nauc_mrr_at_5_max": 0.545726, + "nauc_mrr_at_5_std": 0.295183, + "nauc_mrr_at_5_diff1": 0.553744, + "nauc_mrr_at_10_max": 0.556467, + "nauc_mrr_at_10_std": 0.293154, + "nauc_mrr_at_10_diff1": 0.545328, + "nauc_mrr_at_20_max": 0.5513, + "nauc_mrr_at_20_std": 0.296253, + "nauc_mrr_at_20_diff1": 0.551298, + "nauc_mrr_at_100_max": 0.553271, + "nauc_mrr_at_100_std": 0.300889, + "nauc_mrr_at_100_diff1": 0.55098, + "nauc_mrr_at_1000_max": 0.553271, + "nauc_mrr_at_1000_std": 0.300889, + "nauc_mrr_at_1000_diff1": 0.55098, + "main_score": 0.54, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_2048": [ + { + "ndcg_at_1": 0.36, + "ndcg_at_3": 0.50571, + "ndcg_at_5": 0.56338, + "ndcg_at_10": 0.59545, + "ndcg_at_20": 0.62144, + "ndcg_at_100": 0.6296, + "ndcg_at_1000": 0.6296, + "map_at_1": 0.36, + "map_at_3": 0.46667, + "map_at_5": 0.49867, + "map_at_10": 0.51178, + "map_at_20": 0.51927, + "map_at_100": 0.52066, + "map_at_1000": 0.52066, + "recall_at_1": 0.36, + "recall_at_3": 0.62, + "recall_at_5": 0.76, + "recall_at_10": 0.86, + "recall_at_20": 0.96, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.36, + "precision_at_3": 0.20667, + "precision_at_5": 0.152, + "precision_at_10": 0.086, + "precision_at_20": 0.048, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.36, + "mrr_at_3": 0.466667, + "mrr_at_5": 0.498667, + "mrr_at_10": 0.511778, + "mrr_at_20": 0.519275, + "mrr_at_100": 0.520663, + "mrr_at_1000": 0.520663, + "nauc_ndcg_at_1_max": 0.606912, + "nauc_ndcg_at_1_std": 0.455099, + "nauc_ndcg_at_1_diff1": 0.594526, + "nauc_ndcg_at_3_max": 0.462994, + "nauc_ndcg_at_3_std": 0.470938, + "nauc_ndcg_at_3_diff1": 0.528057, + "nauc_ndcg_at_5_max": 0.469999, + "nauc_ndcg_at_5_std": 0.488433, + "nauc_ndcg_at_5_diff1": 0.537613, + "nauc_ndcg_at_10_max": 0.487962, + "nauc_ndcg_at_10_std": 0.483377, + "nauc_ndcg_at_10_diff1": 0.555945, + "nauc_ndcg_at_20_max": 0.479222, + "nauc_ndcg_at_20_std": 0.485021, + "nauc_ndcg_at_20_diff1": 0.549199, + "nauc_ndcg_at_100_max": 0.501805, + "nauc_ndcg_at_100_std": 0.471755, + "nauc_ndcg_at_100_diff1": 0.552281, + "nauc_ndcg_at_1000_max": 0.501805, + "nauc_ndcg_at_1000_std": 0.471755, + "nauc_ndcg_at_1000_diff1": 0.552281, + "nauc_map_at_1_max": 0.606912, + "nauc_map_at_1_std": 0.455099, + "nauc_map_at_1_diff1": 0.594526, + "nauc_map_at_3_max": 0.504864, + "nauc_map_at_3_std": 0.467561, + "nauc_map_at_3_diff1": 0.5509, + "nauc_map_at_5_max": 0.509174, + "nauc_map_at_5_std": 0.473046, + "nauc_map_at_5_diff1": 0.555388, + "nauc_map_at_10_max": 0.513457, + "nauc_map_at_10_std": 0.470591, + "nauc_map_at_10_diff1": 0.559263, + "nauc_map_at_20_max": 0.512357, + "nauc_map_at_20_std": 0.470328, + "nauc_map_at_20_diff1": 0.557678, + "nauc_map_at_100_max": 0.515528, + "nauc_map_at_100_std": 0.468417, + "nauc_map_at_100_diff1": 0.55808, + "nauc_map_at_1000_max": 0.515528, + "nauc_map_at_1000_std": 0.468417, + "nauc_map_at_1000_diff1": 0.55808, + "nauc_recall_at_1_max": 0.606912, + "nauc_recall_at_1_std": 0.455099, + "nauc_recall_at_1_diff1": 0.594526, + "nauc_recall_at_3_max": 0.326031, + "nauc_recall_at_3_std": 0.481494, + "nauc_recall_at_3_diff1": 0.451928, + "nauc_recall_at_5_max": 0.304967, + "nauc_recall_at_5_std": 0.563425, + "nauc_recall_at_5_diff1": 0.463658, + "nauc_recall_at_10_max": 0.358321, + "nauc_recall_at_10_std": 0.578911, + "nauc_recall_at_10_diff1": 0.571914, + "nauc_recall_at_20_max": -0.171335, + "nauc_recall_at_20_std": 0.861111, + "nauc_recall_at_20_diff1": 0.456116, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.606912, + "nauc_precision_at_1_std": 0.455099, + "nauc_precision_at_1_diff1": 0.594526, + "nauc_precision_at_3_max": 0.326031, + "nauc_precision_at_3_std": 0.481494, + "nauc_precision_at_3_diff1": 0.451928, + "nauc_precision_at_5_max": 0.304967, + "nauc_precision_at_5_std": 0.563425, + "nauc_precision_at_5_diff1": 0.463658, + "nauc_precision_at_10_max": 0.358321, + "nauc_precision_at_10_std": 0.578911, + "nauc_precision_at_10_diff1": 0.571914, + "nauc_precision_at_20_max": -0.171335, + "nauc_precision_at_20_std": 0.861111, + "nauc_precision_at_20_diff1": 0.456116, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.606912, + "nauc_mrr_at_1_std": 0.455099, + "nauc_mrr_at_1_diff1": 0.594526, + "nauc_mrr_at_3_max": 0.504864, + "nauc_mrr_at_3_std": 0.467561, + "nauc_mrr_at_3_diff1": 0.5509, + "nauc_mrr_at_5_max": 0.509174, + "nauc_mrr_at_5_std": 0.473046, + "nauc_mrr_at_5_diff1": 0.555388, + "nauc_mrr_at_10_max": 0.513457, + "nauc_mrr_at_10_std": 0.470591, + "nauc_mrr_at_10_diff1": 0.559263, + "nauc_mrr_at_20_max": 0.512357, + "nauc_mrr_at_20_std": 0.470328, + "nauc_mrr_at_20_diff1": 0.557678, + "nauc_mrr_at_100_max": 0.515528, + "nauc_mrr_at_100_std": 0.468417, + "nauc_mrr_at_100_diff1": 0.55808, + "nauc_mrr_at_1000_max": 0.515528, + "nauc_mrr_at_1000_std": 0.468417, + "nauc_mrr_at_1000_diff1": 0.55808, + "main_score": 0.36, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_4096": [ + { + "ndcg_at_1": 0.36, + "ndcg_at_3": 0.5288, + "ndcg_at_5": 0.57961, + "ndcg_at_10": 0.63267, + "ndcg_at_20": 0.64824, + "ndcg_at_100": 0.65267, + "ndcg_at_1000": 0.65267, + "map_at_1": 0.36, + "map_at_3": 0.49, + "map_at_5": 0.519, + "map_at_10": 0.54166, + "map_at_20": 0.54614, + "map_at_100": 0.54705, + "map_at_1000": 0.54705, + "recall_at_1": 0.36, + "recall_at_3": 0.64, + "recall_at_5": 0.76, + "recall_at_10": 0.92, + "recall_at_20": 0.98, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.36, + "precision_at_3": 0.21333, + "precision_at_5": 0.152, + "precision_at_10": 0.092, + "precision_at_20": 0.049, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.36, + "mrr_at_3": 0.49, + "mrr_at_5": 0.519, + "mrr_at_10": 0.541659, + "mrr_at_20": 0.546144, + "mrr_at_100": 0.547053, + "mrr_at_1000": 0.547053, + "nauc_ndcg_at_1_max": 0.279562, + "nauc_ndcg_at_1_std": 0.091009, + "nauc_ndcg_at_1_diff1": 0.322548, + "nauc_ndcg_at_3_max": 0.329125, + "nauc_ndcg_at_3_std": 0.144915, + "nauc_ndcg_at_3_diff1": 0.282331, + "nauc_ndcg_at_5_max": 0.245668, + "nauc_ndcg_at_5_std": 0.142749, + "nauc_ndcg_at_5_diff1": 0.267273, + "nauc_ndcg_at_10_max": 0.272644, + "nauc_ndcg_at_10_std": 0.125175, + "nauc_ndcg_at_10_diff1": 0.264248, + "nauc_ndcg_at_20_max": 0.276214, + "nauc_ndcg_at_20_std": 0.138134, + "nauc_ndcg_at_20_diff1": 0.290764, + "nauc_ndcg_at_100_max": 0.292902, + "nauc_ndcg_at_100_std": 0.133763, + "nauc_ndcg_at_100_diff1": 0.294102, + "nauc_ndcg_at_1000_max": 0.292902, + "nauc_ndcg_at_1000_std": 0.133763, + "nauc_ndcg_at_1000_diff1": 0.294102, + "nauc_map_at_1_max": 0.279562, + "nauc_map_at_1_std": 0.091009, + "nauc_map_at_1_diff1": 0.322548, + "nauc_map_at_3_max": 0.325174, + "nauc_map_at_3_std": 0.13623, + "nauc_map_at_3_diff1": 0.298292, + "nauc_map_at_5_max": 0.281331, + "nauc_map_at_5_std": 0.135761, + "nauc_map_at_5_diff1": 0.292412, + "nauc_map_at_10_max": 0.294801, + "nauc_map_at_10_std": 0.129862, + "nauc_map_at_10_diff1": 0.294092, + "nauc_map_at_20_max": 0.295213, + "nauc_map_at_20_std": 0.132398, + "nauc_map_at_20_diff1": 0.29962, + "nauc_map_at_100_max": 0.297941, + "nauc_map_at_100_std": 0.131681, + "nauc_map_at_100_diff1": 0.300181, + "nauc_map_at_1000_max": 0.297941, + "nauc_map_at_1000_std": 0.131681, + "nauc_map_at_1000_diff1": 0.300181, + "nauc_recall_at_1_max": 0.279562, + "nauc_recall_at_1_std": 0.091009, + "nauc_recall_at_1_diff1": 0.322548, + "nauc_recall_at_3_max": 0.337783, + "nauc_recall_at_3_std": 0.170616, + "nauc_recall_at_3_diff1": 0.227967, + "nauc_recall_at_5_max": 0.075712, + "nauc_recall_at_5_std": 0.16583, + "nauc_recall_at_5_diff1": 0.154625, + "nauc_recall_at_10_max": 0.005836, + "nauc_recall_at_10_std": 0.029645, + "nauc_recall_at_10_diff1": -0.085201, + "nauc_recall_at_20_max": -0.563492, + "nauc_recall_at_20_std": 0.358077, + "nauc_recall_at_20_diff1": 0.122782, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.279562, + "nauc_precision_at_1_std": 0.091009, + "nauc_precision_at_1_diff1": 0.322548, + "nauc_precision_at_3_max": 0.337783, + "nauc_precision_at_3_std": 0.170616, + "nauc_precision_at_3_diff1": 0.227967, + "nauc_precision_at_5_max": 0.075712, + "nauc_precision_at_5_std": 0.16583, + "nauc_precision_at_5_diff1": 0.154625, + "nauc_precision_at_10_max": 0.005836, + "nauc_precision_at_10_std": 0.029645, + "nauc_precision_at_10_diff1": -0.085201, + "nauc_precision_at_20_max": -0.563492, + "nauc_precision_at_20_std": 0.358077, + "nauc_precision_at_20_diff1": 0.122782, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.279562, + "nauc_mrr_at_1_std": 0.091009, + "nauc_mrr_at_1_diff1": 0.322548, + "nauc_mrr_at_3_max": 0.325174, + "nauc_mrr_at_3_std": 0.13623, + "nauc_mrr_at_3_diff1": 0.298292, + "nauc_mrr_at_5_max": 0.281331, + "nauc_mrr_at_5_std": 0.135761, + "nauc_mrr_at_5_diff1": 0.292412, + "nauc_mrr_at_10_max": 0.294801, + "nauc_mrr_at_10_std": 0.129862, + "nauc_mrr_at_10_diff1": 0.294092, + "nauc_mrr_at_20_max": 0.295213, + "nauc_mrr_at_20_std": 0.132398, + "nauc_mrr_at_20_diff1": 0.29962, + "nauc_mrr_at_100_max": 0.297941, + "nauc_mrr_at_100_std": 0.131681, + "nauc_mrr_at_100_diff1": 0.300181, + "nauc_mrr_at_1000_max": 0.297941, + "nauc_mrr_at_1000_std": 0.131681, + "nauc_mrr_at_1000_diff1": 0.300181, + "main_score": 0.36, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_8192": [ + { + "ndcg_at_1": 0.34, + "ndcg_at_3": 0.43047, + "ndcg_at_5": 0.47091, + "ndcg_at_10": 0.52351, + "ndcg_at_20": 0.53861, + "ndcg_at_100": 0.57375, + "ndcg_at_1000": 0.57375, + "map_at_1": 0.34, + "map_at_3": 0.40667, + "map_at_5": 0.42867, + "map_at_10": 0.45085, + "map_at_20": 0.45494, + "map_at_100": 0.46051, + "map_at_1000": 0.46051, + "recall_at_1": 0.34, + "recall_at_3": 0.5, + "recall_at_5": 0.6, + "recall_at_10": 0.76, + "recall_at_20": 0.82, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.34, + "precision_at_3": 0.16667, + "precision_at_5": 0.12, + "precision_at_10": 0.076, + "precision_at_20": 0.041, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.34, + "mrr_at_3": 0.406667, + "mrr_at_5": 0.428667, + "mrr_at_10": 0.450849, + "mrr_at_20": 0.454942, + "mrr_at_100": 0.460507, + "mrr_at_1000": 0.460507, + "nauc_ndcg_at_1_max": 0.071173, + "nauc_ndcg_at_1_std": -0.056361, + "nauc_ndcg_at_1_diff1": 0.38496, + "nauc_ndcg_at_3_max": 0.160549, + "nauc_ndcg_at_3_std": 0.043262, + "nauc_ndcg_at_3_diff1": 0.41031, + "nauc_ndcg_at_5_max": 0.145813, + "nauc_ndcg_at_5_std": 0.064322, + "nauc_ndcg_at_5_diff1": 0.417249, + "nauc_ndcg_at_10_max": 0.151825, + "nauc_ndcg_at_10_std": 0.136008, + "nauc_ndcg_at_10_diff1": 0.445857, + "nauc_ndcg_at_20_max": 0.136449, + "nauc_ndcg_at_20_std": 0.148118, + "nauc_ndcg_at_20_diff1": 0.447929, + "nauc_ndcg_at_100_max": 0.125388, + "nauc_ndcg_at_100_std": 0.065227, + "nauc_ndcg_at_100_diff1": 0.418599, + "nauc_ndcg_at_1000_max": 0.125388, + "nauc_ndcg_at_1000_std": 0.065227, + "nauc_ndcg_at_1000_diff1": 0.418599, + "nauc_map_at_1_max": 0.071173, + "nauc_map_at_1_std": -0.056361, + "nauc_map_at_1_diff1": 0.38496, + "nauc_map_at_3_max": 0.127431, + "nauc_map_at_3_std": 0.015495, + "nauc_map_at_3_diff1": 0.403965, + "nauc_map_at_5_max": 0.116877, + "nauc_map_at_5_std": 0.024498, + "nauc_map_at_5_diff1": 0.406925, + "nauc_map_at_10_max": 0.116676, + "nauc_map_at_10_std": 0.049082, + "nauc_map_at_10_diff1": 0.416273, + "nauc_map_at_20_max": 0.112084, + "nauc_map_at_20_std": 0.050832, + "nauc_map_at_20_diff1": 0.415966, + "nauc_map_at_100_max": 0.11222, + "nauc_map_at_100_std": 0.038413, + "nauc_map_at_100_diff1": 0.411138, + "nauc_map_at_1000_max": 0.11222, + "nauc_map_at_1000_std": 0.038413, + "nauc_map_at_1000_diff1": 0.411138, + "nauc_recall_at_1_max": 0.071173, + "nauc_recall_at_1_std": -0.056361, + "nauc_recall_at_1_diff1": 0.38496, + "nauc_recall_at_3_max": 0.261666, + "nauc_recall_at_3_std": 0.12557, + "nauc_recall_at_3_diff1": 0.428861, + "nauc_recall_at_5_max": 0.242891, + "nauc_recall_at_5_std": 0.198071, + "nauc_recall_at_5_diff1": 0.452093, + "nauc_recall_at_10_max": 0.323743, + "nauc_recall_at_10_std": 0.581725, + "nauc_recall_at_10_diff1": 0.602319, + "nauc_recall_at_20_max": 0.272111, + "nauc_recall_at_20_std": 0.791882, + "nauc_recall_at_20_diff1": 0.666648, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.071173, + "nauc_precision_at_1_std": -0.056361, + "nauc_precision_at_1_diff1": 0.38496, + "nauc_precision_at_3_max": 0.261666, + "nauc_precision_at_3_std": 0.12557, + "nauc_precision_at_3_diff1": 0.428861, + "nauc_precision_at_5_max": 0.242891, + "nauc_precision_at_5_std": 0.198071, + "nauc_precision_at_5_diff1": 0.452093, + "nauc_precision_at_10_max": 0.323743, + "nauc_precision_at_10_std": 0.581725, + "nauc_precision_at_10_diff1": 0.602319, + "nauc_precision_at_20_max": 0.272111, + "nauc_precision_at_20_std": 0.791882, + "nauc_precision_at_20_diff1": 0.666648, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.071173, + "nauc_mrr_at_1_std": -0.056361, + "nauc_mrr_at_1_diff1": 0.38496, + "nauc_mrr_at_3_max": 0.127431, + "nauc_mrr_at_3_std": 0.015495, + "nauc_mrr_at_3_diff1": 0.403965, + "nauc_mrr_at_5_max": 0.116877, + "nauc_mrr_at_5_std": 0.024498, + "nauc_mrr_at_5_diff1": 0.406925, + "nauc_mrr_at_10_max": 0.116676, + "nauc_mrr_at_10_std": 0.049082, + "nauc_mrr_at_10_diff1": 0.416273, + "nauc_mrr_at_20_max": 0.112084, + "nauc_mrr_at_20_std": 0.050832, + "nauc_mrr_at_20_diff1": 0.415966, + "nauc_mrr_at_100_max": 0.11222, + "nauc_mrr_at_100_std": 0.038413, + "nauc_mrr_at_100_diff1": 0.411138, + "nauc_mrr_at_1000_max": 0.11222, + "nauc_mrr_at_1000_std": 0.038413, + "nauc_mrr_at_1000_diff1": 0.411138, + "main_score": 0.34, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_16384": [ + { + "ndcg_at_1": 0.16, + "ndcg_at_3": 0.27309, + "ndcg_at_5": 0.28171, + "ndcg_at_10": 0.31361, + "ndcg_at_20": 0.32887, + "ndcg_at_100": 0.40675, + "ndcg_at_1000": 0.40675, + "map_at_1": 0.16, + "map_at_3": 0.24333, + "map_at_5": 0.24833, + "map_at_10": 0.26125, + "map_at_20": 0.26546, + "map_at_100": 0.27342, + "map_at_1000": 0.27342, + "recall_at_1": 0.16, + "recall_at_3": 0.36, + "recall_at_5": 0.38, + "recall_at_10": 0.48, + "recall_at_20": 0.54, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.16, + "precision_at_3": 0.12, + "precision_at_5": 0.076, + "precision_at_10": 0.048, + "precision_at_20": 0.027, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.16, + "mrr_at_3": 0.243333, + "mrr_at_5": 0.248333, + "mrr_at_10": 0.261246, + "mrr_at_20": 0.265463, + "mrr_at_100": 0.273616, + "mrr_at_1000": 0.273616, + "nauc_ndcg_at_1_max": 0.079467, + "nauc_ndcg_at_1_std": 0.035528, + "nauc_ndcg_at_1_diff1": 0.229605, + "nauc_ndcg_at_3_max": 0.026972, + "nauc_ndcg_at_3_std": 0.046015, + "nauc_ndcg_at_3_diff1": 0.032545, + "nauc_ndcg_at_5_max": 0.002798, + "nauc_ndcg_at_5_std": 0.035891, + "nauc_ndcg_at_5_diff1": 0.03713, + "nauc_ndcg_at_10_max": -0.088905, + "nauc_ndcg_at_10_std": -0.026503, + "nauc_ndcg_at_10_diff1": 0.03457, + "nauc_ndcg_at_20_max": -0.074222, + "nauc_ndcg_at_20_std": -0.015836, + "nauc_ndcg_at_20_diff1": 0.060141, + "nauc_ndcg_at_100_max": -0.038513, + "nauc_ndcg_at_100_std": -0.00456, + "nauc_ndcg_at_100_diff1": 0.065101, + "nauc_ndcg_at_1000_max": -0.038513, + "nauc_ndcg_at_1000_std": -0.00456, + "nauc_ndcg_at_1000_diff1": 0.065101, + "nauc_map_at_1_max": 0.079467, + "nauc_map_at_1_std": 0.035528, + "nauc_map_at_1_diff1": 0.229605, + "nauc_map_at_3_max": 0.024699, + "nauc_map_at_3_std": 0.038103, + "nauc_map_at_3_diff1": 0.067593, + "nauc_map_at_5_max": 0.009886, + "nauc_map_at_5_std": 0.031919, + "nauc_map_at_5_diff1": 0.070302, + "nauc_map_at_10_max": -0.029502, + "nauc_map_at_10_std": 0.005576, + "nauc_map_at_10_diff1": 0.07274, + "nauc_map_at_20_max": -0.025392, + "nauc_map_at_20_std": 0.00917, + "nauc_map_at_20_diff1": 0.080338, + "nauc_map_at_100_max": -0.020387, + "nauc_map_at_100_std": 0.008638, + "nauc_map_at_100_diff1": 0.080028, + "nauc_map_at_1000_max": -0.020387, + "nauc_map_at_1000_std": 0.008638, + "nauc_map_at_1000_diff1": 0.080028, + "nauc_recall_at_1_max": 0.079467, + "nauc_recall_at_1_std": 0.035528, + "nauc_recall_at_1_diff1": 0.229605, + "nauc_recall_at_3_max": 0.03674, + "nauc_recall_at_3_std": 0.066866, + "nauc_recall_at_3_diff1": -0.049572, + "nauc_recall_at_5_max": -0.012008, + "nauc_recall_at_5_std": 0.046292, + "nauc_recall_at_5_diff1": -0.03956, + "nauc_recall_at_10_max": -0.262967, + "nauc_recall_at_10_std": -0.126204, + "nauc_recall_at_10_diff1": -0.06338, + "nauc_recall_at_20_max": -0.216999, + "nauc_recall_at_20_std": -0.095774, + "nauc_recall_at_20_diff1": 0.022394, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.079467, + "nauc_precision_at_1_std": 0.035528, + "nauc_precision_at_1_diff1": 0.229605, + "nauc_precision_at_3_max": 0.03674, + "nauc_precision_at_3_std": 0.066866, + "nauc_precision_at_3_diff1": -0.049572, + "nauc_precision_at_5_max": -0.012008, + "nauc_precision_at_5_std": 0.046292, + "nauc_precision_at_5_diff1": -0.03956, + "nauc_precision_at_10_max": -0.262967, + "nauc_precision_at_10_std": -0.126204, + "nauc_precision_at_10_diff1": -0.06338, + "nauc_precision_at_20_max": -0.216999, + "nauc_precision_at_20_std": -0.095774, + "nauc_precision_at_20_diff1": 0.022394, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.079467, + "nauc_mrr_at_1_std": 0.035528, + "nauc_mrr_at_1_diff1": 0.229605, + "nauc_mrr_at_3_max": 0.024699, + "nauc_mrr_at_3_std": 0.038103, + "nauc_mrr_at_3_diff1": 0.067593, + "nauc_mrr_at_5_max": 0.009886, + "nauc_mrr_at_5_std": 0.031919, + "nauc_mrr_at_5_diff1": 0.070302, + "nauc_mrr_at_10_max": -0.029502, + "nauc_mrr_at_10_std": 0.005576, + "nauc_mrr_at_10_diff1": 0.07274, + "nauc_mrr_at_20_max": -0.025392, + "nauc_mrr_at_20_std": 0.00917, + "nauc_mrr_at_20_diff1": 0.080338, + "nauc_mrr_at_100_max": -0.021947, + "nauc_mrr_at_100_std": 0.010559, + "nauc_mrr_at_100_diff1": 0.081312, + "nauc_mrr_at_1000_max": -0.021947, + "nauc_mrr_at_1000_std": 0.010559, + "nauc_mrr_at_1000_diff1": 0.081312, + "main_score": 0.16, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_32768": [ + { + "ndcg_at_1": 0.1, + "ndcg_at_3": 0.18047, + "ndcg_at_5": 0.18047, + "ndcg_at_10": 0.18047, + "ndcg_at_20": 0.20015, + "ndcg_at_100": 0.32162, + "ndcg_at_1000": 0.32162, + "map_at_1": 0.1, + "map_at_3": 0.16, + "map_at_5": 0.16, + "map_at_10": 0.16, + "map_at_20": 0.1651, + "map_at_100": 0.18026, + "map_at_1000": 0.18026, + "recall_at_1": 0.1, + "recall_at_3": 0.24, + "recall_at_5": 0.24, + "recall_at_10": 0.24, + "recall_at_20": 0.32, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.1, + "precision_at_3": 0.08, + "precision_at_5": 0.048, + "precision_at_10": 0.024, + "precision_at_20": 0.016, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.1, + "mrr_at_3": 0.16, + "mrr_at_5": 0.16, + "mrr_at_10": 0.16, + "mrr_at_20": 0.165049, + "mrr_at_100": 0.179793, + "mrr_at_1000": 0.179793, + "nauc_ndcg_at_1_max": 0.268098, + "nauc_ndcg_at_1_std": 0.335122, + "nauc_ndcg_at_1_diff1": 0.204239, + "nauc_ndcg_at_3_max": 0.269973, + "nauc_ndcg_at_3_std": 0.470006, + "nauc_ndcg_at_3_diff1": 0.309798, + "nauc_ndcg_at_5_max": 0.269973, + "nauc_ndcg_at_5_std": 0.470006, + "nauc_ndcg_at_5_diff1": 0.309798, + "nauc_ndcg_at_10_max": 0.269973, + "nauc_ndcg_at_10_std": 0.470006, + "nauc_ndcg_at_10_diff1": 0.309798, + "nauc_ndcg_at_20_max": 0.235736, + "nauc_ndcg_at_20_std": 0.420362, + "nauc_ndcg_at_20_diff1": 0.294939, + "nauc_ndcg_at_100_max": 0.255719, + "nauc_ndcg_at_100_std": 0.436387, + "nauc_ndcg_at_100_diff1": 0.290831, + "nauc_ndcg_at_1000_max": 0.255719, + "nauc_ndcg_at_1000_std": 0.436387, + "nauc_ndcg_at_1000_diff1": 0.290831, + "nauc_map_at_1_max": 0.268098, + "nauc_map_at_1_std": 0.335122, + "nauc_map_at_1_diff1": 0.204239, + "nauc_map_at_3_max": 0.269448, + "nauc_map_at_3_std": 0.452492, + "nauc_map_at_3_diff1": 0.292586, + "nauc_map_at_5_max": 0.269448, + "nauc_map_at_5_std": 0.452492, + "nauc_map_at_5_diff1": 0.292586, + "nauc_map_at_10_max": 0.269448, + "nauc_map_at_10_std": 0.452492, + "nauc_map_at_10_diff1": 0.292586, + "nauc_map_at_20_max": 0.259322, + "nauc_map_at_20_std": 0.438437, + "nauc_map_at_20_diff1": 0.288781, + "nauc_map_at_100_max": 0.262494, + "nauc_map_at_100_std": 0.438883, + "nauc_map_at_100_diff1": 0.286981, + "nauc_map_at_1000_max": 0.262494, + "nauc_map_at_1000_std": 0.438883, + "nauc_map_at_1000_diff1": 0.286981, + "nauc_recall_at_1_max": 0.268098, + "nauc_recall_at_1_std": 0.335122, + "nauc_recall_at_1_diff1": 0.204239, + "nauc_recall_at_3_max": 0.271178, + "nauc_recall_at_3_std": 0.506111, + "nauc_recall_at_3_diff1": 0.346783, + "nauc_recall_at_5_max": 0.271178, + "nauc_recall_at_5_std": 0.506111, + "nauc_recall_at_5_diff1": 0.346783, + "nauc_recall_at_10_max": 0.271178, + "nauc_recall_at_10_std": 0.506111, + "nauc_recall_at_10_diff1": 0.346783, + "nauc_recall_at_20_max": 0.167184, + "nauc_recall_at_20_std": 0.349245, + "nauc_recall_at_20_diff1": 0.295596, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.268098, + "nauc_precision_at_1_std": 0.335122, + "nauc_precision_at_1_diff1": 0.204239, + "nauc_precision_at_3_max": 0.271178, + "nauc_precision_at_3_std": 0.506111, + "nauc_precision_at_3_diff1": 0.346783, + "nauc_precision_at_5_max": 0.271178, + "nauc_precision_at_5_std": 0.506111, + "nauc_precision_at_5_diff1": 0.346783, + "nauc_precision_at_10_max": 0.271178, + "nauc_precision_at_10_std": 0.506111, + "nauc_precision_at_10_diff1": 0.346783, + "nauc_precision_at_20_max": 0.167184, + "nauc_precision_at_20_std": 0.349245, + "nauc_precision_at_20_diff1": 0.295596, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.268098, + "nauc_mrr_at_1_std": 0.335122, + "nauc_mrr_at_1_diff1": 0.204239, + "nauc_mrr_at_3_max": 0.269448, + "nauc_mrr_at_3_std": 0.452492, + "nauc_mrr_at_3_diff1": 0.292586, + "nauc_mrr_at_5_max": 0.269448, + "nauc_mrr_at_5_std": 0.452492, + "nauc_mrr_at_5_diff1": 0.292586, + "nauc_mrr_at_10_max": 0.269448, + "nauc_mrr_at_10_std": 0.452492, + "nauc_mrr_at_10_diff1": 0.292586, + "nauc_mrr_at_20_max": 0.257211, + "nauc_mrr_at_20_std": 0.441533, + "nauc_mrr_at_20_diff1": 0.291519, + "nauc_mrr_at_100_max": 0.263365, + "nauc_mrr_at_100_std": 0.440119, + "nauc_mrr_at_100_diff1": 0.290535, + "nauc_mrr_at_1000_max": 0.263365, + "nauc_mrr_at_1000_std": 0.440119, + "nauc_mrr_at_1000_diff1": 0.290535, + "main_score": 0.1, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 24.460757970809937, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBPasskeyRetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBPasskeyRetrieval.json new file mode 100644 index 0000000000..bf41bae56d --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBPasskeyRetrieval.json @@ -0,0 +1,1201 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBPasskeyRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test_256": [ + { + "ndcg_at_1": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "map_at_1": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "map_at_10": 1.0, + "map_at_20": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "recall_at_1": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 1.0, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "mrr_at_10": 1.0, + "mrr_at_20": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_1000_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "main_score": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_512": [ + { + "ndcg_at_1": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "map_at_1": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "map_at_10": 1.0, + "map_at_20": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "recall_at_1": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 1.0, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "mrr_at_10": 1.0, + "mrr_at_20": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_1000_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "main_score": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_1024": [ + { + "ndcg_at_1": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "map_at_1": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "map_at_10": 1.0, + "map_at_20": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "recall_at_1": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 1.0, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "mrr_at_10": 1.0, + "mrr_at_20": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_1000_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "main_score": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_2048": [ + { + "ndcg_at_1": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "map_at_1": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "map_at_10": 1.0, + "map_at_20": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "recall_at_1": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 1.0, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "mrr_at_10": 1.0, + "mrr_at_20": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_1000_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "main_score": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_4096": [ + { + "ndcg_at_1": 0.98, + "ndcg_at_3": 0.99262, + "ndcg_at_5": 0.99262, + "ndcg_at_10": 0.99262, + "ndcg_at_20": 0.99262, + "ndcg_at_100": 0.99262, + "ndcg_at_1000": 0.99262, + "map_at_1": 0.98, + "map_at_3": 0.99, + "map_at_5": 0.99, + "map_at_10": 0.99, + "map_at_20": 0.99, + "map_at_100": 0.99, + "map_at_1000": 0.99, + "recall_at_1": 0.98, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.98, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.98, + "mrr_at_3": 0.99, + "mrr_at_5": 0.99, + "mrr_at_10": 0.99, + "mrr_at_20": 0.99, + "mrr_at_100": 0.99, + "mrr_at_1000": 0.99, + "nauc_ndcg_at_1_max": 0.554155, + "nauc_ndcg_at_1_std": 1.0, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.554155, + "nauc_ndcg_at_3_std": 1.0, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.554155, + "nauc_ndcg_at_5_std": 1.0, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.554155, + "nauc_ndcg_at_10_std": 1.0, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.554155, + "nauc_ndcg_at_20_std": 1.0, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.554155, + "nauc_ndcg_at_100_std": 1.0, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.554155, + "nauc_ndcg_at_1000_std": 1.0, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.554155, + "nauc_map_at_1_std": 1.0, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.554155, + "nauc_map_at_3_std": 1.0, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.554155, + "nauc_map_at_5_std": 1.0, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.554155, + "nauc_map_at_10_std": 1.0, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.554155, + "nauc_map_at_20_std": 1.0, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.554155, + "nauc_map_at_100_std": 1.0, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.554155, + "nauc_map_at_1000_std": 1.0, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.554155, + "nauc_recall_at_1_std": 1.0, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.554155, + "nauc_precision_at_1_std": 1.0, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.554155, + "nauc_mrr_at_1_std": 1.0, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.554155, + "nauc_mrr_at_3_std": 1.0, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.554155, + "nauc_mrr_at_5_std": 1.0, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.554155, + "nauc_mrr_at_10_std": 1.0, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.554155, + "nauc_mrr_at_20_std": 1.0, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.554155, + "nauc_mrr_at_100_std": 1.0, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.554155, + "nauc_mrr_at_1000_std": 1.0, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.98, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_8192": [ + { + "ndcg_at_1": 0.8, + "ndcg_at_3": 0.89309, + "ndcg_at_5": 0.89309, + "ndcg_at_10": 0.8994, + "ndcg_at_20": 0.8994, + "ndcg_at_100": 0.90327, + "ndcg_at_1000": 0.90327, + "map_at_1": 0.8, + "map_at_3": 0.87, + "map_at_5": 0.87, + "map_at_10": 0.8725, + "map_at_20": 0.8725, + "map_at_100": 0.87307, + "map_at_1000": 0.87307, + "recall_at_1": 0.8, + "recall_at_3": 0.96, + "recall_at_5": 0.96, + "recall_at_10": 0.98, + "recall_at_20": 0.98, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.8, + "precision_at_3": 0.32, + "precision_at_5": 0.192, + "precision_at_10": 0.098, + "precision_at_20": 0.049, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.8, + "mrr_at_3": 0.87, + "mrr_at_5": 0.87, + "mrr_at_10": 0.8725, + "mrr_at_20": 0.8725, + "mrr_at_100": 0.873071, + "mrr_at_1000": 0.873071, + "nauc_ndcg_at_1_max": 0.679121, + "nauc_ndcg_at_1_std": 0.565335, + "nauc_ndcg_at_1_diff1": 0.895854, + "nauc_ndcg_at_3_max": 0.712053, + "nauc_ndcg_at_3_std": 0.503503, + "nauc_ndcg_at_3_diff1": 0.835477, + "nauc_ndcg_at_5_max": 0.712053, + "nauc_ndcg_at_5_std": 0.503503, + "nauc_ndcg_at_5_diff1": 0.835477, + "nauc_ndcg_at_10_max": 0.693084, + "nauc_ndcg_at_10_std": 0.470795, + "nauc_ndcg_at_10_diff1": 0.824639, + "nauc_ndcg_at_20_max": 0.693084, + "nauc_ndcg_at_20_std": 0.470795, + "nauc_ndcg_at_20_diff1": 0.824639, + "nauc_ndcg_at_100_max": 0.691858, + "nauc_ndcg_at_100_std": 0.497824, + "nauc_ndcg_at_100_diff1": 0.844277, + "nauc_ndcg_at_1000_max": 0.691858, + "nauc_ndcg_at_1000_std": 0.497824, + "nauc_ndcg_at_1000_diff1": 0.844277, + "nauc_map_at_1_max": 0.679121, + "nauc_map_at_1_std": 0.565335, + "nauc_map_at_1_diff1": 0.895854, + "nauc_map_at_3_max": 0.695909, + "nauc_map_at_3_std": 0.515087, + "nauc_map_at_3_diff1": 0.853601, + "nauc_map_at_5_max": 0.695909, + "nauc_map_at_5_std": 0.515087, + "nauc_map_at_5_diff1": 0.853601, + "nauc_map_at_10_max": 0.689624, + "nauc_map_at_10_std": 0.505066, + "nauc_map_at_10_diff1": 0.850575, + "nauc_map_at_20_max": 0.689624, + "nauc_map_at_20_std": 0.505066, + "nauc_map_at_20_diff1": 0.850575, + "nauc_map_at_100_max": 0.689469, + "nauc_map_at_100_std": 0.508276, + "nauc_map_at_100_diff1": 0.852913, + "nauc_map_at_1000_max": 0.689469, + "nauc_map_at_1000_std": 0.508276, + "nauc_map_at_1000_diff1": 0.852913, + "nauc_recall_at_1_max": 0.679121, + "nauc_recall_at_1_std": 0.565335, + "nauc_recall_at_1_diff1": 0.895854, + "nauc_recall_at_3_max": 0.861111, + "nauc_recall_at_3_std": 0.414332, + "nauc_recall_at_3_diff1": 0.679038, + "nauc_recall_at_5_max": 0.861111, + "nauc_recall_at_5_std": 0.414332, + "nauc_recall_at_5_diff1": 0.679038, + "nauc_recall_at_10_max": 0.722222, + "nauc_recall_at_10_std": -0.171335, + "nauc_recall_at_10_diff1": 0.358077, + "nauc_recall_at_20_max": 0.722222, + "nauc_recall_at_20_std": -0.171335, + "nauc_recall_at_20_diff1": 0.358077, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.679121, + "nauc_precision_at_1_std": 0.565335, + "nauc_precision_at_1_diff1": 0.895854, + "nauc_precision_at_3_max": 0.861111, + "nauc_precision_at_3_std": 0.414332, + "nauc_precision_at_3_diff1": 0.679038, + "nauc_precision_at_5_max": 0.861111, + "nauc_precision_at_5_std": 0.414332, + "nauc_precision_at_5_diff1": 0.679038, + "nauc_precision_at_10_max": 0.722222, + "nauc_precision_at_10_std": -0.171335, + "nauc_precision_at_10_diff1": 0.358077, + "nauc_precision_at_20_max": 0.722222, + "nauc_precision_at_20_std": -0.171335, + "nauc_precision_at_20_diff1": 0.358077, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.679121, + "nauc_mrr_at_1_std": 0.565335, + "nauc_mrr_at_1_diff1": 0.895854, + "nauc_mrr_at_3_max": 0.695909, + "nauc_mrr_at_3_std": 0.515087, + "nauc_mrr_at_3_diff1": 0.853601, + "nauc_mrr_at_5_max": 0.695909, + "nauc_mrr_at_5_std": 0.515087, + "nauc_mrr_at_5_diff1": 0.853601, + "nauc_mrr_at_10_max": 0.689624, + "nauc_mrr_at_10_std": 0.505066, + "nauc_mrr_at_10_diff1": 0.850575, + "nauc_mrr_at_20_max": 0.689624, + "nauc_mrr_at_20_std": 0.505066, + "nauc_mrr_at_20_diff1": 0.850575, + "nauc_mrr_at_100_max": 0.689469, + "nauc_mrr_at_100_std": 0.508276, + "nauc_mrr_at_100_diff1": 0.852913, + "nauc_mrr_at_1000_max": 0.689469, + "nauc_mrr_at_1000_std": 0.508276, + "nauc_mrr_at_1000_diff1": 0.852913, + "main_score": 0.8, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_16384": [ + { + "ndcg_at_1": 0.52, + "ndcg_at_3": 0.52, + "ndcg_at_5": 0.52, + "ndcg_at_10": 0.52667, + "ndcg_at_20": 0.52667, + "ndcg_at_100": 0.60082, + "ndcg_at_1000": 0.60082, + "map_at_1": 0.52, + "map_at_3": 0.52, + "map_at_5": 0.52, + "map_at_10": 0.52286, + "map_at_20": 0.52286, + "map_at_100": 0.52925, + "map_at_1000": 0.52925, + "recall_at_1": 0.52, + "recall_at_3": 0.52, + "recall_at_5": 0.52, + "recall_at_10": 0.54, + "recall_at_20": 0.54, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.52, + "precision_at_3": 0.17333, + "precision_at_5": 0.104, + "precision_at_10": 0.054, + "precision_at_20": 0.027, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.52, + "mrr_at_3": 0.52, + "mrr_at_5": 0.52, + "mrr_at_10": 0.522857, + "mrr_at_20": 0.522857, + "mrr_at_100": 0.529166, + "mrr_at_1000": 0.529166, + "nauc_ndcg_at_1_max": 0.817972, + "nauc_ndcg_at_1_std": 0.287856, + "nauc_ndcg_at_1_diff1": 0.926873, + "nauc_ndcg_at_3_max": 0.817972, + "nauc_ndcg_at_3_std": 0.287856, + "nauc_ndcg_at_3_diff1": 0.926873, + "nauc_ndcg_at_5_max": 0.817972, + "nauc_ndcg_at_5_std": 0.287856, + "nauc_ndcg_at_5_diff1": 0.926873, + "nauc_ndcg_at_10_max": 0.806697, + "nauc_ndcg_at_10_std": 0.282868, + "nauc_ndcg_at_10_diff1": 0.919238, + "nauc_ndcg_at_20_max": 0.806697, + "nauc_ndcg_at_20_std": 0.282868, + "nauc_ndcg_at_20_diff1": 0.919238, + "nauc_ndcg_at_100_max": 0.808715, + "nauc_ndcg_at_100_std": 0.282761, + "nauc_ndcg_at_100_diff1": 0.920425, + "nauc_ndcg_at_1000_max": 0.808715, + "nauc_ndcg_at_1000_std": 0.282761, + "nauc_ndcg_at_1000_diff1": 0.920425, + "nauc_map_at_1_max": 0.817972, + "nauc_map_at_1_std": 0.287856, + "nauc_map_at_1_diff1": 0.926873, + "nauc_map_at_3_max": 0.817972, + "nauc_map_at_3_std": 0.287856, + "nauc_map_at_3_diff1": 0.926873, + "nauc_map_at_5_max": 0.817972, + "nauc_map_at_5_std": 0.287856, + "nauc_map_at_5_diff1": 0.926873, + "nauc_map_at_10_max": 0.813159, + "nauc_map_at_10_std": 0.285727, + "nauc_map_at_10_diff1": 0.923614, + "nauc_map_at_20_max": 0.813159, + "nauc_map_at_20_std": 0.285727, + "nauc_map_at_20_diff1": 0.923614, + "nauc_map_at_100_max": 0.812819, + "nauc_map_at_100_std": 0.285237, + "nauc_map_at_100_diff1": 0.923313, + "nauc_map_at_1000_max": 0.812819, + "nauc_map_at_1000_std": 0.285237, + "nauc_map_at_1000_diff1": 0.923313, + "nauc_recall_at_1_max": 0.817972, + "nauc_recall_at_1_std": 0.287856, + "nauc_recall_at_1_diff1": 0.926873, + "nauc_recall_at_3_max": 0.817972, + "nauc_recall_at_3_std": 0.287856, + "nauc_recall_at_3_diff1": 0.926873, + "nauc_recall_at_5_max": 0.817972, + "nauc_recall_at_5_std": 0.287856, + "nauc_recall_at_5_diff1": 0.926873, + "nauc_recall_at_10_max": 0.783668, + "nauc_recall_at_10_std": 0.272679, + "nauc_recall_at_10_diff1": 0.903643, + "nauc_recall_at_20_max": 0.783668, + "nauc_recall_at_20_std": 0.272679, + "nauc_recall_at_20_diff1": 0.903643, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.817972, + "nauc_precision_at_1_std": 0.287856, + "nauc_precision_at_1_diff1": 0.926873, + "nauc_precision_at_3_max": 0.817972, + "nauc_precision_at_3_std": 0.287856, + "nauc_precision_at_3_diff1": 0.926873, + "nauc_precision_at_5_max": 0.817972, + "nauc_precision_at_5_std": 0.287856, + "nauc_precision_at_5_diff1": 0.926873, + "nauc_precision_at_10_max": 0.783668, + "nauc_precision_at_10_std": 0.272679, + "nauc_precision_at_10_diff1": 0.903643, + "nauc_precision_at_20_max": 0.783668, + "nauc_precision_at_20_std": 0.272679, + "nauc_precision_at_20_diff1": 0.903643, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.817972, + "nauc_mrr_at_1_std": 0.287856, + "nauc_mrr_at_1_diff1": 0.926873, + "nauc_mrr_at_3_max": 0.817972, + "nauc_mrr_at_3_std": 0.287856, + "nauc_mrr_at_3_diff1": 0.926873, + "nauc_mrr_at_5_max": 0.817972, + "nauc_mrr_at_5_std": 0.287856, + "nauc_mrr_at_5_diff1": 0.926873, + "nauc_mrr_at_10_max": 0.813159, + "nauc_mrr_at_10_std": 0.285727, + "nauc_mrr_at_10_diff1": 0.923614, + "nauc_mrr_at_20_max": 0.813159, + "nauc_mrr_at_20_std": 0.285727, + "nauc_mrr_at_20_diff1": 0.923614, + "nauc_mrr_at_100_max": 0.812926, + "nauc_mrr_at_100_std": 0.284854, + "nauc_mrr_at_100_diff1": 0.923284, + "nauc_mrr_at_1000_max": 0.812926, + "nauc_mrr_at_1000_std": 0.284854, + "nauc_mrr_at_1000_diff1": 0.923284, + "main_score": 0.52, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_32768": [ + { + "ndcg_at_1": 0.24, + "ndcg_at_3": 0.24, + "ndcg_at_5": 0.24, + "ndcg_at_10": 0.24631, + "ndcg_at_20": 0.24631, + "ndcg_at_100": 0.37487, + "ndcg_at_1000": 0.37487, + "map_at_1": 0.24, + "map_at_3": 0.24, + "map_at_5": 0.24, + "map_at_10": 0.2425, + "map_at_20": 0.2425, + "map_at_100": 0.2571, + "map_at_1000": 0.2571, + "recall_at_1": 0.24, + "recall_at_3": 0.24, + "recall_at_5": 0.24, + "recall_at_10": 0.26, + "recall_at_20": 0.26, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.24, + "precision_at_3": 0.08, + "precision_at_5": 0.048, + "precision_at_10": 0.026, + "precision_at_20": 0.013, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.24, + "mrr_at_3": 0.24, + "mrr_at_5": 0.24, + "mrr_at_10": 0.2425, + "mrr_at_20": 0.2425, + "mrr_at_100": 0.255795, + "mrr_at_1000": 0.255795, + "nauc_ndcg_at_1_max": 0.498023, + "nauc_ndcg_at_1_std": 0.777318, + "nauc_ndcg_at_1_diff1": 0.363887, + "nauc_ndcg_at_3_max": 0.498023, + "nauc_ndcg_at_3_std": 0.777318, + "nauc_ndcg_at_3_diff1": 0.363887, + "nauc_ndcg_at_5_max": 0.498023, + "nauc_ndcg_at_5_std": 0.777318, + "nauc_ndcg_at_5_diff1": 0.363887, + "nauc_ndcg_at_10_max": 0.477953, + "nauc_ndcg_at_10_std": 0.766095, + "nauc_ndcg_at_10_diff1": 0.37106, + "nauc_ndcg_at_20_max": 0.477953, + "nauc_ndcg_at_20_std": 0.766095, + "nauc_ndcg_at_20_diff1": 0.37106, + "nauc_ndcg_at_100_max": 0.474578, + "nauc_ndcg_at_100_std": 0.757879, + "nauc_ndcg_at_100_diff1": 0.351574, + "nauc_ndcg_at_1000_max": 0.474578, + "nauc_ndcg_at_1000_std": 0.757879, + "nauc_ndcg_at_1000_diff1": 0.351574, + "nauc_map_at_1_max": 0.498023, + "nauc_map_at_1_std": 0.777318, + "nauc_map_at_1_diff1": 0.363887, + "nauc_map_at_3_max": 0.498023, + "nauc_map_at_3_std": 0.777318, + "nauc_map_at_3_diff1": 0.363887, + "nauc_map_at_5_max": 0.498023, + "nauc_map_at_5_std": 0.777318, + "nauc_map_at_5_diff1": 0.363887, + "nauc_map_at_10_max": 0.490016, + "nauc_map_at_10_std": 0.772841, + "nauc_map_at_10_diff1": 0.366749, + "nauc_map_at_20_max": 0.490016, + "nauc_map_at_20_std": 0.772841, + "nauc_map_at_20_diff1": 0.366749, + "nauc_map_at_100_max": 0.486033, + "nauc_map_at_100_std": 0.7679, + "nauc_map_at_100_diff1": 0.359581, + "nauc_map_at_1000_max": 0.486033, + "nauc_map_at_1000_std": 0.7679, + "nauc_map_at_1000_diff1": 0.359581, + "nauc_recall_at_1_max": 0.498023, + "nauc_recall_at_1_std": 0.777318, + "nauc_recall_at_1_diff1": 0.363887, + "nauc_recall_at_3_max": 0.498023, + "nauc_recall_at_3_std": 0.777318, + "nauc_recall_at_3_diff1": 0.363887, + "nauc_recall_at_5_max": 0.498023, + "nauc_recall_at_5_std": 0.777318, + "nauc_recall_at_5_diff1": 0.363887, + "nauc_recall_at_10_max": 0.435923, + "nauc_recall_at_10_std": 0.742592, + "nauc_recall_at_10_diff1": 0.386083, + "nauc_recall_at_20_max": 0.435923, + "nauc_recall_at_20_std": 0.742592, + "nauc_recall_at_20_diff1": 0.386083, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.498023, + "nauc_precision_at_1_std": 0.777318, + "nauc_precision_at_1_diff1": 0.363887, + "nauc_precision_at_3_max": 0.498023, + "nauc_precision_at_3_std": 0.777318, + "nauc_precision_at_3_diff1": 0.363887, + "nauc_precision_at_5_max": 0.498023, + "nauc_precision_at_5_std": 0.777318, + "nauc_precision_at_5_diff1": 0.363887, + "nauc_precision_at_10_max": 0.435923, + "nauc_precision_at_10_std": 0.742592, + "nauc_precision_at_10_diff1": 0.386083, + "nauc_precision_at_20_max": 0.435923, + "nauc_precision_at_20_std": 0.742592, + "nauc_precision_at_20_diff1": 0.386083, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.498023, + "nauc_mrr_at_1_std": 0.777318, + "nauc_mrr_at_1_diff1": 0.363887, + "nauc_mrr_at_3_max": 0.498023, + "nauc_mrr_at_3_std": 0.777318, + "nauc_mrr_at_3_diff1": 0.363887, + "nauc_mrr_at_5_max": 0.498023, + "nauc_mrr_at_5_std": 0.777318, + "nauc_mrr_at_5_diff1": 0.363887, + "nauc_mrr_at_10_max": 0.490016, + "nauc_mrr_at_10_std": 0.772841, + "nauc_mrr_at_10_diff1": 0.366749, + "nauc_mrr_at_20_max": 0.490016, + "nauc_mrr_at_20_std": 0.772841, + "nauc_mrr_at_20_diff1": 0.366749, + "nauc_mrr_at_100_max": 0.486329, + "nauc_mrr_at_100_std": 0.766622, + "nauc_mrr_at_100_diff1": 0.362755, + "nauc_mrr_at_1000_max": 0.486329, + "nauc_mrr_at_1000_std": 0.766622, + "nauc_mrr_at_1000_diff1": 0.362755, + "main_score": 0.24, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 23.525697946548462, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBQMSumRetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBQMSumRetrieval.json new file mode 100644 index 0000000000..de91592fec --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBQMSumRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBQMSumRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.26457, + "ndcg_at_3": 0.35149, + "ndcg_at_5": 0.37868, + "ndcg_at_10": 0.41576, + "ndcg_at_20": 0.44026, + "ndcg_at_100": 0.49232, + "ndcg_at_1000": 0.49578, + "map_at_1": 0.26457, + "map_at_3": 0.32995, + "map_at_5": 0.34501, + "map_at_10": 0.36045, + "map_at_20": 0.36725, + "map_at_100": 0.37408, + "map_at_1000": 0.37429, + "recall_at_1": 0.26457, + "recall_at_3": 0.41388, + "recall_at_5": 0.48003, + "recall_at_10": 0.59398, + "recall_at_20": 0.69024, + "recall_at_100": 0.97642, + "recall_at_1000": 1.0, + "precision_at_1": 0.26457, + "precision_at_3": 0.13796, + "precision_at_5": 0.09601, + "precision_at_10": 0.0594, + "precision_at_20": 0.03451, + "precision_at_100": 0.00976, + "precision_at_1000": 0.001, + "mrr_at_1": 0.264571, + "mrr_at_3": 0.32995, + "mrr_at_5": 0.345012, + "mrr_at_10": 0.360447, + "mrr_at_20": 0.367254, + "mrr_at_100": 0.37408, + "mrr_at_1000": 0.374291, + "nauc_ndcg_at_1_max": 0.028993, + "nauc_ndcg_at_1_std": -0.378069, + "nauc_ndcg_at_1_diff1": 0.529259, + "nauc_ndcg_at_3_max": 0.001259, + "nauc_ndcg_at_3_std": -0.415083, + "nauc_ndcg_at_3_diff1": 0.453145, + "nauc_ndcg_at_5_max": -0.018633, + "nauc_ndcg_at_5_std": -0.432603, + "nauc_ndcg_at_5_diff1": 0.442576, + "nauc_ndcg_at_10_max": -0.035057, + "nauc_ndcg_at_10_std": -0.458169, + "nauc_ndcg_at_10_diff1": 0.444695, + "nauc_ndcg_at_20_max": -0.043889, + "nauc_ndcg_at_20_std": -0.465684, + "nauc_ndcg_at_20_diff1": 0.443455, + "nauc_ndcg_at_100_max": -0.020491, + "nauc_ndcg_at_100_std": -0.435695, + "nauc_ndcg_at_100_diff1": 0.454283, + "nauc_ndcg_at_1000_max": -0.020444, + "nauc_ndcg_at_1000_std": -0.43785, + "nauc_ndcg_at_1000_diff1": 0.456057, + "nauc_map_at_1_max": 0.028993, + "nauc_map_at_1_std": -0.378069, + "nauc_map_at_1_diff1": 0.529259, + "nauc_map_at_3_max": 0.008739, + "nauc_map_at_3_std": -0.405327, + "nauc_map_at_3_diff1": 0.469553, + "nauc_map_at_5_max": -0.002439, + "nauc_map_at_5_std": -0.41531, + "nauc_map_at_5_diff1": 0.463559, + "nauc_map_at_10_max": -0.00952, + "nauc_map_at_10_std": -0.426022, + "nauc_map_at_10_diff1": 0.464626, + "nauc_map_at_20_max": -0.011519, + "nauc_map_at_20_std": -0.427788, + "nauc_map_at_20_diff1": 0.464634, + "nauc_map_at_100_max": -0.008716, + "nauc_map_at_100_std": -0.424217, + "nauc_map_at_100_diff1": 0.465903, + "nauc_map_at_1000_max": -0.008697, + "nauc_map_at_1000_std": -0.424308, + "nauc_map_at_1000_diff1": 0.465997, + "nauc_recall_at_1_max": 0.028993, + "nauc_recall_at_1_std": -0.378069, + "nauc_recall_at_1_diff1": 0.529259, + "nauc_recall_at_3_max": -0.019655, + "nauc_recall_at_3_std": -0.442281, + "nauc_recall_at_3_diff1": 0.408649, + "nauc_recall_at_5_max": -0.066905, + "nauc_recall_at_5_std": -0.483483, + "nauc_recall_at_5_diff1": 0.383263, + "nauc_recall_at_10_max": -0.122367, + "nauc_recall_at_10_std": -0.569325, + "nauc_recall_at_10_diff1": 0.38389, + "nauc_recall_at_20_max": -0.1822, + "nauc_recall_at_20_std": -0.62587, + "nauc_recall_at_20_diff1": 0.364899, + "nauc_recall_at_100_max": -0.019587, + "nauc_recall_at_100_std": -0.260361, + "nauc_recall_at_100_diff1": 0.313516, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.028993, + "nauc_precision_at_1_std": -0.378069, + "nauc_precision_at_1_diff1": 0.529259, + "nauc_precision_at_3_max": -0.019655, + "nauc_precision_at_3_std": -0.442281, + "nauc_precision_at_3_diff1": 0.408649, + "nauc_precision_at_5_max": -0.066905, + "nauc_precision_at_5_std": -0.483483, + "nauc_precision_at_5_diff1": 0.383263, + "nauc_precision_at_10_max": -0.122367, + "nauc_precision_at_10_std": -0.569325, + "nauc_precision_at_10_diff1": 0.38389, + "nauc_precision_at_20_max": -0.1822, + "nauc_precision_at_20_std": -0.62587, + "nauc_precision_at_20_diff1": 0.364899, + "nauc_precision_at_100_max": -0.019587, + "nauc_precision_at_100_std": -0.260361, + "nauc_precision_at_100_diff1": 0.313516, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.028993, + "nauc_mrr_at_1_std": -0.378069, + "nauc_mrr_at_1_diff1": 0.529259, + "nauc_mrr_at_3_max": 0.008739, + "nauc_mrr_at_3_std": -0.405327, + "nauc_mrr_at_3_diff1": 0.469553, + "nauc_mrr_at_5_max": -0.002439, + "nauc_mrr_at_5_std": -0.41531, + "nauc_mrr_at_5_diff1": 0.463559, + "nauc_mrr_at_10_max": -0.00952, + "nauc_mrr_at_10_std": -0.426022, + "nauc_mrr_at_10_diff1": 0.464626, + "nauc_mrr_at_20_max": -0.011519, + "nauc_mrr_at_20_std": -0.427788, + "nauc_mrr_at_20_diff1": 0.464634, + "nauc_mrr_at_100_max": -0.008716, + "nauc_mrr_at_100_std": -0.424217, + "nauc_mrr_at_100_diff1": 0.465903, + "nauc_mrr_at_1000_max": -0.008697, + "nauc_mrr_at_1000_std": -0.424308, + "nauc_mrr_at_1000_diff1": 0.465997, + "main_score": 0.41576, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11.951326847076416, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBSummScreenFDRetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBSummScreenFDRetrieval.json new file mode 100644 index 0000000000..2eb42a898e --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBSummScreenFDRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBSummScreenFDRetrieval", + "mteb_version": "1.36.8", + "scores": { + "validation": [ + { + "ndcg_at_1": 0.8869, + "ndcg_at_3": 0.92471, + "ndcg_at_5": 0.93457, + "ndcg_at_10": 0.9365, + "ndcg_at_20": 0.93805, + "ndcg_at_100": 0.9393, + "ndcg_at_1000": 0.94008, + "map_at_1": 0.8869, + "map_at_3": 0.91518, + "map_at_5": 0.92068, + "map_at_10": 0.92148, + "map_at_20": 0.92193, + "map_at_100": 0.92215, + "map_at_1000": 0.92218, + "recall_at_1": 0.8869, + "recall_at_3": 0.95238, + "recall_at_5": 0.97619, + "recall_at_10": 0.98214, + "recall_at_20": 0.9881, + "recall_at_100": 0.99405, + "recall_at_1000": 1.0, + "precision_at_1": 0.8869, + "precision_at_3": 0.31746, + "precision_at_5": 0.19524, + "precision_at_10": 0.09821, + "precision_at_20": 0.0494, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "mrr_at_1": 0.886905, + "mrr_at_3": 0.915179, + "mrr_at_5": 0.920685, + "mrr_at_10": 0.921482, + "mrr_at_20": 0.921928, + "mrr_at_100": 0.922154, + "mrr_at_1000": 0.922185, + "nauc_ndcg_at_1_max": 0.661307, + "nauc_ndcg_at_1_std": 0.046528, + "nauc_ndcg_at_1_diff1": 0.915674, + "nauc_ndcg_at_3_max": 0.656072, + "nauc_ndcg_at_3_std": 0.04769, + "nauc_ndcg_at_3_diff1": 0.915574, + "nauc_ndcg_at_5_max": 0.65211, + "nauc_ndcg_at_5_std": 0.012081, + "nauc_ndcg_at_5_diff1": 0.909969, + "nauc_ndcg_at_10_max": 0.65247, + "nauc_ndcg_at_10_std": 0.01892, + "nauc_ndcg_at_10_diff1": 0.90927, + "nauc_ndcg_at_20_max": 0.671971, + "nauc_ndcg_at_20_std": 0.048495, + "nauc_ndcg_at_20_diff1": 0.912065, + "nauc_ndcg_at_100_max": 0.665182, + "nauc_ndcg_at_100_std": 0.038899, + "nauc_ndcg_at_100_diff1": 0.911541, + "nauc_ndcg_at_1000_max": 0.662568, + "nauc_ndcg_at_1000_std": 0.037286, + "nauc_ndcg_at_1000_diff1": 0.912189, + "nauc_map_at_1_max": 0.661307, + "nauc_map_at_1_std": 0.046528, + "nauc_map_at_1_diff1": 0.915674, + "nauc_map_at_3_max": 0.659641, + "nauc_map_at_3_std": 0.046329, + "nauc_map_at_3_diff1": 0.915473, + "nauc_map_at_5_max": 0.657462, + "nauc_map_at_5_std": 0.030553, + "nauc_map_at_5_diff1": 0.912772, + "nauc_map_at_10_max": 0.657604, + "nauc_map_at_10_std": 0.03289, + "nauc_map_at_10_diff1": 0.912593, + "nauc_map_at_20_max": 0.66229, + "nauc_map_at_20_std": 0.039742, + "nauc_map_at_20_diff1": 0.913221, + "nauc_map_at_100_max": 0.661302, + "nauc_map_at_100_std": 0.038245, + "nauc_map_at_100_diff1": 0.913131, + "nauc_map_at_1000_max": 0.661218, + "nauc_map_at_1000_std": 0.038182, + "nauc_map_at_1000_diff1": 0.913149, + "nauc_recall_at_1_max": 0.661307, + "nauc_recall_at_1_std": 0.046528, + "nauc_recall_at_1_diff1": 0.915674, + "nauc_recall_at_3_max": 0.636516, + "nauc_recall_at_3_std": 0.055341, + "nauc_recall_at_3_diff1": 0.916185, + "nauc_recall_at_5_max": 0.599139, + "nauc_recall_at_5_std": -0.181186, + "nauc_recall_at_5_diff1": 0.881455, + "nauc_recall_at_10_max": 0.586167, + "nauc_recall_at_10_std": -0.168173, + "nauc_recall_at_10_diff1": 0.863755, + "nauc_recall_at_20_max": 0.93054, + "nauc_recall_at_20_std": 0.324442, + "nauc_recall_at_20_diff1": 0.897817, + "nauc_recall_at_100_max": 0.861079, + "nauc_recall_at_100_std": 0.152848, + "nauc_recall_at_100_diff1": 0.861079, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.661307, + "nauc_precision_at_1_std": 0.046528, + "nauc_precision_at_1_diff1": 0.915674, + "nauc_precision_at_3_max": 0.636516, + "nauc_precision_at_3_std": 0.055341, + "nauc_precision_at_3_diff1": 0.916185, + "nauc_precision_at_5_max": 0.599139, + "nauc_precision_at_5_std": -0.181186, + "nauc_precision_at_5_diff1": 0.881455, + "nauc_precision_at_10_max": 0.586167, + "nauc_precision_at_10_std": -0.168173, + "nauc_precision_at_10_diff1": 0.863755, + "nauc_precision_at_20_max": 0.93054, + "nauc_precision_at_20_std": 0.324442, + "nauc_precision_at_20_diff1": 0.897817, + "nauc_precision_at_100_max": 0.861079, + "nauc_precision_at_100_std": 0.152848, + "nauc_precision_at_100_diff1": 0.861079, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.661307, + "nauc_mrr_at_1_std": 0.046528, + "nauc_mrr_at_1_diff1": 0.915674, + "nauc_mrr_at_3_max": 0.659641, + "nauc_mrr_at_3_std": 0.046329, + "nauc_mrr_at_3_diff1": 0.915473, + "nauc_mrr_at_5_max": 0.657462, + "nauc_mrr_at_5_std": 0.030553, + "nauc_mrr_at_5_diff1": 0.912772, + "nauc_mrr_at_10_max": 0.657604, + "nauc_mrr_at_10_std": 0.03289, + "nauc_mrr_at_10_diff1": 0.912593, + "nauc_mrr_at_20_max": 0.66229, + "nauc_mrr_at_20_std": 0.039742, + "nauc_mrr_at_20_diff1": 0.913221, + "nauc_mrr_at_100_max": 0.661302, + "nauc_mrr_at_100_std": 0.038245, + "nauc_mrr_at_100_diff1": 0.913131, + "nauc_mrr_at_1000_max": 0.661218, + "nauc_mrr_at_1000_std": 0.038182, + "nauc_mrr_at_1000_diff1": 0.913149, + "main_score": 0.9365, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 15.555839776992798, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBWikimQARetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBWikimQARetrieval.json new file mode 100644 index 0000000000..1d5fab1c4c --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/LEMBWikimQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBWikimQARetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.80333, + "ndcg_at_3": 0.84067, + "ndcg_at_5": 0.84483, + "ndcg_at_10": 0.859, + "ndcg_at_20": 0.86724, + "ndcg_at_100": 0.87211, + "ndcg_at_1000": 0.87484, + "map_at_1": 0.80333, + "map_at_3": 0.83167, + "map_at_5": 0.834, + "map_at_10": 0.83995, + "map_at_20": 0.8421, + "map_at_100": 0.84274, + "map_at_1000": 0.84287, + "recall_at_1": 0.80333, + "recall_at_3": 0.86667, + "recall_at_5": 0.87667, + "recall_at_10": 0.92, + "recall_at_20": 0.95333, + "recall_at_100": 0.98, + "recall_at_1000": 1.0, + "precision_at_1": 0.80333, + "precision_at_3": 0.28889, + "precision_at_5": 0.17533, + "precision_at_10": 0.092, + "precision_at_20": 0.04767, + "precision_at_100": 0.0098, + "precision_at_1000": 0.001, + "mrr_at_1": 0.803333, + "mrr_at_3": 0.831667, + "mrr_at_5": 0.834, + "mrr_at_10": 0.839947, + "mrr_at_20": 0.842103, + "mrr_at_100": 0.84274, + "mrr_at_1000": 0.84287, + "nauc_ndcg_at_1_max": 0.7962, + "nauc_ndcg_at_1_std": 0.112914, + "nauc_ndcg_at_1_diff1": 0.900184, + "nauc_ndcg_at_3_max": 0.826663, + "nauc_ndcg_at_3_std": 0.146003, + "nauc_ndcg_at_3_diff1": 0.891005, + "nauc_ndcg_at_5_max": 0.828192, + "nauc_ndcg_at_5_std": 0.148444, + "nauc_ndcg_at_5_diff1": 0.8885, + "nauc_ndcg_at_10_max": 0.826694, + "nauc_ndcg_at_10_std": 0.130812, + "nauc_ndcg_at_10_diff1": 0.884516, + "nauc_ndcg_at_20_max": 0.823282, + "nauc_ndcg_at_20_std": 0.117708, + "nauc_ndcg_at_20_diff1": 0.887258, + "nauc_ndcg_at_100_max": 0.8211, + "nauc_ndcg_at_100_std": 0.139198, + "nauc_ndcg_at_100_diff1": 0.888254, + "nauc_ndcg_at_1000_max": 0.820056, + "nauc_ndcg_at_1000_std": 0.131832, + "nauc_ndcg_at_1000_diff1": 0.888563, + "nauc_map_at_1_max": 0.7962, + "nauc_map_at_1_std": 0.112914, + "nauc_map_at_1_diff1": 0.900184, + "nauc_map_at_3_max": 0.819187, + "nauc_map_at_3_std": 0.139945, + "nauc_map_at_3_diff1": 0.892881, + "nauc_map_at_5_max": 0.819927, + "nauc_map_at_5_std": 0.141319, + "nauc_map_at_5_diff1": 0.891587, + "nauc_map_at_10_max": 0.818386, + "nauc_map_at_10_std": 0.133727, + "nauc_map_at_10_diff1": 0.889624, + "nauc_map_at_20_max": 0.817456, + "nauc_map_at_20_std": 0.13064, + "nauc_map_at_20_diff1": 0.890154, + "nauc_map_at_100_max": 0.817187, + "nauc_map_at_100_std": 0.132775, + "nauc_map_at_100_diff1": 0.890236, + "nauc_map_at_1000_max": 0.817134, + "nauc_map_at_1000_std": 0.132444, + "nauc_map_at_1000_diff1": 0.890275, + "nauc_recall_at_1_max": 0.7962, + "nauc_recall_at_1_std": 0.112914, + "nauc_recall_at_1_diff1": 0.900184, + "nauc_recall_at_3_max": 0.852811, + "nauc_recall_at_3_std": 0.165999, + "nauc_recall_at_3_diff1": 0.884665, + "nauc_recall_at_5_max": 0.859058, + "nauc_recall_at_5_std": 0.173765, + "nauc_recall_at_5_diff1": 0.876782, + "nauc_recall_at_10_max": 0.873852, + "nauc_recall_at_10_std": 0.103661, + "nauc_recall_at_10_diff1": 0.856248, + "nauc_recall_at_20_max": 0.871282, + "nauc_recall_at_20_std": -0.054655, + "nauc_recall_at_20_diff1": 0.869781, + "nauc_recall_at_100_max": 0.863834, + "nauc_recall_at_100_std": 0.447323, + "nauc_recall_at_100_diff1": 0.88212, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.7962, + "nauc_precision_at_1_std": 0.112914, + "nauc_precision_at_1_diff1": 0.900184, + "nauc_precision_at_3_max": 0.852811, + "nauc_precision_at_3_std": 0.165999, + "nauc_precision_at_3_diff1": 0.884665, + "nauc_precision_at_5_max": 0.859058, + "nauc_precision_at_5_std": 0.173765, + "nauc_precision_at_5_diff1": 0.876782, + "nauc_precision_at_10_max": 0.873852, + "nauc_precision_at_10_std": 0.103661, + "nauc_precision_at_10_diff1": 0.856248, + "nauc_precision_at_20_max": 0.871282, + "nauc_precision_at_20_std": -0.054655, + "nauc_precision_at_20_diff1": 0.869781, + "nauc_precision_at_100_max": 0.863834, + "nauc_precision_at_100_std": 0.447323, + "nauc_precision_at_100_diff1": 0.88212, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.7962, + "nauc_mrr_at_1_std": 0.112914, + "nauc_mrr_at_1_diff1": 0.900184, + "nauc_mrr_at_3_max": 0.819187, + "nauc_mrr_at_3_std": 0.139945, + "nauc_mrr_at_3_diff1": 0.892881, + "nauc_mrr_at_5_max": 0.819927, + "nauc_mrr_at_5_std": 0.141319, + "nauc_mrr_at_5_diff1": 0.891587, + "nauc_mrr_at_10_max": 0.818386, + "nauc_mrr_at_10_std": 0.133727, + "nauc_mrr_at_10_diff1": 0.889624, + "nauc_mrr_at_20_max": 0.817456, + "nauc_mrr_at_20_std": 0.13064, + "nauc_mrr_at_20_diff1": 0.890154, + "nauc_mrr_at_100_max": 0.817187, + "nauc_mrr_at_100_std": 0.132775, + "nauc_mrr_at_100_diff1": 0.890236, + "nauc_mrr_at_1000_max": 0.817134, + "nauc_mrr_at_1000_std": 0.132444, + "nauc_mrr_at_1000_diff1": 0.890275, + "main_score": 0.859, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 13.125711917877197, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MSMARCO.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MSMARCO.json new file mode 100644 index 0000000000..d53fe5dfb1 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MSMARCO.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c5a29a104738b98a9e76336939199e264163d4a0", + "task_name": "MSMARCO", + "mteb_version": "1.36.8", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.15516, + "ndcg_at_3": 0.24287, + "ndcg_at_5": 0.28187, + "ndcg_at_10": 0.32142, + "ndcg_at_20": 0.3496, + "ndcg_at_100": 0.38671, + "ndcg_at_1000": 0.40416, + "map_at_1": 0.15111, + "map_at_3": 0.2189, + "map_at_5": 0.2406, + "map_at_10": 0.25719, + "map_at_20": 0.26512, + "map_at_100": 0.27028, + "map_at_1000": 0.27097, + "recall_at_1": 0.15111, + "recall_at_3": 0.30734, + "recall_at_5": 0.4013, + "recall_at_10": 0.52202, + "recall_at_20": 0.63122, + "recall_at_100": 0.82907, + "recall_at_1000": 0.96309, + "precision_at_1": 0.15516, + "precision_at_3": 0.10597, + "precision_at_5": 0.0833, + "precision_at_10": 0.05438, + "precision_at_20": 0.03303, + "precision_at_100": 0.00874, + "precision_at_1000": 0.00102, + "mrr_at_1": 0.155158, + "mrr_at_3": 0.223926, + "mrr_at_5": 0.245645, + "mrr_at_10": 0.261793, + "mrr_at_20": 0.269532, + "mrr_at_100": 0.274443, + "mrr_at_1000": 0.275081, + "nauc_ndcg_at_1_max": -0.002083, + "nauc_ndcg_at_1_std": -0.146845, + "nauc_ndcg_at_1_diff1": 0.279912, + "nauc_ndcg_at_3_max": -0.003006, + "nauc_ndcg_at_3_std": -0.161265, + "nauc_ndcg_at_3_diff1": 0.240907, + "nauc_ndcg_at_5_max": -0.006505, + "nauc_ndcg_at_5_std": -0.16537, + "nauc_ndcg_at_5_diff1": 0.235789, + "nauc_ndcg_at_10_max": -0.004945, + "nauc_ndcg_at_10_std": -0.163183, + "nauc_ndcg_at_10_diff1": 0.235371, + "nauc_ndcg_at_20_max": 0.004832, + "nauc_ndcg_at_20_std": -0.144333, + "nauc_ndcg_at_20_diff1": 0.234703, + "nauc_ndcg_at_100_max": 0.015138, + "nauc_ndcg_at_100_std": -0.11674, + "nauc_ndcg_at_100_diff1": 0.231754, + "nauc_ndcg_at_1000_max": 0.014672, + "nauc_ndcg_at_1000_std": -0.124334, + "nauc_ndcg_at_1000_diff1": 0.234552, + "nauc_map_at_1_max": -0.0015, + "nauc_map_at_1_std": -0.146879, + "nauc_map_at_1_diff1": 0.283443, + "nauc_map_at_3_max": -0.003652, + "nauc_map_at_3_std": -0.158392, + "nauc_map_at_3_diff1": 0.250364, + "nauc_map_at_5_max": -0.005505, + "nauc_map_at_5_std": -0.16093, + "nauc_map_at_5_diff1": 0.247168, + "nauc_map_at_10_max": -0.004791, + "nauc_map_at_10_std": -0.160151, + "nauc_map_at_10_diff1": 0.246732, + "nauc_map_at_20_max": -0.002097, + "nauc_map_at_20_std": -0.154894, + "nauc_map_at_20_diff1": 0.246528, + "nauc_map_at_100_max": -0.000422, + "nauc_map_at_100_std": -0.151015, + "nauc_map_at_100_diff1": 0.246175, + "nauc_map_at_1000_max": -0.000366, + "nauc_map_at_1000_std": -0.151064, + "nauc_map_at_1000_diff1": 0.24631, + "nauc_recall_at_1_max": -0.0015, + "nauc_recall_at_1_std": -0.146879, + "nauc_recall_at_1_diff1": 0.283443, + "nauc_recall_at_3_max": -0.002648, + "nauc_recall_at_3_std": -0.168767, + "nauc_recall_at_3_diff1": 0.21919, + "nauc_recall_at_5_max": -0.009896, + "nauc_recall_at_5_std": -0.176951, + "nauc_recall_at_5_diff1": 0.210143, + "nauc_recall_at_10_max": -0.007059, + "nauc_recall_at_10_std": -0.171639, + "nauc_recall_at_10_diff1": 0.207849, + "nauc_recall_at_20_max": 0.028239, + "nauc_recall_at_20_std": -0.101008, + "nauc_recall_at_20_diff1": 0.200827, + "nauc_recall_at_100_max": 0.124945, + "nauc_recall_at_100_std": 0.164685, + "nauc_recall_at_100_diff1": 0.151627, + "nauc_recall_at_1000_max": 0.408193, + "nauc_recall_at_1000_std": 0.599883, + "nauc_recall_at_1000_diff1": 0.035337, + "nauc_precision_at_1_max": -0.002083, + "nauc_precision_at_1_std": -0.146845, + "nauc_precision_at_1_diff1": 0.279912, + "nauc_precision_at_3_max": -0.003423, + "nauc_precision_at_3_std": -0.169856, + "nauc_precision_at_3_diff1": 0.21718, + "nauc_precision_at_5_max": -0.00932, + "nauc_precision_at_5_std": -0.176556, + "nauc_precision_at_5_diff1": 0.204975, + "nauc_precision_at_10_max": -0.00296, + "nauc_precision_at_10_std": -0.163435, + "nauc_precision_at_10_diff1": 0.195614, + "nauc_precision_at_20_max": 0.040716, + "nauc_precision_at_20_std": -0.084008, + "nauc_precision_at_20_diff1": 0.183193, + "nauc_precision_at_100_max": 0.141963, + "nauc_precision_at_100_std": 0.164304, + "nauc_precision_at_100_diff1": 0.102619, + "nauc_precision_at_1000_max": 0.238428, + "nauc_precision_at_1000_std": 0.248816, + "nauc_precision_at_1000_diff1": -0.019379, + "nauc_mrr_at_1_max": -0.002083, + "nauc_mrr_at_1_std": -0.146845, + "nauc_mrr_at_1_diff1": 0.279912, + "nauc_mrr_at_3_max": -0.002063, + "nauc_mrr_at_3_std": -0.156091, + "nauc_mrr_at_3_diff1": 0.246874, + "nauc_mrr_at_5_max": -0.004473, + "nauc_mrr_at_5_std": -0.158296, + "nauc_mrr_at_5_diff1": 0.243672, + "nauc_mrr_at_10_max": -0.003422, + "nauc_mrr_at_10_std": -0.156994, + "nauc_mrr_at_10_diff1": 0.243769, + "nauc_mrr_at_20_max": -0.000591, + "nauc_mrr_at_20_std": -0.152012, + "nauc_mrr_at_20_diff1": 0.24384, + "nauc_mrr_at_100_max": 0.000643, + "nauc_mrr_at_100_std": -0.148682, + "nauc_mrr_at_100_diff1": 0.243391, + "nauc_mrr_at_1000_max": 0.000622, + "nauc_mrr_at_1000_std": -0.148777, + "nauc_mrr_at_1000_diff1": 0.243515, + "main_score": 0.32142, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6282.300637722015, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MTOPDomainClassification.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MTOPDomainClassification.json new file mode 100644 index 0000000000..5526b3e3f1 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MTOPDomainClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf", + "task_name": "MTOPDomainClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.923484, + "f1": 0.919343, + "f1_weighted": 0.923892, + "scores_per_experiment": [ + { + "accuracy": 0.920429, + "f1": 0.914886, + "f1_weighted": 0.920459 + }, + { + "accuracy": 0.930461, + "f1": 0.928368, + "f1_weighted": 0.930641 + }, + { + "accuracy": 0.924077, + "f1": 0.920221, + "f1_weighted": 0.923937 + }, + { + "accuracy": 0.931601, + "f1": 0.927188, + "f1_weighted": 0.931934 + }, + { + "accuracy": 0.914501, + "f1": 0.908067, + "f1_weighted": 0.915685 + }, + { + "accuracy": 0.925217, + "f1": 0.924044, + "f1_weighted": 0.925042 + }, + { + "accuracy": 0.913361, + "f1": 0.906999, + "f1_weighted": 0.913483 + }, + { + "accuracy": 0.928865, + "f1": 0.923681, + "f1_weighted": 0.92936 + }, + { + "accuracy": 0.917009, + "f1": 0.914747, + "f1_weighted": 0.918269 + }, + { + "accuracy": 0.929321, + "f1": 0.925228, + "f1_weighted": 0.93011 + } + ], + "main_score": 0.923484, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.76896333694458, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MTOPIntentClassification.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MTOPIntentClassification.json new file mode 100644 index 0000000000..8a7f2ede1d --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MTOPIntentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba", + "task_name": "MTOPIntentClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.710397, + "f1": 0.524524, + "f1_weighted": 0.742848, + "scores_per_experiment": [ + { + "accuracy": 0.701778, + "f1": 0.505562, + "f1_weighted": 0.739801 + }, + { + "accuracy": 0.729594, + "f1": 0.527908, + "f1_weighted": 0.762585 + }, + { + "accuracy": 0.71409, + "f1": 0.526676, + "f1_weighted": 0.747047 + }, + { + "accuracy": 0.724578, + "f1": 0.518617, + "f1_weighted": 0.754242 + }, + { + "accuracy": 0.70383, + "f1": 0.525107, + "f1_weighted": 0.734525 + }, + { + "accuracy": 0.677383, + "f1": 0.507047, + "f1_weighted": 0.710717 + }, + { + "accuracy": 0.71865, + "f1": 0.533099, + "f1_weighted": 0.753048 + }, + { + "accuracy": 0.733242, + "f1": 0.538501, + "f1_weighted": 0.763259 + }, + { + "accuracy": 0.706794, + "f1": 0.542212, + "f1_weighted": 0.737138 + }, + { + "accuracy": 0.694026, + "f1": 0.520506, + "f1_weighted": 0.726119 + } + ], + "main_score": 0.710397, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 42.898452281951904, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MassiveIntentClassification.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MassiveIntentClassification.json new file mode 100644 index 0000000000..2ac57cc789 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MassiveIntentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "task_name": "MassiveIntentClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.705649, + "f1": 0.679231, + "f1_weighted": 0.706344, + "scores_per_experiment": [ + { + "accuracy": 0.70612, + "f1": 0.683533, + "f1_weighted": 0.706216 + }, + { + "accuracy": 0.724613, + "f1": 0.698989, + "f1_weighted": 0.728339 + }, + { + "accuracy": 0.707128, + "f1": 0.678494, + "f1_weighted": 0.704386 + }, + { + "accuracy": 0.729993, + "f1": 0.690989, + "f1_weighted": 0.73371 + }, + { + "accuracy": 0.702085, + "f1": 0.671025, + "f1_weighted": 0.697054 + }, + { + "accuracy": 0.686954, + "f1": 0.670726, + "f1_weighted": 0.689964 + }, + { + "accuracy": 0.694687, + "f1": 0.6674, + "f1_weighted": 0.694828 + }, + { + "accuracy": 0.688971, + "f1": 0.666712, + "f1_weighted": 0.692249 + }, + { + "accuracy": 0.701748, + "f1": 0.678964, + "f1_weighted": 0.704016 + }, + { + "accuracy": 0.71419, + "f1": 0.685477, + "f1_weighted": 0.712676 + } + ], + "main_score": 0.705649, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11.031490087509155, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MassiveScenarioClassification.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MassiveScenarioClassification.json new file mode 100644 index 0000000000..128d9641a0 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MassiveScenarioClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "task_name": "MassiveScenarioClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.752421, + "f1": 0.750977, + "f1_weighted": 0.751916, + "scores_per_experiment": [ + { + "accuracy": 0.776732, + "f1": 0.776361, + "f1_weighted": 0.773859 + }, + { + "accuracy": 0.761937, + "f1": 0.75831, + "f1_weighted": 0.761581 + }, + { + "accuracy": 0.742098, + "f1": 0.737792, + "f1_weighted": 0.738895 + }, + { + "accuracy": 0.754876, + "f1": 0.752713, + "f1_weighted": 0.757934 + }, + { + "accuracy": 0.777068, + "f1": 0.772874, + "f1_weighted": 0.773983 + }, + { + "accuracy": 0.746133, + "f1": 0.744752, + "f1_weighted": 0.747069 + }, + { + "accuracy": 0.736046, + "f1": 0.736566, + "f1_weighted": 0.738807 + }, + { + "accuracy": 0.738063, + "f1": 0.735549, + "f1_weighted": 0.738781 + }, + { + "accuracy": 0.745797, + "f1": 0.748593, + "f1_weighted": 0.743422 + }, + { + "accuracy": 0.745461, + "f1": 0.746263, + "f1_weighted": 0.744824 + } + ], + "main_score": 0.752421, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 8.333421230316162, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MedrxivClusteringP2P.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MedrxivClusteringP2P.json new file mode 100644 index 0000000000..ca1cdc0873 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MedrxivClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "task_name": "MedrxivClusteringP2P", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.330321, + "v_measure_std": 0.015737, + "v_measures": [ + 0.322126, + 0.313074, + 0.307733, + 0.321082, + 0.315534, + 0.351253, + 0.340156, + 0.34117, + 0.355076, + 0.33601 + ], + "main_score": 0.330321, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 110.42463564872742, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MedrxivClusteringP2P.v2.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MedrxivClusteringP2P.v2.json new file mode 100644 index 0000000000..ebdc058ef6 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MedrxivClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "task_name": "MedrxivClusteringP2P.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.358515, + 0.360143, + 0.356019, + 0.354544, + 0.357126, + 0.366134, + 0.369494, + 0.372907, + 0.359399, + 0.360493 + ] + }, + "v_measure": 0.361477, + "v_measure_std": 0.005738, + "main_score": 0.361477, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 44.493255376815796, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MedrxivClusteringS2S.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MedrxivClusteringS2S.json new file mode 100644 index 0000000000..098b0c9d3f --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MedrxivClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "task_name": "MedrxivClusteringS2S", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.322414, + "v_measure_std": 0.017352, + "v_measures": [ + 0.308241, + 0.308172, + 0.301019, + 0.307454, + 0.307194, + 0.350259, + 0.327298, + 0.338018, + 0.346914, + 0.329568 + ], + "main_score": 0.322414, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 40.25102972984314, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MedrxivClusteringS2S.v2.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MedrxivClusteringS2S.v2.json new file mode 100644 index 0000000000..3f7835cf23 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MedrxivClusteringS2S.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "task_name": "MedrxivClusteringS2S.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.357492, + 0.3628, + 0.360006, + 0.35337, + 0.350596, + 0.353843, + 0.359466, + 0.34936, + 0.367261, + 0.359859 + ] + }, + "v_measure": 0.357405, + "v_measure_std": 0.005323, + "main_score": 0.357405, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 36.269169330596924, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MindSmallReranking.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MindSmallReranking.json new file mode 100644 index 0000000000..836b13db3b --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MindSmallReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "59042f120c80e8afa9cdbb224f67076cec0fc9a7", + "task_name": "MindSmallReranking", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "map": 0.317194, + "mrr": 0.328632, + "nAUC_map_max": -0.208997, + "nAUC_map_std": -0.043772, + "nAUC_map_diff1": 0.118697, + "nAUC_mrr_max": -0.156486, + "nAUC_mrr_std": -0.022578, + "nAUC_mrr_diff1": 0.114318, + "main_score": 0.317194, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1420.4328217506409, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MultiLongDocRetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MultiLongDocRetrieval.json new file mode 100644 index 0000000000..6508422fbb --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/MultiLongDocRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", + "task_name": "MultiLongDocRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.315, + "ndcg_at_3": 0.38199, + "ndcg_at_5": 0.40072, + "ndcg_at_10": 0.41556, + "ndcg_at_20": 0.42712, + "ndcg_at_100": 0.44692, + "ndcg_at_1000": 0.46553, + "map_at_1": 0.315, + "map_at_3": 0.36542, + "map_at_5": 0.37592, + "map_at_10": 0.38198, + "map_at_20": 0.38525, + "map_at_100": 0.38795, + "map_at_1000": 0.38853, + "recall_at_1": 0.315, + "recall_at_3": 0.43, + "recall_at_5": 0.475, + "recall_at_10": 0.52125, + "recall_at_20": 0.56625, + "recall_at_100": 0.67375, + "recall_at_1000": 0.82625, + "precision_at_1": 0.315, + "precision_at_3": 0.14333, + "precision_at_5": 0.095, + "precision_at_10": 0.05213, + "precision_at_20": 0.02831, + "precision_at_100": 0.00674, + "precision_at_1000": 0.00083, + "mrr_at_1": 0.315, + "mrr_at_3": 0.365417, + "mrr_at_5": 0.375917, + "mrr_at_10": 0.381979, + "mrr_at_20": 0.385253, + "mrr_at_100": 0.387945, + "mrr_at_1000": 0.388526, + "nauc_ndcg_at_1_max": 0.459504, + "nauc_ndcg_at_1_std": 0.008352, + "nauc_ndcg_at_1_diff1": 0.537444, + "nauc_ndcg_at_3_max": 0.483943, + "nauc_ndcg_at_3_std": 0.042827, + "nauc_ndcg_at_3_diff1": 0.490859, + "nauc_ndcg_at_5_max": 0.470679, + "nauc_ndcg_at_5_std": 0.044199, + "nauc_ndcg_at_5_diff1": 0.477899, + "nauc_ndcg_at_10_max": 0.474062, + "nauc_ndcg_at_10_std": 0.056459, + "nauc_ndcg_at_10_diff1": 0.475334, + "nauc_ndcg_at_20_max": 0.474394, + "nauc_ndcg_at_20_std": 0.060936, + "nauc_ndcg_at_20_diff1": 0.476709, + "nauc_ndcg_at_100_max": 0.478445, + "nauc_ndcg_at_100_std": 0.079765, + "nauc_ndcg_at_100_diff1": 0.477854, + "nauc_ndcg_at_1000_max": 0.478351, + "nauc_ndcg_at_1000_std": 0.077645, + "nauc_ndcg_at_1000_diff1": 0.477574, + "nauc_map_at_1_max": 0.459504, + "nauc_map_at_1_std": 0.008352, + "nauc_map_at_1_diff1": 0.537444, + "nauc_map_at_3_max": 0.478435, + "nauc_map_at_3_std": 0.033171, + "nauc_map_at_3_diff1": 0.50141, + "nauc_map_at_5_max": 0.471, + "nauc_map_at_5_std": 0.033772, + "nauc_map_at_5_diff1": 0.494337, + "nauc_map_at_10_max": 0.47222, + "nauc_map_at_10_std": 0.038485, + "nauc_map_at_10_diff1": 0.493379, + "nauc_map_at_20_max": 0.472231, + "nauc_map_at_20_std": 0.039546, + "nauc_map_at_20_diff1": 0.493672, + "nauc_map_at_100_max": 0.472898, + "nauc_map_at_100_std": 0.042318, + "nauc_map_at_100_diff1": 0.493725, + "nauc_map_at_1000_max": 0.472893, + "nauc_map_at_1000_std": 0.042241, + "nauc_map_at_1000_diff1": 0.493677, + "nauc_recall_at_1_max": 0.459504, + "nauc_recall_at_1_std": 0.008352, + "nauc_recall_at_1_diff1": 0.537444, + "nauc_recall_at_3_max": 0.499491, + "nauc_recall_at_3_std": 0.070889, + "nauc_recall_at_3_diff1": 0.461055, + "nauc_recall_at_5_max": 0.467777, + "nauc_recall_at_5_std": 0.075176, + "nauc_recall_at_5_diff1": 0.428815, + "nauc_recall_at_10_max": 0.479366, + "nauc_recall_at_10_std": 0.116366, + "nauc_recall_at_10_diff1": 0.41908, + "nauc_recall_at_20_max": 0.481609, + "nauc_recall_at_20_std": 0.138696, + "nauc_recall_at_20_diff1": 0.423256, + "nauc_recall_at_100_max": 0.507715, + "nauc_recall_at_100_std": 0.272081, + "nauc_recall_at_100_diff1": 0.423638, + "nauc_recall_at_1000_max": 0.525934, + "nauc_recall_at_1000_std": 0.36375, + "nauc_recall_at_1000_diff1": 0.391166, + "nauc_precision_at_1_max": 0.459504, + "nauc_precision_at_1_std": 0.008352, + "nauc_precision_at_1_diff1": 0.537444, + "nauc_precision_at_3_max": 0.499491, + "nauc_precision_at_3_std": 0.070889, + "nauc_precision_at_3_diff1": 0.461055, + "nauc_precision_at_5_max": 0.467777, + "nauc_precision_at_5_std": 0.075176, + "nauc_precision_at_5_diff1": 0.428815, + "nauc_precision_at_10_max": 0.479366, + "nauc_precision_at_10_std": 0.116366, + "nauc_precision_at_10_diff1": 0.41908, + "nauc_precision_at_20_max": 0.481609, + "nauc_precision_at_20_std": 0.138696, + "nauc_precision_at_20_diff1": 0.423256, + "nauc_precision_at_100_max": 0.507715, + "nauc_precision_at_100_std": 0.272081, + "nauc_precision_at_100_diff1": 0.423638, + "nauc_precision_at_1000_max": 0.525934, + "nauc_precision_at_1000_std": 0.36375, + "nauc_precision_at_1000_diff1": 0.391166, + "nauc_mrr_at_1_max": 0.459504, + "nauc_mrr_at_1_std": 0.008352, + "nauc_mrr_at_1_diff1": 0.537444, + "nauc_mrr_at_3_max": 0.478435, + "nauc_mrr_at_3_std": 0.033171, + "nauc_mrr_at_3_diff1": 0.50141, + "nauc_mrr_at_5_max": 0.471, + "nauc_mrr_at_5_std": 0.033772, + "nauc_mrr_at_5_diff1": 0.494337, + "nauc_mrr_at_10_max": 0.47222, + "nauc_mrr_at_10_std": 0.038485, + "nauc_mrr_at_10_diff1": 0.493379, + "nauc_mrr_at_20_max": 0.472231, + "nauc_mrr_at_20_std": 0.039546, + "nauc_mrr_at_20_diff1": 0.493672, + "nauc_mrr_at_100_max": 0.472898, + "nauc_mrr_at_100_std": 0.042318, + "nauc_mrr_at_100_diff1": 0.493725, + "nauc_mrr_at_1000_max": 0.472893, + "nauc_mrr_at_1000_std": 0.042241, + "nauc_mrr_at_1000_diff1": 0.493677, + "main_score": 0.41556, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3224.701375722885, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/NFCorpus.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/NFCorpus.json new file mode 100644 index 0000000000..0651805b91 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/NFCorpus.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814", + "task_name": "NFCorpus", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.44427, + "ndcg_at_3": 0.42593, + "ndcg_at_5": 0.40804, + "ndcg_at_10": 0.37492, + "ndcg_at_20": 0.35135, + "ndcg_at_100": 0.34959, + "ndcg_at_1000": 0.4383, + "map_at_1": 0.05827, + "map_at_3": 0.09922, + "map_at_5": 0.1184, + "map_at_10": 0.14033, + "map_at_20": 0.15852, + "map_at_100": 0.18526, + "map_at_1000": 0.20199, + "recall_at_1": 0.05827, + "recall_at_3": 0.11108, + "recall_at_5": 0.14145, + "recall_at_10": 0.18154, + "recall_at_20": 0.22449, + "recall_at_100": 0.35774, + "recall_at_1000": 0.68543, + "precision_at_1": 0.4613, + "precision_at_3": 0.40867, + "precision_at_5": 0.36409, + "precision_at_10": 0.28545, + "precision_at_20": 0.21455, + "precision_at_100": 0.09331, + "precision_at_1000": 0.02252, + "mrr_at_1": 0.464396, + "mrr_at_3": 0.547472, + "mrr_at_5": 0.55645, + "mrr_at_10": 0.561913, + "mrr_at_20": 0.565715, + "mrr_at_100": 0.568514, + "mrr_at_1000": 0.568855, + "nauc_ndcg_at_1_max": 0.399217, + "nauc_ndcg_at_1_std": 0.247822, + "nauc_ndcg_at_1_diff1": 0.302679, + "nauc_ndcg_at_3_max": 0.410624, + "nauc_ndcg_at_3_std": 0.276392, + "nauc_ndcg_at_3_diff1": 0.220636, + "nauc_ndcg_at_5_max": 0.427934, + "nauc_ndcg_at_5_std": 0.298907, + "nauc_ndcg_at_5_diff1": 0.203877, + "nauc_ndcg_at_10_max": 0.418955, + "nauc_ndcg_at_10_std": 0.309136, + "nauc_ndcg_at_10_diff1": 0.164516, + "nauc_ndcg_at_20_max": 0.40312, + "nauc_ndcg_at_20_std": 0.306113, + "nauc_ndcg_at_20_diff1": 0.141919, + "nauc_ndcg_at_100_max": 0.423664, + "nauc_ndcg_at_100_std": 0.322324, + "nauc_ndcg_at_100_diff1": 0.161205, + "nauc_ndcg_at_1000_max": 0.475444, + "nauc_ndcg_at_1000_std": 0.355689, + "nauc_ndcg_at_1000_diff1": 0.169329, + "nauc_map_at_1_max": 0.073412, + "nauc_map_at_1_std": -0.190612, + "nauc_map_at_1_diff1": 0.424569, + "nauc_map_at_3_max": 0.138126, + "nauc_map_at_3_std": -0.132413, + "nauc_map_at_3_diff1": 0.306974, + "nauc_map_at_5_max": 0.192656, + "nauc_map_at_5_std": -0.08447, + "nauc_map_at_5_diff1": 0.27014, + "nauc_map_at_10_max": 0.24316, + "nauc_map_at_10_std": -0.012057, + "nauc_map_at_10_diff1": 0.229104, + "nauc_map_at_20_max": 0.271023, + "nauc_map_at_20_std": 0.039399, + "nauc_map_at_20_diff1": 0.202602, + "nauc_map_at_100_max": 0.310624, + "nauc_map_at_100_std": 0.124628, + "nauc_map_at_100_diff1": 0.187939, + "nauc_map_at_1000_max": 0.317333, + "nauc_map_at_1000_std": 0.160932, + "nauc_map_at_1000_diff1": 0.177887, + "nauc_recall_at_1_max": 0.073412, + "nauc_recall_at_1_std": -0.190612, + "nauc_recall_at_1_diff1": 0.424569, + "nauc_recall_at_3_max": 0.126575, + "nauc_recall_at_3_std": -0.134701, + "nauc_recall_at_3_diff1": 0.268091, + "nauc_recall_at_5_max": 0.195539, + "nauc_recall_at_5_std": -0.066974, + "nauc_recall_at_5_diff1": 0.20145, + "nauc_recall_at_10_max": 0.236871, + "nauc_recall_at_10_std": 0.017612, + "nauc_recall_at_10_diff1": 0.143317, + "nauc_recall_at_20_max": 0.247154, + "nauc_recall_at_20_std": 0.064504, + "nauc_recall_at_20_diff1": 0.113423, + "nauc_recall_at_100_max": 0.300916, + "nauc_recall_at_100_std": 0.235463, + "nauc_recall_at_100_diff1": 0.09438, + "nauc_recall_at_1000_max": 0.237703, + "nauc_recall_at_1000_std": 0.217762, + "nauc_recall_at_1000_diff1": -0.003052, + "nauc_precision_at_1_max": 0.420227, + "nauc_precision_at_1_std": 0.26231, + "nauc_precision_at_1_diff1": 0.295668, + "nauc_precision_at_3_max": 0.434452, + "nauc_precision_at_3_std": 0.341823, + "nauc_precision_at_3_diff1": 0.117007, + "nauc_precision_at_5_max": 0.452393, + "nauc_precision_at_5_std": 0.393613, + "nauc_precision_at_5_diff1": 0.072001, + "nauc_precision_at_10_max": 0.416245, + "nauc_precision_at_10_std": 0.438693, + "nauc_precision_at_10_diff1": 0.002918, + "nauc_precision_at_20_max": 0.346158, + "nauc_precision_at_20_std": 0.457348, + "nauc_precision_at_20_diff1": -0.040875, + "nauc_precision_at_100_max": 0.218707, + "nauc_precision_at_100_std": 0.482504, + "nauc_precision_at_100_diff1": -0.074214, + "nauc_precision_at_1000_max": 0.070074, + "nauc_precision_at_1000_std": 0.365029, + "nauc_precision_at_1000_diff1": -0.081932, + "nauc_mrr_at_1_max": 0.415844, + "nauc_mrr_at_1_std": 0.257759, + "nauc_mrr_at_1_diff1": 0.287264, + "nauc_mrr_at_3_max": 0.464637, + "nauc_mrr_at_3_std": 0.311985, + "nauc_mrr_at_3_diff1": 0.2722, + "nauc_mrr_at_5_max": 0.478766, + "nauc_mrr_at_5_std": 0.319555, + "nauc_mrr_at_5_diff1": 0.267158, + "nauc_mrr_at_10_max": 0.476372, + "nauc_mrr_at_10_std": 0.320721, + "nauc_mrr_at_10_diff1": 0.263116, + "nauc_mrr_at_20_max": 0.478461, + "nauc_mrr_at_20_std": 0.323044, + "nauc_mrr_at_20_diff1": 0.265754, + "nauc_mrr_at_100_max": 0.477813, + "nauc_mrr_at_100_std": 0.323208, + "nauc_mrr_at_100_diff1": 0.265645, + "nauc_mrr_at_1000_max": 0.477557, + "nauc_mrr_at_1000_std": 0.322874, + "nauc_mrr_at_1000_diff1": 0.265879, + "main_score": 0.37492, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 8.255230188369751, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/NQ.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/NQ.json new file mode 100644 index 0000000000..1ce2b9c5b2 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/NQ.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31", + "task_name": "NQ", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.37601, + "ndcg_at_3": 0.4945, + "ndcg_at_5": 0.54593, + "ndcg_at_10": 0.58216, + "ndcg_at_20": 0.60105, + "ndcg_at_100": 0.61783, + "ndcg_at_1000": 0.6223, + "map_at_1": 0.33447, + "map_at_3": 0.45188, + "map_at_5": 0.48333, + "map_at_10": 0.50074, + "map_at_20": 0.50678, + "map_at_100": 0.50967, + "map_at_1000": 0.50986, + "recall_at_1": 0.33447, + "recall_at_3": 0.58242, + "recall_at_5": 0.70017, + "recall_at_10": 0.8034, + "recall_at_20": 0.87285, + "recall_at_100": 0.95524, + "recall_at_1000": 0.98851, + "precision_at_1": 0.37601, + "precision_at_3": 0.22557, + "precision_at_5": 0.16547, + "precision_at_10": 0.09629, + "precision_at_20": 0.05277, + "precision_at_100": 0.01163, + "precision_at_1000": 0.0012, + "mrr_at_1": 0.376014, + "mrr_at_3": 0.487543, + "mrr_at_5": 0.512558, + "mrr_at_10": 0.525075, + "mrr_at_20": 0.529252, + "mrr_at_100": 0.531244, + "mrr_at_1000": 0.531389, + "nauc_ndcg_at_1_max": 0.2499, + "nauc_ndcg_at_1_std": 0.01131, + "nauc_ndcg_at_1_diff1": 0.345154, + "nauc_ndcg_at_3_max": 0.296004, + "nauc_ndcg_at_3_std": 0.013553, + "nauc_ndcg_at_3_diff1": 0.289159, + "nauc_ndcg_at_5_max": 0.322312, + "nauc_ndcg_at_5_std": 0.028967, + "nauc_ndcg_at_5_diff1": 0.281634, + "nauc_ndcg_at_10_max": 0.329926, + "nauc_ndcg_at_10_std": 0.043764, + "nauc_ndcg_at_10_diff1": 0.287479, + "nauc_ndcg_at_20_max": 0.332381, + "nauc_ndcg_at_20_std": 0.051296, + "nauc_ndcg_at_20_diff1": 0.293278, + "nauc_ndcg_at_100_max": 0.323432, + "nauc_ndcg_at_100_std": 0.05239, + "nauc_ndcg_at_100_diff1": 0.295435, + "nauc_ndcg_at_1000_max": 0.316989, + "nauc_ndcg_at_1000_std": 0.044689, + "nauc_ndcg_at_1000_diff1": 0.295884, + "nauc_map_at_1_max": 0.233127, + "nauc_map_at_1_std": -0.011976, + "nauc_map_at_1_diff1": 0.34388, + "nauc_map_at_3_max": 0.282491, + "nauc_map_at_3_std": 0.002741, + "nauc_map_at_3_diff1": 0.301026, + "nauc_map_at_5_max": 0.29796, + "nauc_map_at_5_std": 0.012367, + "nauc_map_at_5_diff1": 0.297607, + "nauc_map_at_10_max": 0.300943, + "nauc_map_at_10_std": 0.018693, + "nauc_map_at_10_diff1": 0.300111, + "nauc_map_at_20_max": 0.301707, + "nauc_map_at_20_std": 0.021256, + "nauc_map_at_20_diff1": 0.30136, + "nauc_map_at_100_max": 0.300308, + "nauc_map_at_100_std": 0.021569, + "nauc_map_at_100_diff1": 0.301475, + "nauc_map_at_1000_max": 0.300081, + "nauc_map_at_1000_std": 0.021307, + "nauc_map_at_1000_diff1": 0.301542, + "nauc_recall_at_1_max": 0.233127, + "nauc_recall_at_1_std": -0.011976, + "nauc_recall_at_1_diff1": 0.34388, + "nauc_recall_at_3_max": 0.320008, + "nauc_recall_at_3_std": 0.019494, + "nauc_recall_at_3_diff1": 0.242728, + "nauc_recall_at_5_max": 0.39394, + "nauc_recall_at_5_std": 0.057617, + "nauc_recall_at_5_diff1": 0.208797, + "nauc_recall_at_10_max": 0.46205, + "nauc_recall_at_10_std": 0.133054, + "nauc_recall_at_10_diff1": 0.216125, + "nauc_recall_at_20_max": 0.547431, + "nauc_recall_at_20_std": 0.224621, + "nauc_recall_at_20_diff1": 0.241312, + "nauc_recall_at_100_max": 0.698178, + "nauc_recall_at_100_std": 0.532325, + "nauc_recall_at_100_diff1": 0.246482, + "nauc_recall_at_1000_max": 0.767492, + "nauc_recall_at_1000_std": 0.659522, + "nauc_recall_at_1000_diff1": 0.168545, + "nauc_precision_at_1_max": 0.2499, + "nauc_precision_at_1_std": 0.01131, + "nauc_precision_at_1_diff1": 0.345154, + "nauc_precision_at_3_max": 0.315689, + "nauc_precision_at_3_std": 0.061896, + "nauc_precision_at_3_diff1": 0.188282, + "nauc_precision_at_5_max": 0.31694, + "nauc_precision_at_5_std": 0.109209, + "nauc_precision_at_5_diff1": 0.112079, + "nauc_precision_at_10_max": 0.273877, + "nauc_precision_at_10_std": 0.154243, + "nauc_precision_at_10_diff1": 0.060301, + "nauc_precision_at_20_max": 0.233688, + "nauc_precision_at_20_std": 0.189781, + "nauc_precision_at_20_diff1": 0.022338, + "nauc_precision_at_100_max": 0.117828, + "nauc_precision_at_100_std": 0.198977, + "nauc_precision_at_100_diff1": -0.048729, + "nauc_precision_at_1000_max": 0.043448, + "nauc_precision_at_1000_std": 0.150832, + "nauc_precision_at_1000_diff1": -0.081867, + "nauc_mrr_at_1_max": 0.2499, + "nauc_mrr_at_1_std": 0.01131, + "nauc_mrr_at_1_diff1": 0.345154, + "nauc_mrr_at_3_max": 0.287975, + "nauc_mrr_at_3_std": 0.029507, + "nauc_mrr_at_3_diff1": 0.304427, + "nauc_mrr_at_5_max": 0.298263, + "nauc_mrr_at_5_std": 0.035241, + "nauc_mrr_at_5_diff1": 0.301038, + "nauc_mrr_at_10_max": 0.299103, + "nauc_mrr_at_10_std": 0.038328, + "nauc_mrr_at_10_diff1": 0.304039, + "nauc_mrr_at_20_max": 0.298734, + "nauc_mrr_at_20_std": 0.038623, + "nauc_mrr_at_20_diff1": 0.306043, + "nauc_mrr_at_100_max": 0.297511, + "nauc_mrr_at_100_std": 0.038386, + "nauc_mrr_at_100_diff1": 0.306347, + "nauc_mrr_at_1000_max": 0.297315, + "nauc_mrr_at_1000_std": 0.038185, + "nauc_mrr_at_1000_diff1": 0.30633, + "main_score": 0.58216, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2270.5925137996674, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/QuoraRetrieval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/QuoraRetrieval.json new file mode 100644 index 0000000000..e7d8097f51 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/QuoraRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e4e08e0b7dbe3c8700f0daef558ff32256715259", + "task_name": "QuoraRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.8068, + "ndcg_at_3": 0.84895, + "ndcg_at_5": 0.86586, + "ndcg_at_10": 0.87839, + "ndcg_at_20": 0.88506, + "ndcg_at_100": 0.89081, + "ndcg_at_1000": 0.89206, + "map_at_1": 0.7017, + "map_at_3": 0.81018, + "map_at_5": 0.82945, + "map_at_10": 0.84014, + "map_at_20": 0.84429, + "map_at_100": 0.8465, + "map_at_1000": 0.84668, + "recall_at_1": 0.7017, + "recall_at_3": 0.86692, + "recall_at_5": 0.91442, + "recall_at_10": 0.95118, + "recall_at_20": 0.97258, + "recall_at_100": 0.99373, + "recall_at_1000": 0.99958, + "precision_at_1": 0.8068, + "precision_at_3": 0.37097, + "precision_at_5": 0.2447, + "precision_at_10": 0.1333, + "precision_at_20": 0.07075, + "precision_at_100": 0.01524, + "precision_at_1000": 0.00157, + "mrr_at_1": 0.8064, + "mrr_at_3": 0.859883, + "mrr_at_5": 0.867108, + "mrr_at_10": 0.870199, + "mrr_at_20": 0.870956, + "mrr_at_100": 0.871217, + "mrr_at_1000": 0.871229, + "nauc_ndcg_at_1_max": 0.380824, + "nauc_ndcg_at_1_std": -0.346737, + "nauc_ndcg_at_1_diff1": 0.773285, + "nauc_ndcg_at_3_max": 0.363576, + "nauc_ndcg_at_3_std": -0.392248, + "nauc_ndcg_at_3_diff1": 0.755609, + "nauc_ndcg_at_5_max": 0.364513, + "nauc_ndcg_at_5_std": -0.408992, + "nauc_ndcg_at_5_diff1": 0.760823, + "nauc_ndcg_at_10_max": 0.370009, + "nauc_ndcg_at_10_std": -0.399446, + "nauc_ndcg_at_10_diff1": 0.763502, + "nauc_ndcg_at_20_max": 0.372721, + "nauc_ndcg_at_20_std": -0.386662, + "nauc_ndcg_at_20_diff1": 0.762316, + "nauc_ndcg_at_100_max": 0.375741, + "nauc_ndcg_at_100_std": -0.368968, + "nauc_ndcg_at_100_diff1": 0.761114, + "nauc_ndcg_at_1000_max": 0.3762, + "nauc_ndcg_at_1000_std": -0.368262, + "nauc_ndcg_at_1000_diff1": 0.760624, + "nauc_map_at_1_max": 0.281469, + "nauc_map_at_1_std": -0.372441, + "nauc_map_at_1_diff1": 0.796762, + "nauc_map_at_3_max": 0.339024, + "nauc_map_at_3_std": -0.41969, + "nauc_map_at_3_diff1": 0.770748, + "nauc_map_at_5_max": 0.350591, + "nauc_map_at_5_std": -0.422768, + "nauc_map_at_5_diff1": 0.768903, + "nauc_map_at_10_max": 0.358694, + "nauc_map_at_10_std": -0.409432, + "nauc_map_at_10_diff1": 0.766995, + "nauc_map_at_20_max": 0.361207, + "nauc_map_at_20_std": -0.399375, + "nauc_map_at_20_diff1": 0.765085, + "nauc_map_at_100_max": 0.362429, + "nauc_map_at_100_std": -0.391798, + "nauc_map_at_100_diff1": 0.764236, + "nauc_map_at_1000_max": 0.362571, + "nauc_map_at_1000_std": -0.391302, + "nauc_map_at_1000_diff1": 0.764163, + "nauc_recall_at_1_max": 0.281469, + "nauc_recall_at_1_std": -0.372441, + "nauc_recall_at_1_diff1": 0.796762, + "nauc_recall_at_3_max": 0.311829, + "nauc_recall_at_3_std": -0.463865, + "nauc_recall_at_3_diff1": 0.731682, + "nauc_recall_at_5_max": 0.314533, + "nauc_recall_at_5_std": -0.526949, + "nauc_recall_at_5_diff1": 0.72399, + "nauc_recall_at_10_max": 0.328967, + "nauc_recall_at_10_std": -0.536849, + "nauc_recall_at_10_diff1": 0.733397, + "nauc_recall_at_20_max": 0.338942, + "nauc_recall_at_20_std": -0.490958, + "nauc_recall_at_20_diff1": 0.732807, + "nauc_recall_at_100_max": 0.373728, + "nauc_recall_at_100_std": -0.150208, + "nauc_recall_at_100_diff1": 0.740886, + "nauc_recall_at_1000_max": 0.614527, + "nauc_recall_at_1000_std": 0.57631, + "nauc_recall_at_1000_diff1": 0.701046, + "nauc_precision_at_1_max": 0.380824, + "nauc_precision_at_1_std": -0.346737, + "nauc_precision_at_1_diff1": 0.773285, + "nauc_precision_at_3_max": 0.089562, + "nauc_precision_at_3_std": 0.054197, + "nauc_precision_at_3_diff1": -0.156136, + "nauc_precision_at_5_max": 0.025786, + "nauc_precision_at_5_std": 0.146404, + "nauc_precision_at_5_diff1": -0.293457, + "nauc_precision_at_10_max": -0.016189, + "nauc_precision_at_10_std": 0.24081, + "nauc_precision_at_10_diff1": -0.37218, + "nauc_precision_at_20_max": -0.041225, + "nauc_precision_at_20_std": 0.300649, + "nauc_precision_at_20_diff1": -0.40868, + "nauc_precision_at_100_max": -0.065873, + "nauc_precision_at_100_std": 0.359159, + "nauc_precision_at_100_diff1": -0.433074, + "nauc_precision_at_1000_max": -0.072103, + "nauc_precision_at_1000_std": 0.369788, + "nauc_precision_at_1000_diff1": -0.438284, + "nauc_mrr_at_1_max": 0.380404, + "nauc_mrr_at_1_std": -0.347125, + "nauc_mrr_at_1_diff1": 0.774008, + "nauc_mrr_at_3_max": 0.389681, + "nauc_mrr_at_3_std": -0.353055, + "nauc_mrr_at_3_diff1": 0.760246, + "nauc_mrr_at_5_max": 0.388675, + "nauc_mrr_at_5_std": -0.355436, + "nauc_mrr_at_5_diff1": 0.762395, + "nauc_mrr_at_10_max": 0.388475, + "nauc_mrr_at_10_std": -0.353121, + "nauc_mrr_at_10_diff1": 0.763696, + "nauc_mrr_at_20_max": 0.388404, + "nauc_mrr_at_20_std": -0.351857, + "nauc_mrr_at_20_diff1": 0.763661, + "nauc_mrr_at_100_max": 0.388275, + "nauc_mrr_at_100_std": -0.351756, + "nauc_mrr_at_100_diff1": 0.763617, + "nauc_mrr_at_1000_max": 0.388287, + "nauc_mrr_at_1000_std": -0.351802, + "nauc_mrr_at_1000_diff1": 0.763625, + "main_score": 0.87839, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 294.6119215488434, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/RedditClustering.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/RedditClustering.json new file mode 100644 index 0000000000..f21d4fe98d --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/RedditClustering.json @@ -0,0 +1,47 @@ +{ + "dataset_revision": "24640382cdbf8abc73003fb0fa6d111a705499eb", + "task_name": "RedditClustering", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.532248, + "v_measure_std": 0.04809, + "v_measures": [ + 0.574456, + 0.626452, + 0.485746, + 0.515459, + 0.52651, + 0.488402, + 0.552689, + 0.488453, + 0.504659, + 0.511683, + 0.468962, + 0.548897, + 0.531215, + 0.540767, + 0.626728, + 0.465505, + 0.540594, + 0.579431, + 0.519008, + 0.509214, + 0.498281, + 0.526491, + 0.64744, + 0.555942, + 0.473227 + ], + "main_score": 0.532248, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 323.9741518497467, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/RedditClusteringP2P.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/RedditClusteringP2P.json new file mode 100644 index 0000000000..41507b62f1 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/RedditClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "385e3cb46b4cfa89021f56c4380204149d0efe33", + "task_name": "RedditClusteringP2P", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.564151, + "v_measure_std": 0.119522, + "v_measures": [ + 0.623999, + 0.601884, + 0.606223, + 0.338997, + 0.615474, + 0.541327, + 0.333175, + 0.669447, + 0.631958, + 0.679025 + ], + "main_score": 0.564151, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 643.6604087352753, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SCIDOCS.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SCIDOCS.json new file mode 100644 index 0000000000..aab1ba7f19 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SCIDOCS.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88", + "task_name": "SCIDOCS", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.279, + "ndcg_at_3": 0.23708, + "ndcg_at_5": 0.20643, + "ndcg_at_10": 0.2495, + "ndcg_at_20": 0.28484, + "ndcg_at_100": 0.35015, + "ndcg_at_1000": 0.40686, + "map_at_1": 0.05658, + "map_at_3": 0.1089, + "map_at_5": 0.13117, + "map_at_10": 0.15455, + "map_at_20": 0.16858, + "map_at_100": 0.18278, + "map_at_1000": 0.18663, + "recall_at_1": 0.05658, + "recall_at_3": 0.13663, + "recall_at_5": 0.18517, + "recall_at_10": 0.2641, + "recall_at_20": 0.34718, + "recall_at_100": 0.55913, + "recall_at_1000": 0.83237, + "precision_at_1": 0.279, + "precision_at_3": 0.22433, + "precision_at_5": 0.1824, + "precision_at_10": 0.1303, + "precision_at_20": 0.0856, + "precision_at_100": 0.02754, + "precision_at_1000": 0.0041, + "mrr_at_1": 0.279, + "mrr_at_3": 0.369, + "mrr_at_5": 0.3886, + "mrr_at_10": 0.404029, + "mrr_at_20": 0.410485, + "mrr_at_100": 0.414657, + "mrr_at_1000": 0.41497, + "nauc_ndcg_at_1_max": 0.215046, + "nauc_ndcg_at_1_std": 0.058185, + "nauc_ndcg_at_1_diff1": 0.199122, + "nauc_ndcg_at_3_max": 0.248896, + "nauc_ndcg_at_3_std": 0.123876, + "nauc_ndcg_at_3_diff1": 0.148048, + "nauc_ndcg_at_5_max": 0.262262, + "nauc_ndcg_at_5_std": 0.160217, + "nauc_ndcg_at_5_diff1": 0.145314, + "nauc_ndcg_at_10_max": 0.286715, + "nauc_ndcg_at_10_std": 0.217675, + "nauc_ndcg_at_10_diff1": 0.113962, + "nauc_ndcg_at_20_max": 0.300561, + "nauc_ndcg_at_20_std": 0.241141, + "nauc_ndcg_at_20_diff1": 0.114347, + "nauc_ndcg_at_100_max": 0.312962, + "nauc_ndcg_at_100_std": 0.290551, + "nauc_ndcg_at_100_diff1": 0.125487, + "nauc_ndcg_at_1000_max": 0.306728, + "nauc_ndcg_at_1000_std": 0.277248, + "nauc_ndcg_at_1000_diff1": 0.121425, + "nauc_map_at_1_max": 0.213355, + "nauc_map_at_1_std": 0.055657, + "nauc_map_at_1_diff1": 0.197983, + "nauc_map_at_3_max": 0.240955, + "nauc_map_at_3_std": 0.11001, + "nauc_map_at_3_diff1": 0.143695, + "nauc_map_at_5_max": 0.25073, + "nauc_map_at_5_std": 0.144941, + "nauc_map_at_5_diff1": 0.138713, + "nauc_map_at_10_max": 0.273587, + "nauc_map_at_10_std": 0.196368, + "nauc_map_at_10_diff1": 0.115564, + "nauc_map_at_20_max": 0.281346, + "nauc_map_at_20_std": 0.214234, + "nauc_map_at_20_diff1": 0.110551, + "nauc_map_at_100_max": 0.288759, + "nauc_map_at_100_std": 0.231706, + "nauc_map_at_100_diff1": 0.116265, + "nauc_map_at_1000_max": 0.288284, + "nauc_map_at_1000_std": 0.232158, + "nauc_map_at_1000_diff1": 0.115887, + "nauc_recall_at_1_max": 0.213355, + "nauc_recall_at_1_std": 0.055657, + "nauc_recall_at_1_diff1": 0.197983, + "nauc_recall_at_3_max": 0.253065, + "nauc_recall_at_3_std": 0.143975, + "nauc_recall_at_3_diff1": 0.121649, + "nauc_recall_at_5_max": 0.268346, + "nauc_recall_at_5_std": 0.200178, + "nauc_recall_at_5_diff1": 0.118885, + "nauc_recall_at_10_max": 0.293811, + "nauc_recall_at_10_std": 0.287497, + "nauc_recall_at_10_diff1": 0.055076, + "nauc_recall_at_20_max": 0.302809, + "nauc_recall_at_20_std": 0.316064, + "nauc_recall_at_20_diff1": 0.052598, + "nauc_recall_at_100_max": 0.296175, + "nauc_recall_at_100_std": 0.428048, + "nauc_recall_at_100_diff1": 0.064865, + "nauc_recall_at_1000_max": 0.251434, + "nauc_recall_at_1000_std": 0.42926, + "nauc_recall_at_1000_diff1": 0.000258, + "nauc_precision_at_1_max": 0.215046, + "nauc_precision_at_1_std": 0.058185, + "nauc_precision_at_1_diff1": 0.199122, + "nauc_precision_at_3_max": 0.25773, + "nauc_precision_at_3_std": 0.148618, + "nauc_precision_at_3_diff1": 0.124551, + "nauc_precision_at_5_max": 0.274247, + "nauc_precision_at_5_std": 0.204766, + "nauc_precision_at_5_diff1": 0.122956, + "nauc_precision_at_10_max": 0.298923, + "nauc_precision_at_10_std": 0.290761, + "nauc_precision_at_10_diff1": 0.058647, + "nauc_precision_at_20_max": 0.306394, + "nauc_precision_at_20_std": 0.317377, + "nauc_precision_at_20_diff1": 0.05625, + "nauc_precision_at_100_max": 0.303468, + "nauc_precision_at_100_std": 0.425013, + "nauc_precision_at_100_diff1": 0.071078, + "nauc_precision_at_1000_max": 0.261582, + "nauc_precision_at_1000_std": 0.404186, + "nauc_precision_at_1000_diff1": 0.011816, + "nauc_mrr_at_1_max": 0.215046, + "nauc_mrr_at_1_std": 0.058185, + "nauc_mrr_at_1_diff1": 0.199122, + "nauc_mrr_at_3_max": 0.245316, + "nauc_mrr_at_3_std": 0.106071, + "nauc_mrr_at_3_diff1": 0.176741, + "nauc_mrr_at_5_max": 0.252147, + "nauc_mrr_at_5_std": 0.117153, + "nauc_mrr_at_5_diff1": 0.181355, + "nauc_mrr_at_10_max": 0.253648, + "nauc_mrr_at_10_std": 0.122909, + "nauc_mrr_at_10_diff1": 0.172722, + "nauc_mrr_at_20_max": 0.254319, + "nauc_mrr_at_20_std": 0.121504, + "nauc_mrr_at_20_diff1": 0.177222, + "nauc_mrr_at_100_max": 0.252834, + "nauc_mrr_at_100_std": 0.12071, + "nauc_mrr_at_100_diff1": 0.176885, + "nauc_mrr_at_1000_max": 0.252628, + "nauc_mrr_at_1000_std": 0.120356, + "nauc_mrr_at_1000_diff1": 0.176884, + "main_score": 0.2495, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 34.90246033668518, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SICK-R.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SICK-R.json new file mode 100644 index 0000000000..d5035bafa6 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SICK-R.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "20a6d6f312dd54037fe07a32d58e5e168867909d", + "task_name": "SICK-R", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.793382, + "spearman": 0.713409, + "cosine_pearson": 0.793382, + "cosine_spearman": 0.713409, + "manhattan_pearson": 0.75573, + "manhattan_spearman": 0.713205, + "euclidean_pearson": 0.755814, + "euclidean_spearman": 0.71316, + "main_score": 0.713409, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.1221141815185547, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS12.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS12.json new file mode 100644 index 0000000000..31c5263822 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS12.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384", + "task_name": "STS12", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.747855, + "spearman": 0.670192, + "cosine_pearson": 0.747855, + "cosine_spearman": 0.670177, + "manhattan_pearson": 0.701336, + "manhattan_spearman": 0.666927, + "euclidean_pearson": 0.701839, + "euclidean_spearman": 0.667653, + "main_score": 0.670177, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.0979928970336914, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS13.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS13.json new file mode 100644 index 0000000000..c139063721 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS13.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca", + "task_name": "STS13", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.839751, + "spearman": 0.840915, + "cosine_pearson": 0.839751, + "cosine_spearman": 0.840914, + "manhattan_pearson": 0.829973, + "manhattan_spearman": 0.837752, + "euclidean_pearson": 0.830587, + "euclidean_spearman": 0.838695, + "main_score": 0.840914, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.5298104286193848, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS14.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS14.json new file mode 100644 index 0000000000..7b8b2224c4 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS14.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375", + "task_name": "STS14", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.803215, + "spearman": 0.747689, + "cosine_pearson": 0.803215, + "cosine_spearman": 0.747689, + "manhattan_pearson": 0.785906, + "manhattan_spearman": 0.746759, + "euclidean_pearson": 0.786412, + "euclidean_spearman": 0.747121, + "main_score": 0.747689, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.259183406829834, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS15.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS15.json new file mode 100644 index 0000000000..1c8c3a0495 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS15.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3", + "task_name": "STS15", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.843829, + "spearman": 0.853734, + "cosine_pearson": 0.843829, + "cosine_spearman": 0.853734, + "manhattan_pearson": 0.846819, + "manhattan_spearman": 0.853473, + "euclidean_pearson": 0.847335, + "euclidean_spearman": 0.854124, + "main_score": 0.853734, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.027928113937378, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS16.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS16.json new file mode 100644 index 0000000000..a3e8d09ddf --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS16.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "4d8694f8f0e0100860b497b999b3dbed754a0513", + "task_name": "STS16", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.780777, + "spearman": 0.788774, + "cosine_pearson": 0.780777, + "cosine_spearman": 0.788774, + "manhattan_pearson": 0.78467, + "manhattan_spearman": 0.786648, + "euclidean_pearson": 0.785193, + "euclidean_spearman": 0.787054, + "main_score": 0.788774, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.3136167526245117, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS17.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS17.json new file mode 100644 index 0000000000..3093bbc84e --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS17.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "faeb762787bd10488a50c8b5be4a3b82e411949c", + "task_name": "STS17", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.857018, + "spearman": 0.862089, + "cosine_pearson": 0.857018, + "cosine_spearman": 0.862089, + "manhattan_pearson": 0.866898, + "manhattan_spearman": 0.862179, + "euclidean_pearson": 0.867594, + "euclidean_spearman": 0.862031, + "main_score": 0.862089, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.12307119369506836, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS22.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS22.json new file mode 100644 index 0000000000..e28b664e8a --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "task_name": "STS22", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.686109, + "spearman": 0.684957, + "cosine_pearson": 0.686109, + "cosine_spearman": 0.684957, + "manhattan_pearson": 0.698282, + "manhattan_spearman": 0.685329, + "euclidean_pearson": 0.699508, + "euclidean_spearman": 0.68539, + "main_score": 0.684957, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.092864751815796, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS22.v2.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS22.v2.json new file mode 100644 index 0000000000..6df5abf222 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STS22.v2.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d31f33a128469b20e357535c39b82fb3c3f6f2bd", + "task_name": "STS22.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.686145, + "spearman": 0.684735, + "cosine_pearson": 0.686145, + "cosine_spearman": 0.684735, + "manhattan_pearson": 0.698319, + "manhattan_spearman": 0.685374, + "euclidean_pearson": 0.699537, + "euclidean_spearman": 0.685479, + "main_score": 0.684735, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.6958348751068115, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STSBenchmark.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STSBenchmark.json new file mode 100644 index 0000000000..40d47b5e97 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/STSBenchmark.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831", + "task_name": "STSBenchmark", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.804005, + "spearman": 0.791728, + "cosine_pearson": 0.804005, + "cosine_spearman": 0.791728, + "manhattan_pearson": 0.796916, + "manhattan_spearman": 0.789495, + "euclidean_pearson": 0.797789, + "euclidean_spearman": 0.790297, + "main_score": 0.791728, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.48796796798706055, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SciDocsRR.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SciDocsRR.json new file mode 100644 index 0000000000..5d9d22ecc5 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SciDocsRR.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab", + "task_name": "SciDocsRR", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "map": 0.881636, + "mrr": 0.96579, + "nAUC_map_max": 0.548469, + "nAUC_map_std": 0.686123, + "nAUC_map_diff1": 0.013802, + "nAUC_mrr_max": 0.893648, + "nAUC_mrr_std": 0.842765, + "nAUC_mrr_diff1": 0.481009, + "main_score": 0.881636, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 41.930646657943726, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SciFact.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SciFact.json new file mode 100644 index 0000000000..9673831d36 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SciFact.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "0228b52cf27578f30900b9e5271d331663a030d7", + "task_name": "SciFact", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.62667, + "ndcg_at_3": 0.72006, + "ndcg_at_5": 0.74022, + "ndcg_at_10": 0.75797, + "ndcg_at_20": 0.76984, + "ndcg_at_100": 0.77827, + "ndcg_at_1000": 0.7809, + "map_at_1": 0.59594, + "map_at_3": 0.6868, + "map_at_5": 0.70205, + "map_at_10": 0.71036, + "map_at_20": 0.71457, + "map_at_100": 0.71576, + "map_at_1000": 0.71587, + "recall_at_1": 0.59594, + "recall_at_3": 0.78906, + "recall_at_5": 0.83911, + "recall_at_10": 0.89172, + "recall_at_20": 0.935, + "recall_at_100": 0.98, + "recall_at_1000": 1.0, + "precision_at_1": 0.62667, + "precision_at_3": 0.28778, + "precision_at_5": 0.188, + "precision_at_10": 0.10067, + "precision_at_20": 0.05317, + "precision_at_100": 0.0111, + "precision_at_1000": 0.00113, + "mrr_at_1": 0.626667, + "mrr_at_3": 0.704444, + "mrr_at_5": 0.714278, + "mrr_at_10": 0.720328, + "mrr_at_20": 0.723038, + "mrr_at_100": 0.724151, + "mrr_at_1000": 0.72426, + "nauc_ndcg_at_1_max": 0.586771, + "nauc_ndcg_at_1_std": 0.022549, + "nauc_ndcg_at_1_diff1": 0.783165, + "nauc_ndcg_at_3_max": 0.599084, + "nauc_ndcg_at_3_std": -0.012818, + "nauc_ndcg_at_3_diff1": 0.740472, + "nauc_ndcg_at_5_max": 0.614461, + "nauc_ndcg_at_5_std": -0.011827, + "nauc_ndcg_at_5_diff1": 0.752332, + "nauc_ndcg_at_10_max": 0.628557, + "nauc_ndcg_at_10_std": 0.026731, + "nauc_ndcg_at_10_diff1": 0.74646, + "nauc_ndcg_at_20_max": 0.628054, + "nauc_ndcg_at_20_std": 0.037416, + "nauc_ndcg_at_20_diff1": 0.756515, + "nauc_ndcg_at_100_max": 0.619331, + "nauc_ndcg_at_100_std": 0.030788, + "nauc_ndcg_at_100_diff1": 0.757004, + "nauc_ndcg_at_1000_max": 0.615599, + "nauc_ndcg_at_1000_std": 0.027541, + "nauc_ndcg_at_1000_diff1": 0.756834, + "nauc_map_at_1_max": 0.536828, + "nauc_map_at_1_std": -0.063699, + "nauc_map_at_1_diff1": 0.796977, + "nauc_map_at_3_max": 0.575186, + "nauc_map_at_3_std": -0.041184, + "nauc_map_at_3_diff1": 0.760214, + "nauc_map_at_5_max": 0.593737, + "nauc_map_at_5_std": -0.022319, + "nauc_map_at_5_diff1": 0.76345, + "nauc_map_at_10_max": 0.600932, + "nauc_map_at_10_std": -8.7e-05, + "nauc_map_at_10_diff1": 0.761113, + "nauc_map_at_20_max": 0.601449, + "nauc_map_at_20_std": 0.006711, + "nauc_map_at_20_diff1": 0.762773, + "nauc_map_at_100_max": 0.60036, + "nauc_map_at_100_std": 0.005476, + "nauc_map_at_100_diff1": 0.762905, + "nauc_map_at_1000_max": 0.600229, + "nauc_map_at_1000_std": 0.005432, + "nauc_map_at_1000_diff1": 0.762896, + "nauc_recall_at_1_max": 0.536828, + "nauc_recall_at_1_std": -0.063699, + "nauc_recall_at_1_diff1": 0.796977, + "nauc_recall_at_3_max": 0.595037, + "nauc_recall_at_3_std": -0.075362, + "nauc_recall_at_3_diff1": 0.681326, + "nauc_recall_at_5_max": 0.662481, + "nauc_recall_at_5_std": -0.065317, + "nauc_recall_at_5_diff1": 0.704836, + "nauc_recall_at_10_max": 0.755116, + "nauc_recall_at_10_std": 0.089812, + "nauc_recall_at_10_diff1": 0.660008, + "nauc_recall_at_20_max": 0.834147, + "nauc_recall_at_20_std": 0.203488, + "nauc_recall_at_20_diff1": 0.747384, + "nauc_recall_at_100_max": 0.879396, + "nauc_recall_at_100_std": 0.300187, + "nauc_recall_at_100_diff1": 0.765017, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.586771, + "nauc_precision_at_1_std": 0.022549, + "nauc_precision_at_1_diff1": 0.783165, + "nauc_precision_at_3_max": 0.494046, + "nauc_precision_at_3_std": 0.213963, + "nauc_precision_at_3_diff1": 0.345286, + "nauc_precision_at_5_max": 0.425883, + "nauc_precision_at_5_std": 0.321818, + "nauc_precision_at_5_diff1": 0.177114, + "nauc_precision_at_10_max": 0.365849, + "nauc_precision_at_10_std": 0.475769, + "nauc_precision_at_10_diff1": 0.01256, + "nauc_precision_at_20_max": 0.271983, + "nauc_precision_at_20_std": 0.556646, + "nauc_precision_at_20_diff1": -0.099224, + "nauc_precision_at_100_max": 0.159767, + "nauc_precision_at_100_std": 0.607282, + "nauc_precision_at_100_diff1": -0.253279, + "nauc_precision_at_1000_max": 0.096138, + "nauc_precision_at_1000_std": 0.634435, + "nauc_precision_at_1000_diff1": -0.343316, + "nauc_mrr_at_1_max": 0.586771, + "nauc_mrr_at_1_std": 0.022549, + "nauc_mrr_at_1_diff1": 0.783165, + "nauc_mrr_at_3_max": 0.609936, + "nauc_mrr_at_3_std": 0.027478, + "nauc_mrr_at_3_diff1": 0.7487, + "nauc_mrr_at_5_max": 0.615801, + "nauc_mrr_at_5_std": 0.030691, + "nauc_mrr_at_5_diff1": 0.75467, + "nauc_mrr_at_10_max": 0.617719, + "nauc_mrr_at_10_std": 0.036725, + "nauc_mrr_at_10_diff1": 0.753359, + "nauc_mrr_at_20_max": 0.616316, + "nauc_mrr_at_20_std": 0.03516, + "nauc_mrr_at_20_diff1": 0.75669, + "nauc_mrr_at_100_max": 0.615413, + "nauc_mrr_at_100_std": 0.03429, + "nauc_mrr_at_100_diff1": 0.756841, + "nauc_mrr_at_1000_max": 0.615287, + "nauc_mrr_at_1000_std": 0.034258, + "nauc_mrr_at_1000_diff1": 0.756829, + "main_score": 0.75797, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.197069883346558, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SprintDuplicateQuestions.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SprintDuplicateQuestions.json new file mode 100644 index 0000000000..7dd35a141b --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SprintDuplicateQuestions.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", + "task_name": "SprintDuplicateQuestions", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "similarity_accuracy": 0.997713, + "similarity_accuracy_threshold": 0.924536, + "similarity_f1": 0.879499, + "similarity_f1_threshold": 0.924342, + "similarity_precision": 0.919302, + "similarity_recall": 0.843, + "similarity_ap": 0.945605, + "cosine_accuracy": 0.997713, + "cosine_accuracy_threshold": 0.924536, + "cosine_f1": 0.879499, + "cosine_f1_threshold": 0.924342, + "cosine_precision": 0.919302, + "cosine_recall": 0.843, + "cosine_ap": 0.945605, + "manhattan_accuracy": 0.997713, + "manhattan_accuracy_threshold": 262.898926, + "manhattan_f1": 0.881112, + "manhattan_f1_threshold": 264.546112, + "manhattan_precision": 0.907741, + "manhattan_recall": 0.856, + "manhattan_ap": 0.944733, + "euclidean_accuracy": 0.997693, + "euclidean_accuracy_threshold": 11.659216, + "euclidean_f1": 0.878301, + "euclidean_f1_threshold": 11.84614, + "euclidean_precision": 0.910849, + "euclidean_recall": 0.848, + "euclidean_ap": 0.945027, + "dot_accuracy": 0.997733, + "dot_accuracy_threshold": 846.574097, + "dot_f1": 0.881687, + "dot_f1_threshold": 841.658325, + "dot_precision": 0.907839, + "dot_recall": 0.857, + "dot_ap": 0.946306, + "max_accuracy": 0.997733, + "max_f1": 0.881687, + "max_precision": 0.919302, + "max_recall": 0.857, + "max_ap": 0.946306, + "main_score": 0.946306, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.270533323287964, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackExchangeClustering.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackExchangeClustering.json new file mode 100644 index 0000000000..47a691ec40 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackExchangeClustering.json @@ -0,0 +1,47 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "task_name": "StackExchangeClustering", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.676975, + "v_measure_std": 0.037597, + "v_measures": [ + 0.729234, + 0.686492, + 0.603317, + 0.691289, + 0.714518, + 0.644425, + 0.654738, + 0.750063, + 0.67204, + 0.670283, + 0.686627, + 0.77066, + 0.718772, + 0.694208, + 0.620742, + 0.647892, + 0.66935, + 0.662986, + 0.645831, + 0.643064, + 0.67714, + 0.663884, + 0.655136, + 0.692646, + 0.65903 + ], + "main_score": 0.676975, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 250.2674036026001, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackExchangeClustering.v2.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackExchangeClustering.v2.json new file mode 100644 index 0000000000..e49e85be73 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackExchangeClustering.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "task_name": "StackExchangeClustering.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.601531, + 0.605505, + 0.602132, + 0.593863, + 0.594287, + 0.583135, + 0.59492, + 0.602533, + 0.584135, + 0.595543 + ] + }, + "v_measure": 0.595758, + "v_measure_std": 0.007182, + "main_score": 0.595758, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 51.34344172477722, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackExchangeClusteringP2P.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackExchangeClusteringP2P.json new file mode 100644 index 0000000000..72f6c0079e --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackExchangeClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "task_name": "StackExchangeClusteringP2P", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.341557, + "v_measure_std": 0.01628, + "v_measures": [ + 0.322428, + 0.328063, + 0.326201, + 0.324402, + 0.32828, + 0.367836, + 0.351729, + 0.352504, + 0.357833, + 0.356298 + ], + "main_score": 0.341557, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 162.53429889678955, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackExchangeClusteringP2P.v2.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackExchangeClusteringP2P.v2.json new file mode 100644 index 0000000000..153b063150 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackExchangeClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "task_name": "StackExchangeClusteringP2P.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.400632, + 0.401515, + 0.401029, + 0.397482, + 0.395508, + 0.405707, + 0.405692, + 0.402264, + 0.410289, + 0.393585 + ] + }, + "v_measure": 0.40137, + "v_measure_std": 0.004781, + "main_score": 0.40137, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 54.38410210609436, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackOverflowDupQuestions.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackOverflowDupQuestions.json new file mode 100644 index 0000000000..4ef1522751 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackOverflowDupQuestions.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69", + "task_name": "StackOverflowDupQuestions", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "map": 0.54273, + "mrr": 0.55357, + "nAUC_map_max": 0.123628, + "nAUC_map_std": 0.075668, + "nAUC_map_diff1": 0.394999, + "nAUC_mrr_max": 0.13141, + "nAUC_mrr_std": 0.0787, + "nAUC_mrr_diff1": 0.393299, + "main_score": 0.54273, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 37.454416275024414, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackOverflowQA.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackOverflowQA.json new file mode 100644 index 0000000000..0cabf147e8 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/StackOverflowQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "db8f169f3894c14a00251061f957b2063eef2bd5", + "task_name": "StackOverflowQA", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.86259, + "ndcg_at_3": 0.9056, + "ndcg_at_5": 0.91326, + "ndcg_at_10": 0.91804, + "ndcg_at_20": 0.92085, + "ndcg_at_100": 0.92326, + "ndcg_at_1000": 0.9242, + "map_at_1": 0.86259, + "map_at_3": 0.8956, + "map_at_5": 0.89987, + "map_at_10": 0.90178, + "map_at_20": 0.90257, + "map_at_100": 0.9029, + "map_at_1000": 0.90294, + "recall_at_1": 0.86259, + "recall_at_3": 0.9343, + "recall_at_5": 0.95286, + "recall_at_10": 0.9679, + "recall_at_20": 0.97894, + "recall_at_100": 0.99198, + "recall_at_1000": 0.999, + "precision_at_1": 0.86259, + "precision_at_3": 0.31143, + "precision_at_5": 0.19057, + "precision_at_10": 0.09679, + "precision_at_20": 0.04895, + "precision_at_100": 0.00992, + "precision_at_1000": 0.001, + "mrr_at_1": 0.862588, + "mrr_at_3": 0.895603, + "mrr_at_5": 0.899866, + "mrr_at_10": 0.901784, + "mrr_at_20": 0.902567, + "mrr_at_100": 0.902901, + "mrr_at_1000": 0.902942, + "nauc_ndcg_at_1_max": 0.808088, + "nauc_ndcg_at_1_std": 0.036633, + "nauc_ndcg_at_1_diff1": 0.904628, + "nauc_ndcg_at_3_max": 0.825202, + "nauc_ndcg_at_3_std": 0.042156, + "nauc_ndcg_at_3_diff1": 0.886162, + "nauc_ndcg_at_5_max": 0.824755, + "nauc_ndcg_at_5_std": 0.037999, + "nauc_ndcg_at_5_diff1": 0.88939, + "nauc_ndcg_at_10_max": 0.824312, + "nauc_ndcg_at_10_std": 0.04647, + "nauc_ndcg_at_10_diff1": 0.890755, + "nauc_ndcg_at_20_max": 0.822965, + "nauc_ndcg_at_20_std": 0.048105, + "nauc_ndcg_at_20_diff1": 0.890171, + "nauc_ndcg_at_100_max": 0.823692, + "nauc_ndcg_at_100_std": 0.04844, + "nauc_ndcg_at_100_diff1": 0.89227, + "nauc_ndcg_at_1000_max": 0.822062, + "nauc_ndcg_at_1000_std": 0.047196, + "nauc_ndcg_at_1000_diff1": 0.891904, + "nauc_map_at_1_max": 0.808088, + "nauc_map_at_1_std": 0.036633, + "nauc_map_at_1_diff1": 0.904628, + "nauc_map_at_3_max": 0.820219, + "nauc_map_at_3_std": 0.041815, + "nauc_map_at_3_diff1": 0.89106, + "nauc_map_at_5_max": 0.81981, + "nauc_map_at_5_std": 0.04039, + "nauc_map_at_5_diff1": 0.892798, + "nauc_map_at_10_max": 0.819368, + "nauc_map_at_10_std": 0.043375, + "nauc_map_at_10_diff1": 0.893208, + "nauc_map_at_20_max": 0.819005, + "nauc_map_at_20_std": 0.043667, + "nauc_map_at_20_diff1": 0.893099, + "nauc_map_at_100_max": 0.81922, + "nauc_map_at_100_std": 0.044051, + "nauc_map_at_100_diff1": 0.893366, + "nauc_map_at_1000_max": 0.819163, + "nauc_map_at_1000_std": 0.043997, + "nauc_map_at_1000_diff1": 0.893356, + "nauc_recall_at_1_max": 0.808088, + "nauc_recall_at_1_std": 0.036633, + "nauc_recall_at_1_diff1": 0.904628, + "nauc_recall_at_3_max": 0.847271, + "nauc_recall_at_3_std": 0.042787, + "nauc_recall_at_3_diff1": 0.8648, + "nauc_recall_at_5_max": 0.853795, + "nauc_recall_at_5_std": 0.019759, + "nauc_recall_at_5_diff1": 0.870795, + "nauc_recall_at_10_max": 0.867094, + "nauc_recall_at_10_std": 0.076358, + "nauc_recall_at_10_diff1": 0.874672, + "nauc_recall_at_20_max": 0.8706, + "nauc_recall_at_20_std": 0.117343, + "nauc_recall_at_20_diff1": 0.857758, + "nauc_recall_at_100_max": 0.959159, + "nauc_recall_at_100_std": 0.180359, + "nauc_recall_at_100_diff1": 0.905768, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.456118, + "nauc_recall_at_1000_diff1": 0.795772, + "nauc_precision_at_1_max": 0.808088, + "nauc_precision_at_1_std": 0.036633, + "nauc_precision_at_1_diff1": 0.904628, + "nauc_precision_at_3_max": 0.847271, + "nauc_precision_at_3_std": 0.042787, + "nauc_precision_at_3_diff1": 0.8648, + "nauc_precision_at_5_max": 0.853795, + "nauc_precision_at_5_std": 0.019759, + "nauc_precision_at_5_diff1": 0.870795, + "nauc_precision_at_10_max": 0.867094, + "nauc_precision_at_10_std": 0.076358, + "nauc_precision_at_10_diff1": 0.874672, + "nauc_precision_at_20_max": 0.8706, + "nauc_precision_at_20_std": 0.117343, + "nauc_precision_at_20_diff1": 0.857758, + "nauc_precision_at_100_max": 0.959159, + "nauc_precision_at_100_std": 0.180359, + "nauc_precision_at_100_diff1": 0.905768, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 0.456118, + "nauc_precision_at_1000_diff1": 0.795772, + "nauc_mrr_at_1_max": 0.808088, + "nauc_mrr_at_1_std": 0.036633, + "nauc_mrr_at_1_diff1": 0.904628, + "nauc_mrr_at_3_max": 0.820219, + "nauc_mrr_at_3_std": 0.041815, + "nauc_mrr_at_3_diff1": 0.89106, + "nauc_mrr_at_5_max": 0.81981, + "nauc_mrr_at_5_std": 0.04039, + "nauc_mrr_at_5_diff1": 0.892798, + "nauc_mrr_at_10_max": 0.819368, + "nauc_mrr_at_10_std": 0.043375, + "nauc_mrr_at_10_diff1": 0.893208, + "nauc_mrr_at_20_max": 0.819005, + "nauc_mrr_at_20_std": 0.043667, + "nauc_mrr_at_20_diff1": 0.893099, + "nauc_mrr_at_100_max": 0.81922, + "nauc_mrr_at_100_std": 0.044051, + "nauc_mrr_at_100_diff1": 0.893366, + "nauc_mrr_at_1000_max": 0.819163, + "nauc_mrr_at_1000_std": 0.043997, + "nauc_mrr_at_1000_diff1": 0.893356, + "main_score": 0.91804, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 45.073009967803955, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SummEval.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SummEval.json new file mode 100644 index 0000000000..00509ca958 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SummEval.json @@ -0,0 +1,24 @@ +{ + "dataset_revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c", + "task_name": "SummEval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.302324, + "spearman": 0.315197, + "cosine_spearman": 0.315197, + "cosine_pearson": 0.302324, + "dot_spearman": 0.314674, + "dot_pearson": 0.297233, + "main_score": 0.315197, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.8386435508728027, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SummEvalSummarization.v2.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SummEvalSummarization.v2.json new file mode 100644 index 0000000000..dbefe46d14 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SummEvalSummarization.v2.json @@ -0,0 +1,24 @@ +{ + "dataset_revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c", + "task_name": "SummEvalSummarization.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.340939, + "spearman": 0.293065, + "cosine_spearman": 0.293065, + "cosine_pearson": 0.340939, + "dot_spearman": 0.304426, + "dot_pearson": 0.347518, + "main_score": 0.293065, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.100385904312134, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SyntheticText2SQL.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SyntheticText2SQL.json new file mode 100644 index 0000000000..d76673015a --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/SyntheticText2SQL.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "686b87296c3a0191b5d9415a00526c62db9fce09", + "task_name": "SyntheticText2SQL", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.074, + "ndcg_at_3": 0.43779, + "ndcg_at_5": 0.47227, + "ndcg_at_10": 0.49552, + "ndcg_at_20": 0.51016, + "ndcg_at_100": 0.52579, + "ndcg_at_1000": 0.53006, + "map_at_1": 0.074, + "map_at_3": 0.35689, + "map_at_5": 0.37614, + "map_at_10": 0.38581, + "map_at_20": 0.38983, + "map_at_100": 0.39205, + "map_at_1000": 0.39222, + "recall_at_1": 0.074, + "recall_at_3": 0.66843, + "recall_at_5": 0.75167, + "recall_at_10": 0.82311, + "recall_at_20": 0.88105, + "recall_at_100": 0.96445, + "recall_at_1000": 0.99727, + "precision_at_1": 0.074, + "precision_at_3": 0.22281, + "precision_at_5": 0.15033, + "precision_at_10": 0.08231, + "precision_at_20": 0.04405, + "precision_at_100": 0.00964, + "precision_at_1000": 0.001, + "mrr_at_1": 0.305589, + "mrr_at_3": 0.486384, + "mrr_at_5": 0.502433, + "mrr_at_10": 0.511464, + "mrr_at_20": 0.515341, + "mrr_at_100": 0.517501, + "mrr_at_1000": 0.517676, + "nauc_ndcg_at_1_max": 0.116088, + "nauc_ndcg_at_1_std": -0.105734, + "nauc_ndcg_at_1_diff1": 0.311398, + "nauc_ndcg_at_3_max": 0.316021, + "nauc_ndcg_at_3_std": -0.20413, + "nauc_ndcg_at_3_diff1": -0.574401, + "nauc_ndcg_at_5_max": 0.315297, + "nauc_ndcg_at_5_std": -0.205499, + "nauc_ndcg_at_5_diff1": -0.527972, + "nauc_ndcg_at_10_max": 0.30916, + "nauc_ndcg_at_10_std": -0.196638, + "nauc_ndcg_at_10_diff1": -0.495385, + "nauc_ndcg_at_20_max": 0.306055, + "nauc_ndcg_at_20_std": -0.19656, + "nauc_ndcg_at_20_diff1": -0.476683, + "nauc_ndcg_at_100_max": 0.300777, + "nauc_ndcg_at_100_std": -0.18954, + "nauc_ndcg_at_100_diff1": -0.453449, + "nauc_ndcg_at_1000_max": 0.295407, + "nauc_ndcg_at_1000_std": -0.18966, + "nauc_ndcg_at_1000_diff1": -0.450541, + "nauc_map_at_1_max": 0.116088, + "nauc_map_at_1_std": -0.105734, + "nauc_map_at_1_diff1": 0.311398, + "nauc_map_at_3_max": 0.290117, + "nauc_map_at_3_std": -0.190798, + "nauc_map_at_3_diff1": -0.473044, + "nauc_map_at_5_max": 0.287859, + "nauc_map_at_5_std": -0.190985, + "nauc_map_at_5_diff1": -0.439743, + "nauc_map_at_10_max": 0.284014, + "nauc_map_at_10_std": -0.186795, + "nauc_map_at_10_diff1": -0.42283, + "nauc_map_at_20_max": 0.282892, + "nauc_map_at_20_std": -0.186588, + "nauc_map_at_20_diff1": -0.416685, + "nauc_map_at_100_max": 0.282003, + "nauc_map_at_100_std": -0.185759, + "nauc_map_at_100_diff1": -0.412845, + "nauc_map_at_1000_max": 0.281818, + "nauc_map_at_1000_std": -0.185728, + "nauc_map_at_1000_diff1": -0.412692, + "nauc_recall_at_1_max": 0.116088, + "nauc_recall_at_1_std": -0.105734, + "nauc_recall_at_1_diff1": 0.311398, + "nauc_recall_at_3_max": 0.371643, + "nauc_recall_at_3_std": -0.232967, + "nauc_recall_at_3_diff1": -0.787237, + "nauc_recall_at_5_max": 0.384465, + "nauc_recall_at_5_std": -0.24226, + "nauc_recall_at_5_diff1": -0.735804, + "nauc_recall_at_10_max": 0.387652, + "nauc_recall_at_10_std": -0.223715, + "nauc_recall_at_10_diff1": -0.692566, + "nauc_recall_at_20_max": 0.401395, + "nauc_recall_at_20_std": -0.235054, + "nauc_recall_at_20_diff1": -0.666702, + "nauc_recall_at_100_max": 0.49003, + "nauc_recall_at_100_std": -0.145235, + "nauc_recall_at_100_diff1": -0.565282, + "nauc_recall_at_1000_max": 0.358058, + "nauc_recall_at_1000_std": 0.230798, + "nauc_recall_at_1000_diff1": -0.704562, + "nauc_precision_at_1_max": 0.116088, + "nauc_precision_at_1_std": -0.105734, + "nauc_precision_at_1_diff1": 0.311398, + "nauc_precision_at_3_max": 0.371643, + "nauc_precision_at_3_std": -0.232967, + "nauc_precision_at_3_diff1": -0.787237, + "nauc_precision_at_5_max": 0.384465, + "nauc_precision_at_5_std": -0.24226, + "nauc_precision_at_5_diff1": -0.735804, + "nauc_precision_at_10_max": 0.387652, + "nauc_precision_at_10_std": -0.223715, + "nauc_precision_at_10_diff1": -0.692566, + "nauc_precision_at_20_max": 0.401395, + "nauc_precision_at_20_std": -0.235054, + "nauc_precision_at_20_diff1": -0.666702, + "nauc_precision_at_100_max": 0.49003, + "nauc_precision_at_100_std": -0.145235, + "nauc_precision_at_100_diff1": -0.565282, + "nauc_precision_at_1000_max": 0.358058, + "nauc_precision_at_1000_std": 0.230798, + "nauc_precision_at_1000_diff1": -0.704562, + "nauc_mrr_at_1_max": 0.16362, + "nauc_mrr_at_1_std": -0.121982, + "nauc_mrr_at_1_diff1": -0.370833, + "nauc_mrr_at_3_max": 0.254998, + "nauc_mrr_at_3_std": -0.173288, + "nauc_mrr_at_3_diff1": -0.558326, + "nauc_mrr_at_5_max": 0.252975, + "nauc_mrr_at_5_std": -0.170878, + "nauc_mrr_at_5_diff1": -0.544291, + "nauc_mrr_at_10_max": 0.250242, + "nauc_mrr_at_10_std": -0.167143, + "nauc_mrr_at_10_diff1": -0.536404, + "nauc_mrr_at_20_max": 0.24918, + "nauc_mrr_at_20_std": -0.16697, + "nauc_mrr_at_20_diff1": -0.533971, + "nauc_mrr_at_100_max": 0.248378, + "nauc_mrr_at_100_std": -0.166161, + "nauc_mrr_at_100_diff1": -0.532486, + "nauc_mrr_at_1000_max": 0.248238, + "nauc_mrr_at_1000_std": -0.16613, + "nauc_mrr_at_1000_diff1": -0.53246, + "main_score": 0.49552, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "sql-Code" + ] + } + ] + }, + "evaluation_time": 42.25894832611084, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TRECCOVID.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TRECCOVID.json new file mode 100644 index 0000000000..e1b35f8e17 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TRECCOVID.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bb9466bac8153a0349341eb1b22e06409e78ef4e", + "task_name": "TRECCOVID", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.76, + "ndcg_at_3": 0.73592, + "ndcg_at_5": 0.71633, + "ndcg_at_10": 0.70565, + "ndcg_at_20": 0.67356, + "ndcg_at_100": 0.56253, + "ndcg_at_1000": 0.53139, + "map_at_1": 0.00227, + "map_at_3": 0.00575, + "map_at_5": 0.00909, + "map_at_10": 0.01722, + "map_at_20": 0.031, + "map_at_100": 0.10486, + "map_at_1000": 0.27314, + "recall_at_1": 0.00227, + "recall_at_3": 0.00611, + "recall_at_5": 0.00998, + "recall_at_10": 0.01982, + "recall_at_20": 0.03698, + "recall_at_100": 0.14161, + "recall_at_1000": 0.50987, + "precision_at_1": 0.82, + "precision_at_3": 0.77333, + "precision_at_5": 0.748, + "precision_at_10": 0.742, + "precision_at_20": 0.71, + "precision_at_100": 0.5772, + "precision_at_1000": 0.23638, + "mrr_at_1": 0.82, + "mrr_at_3": 0.906667, + "mrr_at_5": 0.906667, + "mrr_at_10": 0.906667, + "mrr_at_20": 0.906667, + "mrr_at_100": 0.906667, + "mrr_at_1000": 0.906667, + "nauc_ndcg_at_1_max": 0.131618, + "nauc_ndcg_at_1_std": 0.153595, + "nauc_ndcg_at_1_diff1": 0.1079, + "nauc_ndcg_at_3_max": 0.271655, + "nauc_ndcg_at_3_std": 0.325764, + "nauc_ndcg_at_3_diff1": 0.278115, + "nauc_ndcg_at_5_max": 0.362723, + "nauc_ndcg_at_5_std": 0.449215, + "nauc_ndcg_at_5_diff1": 0.184047, + "nauc_ndcg_at_10_max": 0.344907, + "nauc_ndcg_at_10_std": 0.531081, + "nauc_ndcg_at_10_diff1": 0.164505, + "nauc_ndcg_at_20_max": 0.328589, + "nauc_ndcg_at_20_std": 0.575028, + "nauc_ndcg_at_20_diff1": 0.080226, + "nauc_ndcg_at_100_max": 0.284472, + "nauc_ndcg_at_100_std": 0.682092, + "nauc_ndcg_at_100_diff1": -0.065767, + "nauc_ndcg_at_1000_max": 0.352217, + "nauc_ndcg_at_1000_std": 0.748973, + "nauc_ndcg_at_1000_diff1": 0.114642, + "nauc_map_at_1_max": 0.151084, + "nauc_map_at_1_std": 0.020616, + "nauc_map_at_1_diff1": 0.414564, + "nauc_map_at_3_max": 0.306636, + "nauc_map_at_3_std": 0.185598, + "nauc_map_at_3_diff1": 0.506216, + "nauc_map_at_5_max": 0.352601, + "nauc_map_at_5_std": 0.24106, + "nauc_map_at_5_diff1": 0.479498, + "nauc_map_at_10_max": 0.334092, + "nauc_map_at_10_std": 0.276497, + "nauc_map_at_10_diff1": 0.459777, + "nauc_map_at_20_max": 0.323377, + "nauc_map_at_20_std": 0.311888, + "nauc_map_at_20_diff1": 0.416416, + "nauc_map_at_100_max": 0.352967, + "nauc_map_at_100_std": 0.559527, + "nauc_map_at_100_diff1": 0.201746, + "nauc_map_at_1000_max": 0.381513, + "nauc_map_at_1000_std": 0.752691, + "nauc_map_at_1000_diff1": 0.052918, + "nauc_recall_at_1_max": 0.151084, + "nauc_recall_at_1_std": 0.020616, + "nauc_recall_at_1_diff1": 0.414564, + "nauc_recall_at_3_max": 0.26629, + "nauc_recall_at_3_std": 0.170495, + "nauc_recall_at_3_diff1": 0.528673, + "nauc_recall_at_5_max": 0.306003, + "nauc_recall_at_5_std": 0.215306, + "nauc_recall_at_5_diff1": 0.477078, + "nauc_recall_at_10_max": 0.261108, + "nauc_recall_at_10_std": 0.214687, + "nauc_recall_at_10_diff1": 0.445521, + "nauc_recall_at_20_max": 0.242923, + "nauc_recall_at_20_std": 0.233935, + "nauc_recall_at_20_diff1": 0.400654, + "nauc_recall_at_100_max": 0.292977, + "nauc_recall_at_100_std": 0.456811, + "nauc_recall_at_100_diff1": 0.279215, + "nauc_recall_at_1000_max": 0.345081, + "nauc_recall_at_1000_std": 0.676203, + "nauc_recall_at_1000_diff1": 0.190628, + "nauc_precision_at_1_max": 0.334894, + "nauc_precision_at_1_std": 0.17188, + "nauc_precision_at_1_diff1": 0.187355, + "nauc_precision_at_3_max": 0.427349, + "nauc_precision_at_3_std": 0.399045, + "nauc_precision_at_3_diff1": 0.367942, + "nauc_precision_at_5_max": 0.542982, + "nauc_precision_at_5_std": 0.563048, + "nauc_precision_at_5_diff1": 0.195486, + "nauc_precision_at_10_max": 0.467697, + "nauc_precision_at_10_std": 0.593919, + "nauc_precision_at_10_diff1": 0.180466, + "nauc_precision_at_20_max": 0.396172, + "nauc_precision_at_20_std": 0.593066, + "nauc_precision_at_20_diff1": 0.098925, + "nauc_precision_at_100_max": 0.304302, + "nauc_precision_at_100_std": 0.686938, + "nauc_precision_at_100_diff1": -0.095393, + "nauc_precision_at_1000_max": 0.202241, + "nauc_precision_at_1000_std": 0.477545, + "nauc_precision_at_1000_diff1": -0.183182, + "nauc_mrr_at_1_max": 0.334894, + "nauc_mrr_at_1_std": 0.17188, + "nauc_mrr_at_1_diff1": 0.187355, + "nauc_mrr_at_3_max": 0.360057, + "nauc_mrr_at_3_std": 0.170023, + "nauc_mrr_at_3_diff1": 0.207591, + "nauc_mrr_at_5_max": 0.360057, + "nauc_mrr_at_5_std": 0.170023, + "nauc_mrr_at_5_diff1": 0.207591, + "nauc_mrr_at_10_max": 0.360057, + "nauc_mrr_at_10_std": 0.170023, + "nauc_mrr_at_10_diff1": 0.207591, + "nauc_mrr_at_20_max": 0.360057, + "nauc_mrr_at_20_std": 0.170023, + "nauc_mrr_at_20_diff1": 0.207591, + "nauc_mrr_at_100_max": 0.360057, + "nauc_mrr_at_100_std": 0.170023, + "nauc_mrr_at_100_diff1": 0.207591, + "nauc_mrr_at_1000_max": 0.360057, + "nauc_mrr_at_1000_std": 0.170023, + "nauc_mrr_at_1000_diff1": 0.207591, + "main_score": 0.70565, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 219.36413741111755, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/Touche2020.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/Touche2020.json new file mode 100644 index 0000000000..cac4330d46 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/Touche2020.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f", + "task_name": "Touche2020", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.22449, + "ndcg_at_3": 0.25481, + "ndcg_at_5": 0.23456, + "ndcg_at_10": 0.22897, + "ndcg_at_20": 0.24802, + "ndcg_at_100": 0.35624, + "ndcg_at_1000": 0.47949, + "map_at_1": 0.01938, + "map_at_3": 0.05144, + "map_at_5": 0.06845, + "map_at_10": 0.09395, + "map_at_20": 0.12254, + "map_at_100": 0.15698, + "map_at_1000": 0.17426, + "recall_at_1": 0.01938, + "recall_at_3": 0.06523, + "recall_at_5": 0.09697, + "recall_at_10": 0.15755, + "recall_at_20": 0.24639, + "recall_at_100": 0.48562, + "recall_at_1000": 0.86259, + "precision_at_1": 0.2449, + "precision_at_3": 0.29252, + "precision_at_5": 0.25306, + "precision_at_10": 0.2102, + "precision_at_20": 0.16837, + "precision_at_100": 0.07714, + "precision_at_1000": 0.0159, + "mrr_at_1": 0.244898, + "mrr_at_3": 0.394558, + "mrr_at_5": 0.403741, + "mrr_at_10": 0.421453, + "mrr_at_20": 0.42584, + "mrr_at_100": 0.431815, + "mrr_at_1000": 0.431815, + "nauc_ndcg_at_1_max": -0.173689, + "nauc_ndcg_at_1_std": 0.100658, + "nauc_ndcg_at_1_diff1": -0.071161, + "nauc_ndcg_at_3_max": -0.134476, + "nauc_ndcg_at_3_std": -0.011951, + "nauc_ndcg_at_3_diff1": -0.051543, + "nauc_ndcg_at_5_max": -0.103075, + "nauc_ndcg_at_5_std": -0.02582, + "nauc_ndcg_at_5_diff1": -0.060242, + "nauc_ndcg_at_10_max": -0.075527, + "nauc_ndcg_at_10_std": -0.002417, + "nauc_ndcg_at_10_diff1": -0.059239, + "nauc_ndcg_at_20_max": -0.138525, + "nauc_ndcg_at_20_std": 0.012754, + "nauc_ndcg_at_20_diff1": -0.056066, + "nauc_ndcg_at_100_max": -0.04096, + "nauc_ndcg_at_100_std": 0.293202, + "nauc_ndcg_at_100_diff1": 0.048964, + "nauc_ndcg_at_1000_max": -0.035238, + "nauc_ndcg_at_1000_std": 0.327766, + "nauc_ndcg_at_1000_diff1": 0.04864, + "nauc_map_at_1_max": -0.102115, + "nauc_map_at_1_std": -0.008822, + "nauc_map_at_1_diff1": -0.05653, + "nauc_map_at_3_max": -0.138277, + "nauc_map_at_3_std": -0.075045, + "nauc_map_at_3_diff1": -0.041315, + "nauc_map_at_5_max": -0.119451, + "nauc_map_at_5_std": -0.077162, + "nauc_map_at_5_diff1": -0.052736, + "nauc_map_at_10_max": -0.050581, + "nauc_map_at_10_std": -0.035165, + "nauc_map_at_10_diff1": -0.090231, + "nauc_map_at_20_max": -0.072771, + "nauc_map_at_20_std": -0.024161, + "nauc_map_at_20_diff1": -0.077479, + "nauc_map_at_100_max": -0.041482, + "nauc_map_at_100_std": 0.11569, + "nauc_map_at_100_diff1": -0.03478, + "nauc_map_at_1000_max": -0.035132, + "nauc_map_at_1000_std": 0.150517, + "nauc_map_at_1000_diff1": -0.02079, + "nauc_recall_at_1_max": -0.102115, + "nauc_recall_at_1_std": -0.008822, + "nauc_recall_at_1_diff1": -0.05653, + "nauc_recall_at_3_max": -0.156817, + "nauc_recall_at_3_std": -0.114951, + "nauc_recall_at_3_diff1": -0.060491, + "nauc_recall_at_5_max": -0.15452, + "nauc_recall_at_5_std": -0.145911, + "nauc_recall_at_5_diff1": -0.092026, + "nauc_recall_at_10_max": -0.105407, + "nauc_recall_at_10_std": -0.081714, + "nauc_recall_at_10_diff1": -0.087101, + "nauc_recall_at_20_max": -0.161542, + "nauc_recall_at_20_std": -0.041657, + "nauc_recall_at_20_diff1": -0.064906, + "nauc_recall_at_100_max": -0.032602, + "nauc_recall_at_100_std": 0.441426, + "nauc_recall_at_100_diff1": 0.094435, + "nauc_recall_at_1000_max": -0.011698, + "nauc_recall_at_1000_std": 0.726291, + "nauc_recall_at_1000_diff1": 0.168559, + "nauc_precision_at_1_max": -0.148565, + "nauc_precision_at_1_std": 0.1035, + "nauc_precision_at_1_diff1": -0.036396, + "nauc_precision_at_3_max": -0.114288, + "nauc_precision_at_3_std": -0.042929, + "nauc_precision_at_3_diff1": -0.014514, + "nauc_precision_at_5_max": -0.04713, + "nauc_precision_at_5_std": -0.049981, + "nauc_precision_at_5_diff1": -0.019699, + "nauc_precision_at_10_max": 0.031447, + "nauc_precision_at_10_std": 0.068819, + "nauc_precision_at_10_diff1": 0.036891, + "nauc_precision_at_20_max": -0.064137, + "nauc_precision_at_20_std": 0.184969, + "nauc_precision_at_20_diff1": 0.086862, + "nauc_precision_at_100_max": 0.140058, + "nauc_precision_at_100_std": 0.749465, + "nauc_precision_at_100_diff1": 0.27178, + "nauc_precision_at_1000_max": 0.139197, + "nauc_precision_at_1000_std": 0.247753, + "nauc_precision_at_1000_diff1": 0.109727, + "nauc_mrr_at_1_max": -0.148565, + "nauc_mrr_at_1_std": 0.1035, + "nauc_mrr_at_1_diff1": -0.036396, + "nauc_mrr_at_3_max": -0.154321, + "nauc_mrr_at_3_std": 0.024283, + "nauc_mrr_at_3_diff1": -0.049272, + "nauc_mrr_at_5_max": -0.181278, + "nauc_mrr_at_5_std": 0.01151, + "nauc_mrr_at_5_diff1": -0.061316, + "nauc_mrr_at_10_max": -0.161276, + "nauc_mrr_at_10_std": 0.036782, + "nauc_mrr_at_10_diff1": 0.002409, + "nauc_mrr_at_20_max": -0.166195, + "nauc_mrr_at_20_std": 0.036383, + "nauc_mrr_at_20_diff1": -0.003575, + "nauc_mrr_at_100_max": -0.164732, + "nauc_mrr_at_100_std": 0.036779, + "nauc_mrr_at_100_diff1": -0.015309, + "nauc_mrr_at_1000_max": -0.164732, + "nauc_mrr_at_1000_std": 0.036779, + "nauc_mrr_at_1000_diff1": -0.015309, + "main_score": 0.22897, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 852.5746328830719, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/Touche2020Retrieval.v3.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/Touche2020Retrieval.v3.json new file mode 100644 index 0000000000..0a83fea527 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/Touche2020Retrieval.v3.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "431886eaecc48f067a3975b70d0949ea2862463c", + "task_name": "Touche2020Retrieval.v3", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.69388, + "ndcg_at_3": 0.63027, + "ndcg_at_5": 0.58788, + "ndcg_at_10": 0.53431, + "ndcg_at_20": 0.47882, + "ndcg_at_100": 0.57006, + "ndcg_at_1000": 0.69395, + "map_at_1": 0.02755, + "map_at_3": 0.06975, + "map_at_5": 0.10298, + "map_at_10": 0.16509, + "map_at_20": 0.24085, + "map_at_100": 0.34517, + "map_at_1000": 0.37719, + "recall_at_1": 0.02755, + "recall_at_3": 0.07421, + "recall_at_5": 0.11447, + "recall_at_10": 0.20516, + "recall_at_20": 0.32527, + "recall_at_100": 0.61657, + "recall_at_1000": 0.9191, + "precision_at_1": 0.79592, + "precision_at_3": 0.71429, + "precision_at_5": 0.65306, + "precision_at_10": 0.56531, + "precision_at_20": 0.45408, + "precision_at_100": 0.19204, + "precision_at_1000": 0.03169, + "mrr_at_1": 0.795918, + "mrr_at_3": 0.877551, + "mrr_at_5": 0.877551, + "mrr_at_10": 0.880952, + "mrr_at_20": 0.880952, + "mrr_at_100": 0.880952, + "mrr_at_1000": 0.880952, + "nauc_ndcg_at_1_max": 0.213493, + "nauc_ndcg_at_1_std": 0.156652, + "nauc_ndcg_at_1_diff1": 0.124211, + "nauc_ndcg_at_3_max": 0.242144, + "nauc_ndcg_at_3_std": 0.165617, + "nauc_ndcg_at_3_diff1": 0.065054, + "nauc_ndcg_at_5_max": 0.216482, + "nauc_ndcg_at_5_std": 0.219489, + "nauc_ndcg_at_5_diff1": 0.057155, + "nauc_ndcg_at_10_max": 0.145645, + "nauc_ndcg_at_10_std": 0.138792, + "nauc_ndcg_at_10_diff1": 0.130514, + "nauc_ndcg_at_20_max": 0.169857, + "nauc_ndcg_at_20_std": 0.109655, + "nauc_ndcg_at_20_diff1": 0.227705, + "nauc_ndcg_at_100_max": 0.185594, + "nauc_ndcg_at_100_std": 0.339211, + "nauc_ndcg_at_100_diff1": 0.102046, + "nauc_ndcg_at_1000_max": 0.263088, + "nauc_ndcg_at_1000_std": 0.483245, + "nauc_ndcg_at_1000_diff1": 0.145834, + "nauc_map_at_1_max": 0.02503, + "nauc_map_at_1_std": -0.047854, + "nauc_map_at_1_diff1": 0.02552, + "nauc_map_at_3_max": 0.188127, + "nauc_map_at_3_std": -0.023939, + "nauc_map_at_3_diff1": 0.049371, + "nauc_map_at_5_max": 0.121772, + "nauc_map_at_5_std": -0.021893, + "nauc_map_at_5_diff1": 0.060057, + "nauc_map_at_10_max": 0.11441, + "nauc_map_at_10_std": -0.059182, + "nauc_map_at_10_diff1": 0.136669, + "nauc_map_at_20_max": 0.152918, + "nauc_map_at_20_std": -0.003517, + "nauc_map_at_20_diff1": 0.162924, + "nauc_map_at_100_max": 0.197968, + "nauc_map_at_100_std": 0.205578, + "nauc_map_at_100_diff1": 0.103366, + "nauc_map_at_1000_max": 0.210334, + "nauc_map_at_1000_std": 0.247772, + "nauc_map_at_1000_diff1": 0.118004, + "nauc_recall_at_1_max": 0.02503, + "nauc_recall_at_1_std": -0.047854, + "nauc_recall_at_1_diff1": 0.02552, + "nauc_recall_at_3_max": 0.180539, + "nauc_recall_at_3_std": 0.003919, + "nauc_recall_at_3_diff1": 0.040445, + "nauc_recall_at_5_max": 0.105295, + "nauc_recall_at_5_std": -0.001822, + "nauc_recall_at_5_diff1": 0.051064, + "nauc_recall_at_10_max": 0.027797, + "nauc_recall_at_10_std": -0.111062, + "nauc_recall_at_10_diff1": 0.197171, + "nauc_recall_at_20_max": 0.075012, + "nauc_recall_at_20_std": -0.017698, + "nauc_recall_at_20_diff1": 0.191685, + "nauc_recall_at_100_max": 0.100981, + "nauc_recall_at_100_std": 0.344402, + "nauc_recall_at_100_diff1": 0.063116, + "nauc_recall_at_1000_max": 0.18711, + "nauc_recall_at_1000_std": 0.807137, + "nauc_recall_at_1000_diff1": 0.249847, + "nauc_precision_at_1_max": 0.140186, + "nauc_precision_at_1_std": 0.100793, + "nauc_precision_at_1_diff1": -0.029374, + "nauc_precision_at_3_max": 0.285931, + "nauc_precision_at_3_std": 0.160662, + "nauc_precision_at_3_diff1": -0.063617, + "nauc_precision_at_5_max": 0.195271, + "nauc_precision_at_5_std": 0.173401, + "nauc_precision_at_5_diff1": -0.018363, + "nauc_precision_at_10_max": 0.123784, + "nauc_precision_at_10_std": 0.064062, + "nauc_precision_at_10_diff1": 0.085068, + "nauc_precision_at_20_max": 0.173626, + "nauc_precision_at_20_std": 0.230508, + "nauc_precision_at_20_diff1": 0.164644, + "nauc_precision_at_100_max": 0.198583, + "nauc_precision_at_100_std": 0.621642, + "nauc_precision_at_100_diff1": -0.154171, + "nauc_precision_at_1000_max": 0.097183, + "nauc_precision_at_1000_std": 0.25649, + "nauc_precision_at_1000_diff1": -0.045293, + "nauc_mrr_at_1_max": 0.140186, + "nauc_mrr_at_1_std": 0.100793, + "nauc_mrr_at_1_diff1": -0.029374, + "nauc_mrr_at_3_max": 0.223348, + "nauc_mrr_at_3_std": 0.173476, + "nauc_mrr_at_3_diff1": -0.224333, + "nauc_mrr_at_5_max": 0.223348, + "nauc_mrr_at_5_std": 0.173476, + "nauc_mrr_at_5_diff1": -0.224333, + "nauc_mrr_at_10_max": 0.199838, + "nauc_mrr_at_10_std": 0.148456, + "nauc_mrr_at_10_diff1": -0.179565, + "nauc_mrr_at_20_max": 0.199838, + "nauc_mrr_at_20_std": 0.148456, + "nauc_mrr_at_20_diff1": -0.179565, + "nauc_mrr_at_100_max": 0.199838, + "nauc_mrr_at_100_std": 0.148456, + "nauc_mrr_at_100_diff1": -0.179565, + "nauc_mrr_at_1000_max": 0.199838, + "nauc_mrr_at_1000_std": 0.148456, + "nauc_mrr_at_1000_diff1": -0.179565, + "main_score": 0.53431, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 765.1799421310425, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ToxicConversationsClassification.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ToxicConversationsClassification.json new file mode 100644 index 0000000000..00c5387b69 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/ToxicConversationsClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "edfaf9da55d3dd50d43143d90c1ac476895ae6de", + "task_name": "ToxicConversationsClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.620801, + "f1": 0.473896, + "f1_weighted": 0.704236, + "ap": 0.103738, + "ap_weighted": 0.103738, + "scores_per_experiment": [ + { + "accuracy": 0.647949, + "f1": 0.482078, + "f1_weighted": 0.72881, + "ap": 0.097997, + "ap_weighted": 0.097997 + }, + { + "accuracy": 0.710938, + "f1": 0.52373, + "f1_weighted": 0.77509, + "ap": 0.112947, + "ap_weighted": 0.112947 + }, + { + "accuracy": 0.717773, + "f1": 0.515123, + "f1_weighted": 0.778997, + "ap": 0.101989, + "ap_weighted": 0.101989 + }, + { + "accuracy": 0.677246, + "f1": 0.509784, + "f1_weighted": 0.750974, + "ap": 0.113571, + "ap_weighted": 0.113571 + }, + { + "accuracy": 0.479492, + "f1": 0.399371, + "f1_weighted": 0.584036, + "ap": 0.096226, + "ap_weighted": 0.096226 + }, + { + "accuracy": 0.525391, + "f1": 0.426622, + "f1_weighted": 0.626948, + "ap": 0.100008, + "ap_weighted": 0.100008 + }, + { + "accuracy": 0.697266, + "f1": 0.501204, + "f1_weighted": 0.764452, + "ap": 0.097101, + "ap_weighted": 0.097101 + }, + { + "accuracy": 0.570801, + "f1": 0.455322, + "f1_weighted": 0.666442, + "ap": 0.107511, + "ap_weighted": 0.107511 + }, + { + "accuracy": 0.585449, + "f1": 0.46201, + "f1_weighted": 0.678941, + "ap": 0.106917, + "ap_weighted": 0.106917 + }, + { + "accuracy": 0.595703, + "f1": 0.46372, + "f1_weighted": 0.687676, + "ap": 0.103116, + "ap_weighted": 0.103116 + } + ], + "main_score": 0.620801, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 13.800911664962769, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TweetSentimentExtractionClassification.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TweetSentimentExtractionClassification.json new file mode 100644 index 0000000000..61bbfce086 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TweetSentimentExtractionClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a", + "task_name": "TweetSentimentExtractionClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.525608, + "f1": 0.528045, + "f1_weighted": 0.522739, + "scores_per_experiment": [ + { + "accuracy": 0.502547, + "f1": 0.503645, + "f1_weighted": 0.497882 + }, + { + "accuracy": 0.54867, + "f1": 0.552159, + "f1_weighted": 0.54792 + }, + { + "accuracy": 0.536786, + "f1": 0.538481, + "f1_weighted": 0.535326 + }, + { + "accuracy": 0.547255, + "f1": 0.549716, + "f1_weighted": 0.544158 + }, + { + "accuracy": 0.525184, + "f1": 0.526433, + "f1_weighted": 0.517705 + }, + { + "accuracy": 0.533956, + "f1": 0.537887, + "f1_weighted": 0.53303 + }, + { + "accuracy": 0.52575, + "f1": 0.527103, + "f1_weighted": 0.523986 + }, + { + "accuracy": 0.531975, + "f1": 0.536108, + "f1_weighted": 0.528374 + }, + { + "accuracy": 0.500849, + "f1": 0.50426, + "f1_weighted": 0.497928 + }, + { + "accuracy": 0.503113, + "f1": 0.504658, + "f1_weighted": 0.501084 + } + ], + "main_score": 0.525608, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.008575677871704, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TwentyNewsgroupsClustering.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TwentyNewsgroupsClustering.json new file mode 100644 index 0000000000..7aca9c5362 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TwentyNewsgroupsClustering.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "task_name": "TwentyNewsgroupsClustering", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.478958, + "v_measure_std": 0.016634, + "v_measures": [ + 0.475955, + 0.471116, + 0.485157, + 0.503515, + 0.453839, + 0.467922, + 0.483737, + 0.457668, + 0.507367, + 0.483302 + ], + "main_score": 0.478958, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 42.444451570510864, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TwentyNewsgroupsClustering.v2.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TwentyNewsgroupsClustering.v2.json new file mode 100644 index 0000000000..77da28a6e7 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TwentyNewsgroupsClustering.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "task_name": "TwentyNewsgroupsClustering.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.490372, + 0.503239, + 0.494551, + 0.486587, + 0.47975, + 0.479633, + 0.470191, + 0.464658, + 0.449339, + 0.458634 + ] + }, + "v_measure": 0.477695, + "v_measure_std": 0.016056, + "main_score": 0.477695, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 39.6615207195282, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TwitterSemEval2015.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TwitterSemEval2015.json new file mode 100644 index 0000000000..a2a1e54cb6 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TwitterSemEval2015.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", + "task_name": "TwitterSemEval2015", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "similarity_accuracy": 0.820767, + "similarity_accuracy_threshold": 0.920084, + "similarity_f1": 0.570871, + "similarity_f1_threshold": 0.893432, + "similarity_precision": 0.530464, + "similarity_recall": 0.617942, + "similarity_ap": 0.599096, + "cosine_accuracy": 0.820767, + "cosine_accuracy_threshold": 0.920084, + "cosine_f1": 0.570871, + "cosine_f1_threshold": 0.893432, + "cosine_precision": 0.530464, + "cosine_recall": 0.617942, + "cosine_ap": 0.599095, + "manhattan_accuracy": 0.822435, + "manhattan_accuracy_threshold": 266.971619, + "manhattan_f1": 0.573678, + "manhattan_f1_threshold": 307.103088, + "manhattan_precision": 0.545001, + "manhattan_recall": 0.605541, + "manhattan_ap": 0.600577, + "euclidean_accuracy": 0.82178, + "euclidean_accuracy_threshold": 12.162125, + "euclidean_f1": 0.573219, + "euclidean_f1_threshold": 13.990332, + "euclidean_precision": 0.536322, + "euclidean_recall": 0.615567, + "euclidean_ap": 0.60047, + "dot_accuracy": 0.812601, + "dot_accuracy_threshold": 851.180603, + "dot_f1": 0.557311, + "dot_f1_threshold": 823.219788, + "dot_precision": 0.503838, + "dot_recall": 0.623483, + "dot_ap": 0.575996, + "max_accuracy": 0.822435, + "max_f1": 0.573678, + "max_precision": 0.545001, + "max_recall": 0.623483, + "max_ap": 0.600577, + "main_score": 0.600577, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.572017192840576, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TwitterURLCorpus.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TwitterURLCorpus.json new file mode 100644 index 0000000000..fcd86bbab2 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/TwitterURLCorpus.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", + "task_name": "TwitterURLCorpus", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "similarity_accuracy": 0.877692, + "similarity_accuracy_threshold": 0.90208, + "similarity_f1": 0.760826, + "similarity_f1_threshold": 0.890613, + "similarity_precision": 0.711833, + "similarity_recall": 0.817062, + "similarity_ap": 0.833098, + "cosine_accuracy": 0.877692, + "cosine_accuracy_threshold": 0.90208, + "cosine_f1": 0.760826, + "cosine_f1_threshold": 0.890613, + "cosine_precision": 0.711833, + "cosine_recall": 0.817062, + "cosine_ap": 0.833098, + "manhattan_accuracy": 0.877692, + "manhattan_accuracy_threshold": 295.900818, + "manhattan_f1": 0.760842, + "manhattan_f1_threshold": 305.195923, + "manhattan_precision": 0.743136, + "manhattan_recall": 0.779412, + "manhattan_ap": 0.833229, + "euclidean_accuracy": 0.877906, + "euclidean_accuracy_threshold": 13.531165, + "euclidean_f1": 0.761277, + "euclidean_f1_threshold": 14.17765, + "euclidean_precision": 0.714923, + "euclidean_recall": 0.814059, + "euclidean_ap": 0.833369, + "dot_accuracy": 0.873191, + "dot_accuracy_threshold": 832.499207, + "dot_f1": 0.752342, + "dot_f1_threshold": 825.627686, + "dot_precision": 0.719486, + "dot_recall": 0.788343, + "dot_ap": 0.824418, + "max_accuracy": 0.877906, + "max_f1": 0.761277, + "max_precision": 0.743136, + "max_recall": 0.817062, + "max_ap": 0.833369, + "main_score": 0.833369, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.539212226867676, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/model_meta.json b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/model_meta.json new file mode 100644 index 0000000000..869bd27dc3 --- /dev/null +++ b/results/ibm-granite__granite-embedding-english-r2/6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9/model_meta.json @@ -0,0 +1 @@ +{"name": "ibm-granite/granite-embedding-english-r2", "revision": "6e7b8ce0e76270394ac4669ba4bbd7133b60b7f9", "release_date": "2025-08-15", "languages": ["eng-Latn"], "n_parameters": 149000000, "memory_usage_mb": 284.0, "max_tokens": 8192.0, "embed_dim": 768, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/ibm-granite/granite-embedding-english-r2", "similarity_fn_name": "cosine", "use_instructions": false, "training_datasets": {"WikipediaRetrievalMultilingual": [], "WikipediaRerankingMultilingual": [], "StackOverflowDupQuestions": [], "AskUbuntuDupQuestions": [], "StackExchangeClusteringP2P": [], "StackExchangeClusteringP2P.v2": [], "StackExchangeClustering": [], "StackExchangeClustering.v2": [], "NQ": ["test"], "NQ-NL": ["test"], "NQHardNegatives": ["test"], "HotPotQA": ["test"], "HotPotQAHardNegatives": ["test"], "HotPotQA-PL": ["test"], "HotpotQA-NL": ["test"], "FEVER": ["test"], "FEVERHardNegatives": ["test"], "FEVER-NL": ["test"], "MIRACLRetrieval": ["train"], "MIRACLRetrievalHardNegatives": ["train"], "MIRACLReranking": ["train"], "MrTidyRetrieval": ["train"], "DBPedia": ["train"], "DBPedia-NL": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["text"], "loader": "sentence_transformers_loader"} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AmazonCounterfactualClassification.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AmazonCounterfactualClassification.json new file mode 100644 index 0000000000..1b83248c7e --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AmazonCounterfactualClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e8379541af4e31359cca9fbcf4b00f2671dba205", + "task_name": "AmazonCounterfactualClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.617761, + "f1": 0.562275, + "f1_weighted": 0.656681, + "ap": 0.261158, + "ap_weighted": 0.261158, + "scores_per_experiment": [ + { + "accuracy": 0.623881, + "f1": 0.559583, + "f1_weighted": 0.662057, + "ap": 0.251602, + "ap_weighted": 0.251602 + }, + { + "accuracy": 0.61194, + "f1": 0.571426, + "f1_weighted": 0.651668, + "ap": 0.281782, + "ap_weighted": 0.281782 + }, + { + "accuracy": 0.58806, + "f1": 0.545052, + "f1_weighted": 0.630232, + "ap": 0.257778, + "ap_weighted": 0.257778 + }, + { + "accuracy": 0.610448, + "f1": 0.545664, + "f1_weighted": 0.650138, + "ap": 0.241492, + "ap_weighted": 0.241492 + }, + { + "accuracy": 0.602985, + "f1": 0.54783, + "f1_weighted": 0.643998, + "ap": 0.249485, + "ap_weighted": 0.249485 + }, + { + "accuracy": 0.59403, + "f1": 0.541908, + "f1_weighted": 0.636004, + "ap": 0.247541, + "ap_weighted": 0.247541 + }, + { + "accuracy": 0.658209, + "f1": 0.587743, + "f1_weighted": 0.691534, + "ap": 0.269044, + "ap_weighted": 0.269044 + }, + { + "accuracy": 0.658209, + "f1": 0.591354, + "f1_weighted": 0.692007, + "ap": 0.27445, + "ap_weighted": 0.27445 + }, + { + "accuracy": 0.591045, + "f1": 0.545545, + "f1_weighted": 0.633111, + "ap": 0.255795, + "ap_weighted": 0.255795 + }, + { + "accuracy": 0.638806, + "f1": 0.586647, + "f1_weighted": 0.676062, + "ap": 0.282612, + "ap_weighted": 0.282612 + } + ], + "main_score": 0.617761, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.866990804672241, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AmazonPolarityClassification.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AmazonPolarityClassification.json new file mode 100644 index 0000000000..761e774e59 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AmazonPolarityClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e2d317d38cd51312af73b3d32a06d1a08b442046", + "task_name": "AmazonPolarityClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.616928, + "f1": 0.613832, + "f1_weighted": 0.613832, + "ap": 0.573907, + "ap_weighted": 0.573907, + "scores_per_experiment": [ + { + "accuracy": 0.647185, + "f1": 0.647074, + "f1_weighted": 0.647074, + "ap": 0.594515, + "ap_weighted": 0.594515 + }, + { + "accuracy": 0.618498, + "f1": 0.617508, + "f1_weighted": 0.617508, + "ap": 0.571994, + "ap_weighted": 0.571994 + }, + { + "accuracy": 0.62111, + "f1": 0.618624, + "f1_weighted": 0.618624, + "ap": 0.573183, + "ap_weighted": 0.573183 + }, + { + "accuracy": 0.64253, + "f1": 0.628368, + "f1_weighted": 0.628368, + "ap": 0.604591, + "ap_weighted": 0.604591 + }, + { + "accuracy": 0.65873, + "f1": 0.657031, + "f1_weighted": 0.657031, + "ap": 0.601451, + "ap_weighted": 0.601451 + }, + { + "accuracy": 0.617607, + "f1": 0.616858, + "f1_weighted": 0.616858, + "ap": 0.573977, + "ap_weighted": 0.573977 + }, + { + "accuracy": 0.607435, + "f1": 0.604672, + "f1_weighted": 0.604672, + "ap": 0.563606, + "ap_weighted": 0.563606 + }, + { + "accuracy": 0.62829, + "f1": 0.626761, + "f1_weighted": 0.626761, + "ap": 0.583019, + "ap_weighted": 0.583019 + }, + { + "accuracy": 0.554175, + "f1": 0.54988, + "f1_weighted": 0.54988, + "ap": 0.529543, + "ap_weighted": 0.529543 + }, + { + "accuracy": 0.573715, + "f1": 0.571547, + "f1_weighted": 0.571547, + "ap": 0.543193, + "ap_weighted": 0.543193 + } + ], + "main_score": 0.616928, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 275.0725166797638, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AmazonReviewsClassification.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AmazonReviewsClassification.json new file mode 100644 index 0000000000..15358a8ed1 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AmazonReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "1399c76144fd37290681b995c656ef9b2e06e26d", + "task_name": "AmazonReviewsClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.32154, + "f1": 0.318664, + "f1_weighted": 0.318664, + "scores_per_experiment": [ + { + "accuracy": 0.3488, + "f1": 0.344031, + "f1_weighted": 0.344031 + }, + { + "accuracy": 0.328, + "f1": 0.323969, + "f1_weighted": 0.323969 + }, + { + "accuracy": 0.3062, + "f1": 0.303052, + "f1_weighted": 0.303052 + }, + { + "accuracy": 0.3, + "f1": 0.304017, + "f1_weighted": 0.304017 + }, + { + "accuracy": 0.3356, + "f1": 0.333235, + "f1_weighted": 0.333235 + }, + { + "accuracy": 0.324, + "f1": 0.316274, + "f1_weighted": 0.316274 + }, + { + "accuracy": 0.2914, + "f1": 0.287884, + "f1_weighted": 0.287884 + }, + { + "accuracy": 0.3416, + "f1": 0.343373, + "f1_weighted": 0.343373 + }, + { + "accuracy": 0.3222, + "f1": 0.315447, + "f1_weighted": 0.315447 + }, + { + "accuracy": 0.3176, + "f1": 0.315355, + "f1_weighted": 0.315355 + } + ], + "main_score": 0.32154, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.638169527053833, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AppsRetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AppsRetrieval.json new file mode 100644 index 0000000000..f2c11d1daa --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AppsRetrieval.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "f22508f96b7a36c2415181ed8bb76f76e04ae2d5", + "task_name": "AppsRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.08048, + "ndcg_at_3": 0.10979, + "ndcg_at_5": 0.12139, + "ndcg_at_10": 0.13538, + "ndcg_at_20": 0.14754, + "ndcg_at_100": 0.17707, + "ndcg_at_1000": 0.21951, + "map_at_1": 0.08048, + "map_at_3": 0.10266, + "map_at_5": 0.1091, + "map_at_10": 0.11476, + "map_at_20": 0.11803, + "map_at_100": 0.12183, + "map_at_1000": 0.12313, + "recall_at_1": 0.08048, + "recall_at_3": 0.13041, + "recall_at_5": 0.15857, + "recall_at_10": 0.20239, + "recall_at_20": 0.251, + "recall_at_100": 0.41434, + "recall_at_1000": 0.76547, + "precision_at_1": 0.08048, + "precision_at_3": 0.04347, + "precision_at_5": 0.03171, + "precision_at_10": 0.02024, + "precision_at_20": 0.01255, + "precision_at_100": 0.00414, + "precision_at_1000": 0.00077, + "mrr_at_1": 0.080478, + "mrr_at_3": 0.102656, + "mrr_at_5": 0.109097, + "mrr_at_10": 0.114763, + "mrr_at_20": 0.118027, + "mrr_at_100": 0.121826, + "mrr_at_1000": 0.123129, + "nauc_ndcg_at_1_max": 0.261492, + "nauc_ndcg_at_1_std": 0.110498, + "nauc_ndcg_at_1_diff1": 0.41297, + "nauc_ndcg_at_3_max": 0.242483, + "nauc_ndcg_at_3_std": 0.125145, + "nauc_ndcg_at_3_diff1": 0.312743, + "nauc_ndcg_at_5_max": 0.230448, + "nauc_ndcg_at_5_std": 0.129127, + "nauc_ndcg_at_5_diff1": 0.290579, + "nauc_ndcg_at_10_max": 0.216396, + "nauc_ndcg_at_10_std": 0.127506, + "nauc_ndcg_at_10_diff1": 0.268815, + "nauc_ndcg_at_20_max": 0.215861, + "nauc_ndcg_at_20_std": 0.136455, + "nauc_ndcg_at_20_diff1": 0.262266, + "nauc_ndcg_at_100_max": 0.216994, + "nauc_ndcg_at_100_std": 0.151959, + "nauc_ndcg_at_100_diff1": 0.241295, + "nauc_ndcg_at_1000_max": 0.224701, + "nauc_ndcg_at_1000_std": 0.159175, + "nauc_ndcg_at_1000_diff1": 0.243653, + "nauc_map_at_1_max": 0.261492, + "nauc_map_at_1_std": 0.110498, + "nauc_map_at_1_diff1": 0.41297, + "nauc_map_at_3_max": 0.247632, + "nauc_map_at_3_std": 0.123115, + "nauc_map_at_3_diff1": 0.331911, + "nauc_map_at_5_max": 0.24013, + "nauc_map_at_5_std": 0.125563, + "nauc_map_at_5_diff1": 0.317635, + "nauc_map_at_10_max": 0.2331, + "nauc_map_at_10_std": 0.124659, + "nauc_map_at_10_diff1": 0.306718, + "nauc_map_at_20_max": 0.232523, + "nauc_map_at_20_std": 0.127401, + "nauc_map_at_20_diff1": 0.304295, + "nauc_map_at_100_max": 0.23244, + "nauc_map_at_100_std": 0.129634, + "nauc_map_at_100_diff1": 0.300912, + "nauc_map_at_1000_max": 0.232721, + "nauc_map_at_1000_std": 0.130107, + "nauc_map_at_1000_diff1": 0.300881, + "nauc_recall_at_1_max": 0.261492, + "nauc_recall_at_1_std": 0.110498, + "nauc_recall_at_1_diff1": 0.41297, + "nauc_recall_at_3_max": 0.229833, + "nauc_recall_at_3_std": 0.129655, + "nauc_recall_at_3_diff1": 0.267613, + "nauc_recall_at_5_max": 0.208085, + "nauc_recall_at_5_std": 0.136953, + "nauc_recall_at_5_diff1": 0.230666, + "nauc_recall_at_10_max": 0.180813, + "nauc_recall_at_10_std": 0.132938, + "nauc_recall_at_10_diff1": 0.191395, + "nauc_recall_at_20_max": 0.183478, + "nauc_recall_at_20_std": 0.157192, + "nauc_recall_at_20_diff1": 0.180232, + "nauc_recall_at_100_max": 0.19238, + "nauc_recall_at_100_std": 0.209654, + "nauc_recall_at_100_diff1": 0.116608, + "nauc_recall_at_1000_max": 0.238697, + "nauc_recall_at_1000_std": 0.294313, + "nauc_recall_at_1000_diff1": 0.061281, + "nauc_precision_at_1_max": 0.261492, + "nauc_precision_at_1_std": 0.110498, + "nauc_precision_at_1_diff1": 0.41297, + "nauc_precision_at_3_max": 0.229833, + "nauc_precision_at_3_std": 0.129655, + "nauc_precision_at_3_diff1": 0.267613, + "nauc_precision_at_5_max": 0.208085, + "nauc_precision_at_5_std": 0.136953, + "nauc_precision_at_5_diff1": 0.230666, + "nauc_precision_at_10_max": 0.180813, + "nauc_precision_at_10_std": 0.132938, + "nauc_precision_at_10_diff1": 0.191395, + "nauc_precision_at_20_max": 0.183478, + "nauc_precision_at_20_std": 0.157192, + "nauc_precision_at_20_diff1": 0.180232, + "nauc_precision_at_100_max": 0.19238, + "nauc_precision_at_100_std": 0.209654, + "nauc_precision_at_100_diff1": 0.116608, + "nauc_precision_at_1000_max": 0.238697, + "nauc_precision_at_1000_std": 0.294313, + "nauc_precision_at_1000_diff1": 0.061281, + "nauc_mrr_at_1_max": 0.261492, + "nauc_mrr_at_1_std": 0.110498, + "nauc_mrr_at_1_diff1": 0.41297, + "nauc_mrr_at_3_max": 0.247632, + "nauc_mrr_at_3_std": 0.123115, + "nauc_mrr_at_3_diff1": 0.331911, + "nauc_mrr_at_5_max": 0.24013, + "nauc_mrr_at_5_std": 0.125563, + "nauc_mrr_at_5_diff1": 0.317635, + "nauc_mrr_at_10_max": 0.2331, + "nauc_mrr_at_10_std": 0.124659, + "nauc_mrr_at_10_diff1": 0.306718, + "nauc_mrr_at_20_max": 0.232523, + "nauc_mrr_at_20_std": 0.127401, + "nauc_mrr_at_20_diff1": 0.304295, + "nauc_mrr_at_100_max": 0.23244, + "nauc_mrr_at_100_std": 0.129634, + "nauc_mrr_at_100_diff1": 0.300912, + "nauc_mrr_at_1000_max": 0.232721, + "nauc_mrr_at_1000_std": 0.130107, + "nauc_mrr_at_1000_diff1": 0.300881, + "main_score": 0.13538, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 16.15168523788452, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArXivHierarchicalClusteringP2P.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArXivHierarchicalClusteringP2P.json new file mode 100644 index 0000000000..75b2f91155 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArXivHierarchicalClusteringP2P.json @@ -0,0 +1,46 @@ +{ + "dataset_revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8", + "task_name": "ArXivHierarchicalClusteringP2P", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.54505, + 0.492012, + 0.522212, + 0.576162, + 0.524411, + 0.56013, + 0.590431, + 0.599738, + 0.559449, + 0.610874 + ], + "Level 1": [ + 0.595275, + 0.572657, + 0.579162, + 0.58808, + 0.586416, + 0.590454, + 0.579759, + 0.571197, + 0.602416, + 0.57412 + ] + }, + "v_measure": 0.571, + "v_measure_std": 0.029304, + "main_score": 0.571, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 55.31880736351013, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArXivHierarchicalClusteringS2S.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArXivHierarchicalClusteringS2S.json new file mode 100644 index 0000000000..8b6dad0063 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArXivHierarchicalClusteringS2S.json @@ -0,0 +1,46 @@ +{ + "dataset_revision": "b73bd54100e5abfa6e3a23dcafb46fe4d2438dc3", + "task_name": "ArXivHierarchicalClusteringS2S", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.585191, + 0.588482, + 0.608043, + 0.58948, + 0.613268, + 0.608867, + 0.568314, + 0.524086, + 0.595725, + 0.578545 + ], + "Level 1": [ + 0.575347, + 0.596467, + 0.561694, + 0.556155, + 0.536491, + 0.575587, + 0.581125, + 0.57066, + 0.582862, + 0.612019 + ] + }, + "v_measure": 0.58042, + "v_measure_std": 0.023081, + "main_score": 0.58042, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 43.69824004173279, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArguAna.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArguAna.json new file mode 100644 index 0000000000..617efd6e74 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArguAna.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a", + "task_name": "ArguAna", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.29018, + "ndcg_at_3": 0.43782, + "ndcg_at_5": 0.49184, + "ndcg_at_10": 0.54398, + "ndcg_at_20": 0.56711, + "ndcg_at_100": 0.57868, + "ndcg_at_1000": 0.57963, + "map_at_1": 0.29018, + "map_at_3": 0.40007, + "map_at_5": 0.42994, + "map_at_10": 0.45143, + "map_at_20": 0.45803, + "map_at_100": 0.45987, + "map_at_1000": 0.45991, + "recall_at_1": 0.29018, + "recall_at_3": 0.54765, + "recall_at_5": 0.67923, + "recall_at_10": 0.84068, + "recall_at_20": 0.9303, + "recall_at_100": 0.98933, + "recall_at_1000": 0.99644, + "precision_at_1": 0.29018, + "precision_at_3": 0.18255, + "precision_at_5": 0.13585, + "precision_at_10": 0.08407, + "precision_at_20": 0.04651, + "precision_at_100": 0.00989, + "precision_at_1000": 0.001, + "mrr_at_1": 0.29872, + "mrr_at_3": 0.40339, + "mrr_at_5": 0.433049, + "mrr_at_10": 0.454496, + "mrr_at_20": 0.461091, + "mrr_at_100": 0.462931, + "mrr_at_1000": 0.462971, + "nauc_ndcg_at_1_max": -0.076753, + "nauc_ndcg_at_1_std": -0.090642, + "nauc_ndcg_at_1_diff1": 0.177262, + "nauc_ndcg_at_3_max": -0.059856, + "nauc_ndcg_at_3_std": -0.079155, + "nauc_ndcg_at_3_diff1": 0.150345, + "nauc_ndcg_at_5_max": -0.057067, + "nauc_ndcg_at_5_std": -0.081048, + "nauc_ndcg_at_5_diff1": 0.141641, + "nauc_ndcg_at_10_max": -0.043841, + "nauc_ndcg_at_10_std": -0.083919, + "nauc_ndcg_at_10_diff1": 0.146989, + "nauc_ndcg_at_20_max": -0.034172, + "nauc_ndcg_at_20_std": -0.069696, + "nauc_ndcg_at_20_diff1": 0.151765, + "nauc_ndcg_at_100_max": -0.050602, + "nauc_ndcg_at_100_std": -0.07323, + "nauc_ndcg_at_100_diff1": 0.153336, + "nauc_ndcg_at_1000_max": -0.051505, + "nauc_ndcg_at_1000_std": -0.074189, + "nauc_ndcg_at_1000_diff1": 0.152625, + "nauc_map_at_1_max": -0.076753, + "nauc_map_at_1_std": -0.090642, + "nauc_map_at_1_diff1": 0.177262, + "nauc_map_at_3_max": -0.065082, + "nauc_map_at_3_std": -0.080231, + "nauc_map_at_3_diff1": 0.155377, + "nauc_map_at_5_max": -0.063332, + "nauc_map_at_5_std": -0.081155, + "nauc_map_at_5_diff1": 0.150849, + "nauc_map_at_10_max": -0.058514, + "nauc_map_at_10_std": -0.082062, + "nauc_map_at_10_diff1": 0.153285, + "nauc_map_at_20_max": -0.056699, + "nauc_map_at_20_std": -0.078735, + "nauc_map_at_20_diff1": 0.154199, + "nauc_map_at_100_max": -0.058691, + "nauc_map_at_100_std": -0.078787, + "nauc_map_at_100_diff1": 0.154403, + "nauc_map_at_1000_max": -0.058711, + "nauc_map_at_1000_std": -0.078804, + "nauc_map_at_1000_diff1": 0.154383, + "nauc_recall_at_1_max": -0.076753, + "nauc_recall_at_1_std": -0.090642, + "nauc_recall_at_1_diff1": 0.177262, + "nauc_recall_at_3_max": -0.043946, + "nauc_recall_at_3_std": -0.076693, + "nauc_recall_at_3_diff1": 0.136105, + "nauc_recall_at_5_max": -0.034865, + "nauc_recall_at_5_std": -0.082094, + "nauc_recall_at_5_diff1": 0.108886, + "nauc_recall_at_10_max": 0.053421, + "nauc_recall_at_10_std": -0.100496, + "nauc_recall_at_10_diff1": 0.112724, + "nauc_recall_at_20_max": 0.310497, + "nauc_recall_at_20_std": 0.083789, + "nauc_recall_at_20_diff1": 0.151219, + "nauc_recall_at_100_max": 0.220614, + "nauc_recall_at_100_std": 0.349831, + "nauc_recall_at_100_diff1": 0.334232, + "nauc_recall_at_1000_max": 0.251965, + "nauc_recall_at_1000_std": 0.643885, + "nauc_recall_at_1000_diff1": 0.307524, + "nauc_precision_at_1_max": -0.076753, + "nauc_precision_at_1_std": -0.090642, + "nauc_precision_at_1_diff1": 0.177262, + "nauc_precision_at_3_max": -0.043946, + "nauc_precision_at_3_std": -0.076693, + "nauc_precision_at_3_diff1": 0.136105, + "nauc_precision_at_5_max": -0.034865, + "nauc_precision_at_5_std": -0.082094, + "nauc_precision_at_5_diff1": 0.108886, + "nauc_precision_at_10_max": 0.053421, + "nauc_precision_at_10_std": -0.100496, + "nauc_precision_at_10_diff1": 0.112724, + "nauc_precision_at_20_max": 0.310497, + "nauc_precision_at_20_std": 0.083789, + "nauc_precision_at_20_diff1": 0.151219, + "nauc_precision_at_100_max": 0.220614, + "nauc_precision_at_100_std": 0.349831, + "nauc_precision_at_100_diff1": 0.334232, + "nauc_precision_at_1000_max": 0.251965, + "nauc_precision_at_1000_std": 0.643885, + "nauc_precision_at_1000_diff1": 0.307524, + "nauc_mrr_at_1_max": -0.06948, + "nauc_mrr_at_1_std": -0.070849, + "nauc_mrr_at_1_diff1": 0.150036, + "nauc_mrr_at_3_max": -0.071554, + "nauc_mrr_at_3_std": -0.073738, + "nauc_mrr_at_3_diff1": 0.132228, + "nauc_mrr_at_5_max": -0.068895, + "nauc_mrr_at_5_std": -0.073242, + "nauc_mrr_at_5_diff1": 0.12785, + "nauc_mrr_at_10_max": -0.064203, + "nauc_mrr_at_10_std": -0.075071, + "nauc_mrr_at_10_diff1": 0.129157, + "nauc_mrr_at_20_max": -0.062037, + "nauc_mrr_at_20_std": -0.071489, + "nauc_mrr_at_20_diff1": 0.129951, + "nauc_mrr_at_100_max": -0.064043, + "nauc_mrr_at_100_std": -0.071498, + "nauc_mrr_at_100_diff1": 0.130008, + "nauc_mrr_at_1000_max": -0.064064, + "nauc_mrr_at_1000_std": -0.071513, + "nauc_mrr_at_1000_diff1": 0.129985, + "main_score": 0.54398, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11.212677717208862, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArxivClusteringP2P.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArxivClusteringP2P.json new file mode 100644 index 0000000000..d3088e99e4 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArxivClusteringP2P.json @@ -0,0 +1,53 @@ +{ + "dataset_revision": "a122ad7f3f0291bf49cc6f4d32aa80929df69d5d", + "task_name": "ArxivClusteringP2P", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.480239, + "v_measure_std": 0.13752, + "v_measures": [ + 0.482039, + 0.475011, + 0.486723, + 0.486163, + 0.481652, + 0.478428, + 0.478375, + 0.490707, + 0.47526, + 0.478993, + 0.561574, + 0.554451, + 0.561842, + 0.55601, + 0.563024, + 0.561776, + 0.55934, + 0.554039, + 0.555455, + 0.564718, + 0.517616, + 0.298442, + 0.474606, + 0.417139, + 0.359847, + 0.27647, + 0.29516, + 0.232535, + 0.335305, + 1.0, + 0.274707 + ], + "main_score": 0.480239, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 854.6411390304565, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArxivClusteringS2S.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArxivClusteringS2S.json new file mode 100644 index 0000000000..0b74f0e2f4 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ArxivClusteringS2S.json @@ -0,0 +1,53 @@ +{ + "dataset_revision": "f910caf1a6075f7329cdf8c1a6135696f37dbd53", + "task_name": "ArxivClusteringS2S", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.44072, + "v_measure_std": 0.141841, + "v_measures": [ + 0.447554, + 0.441856, + 0.458694, + 0.450227, + 0.45338, + 0.439644, + 0.452035, + 0.451483, + 0.448147, + 0.453271, + 0.512758, + 0.514668, + 0.512595, + 0.510448, + 0.513227, + 0.517551, + 0.512681, + 0.51312, + 0.506963, + 0.510887, + 0.47891, + 0.234514, + 0.391361, + 0.389619, + 0.315153, + 0.234771, + 0.264662, + 0.207953, + 0.293971, + 1.0, + 0.230223 + ], + "main_score": 0.44072, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 371.7148768901825, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AskUbuntuDupQuestions.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AskUbuntuDupQuestions.json new file mode 100644 index 0000000000..5bb130c09a --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/AskUbuntuDupQuestions.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2000358ca161889fa9c082cb41daa8dcfb161a54", + "task_name": "AskUbuntuDupQuestions", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "map": 0.648343, + "mrr": 0.766336, + "nAUC_map_max": 0.252407, + "nAUC_map_std": 0.178444, + "nAUC_map_diff1": 0.160121, + "nAUC_mrr_max": 0.372486, + "nAUC_mrr_std": 0.227515, + "nAUC_mrr_diff1": 0.295027, + "main_score": 0.648343, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.9199328422546387, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/BIOSSES.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/BIOSSES.json new file mode 100644 index 0000000000..6e89cf7aa0 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/BIOSSES.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a", + "task_name": "BIOSSES", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.888633, + "spearman": 0.86502, + "cosine_pearson": 0.888633, + "cosine_spearman": 0.86502, + "manhattan_pearson": 0.86744, + "manhattan_spearman": 0.86249, + "euclidean_pearson": 0.87048, + "euclidean_spearman": 0.865206, + "main_score": 0.86502, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.059772491455078125, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/Banking77Classification.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/Banking77Classification.json new file mode 100644 index 0000000000..90a7d43c16 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/Banking77Classification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "0fd18e25b25c072e09e0d92ab615fda904d66300", + "task_name": "Banking77Classification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.836299, + "f1": 0.83555, + "f1_weighted": 0.83555, + "scores_per_experiment": [ + { + "accuracy": 0.83474, + "f1": 0.833985, + "f1_weighted": 0.833985 + }, + { + "accuracy": 0.836688, + "f1": 0.836156, + "f1_weighted": 0.836156 + }, + { + "accuracy": 0.848701, + "f1": 0.847695, + "f1_weighted": 0.847695 + }, + { + "accuracy": 0.83961, + "f1": 0.840302, + "f1_weighted": 0.840302 + }, + { + "accuracy": 0.834091, + "f1": 0.834766, + "f1_weighted": 0.834766 + }, + { + "accuracy": 0.820779, + "f1": 0.818877, + "f1_weighted": 0.818877 + }, + { + "accuracy": 0.827597, + "f1": 0.826997, + "f1_weighted": 0.826997 + }, + { + "accuracy": 0.827597, + "f1": 0.82578, + "f1_weighted": 0.82578 + }, + { + "accuracy": 0.846753, + "f1": 0.845403, + "f1_weighted": 0.845403 + }, + { + "accuracy": 0.846429, + "f1": 0.845534, + "f1_weighted": 0.845534 + } + ], + "main_score": 0.836299, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 14.51960802078247, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/BiorxivClusteringP2P.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/BiorxivClusteringP2P.json new file mode 100644 index 0000000000..75ca6c83a1 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/BiorxivClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "65b79d1d13f80053f67aca9498d9402c2d9f1f40", + "task_name": "BiorxivClusteringP2P", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.388987, + "v_measure_std": 0.012492, + "v_measures": [ + 0.405493, + 0.388346, + 0.375249, + 0.369816, + 0.37943, + 0.38612, + 0.38669, + 0.413398, + 0.391305, + 0.394019 + ], + "main_score": 0.388987, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 129.80717754364014, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/BiorxivClusteringP2P.v2.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/BiorxivClusteringP2P.v2.json new file mode 100644 index 0000000000..ca44247280 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/BiorxivClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "f5dbc242e11dd8e24def4c4268607a49e02946dc", + "task_name": "BiorxivClusteringP2P.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.41311, + 0.407529, + 0.415565, + 0.394597, + 0.399402, + 0.405229, + 0.404007, + 0.426432, + 0.403915, + 0.418353 + ] + }, + "v_measure": 0.408814, + "v_measure_std": 0.009043, + "main_score": 0.408814, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 47.94275093078613, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/BiorxivClusteringS2S.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/BiorxivClusteringS2S.json new file mode 100644 index 0000000000..d72c3ca07c --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/BiorxivClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "258694dd0231531bc1fd9de6ceb52a0853c6d908", + "task_name": "BiorxivClusteringS2S", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.363321, + "v_measure_std": 0.00984, + "v_measures": [ + 0.375776, + 0.370389, + 0.366738, + 0.342357, + 0.357222, + 0.370233, + 0.356254, + 0.364667, + 0.355544, + 0.374034 + ], + "main_score": 0.363321, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 45.54639959335327, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/COIRCodeSearchNetRetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/COIRCodeSearchNetRetrieval.json new file mode 100644 index 0000000000..c86cb73e21 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/COIRCodeSearchNetRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "4adc7bc41202b5c13543c9c886a25f340634dab3", + "task_name": "COIRCodeSearchNetRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.8908, + "ndcg_at_3": 0.92384, + "ndcg_at_5": 0.92904, + "ndcg_at_10": 0.93331, + "ndcg_at_20": 0.9355, + "ndcg_at_100": 0.9375, + "ndcg_at_1000": 0.93833, + "map_at_1": 0.8908, + "map_at_3": 0.9161, + "map_at_5": 0.91898, + "map_at_10": 0.92076, + "map_at_20": 0.92137, + "map_at_100": 0.92166, + "map_at_1000": 0.92169, + "recall_at_1": 0.8908, + "recall_at_3": 0.94611, + "recall_at_5": 0.95871, + "recall_at_10": 0.97185, + "recall_at_20": 0.98043, + "recall_at_100": 0.99095, + "recall_at_1000": 0.99745, + "precision_at_1": 0.8908, + "precision_at_3": 0.31537, + "precision_at_5": 0.19174, + "precision_at_10": 0.09718, + "precision_at_20": 0.04902, + "precision_at_100": 0.00991, + "precision_at_1000": 0.001, + "mrr_at_1": 0.890803, + "mrr_at_3": 0.916097, + "mrr_at_5": 0.918983, + "mrr_at_10": 0.920759, + "mrr_at_20": 0.921368, + "mrr_at_100": 0.921663, + "mrr_at_1000": 0.921694, + "nauc_ndcg_at_1_max": 0.820454, + "nauc_ndcg_at_1_std": 0.080095, + "nauc_ndcg_at_1_diff1": 0.910238, + "nauc_ndcg_at_3_max": 0.851453, + "nauc_ndcg_at_3_std": 0.080669, + "nauc_ndcg_at_3_diff1": 0.908576, + "nauc_ndcg_at_5_max": 0.852008, + "nauc_ndcg_at_5_std": 0.090083, + "nauc_ndcg_at_5_diff1": 0.909341, + "nauc_ndcg_at_10_max": 0.850516, + "nauc_ndcg_at_10_std": 0.097339, + "nauc_ndcg_at_10_diff1": 0.910065, + "nauc_ndcg_at_20_max": 0.848835, + "nauc_ndcg_at_20_std": 0.094734, + "nauc_ndcg_at_20_diff1": 0.910029, + "nauc_ndcg_at_100_max": 0.847513, + "nauc_ndcg_at_100_std": 0.098854, + "nauc_ndcg_at_100_diff1": 0.909553, + "nauc_ndcg_at_1000_max": 0.846175, + "nauc_ndcg_at_1000_std": 0.094991, + "nauc_ndcg_at_1000_diff1": 0.909432, + "nauc_map_at_1_max": 0.820454, + "nauc_map_at_1_std": 0.080095, + "nauc_map_at_1_diff1": 0.910238, + "nauc_map_at_3_max": 0.843092, + "nauc_map_at_3_std": 0.081169, + "nauc_map_at_3_diff1": 0.908723, + "nauc_map_at_5_max": 0.843052, + "nauc_map_at_5_std": 0.085669, + "nauc_map_at_5_diff1": 0.909134, + "nauc_map_at_10_max": 0.842349, + "nauc_map_at_10_std": 0.08814, + "nauc_map_at_10_diff1": 0.909405, + "nauc_map_at_20_max": 0.841885, + "nauc_map_at_20_std": 0.087418, + "nauc_map_at_20_diff1": 0.909369, + "nauc_map_at_100_max": 0.84169, + "nauc_map_at_100_std": 0.087863, + "nauc_map_at_100_diff1": 0.909297, + "nauc_map_at_1000_max": 0.841647, + "nauc_map_at_1000_std": 0.087755, + "nauc_map_at_1000_diff1": 0.909295, + "nauc_recall_at_1_max": 0.820454, + "nauc_recall_at_1_std": 0.080095, + "nauc_recall_at_1_diff1": 0.910238, + "nauc_recall_at_3_max": 0.887897, + "nauc_recall_at_3_std": 0.078004, + "nauc_recall_at_3_diff1": 0.908157, + "nauc_recall_at_5_max": 0.901402, + "nauc_recall_at_5_std": 0.117023, + "nauc_recall_at_5_diff1": 0.910934, + "nauc_recall_at_10_max": 0.913351, + "nauc_recall_at_10_std": 0.181884, + "nauc_recall_at_10_diff1": 0.9166, + "nauc_recall_at_20_max": 0.919774, + "nauc_recall_at_20_std": 0.187156, + "nauc_recall_at_20_diff1": 0.919682, + "nauc_recall_at_100_max": 0.95596, + "nauc_recall_at_100_std": 0.445813, + "nauc_recall_at_100_diff1": 0.915014, + "nauc_recall_at_1000_max": 0.97893, + "nauc_recall_at_1000_std": 0.574139, + "nauc_recall_at_1000_diff1": 0.902266, + "nauc_precision_at_1_max": 0.820454, + "nauc_precision_at_1_std": 0.080095, + "nauc_precision_at_1_diff1": 0.910238, + "nauc_precision_at_3_max": 0.887897, + "nauc_precision_at_3_std": 0.078004, + "nauc_precision_at_3_diff1": 0.908157, + "nauc_precision_at_5_max": 0.901402, + "nauc_precision_at_5_std": 0.117023, + "nauc_precision_at_5_diff1": 0.910934, + "nauc_precision_at_10_max": 0.913351, + "nauc_precision_at_10_std": 0.181884, + "nauc_precision_at_10_diff1": 0.9166, + "nauc_precision_at_20_max": 0.919774, + "nauc_precision_at_20_std": 0.187156, + "nauc_precision_at_20_diff1": 0.919682, + "nauc_precision_at_100_max": 0.95596, + "nauc_precision_at_100_std": 0.445813, + "nauc_precision_at_100_diff1": 0.915014, + "nauc_precision_at_1000_max": 0.97893, + "nauc_precision_at_1000_std": 0.574139, + "nauc_precision_at_1000_diff1": 0.902266, + "nauc_mrr_at_1_max": 0.820454, + "nauc_mrr_at_1_std": 0.080095, + "nauc_mrr_at_1_diff1": 0.910238, + "nauc_mrr_at_3_max": 0.843092, + "nauc_mrr_at_3_std": 0.081169, + "nauc_mrr_at_3_diff1": 0.908723, + "nauc_mrr_at_5_max": 0.843052, + "nauc_mrr_at_5_std": 0.085669, + "nauc_mrr_at_5_diff1": 0.909134, + "nauc_mrr_at_10_max": 0.842349, + "nauc_mrr_at_10_std": 0.08814, + "nauc_mrr_at_10_diff1": 0.909405, + "nauc_mrr_at_20_max": 0.841885, + "nauc_mrr_at_20_std": 0.087418, + "nauc_mrr_at_20_diff1": 0.909369, + "nauc_mrr_at_100_max": 0.84169, + "nauc_mrr_at_100_std": 0.087863, + "nauc_mrr_at_100_diff1": 0.909297, + "nauc_mrr_at_1000_max": 0.841647, + "nauc_mrr_at_1000_std": 0.087755, + "nauc_mrr_at_1000_diff1": 0.909295, + "main_score": 0.93331, + "hf_subset": "python", + "languages": [ + "python-Code" + ] + }, + { + "ndcg_at_1": 0.3862, + "ndcg_at_3": 0.46236, + "ndcg_at_5": 0.47917, + "ndcg_at_10": 0.50119, + "ndcg_at_20": 0.51485, + "ndcg_at_100": 0.53559, + "ndcg_at_1000": 0.54982, + "map_at_1": 0.3862, + "map_at_3": 0.44363, + "map_at_5": 0.45298, + "map_at_10": 0.46208, + "map_at_20": 0.46581, + "map_at_100": 0.46868, + "map_at_1000": 0.46919, + "recall_at_1": 0.3862, + "recall_at_3": 0.51656, + "recall_at_5": 0.55728, + "recall_at_10": 0.62534, + "recall_at_20": 0.67943, + "recall_at_100": 0.79125, + "recall_at_1000": 0.90428, + "precision_at_1": 0.3862, + "precision_at_3": 0.17219, + "precision_at_5": 0.11146, + "precision_at_10": 0.06253, + "precision_at_20": 0.03397, + "precision_at_100": 0.00791, + "precision_at_1000": 0.0009, + "mrr_at_1": 0.386205, + "mrr_at_3": 0.443533, + "mrr_at_5": 0.452952, + "mrr_at_10": 0.46205, + "mrr_at_20": 0.465787, + "mrr_at_100": 0.468652, + "mrr_at_1000": 0.469165, + "nauc_ndcg_at_1_max": 0.518449, + "nauc_ndcg_at_1_std": 0.018002, + "nauc_ndcg_at_1_diff1": 0.663109, + "nauc_ndcg_at_3_max": 0.53249, + "nauc_ndcg_at_3_std": 0.024065, + "nauc_ndcg_at_3_diff1": 0.604105, + "nauc_ndcg_at_5_max": 0.535757, + "nauc_ndcg_at_5_std": 0.032921, + "nauc_ndcg_at_5_diff1": 0.596583, + "nauc_ndcg_at_10_max": 0.536166, + "nauc_ndcg_at_10_std": 0.049834, + "nauc_ndcg_at_10_diff1": 0.589722, + "nauc_ndcg_at_20_max": 0.535291, + "nauc_ndcg_at_20_std": 0.056611, + "nauc_ndcg_at_20_diff1": 0.587456, + "nauc_ndcg_at_100_max": 0.533909, + "nauc_ndcg_at_100_std": 0.061761, + "nauc_ndcg_at_100_diff1": 0.586695, + "nauc_ndcg_at_1000_max": 0.533248, + "nauc_ndcg_at_1000_std": 0.058004, + "nauc_ndcg_at_1000_diff1": 0.592054, + "nauc_map_at_1_max": 0.518449, + "nauc_map_at_1_std": 0.018002, + "nauc_map_at_1_diff1": 0.663109, + "nauc_map_at_3_max": 0.529732, + "nauc_map_at_3_std": 0.021955, + "nauc_map_at_3_diff1": 0.618814, + "nauc_map_at_5_max": 0.531505, + "nauc_map_at_5_std": 0.026831, + "nauc_map_at_5_diff1": 0.614847, + "nauc_map_at_10_max": 0.53146, + "nauc_map_at_10_std": 0.033788, + "nauc_map_at_10_diff1": 0.612012, + "nauc_map_at_20_max": 0.53116, + "nauc_map_at_20_std": 0.035314, + "nauc_map_at_20_diff1": 0.611612, + "nauc_map_at_100_max": 0.530969, + "nauc_map_at_100_std": 0.03583, + "nauc_map_at_100_diff1": 0.611532, + "nauc_map_at_1000_max": 0.530905, + "nauc_map_at_1000_std": 0.035685, + "nauc_map_at_1000_diff1": 0.611693, + "nauc_recall_at_1_max": 0.518449, + "nauc_recall_at_1_std": 0.018002, + "nauc_recall_at_1_diff1": 0.663109, + "nauc_recall_at_3_max": 0.540375, + "nauc_recall_at_3_std": 0.030585, + "nauc_recall_at_3_diff1": 0.560383, + "nauc_recall_at_5_max": 0.54898, + "nauc_recall_at_5_std": 0.053164, + "nauc_recall_at_5_diff1": 0.539289, + "nauc_recall_at_10_max": 0.552739, + "nauc_recall_at_10_std": 0.112119, + "nauc_recall_at_10_diff1": 0.511787, + "nauc_recall_at_20_max": 0.550755, + "nauc_recall_at_20_std": 0.152988, + "nauc_recall_at_20_diff1": 0.49171, + "nauc_recall_at_100_max": 0.544952, + "nauc_recall_at_100_std": 0.2373, + "nauc_recall_at_100_diff1": 0.449782, + "nauc_recall_at_1000_max": 0.545885, + "nauc_recall_at_1000_std": 0.317446, + "nauc_recall_at_1000_diff1": 0.453396, + "nauc_precision_at_1_max": 0.518449, + "nauc_precision_at_1_std": 0.018002, + "nauc_precision_at_1_diff1": 0.663109, + "nauc_precision_at_3_max": 0.540375, + "nauc_precision_at_3_std": 0.030585, + "nauc_precision_at_3_diff1": 0.560383, + "nauc_precision_at_5_max": 0.54898, + "nauc_precision_at_5_std": 0.053164, + "nauc_precision_at_5_diff1": 0.539289, + "nauc_precision_at_10_max": 0.552739, + "nauc_precision_at_10_std": 0.112119, + "nauc_precision_at_10_diff1": 0.511787, + "nauc_precision_at_20_max": 0.550755, + "nauc_precision_at_20_std": 0.152988, + "nauc_precision_at_20_diff1": 0.49171, + "nauc_precision_at_100_max": 0.544952, + "nauc_precision_at_100_std": 0.2373, + "nauc_precision_at_100_diff1": 0.449782, + "nauc_precision_at_1000_max": 0.545885, + "nauc_precision_at_1000_std": 0.317446, + "nauc_precision_at_1000_diff1": 0.453396, + "nauc_mrr_at_1_max": 0.518449, + "nauc_mrr_at_1_std": 0.018002, + "nauc_mrr_at_1_diff1": 0.663109, + "nauc_mrr_at_3_max": 0.529927, + "nauc_mrr_at_3_std": 0.022111, + "nauc_mrr_at_3_diff1": 0.618909, + "nauc_mrr_at_5_max": 0.531554, + "nauc_mrr_at_5_std": 0.026871, + "nauc_mrr_at_5_diff1": 0.614871, + "nauc_mrr_at_10_max": 0.53151, + "nauc_mrr_at_10_std": 0.033827, + "nauc_mrr_at_10_diff1": 0.612036, + "nauc_mrr_at_20_max": 0.53121, + "nauc_mrr_at_20_std": 0.035354, + "nauc_mrr_at_20_diff1": 0.611637, + "nauc_mrr_at_100_max": 0.531019, + "nauc_mrr_at_100_std": 0.03587, + "nauc_mrr_at_100_diff1": 0.611557, + "nauc_mrr_at_1000_max": 0.530955, + "nauc_mrr_at_1000_std": 0.035725, + "nauc_mrr_at_1000_diff1": 0.611718, + "main_score": 0.50119, + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ] + }, + { + "ndcg_at_1": 0.50295, + "ndcg_at_3": 0.59135, + "ndcg_at_5": 0.61555, + "ndcg_at_10": 0.63618, + "ndcg_at_20": 0.64948, + "ndcg_at_100": 0.66435, + "ndcg_at_1000": 0.67262, + "map_at_1": 0.50295, + "map_at_3": 0.5701, + "map_at_5": 0.58355, + "map_at_10": 0.59215, + "map_at_20": 0.59578, + "map_at_100": 0.59786, + "map_at_1000": 0.59816, + "recall_at_1": 0.50295, + "recall_at_3": 0.65267, + "recall_at_5": 0.71128, + "recall_at_10": 0.77469, + "recall_at_20": 0.82738, + "recall_at_100": 0.90729, + "recall_at_1000": 0.97279, + "precision_at_1": 0.50295, + "precision_at_3": 0.21756, + "precision_at_5": 0.14226, + "precision_at_10": 0.07747, + "precision_at_20": 0.04137, + "precision_at_100": 0.00907, + "precision_at_1000": 0.00097, + "mrr_at_1": 0.502955, + "mrr_at_3": 0.570098, + "mrr_at_5": 0.583555, + "mrr_at_10": 0.59215, + "mrr_at_20": 0.595783, + "mrr_at_100": 0.597859, + "mrr_at_1000": 0.598164, + "nauc_ndcg_at_1_max": 0.426349, + "nauc_ndcg_at_1_std": -0.012849, + "nauc_ndcg_at_1_diff1": 0.637158, + "nauc_ndcg_at_3_max": 0.447623, + "nauc_ndcg_at_3_std": -0.012656, + "nauc_ndcg_at_3_diff1": 0.593062, + "nauc_ndcg_at_5_max": 0.449189, + "nauc_ndcg_at_5_std": -0.008255, + "nauc_ndcg_at_5_diff1": 0.58671, + "nauc_ndcg_at_10_max": 0.451567, + "nauc_ndcg_at_10_std": -0.000455, + "nauc_ndcg_at_10_diff1": 0.586074, + "nauc_ndcg_at_20_max": 0.450443, + "nauc_ndcg_at_20_std": 0.002551, + "nauc_ndcg_at_20_diff1": 0.58331, + "nauc_ndcg_at_100_max": 0.448944, + "nauc_ndcg_at_100_std": 0.006129, + "nauc_ndcg_at_100_diff1": 0.586426, + "nauc_ndcg_at_1000_max": 0.448831, + "nauc_ndcg_at_1000_std": 0.003129, + "nauc_ndcg_at_1000_diff1": 0.591259, + "nauc_map_at_1_max": 0.426349, + "nauc_map_at_1_std": -0.012849, + "nauc_map_at_1_diff1": 0.637158, + "nauc_map_at_3_max": 0.44235, + "nauc_map_at_3_std": -0.013002, + "nauc_map_at_3_diff1": 0.603738, + "nauc_map_at_5_max": 0.442899, + "nauc_map_at_5_std": -0.010898, + "nauc_map_at_5_diff1": 0.600602, + "nauc_map_at_10_max": 0.443803, + "nauc_map_at_10_std": -0.007851, + "nauc_map_at_10_diff1": 0.600793, + "nauc_map_at_20_max": 0.443374, + "nauc_map_at_20_std": -0.007205, + "nauc_map_at_20_diff1": 0.600212, + "nauc_map_at_100_max": 0.443125, + "nauc_map_at_100_std": -0.006834, + "nauc_map_at_100_diff1": 0.600725, + "nauc_map_at_1000_max": 0.44312, + "nauc_map_at_1000_std": -0.006937, + "nauc_map_at_1000_diff1": 0.60089, + "nauc_recall_at_1_max": 0.426349, + "nauc_recall_at_1_std": -0.012849, + "nauc_recall_at_1_diff1": 0.637158, + "nauc_recall_at_3_max": 0.464636, + "nauc_recall_at_3_std": -0.011392, + "nauc_recall_at_3_diff1": 0.558747, + "nauc_recall_at_5_max": 0.472228, + "nauc_recall_at_5_std": 0.002548, + "nauc_recall_at_5_diff1": 0.536003, + "nauc_recall_at_10_max": 0.48653, + "nauc_recall_at_10_std": 0.037236, + "nauc_recall_at_10_diff1": 0.520766, + "nauc_recall_at_20_max": 0.489273, + "nauc_recall_at_20_std": 0.067232, + "nauc_recall_at_20_diff1": 0.486238, + "nauc_recall_at_100_max": 0.497164, + "nauc_recall_at_100_std": 0.169611, + "nauc_recall_at_100_diff1": 0.457324, + "nauc_recall_at_1000_max": 0.599904, + "nauc_recall_at_1000_std": 0.338804, + "nauc_recall_at_1000_diff1": 0.497984, + "nauc_precision_at_1_max": 0.426349, + "nauc_precision_at_1_std": -0.012849, + "nauc_precision_at_1_diff1": 0.637158, + "nauc_precision_at_3_max": 0.464636, + "nauc_precision_at_3_std": -0.011392, + "nauc_precision_at_3_diff1": 0.558747, + "nauc_precision_at_5_max": 0.472228, + "nauc_precision_at_5_std": 0.002548, + "nauc_precision_at_5_diff1": 0.536003, + "nauc_precision_at_10_max": 0.48653, + "nauc_precision_at_10_std": 0.037236, + "nauc_precision_at_10_diff1": 0.520766, + "nauc_precision_at_20_max": 0.489273, + "nauc_precision_at_20_std": 0.067232, + "nauc_precision_at_20_diff1": 0.486238, + "nauc_precision_at_100_max": 0.497164, + "nauc_precision_at_100_std": 0.169611, + "nauc_precision_at_100_diff1": 0.457324, + "nauc_precision_at_1000_max": 0.599904, + "nauc_precision_at_1000_std": 0.338804, + "nauc_precision_at_1000_diff1": 0.497984, + "nauc_mrr_at_1_max": 0.426349, + "nauc_mrr_at_1_std": -0.012849, + "nauc_mrr_at_1_diff1": 0.637158, + "nauc_mrr_at_3_max": 0.44235, + "nauc_mrr_at_3_std": -0.013002, + "nauc_mrr_at_3_diff1": 0.603738, + "nauc_mrr_at_5_max": 0.442899, + "nauc_mrr_at_5_std": -0.010898, + "nauc_mrr_at_5_diff1": 0.600602, + "nauc_mrr_at_10_max": 0.443803, + "nauc_mrr_at_10_std": -0.007851, + "nauc_mrr_at_10_diff1": 0.600793, + "nauc_mrr_at_20_max": 0.443374, + "nauc_mrr_at_20_std": -0.007205, + "nauc_mrr_at_20_diff1": 0.600212, + "nauc_mrr_at_100_max": 0.443125, + "nauc_mrr_at_100_std": -0.006834, + "nauc_mrr_at_100_diff1": 0.600725, + "nauc_mrr_at_1000_max": 0.44312, + "nauc_mrr_at_1000_std": -0.006937, + "nauc_mrr_at_1000_diff1": 0.60089, + "main_score": 0.63618, + "hf_subset": "go", + "languages": [ + "go-Code" + ] + }, + { + "ndcg_at_1": 0.38541, + "ndcg_at_3": 0.46653, + "ndcg_at_5": 0.48714, + "ndcg_at_10": 0.50718, + "ndcg_at_20": 0.52611, + "ndcg_at_100": 0.54789, + "ndcg_at_1000": 0.56209, + "map_at_1": 0.38541, + "map_at_3": 0.4466, + "map_at_5": 0.45806, + "map_at_10": 0.46635, + "map_at_20": 0.47159, + "map_at_100": 0.4746, + "map_at_1000": 0.4751, + "recall_at_1": 0.38541, + "recall_at_3": 0.52419, + "recall_at_5": 0.57415, + "recall_at_10": 0.636, + "recall_at_20": 0.71055, + "recall_at_100": 0.82791, + "recall_at_1000": 0.94211, + "precision_at_1": 0.38541, + "precision_at_3": 0.17473, + "precision_at_5": 0.11483, + "precision_at_10": 0.0636, + "precision_at_20": 0.03553, + "precision_at_100": 0.00828, + "precision_at_1000": 0.00094, + "mrr_at_1": 0.385408, + "mrr_at_3": 0.446603, + "mrr_at_5": 0.458062, + "mrr_at_10": 0.466351, + "mrr_at_20": 0.471587, + "mrr_at_100": 0.474605, + "mrr_at_1000": 0.475099, + "nauc_ndcg_at_1_max": 0.494184, + "nauc_ndcg_at_1_std": 0.008379, + "nauc_ndcg_at_1_diff1": 0.588536, + "nauc_ndcg_at_3_max": 0.489457, + "nauc_ndcg_at_3_std": 0.020796, + "nauc_ndcg_at_3_diff1": 0.521225, + "nauc_ndcg_at_5_max": 0.483105, + "nauc_ndcg_at_5_std": 0.024515, + "nauc_ndcg_at_5_diff1": 0.516164, + "nauc_ndcg_at_10_max": 0.479756, + "nauc_ndcg_at_10_std": 0.021129, + "nauc_ndcg_at_10_diff1": 0.510849, + "nauc_ndcg_at_20_max": 0.473173, + "nauc_ndcg_at_20_std": 0.030686, + "nauc_ndcg_at_20_diff1": 0.512704, + "nauc_ndcg_at_100_max": 0.476468, + "nauc_ndcg_at_100_std": 0.039316, + "nauc_ndcg_at_100_diff1": 0.517761, + "nauc_ndcg_at_1000_max": 0.477026, + "nauc_ndcg_at_1000_std": 0.032422, + "nauc_ndcg_at_1000_diff1": 0.520171, + "nauc_map_at_1_max": 0.494184, + "nauc_map_at_1_std": 0.008379, + "nauc_map_at_1_diff1": 0.588536, + "nauc_map_at_3_max": 0.491112, + "nauc_map_at_3_std": 0.016381, + "nauc_map_at_3_diff1": 0.538069, + "nauc_map_at_5_max": 0.487436, + "nauc_map_at_5_std": 0.018449, + "nauc_map_at_5_diff1": 0.535541, + "nauc_map_at_10_max": 0.485902, + "nauc_map_at_10_std": 0.01661, + "nauc_map_at_10_diff1": 0.533349, + "nauc_map_at_20_max": 0.484074, + "nauc_map_at_20_std": 0.019122, + "nauc_map_at_20_diff1": 0.533773, + "nauc_map_at_100_max": 0.484523, + "nauc_map_at_100_std": 0.020043, + "nauc_map_at_100_diff1": 0.534577, + "nauc_map_at_1000_max": 0.48452, + "nauc_map_at_1000_std": 0.019872, + "nauc_map_at_1000_diff1": 0.534563, + "nauc_recall_at_1_max": 0.494184, + "nauc_recall_at_1_std": 0.008379, + "nauc_recall_at_1_diff1": 0.588536, + "nauc_recall_at_3_max": 0.484314, + "nauc_recall_at_3_std": 0.034533, + "nauc_recall_at_3_diff1": 0.470909, + "nauc_recall_at_5_max": 0.468747, + "nauc_recall_at_5_std": 0.044436, + "nauc_recall_at_5_diff1": 0.454977, + "nauc_recall_at_10_max": 0.457085, + "nauc_recall_at_10_std": 0.037133, + "nauc_recall_at_10_diff1": 0.432075, + "nauc_recall_at_20_max": 0.421746, + "nauc_recall_at_20_std": 0.087733, + "nauc_recall_at_20_diff1": 0.430543, + "nauc_recall_at_100_max": 0.429085, + "nauc_recall_at_100_std": 0.210403, + "nauc_recall_at_100_diff1": 0.436346, + "nauc_recall_at_1000_max": 0.375035, + "nauc_recall_at_1000_std": 0.220039, + "nauc_recall_at_1000_diff1": 0.42049, + "nauc_precision_at_1_max": 0.494184, + "nauc_precision_at_1_std": 0.008379, + "nauc_precision_at_1_diff1": 0.588536, + "nauc_precision_at_3_max": 0.484314, + "nauc_precision_at_3_std": 0.034533, + "nauc_precision_at_3_diff1": 0.470909, + "nauc_precision_at_5_max": 0.468747, + "nauc_precision_at_5_std": 0.044436, + "nauc_precision_at_5_diff1": 0.454977, + "nauc_precision_at_10_max": 0.457085, + "nauc_precision_at_10_std": 0.037133, + "nauc_precision_at_10_diff1": 0.432075, + "nauc_precision_at_20_max": 0.421746, + "nauc_precision_at_20_std": 0.087733, + "nauc_precision_at_20_diff1": 0.430543, + "nauc_precision_at_100_max": 0.429085, + "nauc_precision_at_100_std": 0.210403, + "nauc_precision_at_100_diff1": 0.436346, + "nauc_precision_at_1000_max": 0.375035, + "nauc_precision_at_1000_std": 0.220039, + "nauc_precision_at_1000_diff1": 0.42049, + "nauc_mrr_at_1_max": 0.494184, + "nauc_mrr_at_1_std": 0.008379, + "nauc_mrr_at_1_diff1": 0.588536, + "nauc_mrr_at_3_max": 0.491112, + "nauc_mrr_at_3_std": 0.016381, + "nauc_mrr_at_3_diff1": 0.538069, + "nauc_mrr_at_5_max": 0.487436, + "nauc_mrr_at_5_std": 0.018449, + "nauc_mrr_at_5_diff1": 0.535541, + "nauc_mrr_at_10_max": 0.485902, + "nauc_mrr_at_10_std": 0.01661, + "nauc_mrr_at_10_diff1": 0.533349, + "nauc_mrr_at_20_max": 0.484074, + "nauc_mrr_at_20_std": 0.019122, + "nauc_mrr_at_20_diff1": 0.533773, + "nauc_mrr_at_100_max": 0.484523, + "nauc_mrr_at_100_std": 0.020043, + "nauc_mrr_at_100_diff1": 0.534577, + "nauc_mrr_at_1000_max": 0.48452, + "nauc_mrr_at_1000_std": 0.019872, + "nauc_mrr_at_1000_diff1": 0.534563, + "main_score": 0.50718, + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ] + }, + { + "ndcg_at_1": 0.43898, + "ndcg_at_3": 0.52438, + "ndcg_at_5": 0.54529, + "ndcg_at_10": 0.5648, + "ndcg_at_20": 0.57762, + "ndcg_at_100": 0.59521, + "ndcg_at_1000": 0.60573, + "map_at_1": 0.43898, + "map_at_3": 0.50361, + "map_at_5": 0.51517, + "map_at_10": 0.52324, + "map_at_20": 0.5268, + "map_at_100": 0.52921, + "map_at_1000": 0.52958, + "recall_at_1": 0.43898, + "recall_at_3": 0.58439, + "recall_at_5": 0.63533, + "recall_at_10": 0.69557, + "recall_at_20": 0.74596, + "recall_at_100": 0.84108, + "recall_at_1000": 0.92515, + "precision_at_1": 0.43898, + "precision_at_3": 0.1948, + "precision_at_5": 0.12707, + "precision_at_10": 0.06956, + "precision_at_20": 0.0373, + "precision_at_100": 0.00841, + "precision_at_1000": 0.00093, + "mrr_at_1": 0.438886, + "mrr_at_3": 0.50356, + "mrr_at_5": 0.515121, + "mrr_at_10": 0.523192, + "mrr_at_20": 0.526757, + "mrr_at_100": 0.529162, + "mrr_at_1000": 0.529537, + "nauc_ndcg_at_1_max": 0.374078, + "nauc_ndcg_at_1_std": -0.02786, + "nauc_ndcg_at_1_diff1": 0.63982, + "nauc_ndcg_at_3_max": 0.397467, + "nauc_ndcg_at_3_std": -0.012339, + "nauc_ndcg_at_3_diff1": 0.586321, + "nauc_ndcg_at_5_max": 0.39811, + "nauc_ndcg_at_5_std": -0.003878, + "nauc_ndcg_at_5_diff1": 0.580942, + "nauc_ndcg_at_10_max": 0.398779, + "nauc_ndcg_at_10_std": 0.006678, + "nauc_ndcg_at_10_diff1": 0.574443, + "nauc_ndcg_at_20_max": 0.401882, + "nauc_ndcg_at_20_std": 0.014804, + "nauc_ndcg_at_20_diff1": 0.575159, + "nauc_ndcg_at_100_max": 0.403881, + "nauc_ndcg_at_100_std": 0.021596, + "nauc_ndcg_at_100_diff1": 0.57537, + "nauc_ndcg_at_1000_max": 0.402902, + "nauc_ndcg_at_1000_std": 0.018651, + "nauc_ndcg_at_1000_diff1": 0.578522, + "nauc_map_at_1_max": 0.374078, + "nauc_map_at_1_std": -0.02786, + "nauc_map_at_1_diff1": 0.63982, + "nauc_map_at_3_max": 0.39163, + "nauc_map_at_3_std": -0.016932, + "nauc_map_at_3_diff1": 0.599227, + "nauc_map_at_5_max": 0.391918, + "nauc_map_at_5_std": -0.012441, + "nauc_map_at_5_diff1": 0.596611, + "nauc_map_at_10_max": 0.39214, + "nauc_map_at_10_std": -0.008435, + "nauc_map_at_10_diff1": 0.594229, + "nauc_map_at_20_max": 0.392935, + "nauc_map_at_20_std": -0.0064, + "nauc_map_at_20_diff1": 0.594536, + "nauc_map_at_100_max": 0.393148, + "nauc_map_at_100_std": -0.005508, + "nauc_map_at_100_diff1": 0.594674, + "nauc_map_at_1000_max": 0.393123, + "nauc_map_at_1000_std": -0.005582, + "nauc_map_at_1000_diff1": 0.594786, + "nauc_recall_at_1_max": 0.374078, + "nauc_recall_at_1_std": -0.02786, + "nauc_recall_at_1_diff1": 0.63982, + "nauc_recall_at_3_max": 0.415427, + "nauc_recall_at_3_std": 0.002113, + "nauc_recall_at_3_diff1": 0.546814, + "nauc_recall_at_5_max": 0.418491, + "nauc_recall_at_5_std": 0.025984, + "nauc_recall_at_5_diff1": 0.528709, + "nauc_recall_at_10_max": 0.4233, + "nauc_recall_at_10_std": 0.06923, + "nauc_recall_at_10_diff1": 0.498127, + "nauc_recall_at_20_max": 0.443202, + "nauc_recall_at_20_std": 0.121792, + "nauc_recall_at_20_diff1": 0.490172, + "nauc_recall_at_100_max": 0.484, + "nauc_recall_at_100_std": 0.239896, + "nauc_recall_at_100_diff1": 0.450806, + "nauc_recall_at_1000_max": 0.531926, + "nauc_recall_at_1000_std": 0.361939, + "nauc_recall_at_1000_diff1": 0.419768, + "nauc_precision_at_1_max": 0.374078, + "nauc_precision_at_1_std": -0.02786, + "nauc_precision_at_1_diff1": 0.63982, + "nauc_precision_at_3_max": 0.415427, + "nauc_precision_at_3_std": 0.002113, + "nauc_precision_at_3_diff1": 0.546814, + "nauc_precision_at_5_max": 0.418491, + "nauc_precision_at_5_std": 0.025984, + "nauc_precision_at_5_diff1": 0.528709, + "nauc_precision_at_10_max": 0.4233, + "nauc_precision_at_10_std": 0.06923, + "nauc_precision_at_10_diff1": 0.498127, + "nauc_precision_at_20_max": 0.443202, + "nauc_precision_at_20_std": 0.121792, + "nauc_precision_at_20_diff1": 0.490172, + "nauc_precision_at_100_max": 0.484, + "nauc_precision_at_100_std": 0.239896, + "nauc_precision_at_100_diff1": 0.450806, + "nauc_precision_at_1000_max": 0.531926, + "nauc_precision_at_1000_std": 0.361939, + "nauc_precision_at_1000_diff1": 0.419768, + "nauc_mrr_at_1_max": 0.373923, + "nauc_mrr_at_1_std": -0.028303, + "nauc_mrr_at_1_diff1": 0.640055, + "nauc_mrr_at_3_max": 0.391548, + "nauc_mrr_at_3_std": -0.017164, + "nauc_mrr_at_3_diff1": 0.59935, + "nauc_mrr_at_5_max": 0.391835, + "nauc_mrr_at_5_std": -0.012677, + "nauc_mrr_at_5_diff1": 0.596737, + "nauc_mrr_at_10_max": 0.392056, + "nauc_mrr_at_10_std": -0.008674, + "nauc_mrr_at_10_diff1": 0.594356, + "nauc_mrr_at_20_max": 0.392851, + "nauc_mrr_at_20_std": -0.006641, + "nauc_mrr_at_20_diff1": 0.594665, + "nauc_mrr_at_100_max": 0.393063, + "nauc_mrr_at_100_std": -0.005751, + "nauc_mrr_at_100_diff1": 0.594803, + "nauc_mrr_at_1000_max": 0.393037, + "nauc_mrr_at_1000_std": -0.005825, + "nauc_mrr_at_1000_diff1": 0.594915, + "main_score": 0.5648, + "hf_subset": "java", + "languages": [ + "java-Code" + ] + }, + { + "ndcg_at_1": 0.35379, + "ndcg_at_3": 0.43718, + "ndcg_at_5": 0.46145, + "ndcg_at_10": 0.48494, + "ndcg_at_20": 0.50143, + "ndcg_at_100": 0.52502, + "ndcg_at_1000": 0.53943, + "map_at_1": 0.35379, + "map_at_3": 0.4167, + "map_at_5": 0.43015, + "map_at_10": 0.43992, + "map_at_20": 0.44449, + "map_at_100": 0.44775, + "map_at_1000": 0.44825, + "recall_at_1": 0.35379, + "recall_at_3": 0.49643, + "recall_at_5": 0.55544, + "recall_at_10": 0.62766, + "recall_at_20": 0.69252, + "recall_at_100": 0.81982, + "recall_at_1000": 0.93528, + "precision_at_1": 0.35379, + "precision_at_3": 0.16548, + "precision_at_5": 0.11109, + "precision_at_10": 0.06277, + "precision_at_20": 0.03463, + "precision_at_100": 0.0082, + "precision_at_1000": 0.00094, + "mrr_at_1": 0.353789, + "mrr_at_3": 0.416702, + "mrr_at_5": 0.43015, + "mrr_at_10": 0.439919, + "mrr_at_20": 0.444492, + "mrr_at_100": 0.44775, + "mrr_at_1000": 0.448253, + "nauc_ndcg_at_1_max": 0.334294, + "nauc_ndcg_at_1_std": -0.0371, + "nauc_ndcg_at_1_diff1": 0.578663, + "nauc_ndcg_at_3_max": 0.344521, + "nauc_ndcg_at_3_std": -0.021576, + "nauc_ndcg_at_3_diff1": 0.517994, + "nauc_ndcg_at_5_max": 0.344413, + "nauc_ndcg_at_5_std": -0.012455, + "nauc_ndcg_at_5_diff1": 0.505459, + "nauc_ndcg_at_10_max": 0.346621, + "nauc_ndcg_at_10_std": -0.003171, + "nauc_ndcg_at_10_diff1": 0.49922, + "nauc_ndcg_at_20_max": 0.349445, + "nauc_ndcg_at_20_std": 0.004017, + "nauc_ndcg_at_20_diff1": 0.49798, + "nauc_ndcg_at_100_max": 0.35365, + "nauc_ndcg_at_100_std": 0.014641, + "nauc_ndcg_at_100_diff1": 0.499978, + "nauc_ndcg_at_1000_max": 0.353678, + "nauc_ndcg_at_1000_std": 0.011098, + "nauc_ndcg_at_1000_diff1": 0.504946, + "nauc_map_at_1_max": 0.334294, + "nauc_map_at_1_std": -0.0371, + "nauc_map_at_1_diff1": 0.578663, + "nauc_map_at_3_max": 0.342633, + "nauc_map_at_3_std": -0.025287, + "nauc_map_at_3_diff1": 0.532162, + "nauc_map_at_5_max": 0.342658, + "nauc_map_at_5_std": -0.020294, + "nauc_map_at_5_diff1": 0.525507, + "nauc_map_at_10_max": 0.343554, + "nauc_map_at_10_std": -0.016635, + "nauc_map_at_10_diff1": 0.523157, + "nauc_map_at_20_max": 0.344323, + "nauc_map_at_20_std": -0.014723, + "nauc_map_at_20_diff1": 0.523029, + "nauc_map_at_100_max": 0.344811, + "nauc_map_at_100_std": -0.013521, + "nauc_map_at_100_diff1": 0.523415, + "nauc_map_at_1000_max": 0.344842, + "nauc_map_at_1000_std": -0.013589, + "nauc_map_at_1000_diff1": 0.523591, + "nauc_recall_at_1_max": 0.334294, + "nauc_recall_at_1_std": -0.0371, + "nauc_recall_at_1_diff1": 0.578663, + "nauc_recall_at_3_max": 0.349771, + "nauc_recall_at_3_std": -0.010794, + "nauc_recall_at_3_diff1": 0.476966, + "nauc_recall_at_5_max": 0.349342, + "nauc_recall_at_5_std": 0.012766, + "nauc_recall_at_5_diff1": 0.442598, + "nauc_recall_at_10_max": 0.357316, + "nauc_recall_at_10_std": 0.047274, + "nauc_recall_at_10_diff1": 0.41501, + "nauc_recall_at_20_max": 0.372048, + "nauc_recall_at_20_std": 0.088, + "nauc_recall_at_20_diff1": 0.397268, + "nauc_recall_at_100_max": 0.425591, + "nauc_recall_at_100_std": 0.237778, + "nauc_recall_at_100_diff1": 0.362165, + "nauc_recall_at_1000_max": 0.530576, + "nauc_recall_at_1000_std": 0.448275, + "nauc_recall_at_1000_diff1": 0.322277, + "nauc_precision_at_1_max": 0.334294, + "nauc_precision_at_1_std": -0.0371, + "nauc_precision_at_1_diff1": 0.578663, + "nauc_precision_at_3_max": 0.349771, + "nauc_precision_at_3_std": -0.010794, + "nauc_precision_at_3_diff1": 0.476966, + "nauc_precision_at_5_max": 0.349342, + "nauc_precision_at_5_std": 0.012766, + "nauc_precision_at_5_diff1": 0.442598, + "nauc_precision_at_10_max": 0.357316, + "nauc_precision_at_10_std": 0.047274, + "nauc_precision_at_10_diff1": 0.41501, + "nauc_precision_at_20_max": 0.372048, + "nauc_precision_at_20_std": 0.088, + "nauc_precision_at_20_diff1": 0.397268, + "nauc_precision_at_100_max": 0.425591, + "nauc_precision_at_100_std": 0.237778, + "nauc_precision_at_100_diff1": 0.362165, + "nauc_precision_at_1000_max": 0.530576, + "nauc_precision_at_1000_std": 0.448275, + "nauc_precision_at_1000_diff1": 0.322277, + "nauc_mrr_at_1_max": 0.334294, + "nauc_mrr_at_1_std": -0.0371, + "nauc_mrr_at_1_diff1": 0.578663, + "nauc_mrr_at_3_max": 0.342633, + "nauc_mrr_at_3_std": -0.025287, + "nauc_mrr_at_3_diff1": 0.532162, + "nauc_mrr_at_5_max": 0.342658, + "nauc_mrr_at_5_std": -0.020294, + "nauc_mrr_at_5_diff1": 0.525507, + "nauc_mrr_at_10_max": 0.343554, + "nauc_mrr_at_10_std": -0.016635, + "nauc_mrr_at_10_diff1": 0.523157, + "nauc_mrr_at_20_max": 0.344323, + "nauc_mrr_at_20_std": -0.014723, + "nauc_mrr_at_20_diff1": 0.523029, + "nauc_mrr_at_100_max": 0.344811, + "nauc_mrr_at_100_std": -0.013521, + "nauc_mrr_at_100_diff1": 0.523415, + "nauc_mrr_at_1000_max": 0.344842, + "nauc_mrr_at_1000_std": -0.013589, + "nauc_mrr_at_1000_diff1": 0.523591, + "main_score": 0.48494, + "hf_subset": "php", + "languages": [ + "php-Code" + ] + } + ] + }, + "evaluation_time": 406.73168182373047, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CQADupstackGamingRetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CQADupstackGamingRetrieval.json new file mode 100644 index 0000000000..ccbecf3ca9 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CQADupstackGamingRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4885aa143210c98657558c04aaf3dc47cfb54340", + "task_name": "CQADupstackGamingRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.49718, + "ndcg_at_3": 0.57099, + "ndcg_at_5": 0.59911, + "ndcg_at_10": 0.62437, + "ndcg_at_20": 0.64042, + "ndcg_at_100": 0.6613, + "ndcg_at_1000": 0.66899, + "map_at_1": 0.43792, + "map_at_3": 0.53401, + "map_at_5": 0.55451, + "map_at_10": 0.56774, + "map_at_20": 0.57375, + "map_at_100": 0.57759, + "map_at_1000": 0.57802, + "recall_at_1": 0.43792, + "recall_at_3": 0.61795, + "recall_at_5": 0.68547, + "recall_at_10": 0.75835, + "recall_at_20": 0.81649, + "recall_at_100": 0.91731, + "recall_at_1000": 0.9708, + "precision_at_1": 0.49718, + "precision_at_3": 0.25392, + "precision_at_5": 0.17417, + "precision_at_10": 0.09918, + "precision_at_20": 0.05486, + "precision_at_100": 0.01269, + "precision_at_1000": 0.00137, + "mrr_at_1": 0.497179, + "mrr_at_3": 0.575758, + "mrr_at_5": 0.588924, + "mrr_at_10": 0.597745, + "mrr_at_20": 0.601456, + "mrr_at_100": 0.603711, + "mrr_at_1000": 0.603891, + "nauc_ndcg_at_1_max": 0.423797, + "nauc_ndcg_at_1_std": -0.038729, + "nauc_ndcg_at_1_diff1": 0.579368, + "nauc_ndcg_at_3_max": 0.435493, + "nauc_ndcg_at_3_std": -0.031404, + "nauc_ndcg_at_3_diff1": 0.548577, + "nauc_ndcg_at_5_max": 0.442088, + "nauc_ndcg_at_5_std": -0.022583, + "nauc_ndcg_at_5_diff1": 0.541478, + "nauc_ndcg_at_10_max": 0.457113, + "nauc_ndcg_at_10_std": 0.004274, + "nauc_ndcg_at_10_diff1": 0.535977, + "nauc_ndcg_at_20_max": 0.463343, + "nauc_ndcg_at_20_std": 0.016857, + "nauc_ndcg_at_20_diff1": 0.533201, + "nauc_ndcg_at_100_max": 0.463056, + "nauc_ndcg_at_100_std": 0.020003, + "nauc_ndcg_at_100_diff1": 0.537594, + "nauc_ndcg_at_1000_max": 0.461856, + "nauc_ndcg_at_1000_std": 0.014999, + "nauc_ndcg_at_1000_diff1": 0.542213, + "nauc_map_at_1_max": 0.351369, + "nauc_map_at_1_std": -0.062177, + "nauc_map_at_1_diff1": 0.578384, + "nauc_map_at_3_max": 0.411584, + "nauc_map_at_3_std": -0.053996, + "nauc_map_at_3_diff1": 0.559004, + "nauc_map_at_5_max": 0.421368, + "nauc_map_at_5_std": -0.044566, + "nauc_map_at_5_diff1": 0.552954, + "nauc_map_at_10_max": 0.431699, + "nauc_map_at_10_std": -0.030584, + "nauc_map_at_10_diff1": 0.551363, + "nauc_map_at_20_max": 0.436946, + "nauc_map_at_20_std": -0.023452, + "nauc_map_at_20_diff1": 0.550295, + "nauc_map_at_100_max": 0.438269, + "nauc_map_at_100_std": -0.020811, + "nauc_map_at_100_diff1": 0.550844, + "nauc_map_at_1000_max": 0.438508, + "nauc_map_at_1000_std": -0.020658, + "nauc_map_at_1000_diff1": 0.55108, + "nauc_recall_at_1_max": 0.351369, + "nauc_recall_at_1_std": -0.062177, + "nauc_recall_at_1_diff1": 0.578384, + "nauc_recall_at_3_max": 0.416772, + "nauc_recall_at_3_std": -0.042838, + "nauc_recall_at_3_diff1": 0.510593, + "nauc_recall_at_5_max": 0.432018, + "nauc_recall_at_5_std": -0.012544, + "nauc_recall_at_5_diff1": 0.483594, + "nauc_recall_at_10_max": 0.485854, + "nauc_recall_at_10_std": 0.088116, + "nauc_recall_at_10_diff1": 0.448064, + "nauc_recall_at_20_max": 0.5325, + "nauc_recall_at_20_std": 0.18427, + "nauc_recall_at_20_diff1": 0.415908, + "nauc_recall_at_100_max": 0.585242, + "nauc_recall_at_100_std": 0.353863, + "nauc_recall_at_100_diff1": 0.384046, + "nauc_recall_at_1000_max": 0.732707, + "nauc_recall_at_1000_std": 0.566064, + "nauc_recall_at_1000_diff1": 0.473154, + "nauc_precision_at_1_max": 0.423797, + "nauc_precision_at_1_std": -0.038729, + "nauc_precision_at_1_diff1": 0.579368, + "nauc_precision_at_3_max": 0.419115, + "nauc_precision_at_3_std": 0.046917, + "nauc_precision_at_3_diff1": 0.315282, + "nauc_precision_at_5_max": 0.378557, + "nauc_precision_at_5_std": 0.098138, + "nauc_precision_at_5_diff1": 0.200243, + "nauc_precision_at_10_max": 0.363736, + "nauc_precision_at_10_std": 0.197559, + "nauc_precision_at_10_diff1": 0.102206, + "nauc_precision_at_20_max": 0.341139, + "nauc_precision_at_20_std": 0.266776, + "nauc_precision_at_20_diff1": 0.015639, + "nauc_precision_at_100_max": 0.253722, + "nauc_precision_at_100_std": 0.305487, + "nauc_precision_at_100_diff1": -0.082917, + "nauc_precision_at_1000_max": 0.198171, + "nauc_precision_at_1000_std": 0.284993, + "nauc_precision_at_1000_diff1": -0.113903, + "nauc_mrr_at_1_max": 0.423797, + "nauc_mrr_at_1_std": -0.038729, + "nauc_mrr_at_1_diff1": 0.579368, + "nauc_mrr_at_3_max": 0.449278, + "nauc_mrr_at_3_std": -0.015739, + "nauc_mrr_at_3_diff1": 0.553644, + "nauc_mrr_at_5_max": 0.44893, + "nauc_mrr_at_5_std": -0.013214, + "nauc_mrr_at_5_diff1": 0.552048, + "nauc_mrr_at_10_max": 0.45243, + "nauc_mrr_at_10_std": -0.004754, + "nauc_mrr_at_10_diff1": 0.549742, + "nauc_mrr_at_20_max": 0.452351, + "nauc_mrr_at_20_std": -0.003561, + "nauc_mrr_at_20_diff1": 0.549807, + "nauc_mrr_at_100_max": 0.451838, + "nauc_mrr_at_100_std": -0.004591, + "nauc_mrr_at_100_diff1": 0.550338, + "nauc_mrr_at_1000_max": 0.451777, + "nauc_mrr_at_1000_std": -0.004759, + "nauc_mrr_at_1000_diff1": 0.550432, + "main_score": 0.62437, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 19.061913013458252, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CQADupstackRetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CQADupstackRetrieval.json new file mode 100644 index 0000000000..7f0a797c05 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CQADupstackRetrieval.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "1", + "task_name": "CQADupstackRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_10": 0.478338, + "main_score": 0.478338, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 361.6380100250244, + "kg_co2_emissions": NaN +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CQADupstackUnixRetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CQADupstackUnixRetrieval.json new file mode 100644 index 0000000000..844aee27b7 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CQADupstackUnixRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53", + "task_name": "CQADupstackUnixRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.39459, + "ndcg_at_3": 0.453, + "ndcg_at_5": 0.47892, + "ndcg_at_10": 0.51134, + "ndcg_at_20": 0.53233, + "ndcg_at_100": 0.56209, + "ndcg_at_1000": 0.57793, + "map_at_1": 0.33686, + "map_at_3": 0.4153, + "map_at_5": 0.43445, + "map_at_10": 0.4508, + "map_at_20": 0.45788, + "map_at_100": 0.46288, + "map_at_1000": 0.46373, + "recall_at_1": 0.33686, + "recall_at_3": 0.49246, + "recall_at_5": 0.56036, + "recall_at_10": 0.65429, + "recall_at_20": 0.72764, + "recall_at_100": 0.86745, + "recall_at_1000": 0.97177, + "precision_at_1": 0.39459, + "precision_at_3": 0.20771, + "precision_at_5": 0.14552, + "precision_at_10": 0.08731, + "precision_at_20": 0.04963, + "precision_at_100": 0.01256, + "precision_at_1000": 0.00149, + "mrr_at_1": 0.39459, + "mrr_at_3": 0.463464, + "mrr_at_5": 0.478063, + "mrr_at_10": 0.490645, + "mrr_at_20": 0.495767, + "mrr_at_100": 0.498963, + "mrr_at_1000": 0.499363, + "nauc_ndcg_at_1_max": 0.421641, + "nauc_ndcg_at_1_std": -0.045422, + "nauc_ndcg_at_1_diff1": 0.487875, + "nauc_ndcg_at_3_max": 0.416734, + "nauc_ndcg_at_3_std": -0.014502, + "nauc_ndcg_at_3_diff1": 0.422467, + "nauc_ndcg_at_5_max": 0.410927, + "nauc_ndcg_at_5_std": -0.009626, + "nauc_ndcg_at_5_diff1": 0.411366, + "nauc_ndcg_at_10_max": 0.421622, + "nauc_ndcg_at_10_std": -0.001765, + "nauc_ndcg_at_10_diff1": 0.41256, + "nauc_ndcg_at_20_max": 0.418883, + "nauc_ndcg_at_20_std": 0.001652, + "nauc_ndcg_at_20_diff1": 0.412708, + "nauc_ndcg_at_100_max": 0.418798, + "nauc_ndcg_at_100_std": 0.007439, + "nauc_ndcg_at_100_diff1": 0.416983, + "nauc_ndcg_at_1000_max": 0.420799, + "nauc_ndcg_at_1000_std": 0.003409, + "nauc_ndcg_at_1000_diff1": 0.418303, + "nauc_map_at_1_max": 0.379607, + "nauc_map_at_1_std": -0.046727, + "nauc_map_at_1_diff1": 0.490663, + "nauc_map_at_3_max": 0.407221, + "nauc_map_at_3_std": -0.021387, + "nauc_map_at_3_diff1": 0.442901, + "nauc_map_at_5_max": 0.410473, + "nauc_map_at_5_std": -0.018574, + "nauc_map_at_5_diff1": 0.436096, + "nauc_map_at_10_max": 0.417498, + "nauc_map_at_10_std": -0.016482, + "nauc_map_at_10_diff1": 0.436113, + "nauc_map_at_20_max": 0.417093, + "nauc_map_at_20_std": -0.015499, + "nauc_map_at_20_diff1": 0.43541, + "nauc_map_at_100_max": 0.416862, + "nauc_map_at_100_std": -0.01511, + "nauc_map_at_100_diff1": 0.435334, + "nauc_map_at_1000_max": 0.416893, + "nauc_map_at_1000_std": -0.01508, + "nauc_map_at_1000_diff1": 0.435293, + "nauc_recall_at_1_max": 0.379607, + "nauc_recall_at_1_std": -0.046727, + "nauc_recall_at_1_diff1": 0.490663, + "nauc_recall_at_3_max": 0.39207, + "nauc_recall_at_3_std": 0.004002, + "nauc_recall_at_3_diff1": 0.366255, + "nauc_recall_at_5_max": 0.376865, + "nauc_recall_at_5_std": 0.015179, + "nauc_recall_at_5_diff1": 0.329549, + "nauc_recall_at_10_max": 0.391869, + "nauc_recall_at_10_std": 0.040613, + "nauc_recall_at_10_diff1": 0.314039, + "nauc_recall_at_20_max": 0.376491, + "nauc_recall_at_20_std": 0.06474, + "nauc_recall_at_20_diff1": 0.309795, + "nauc_recall_at_100_max": 0.374809, + "nauc_recall_at_100_std": 0.192545, + "nauc_recall_at_100_diff1": 0.324547, + "nauc_recall_at_1000_max": 0.51495, + "nauc_recall_at_1000_std": 0.538269, + "nauc_recall_at_1000_diff1": 0.221151, + "nauc_precision_at_1_max": 0.421641, + "nauc_precision_at_1_std": -0.045422, + "nauc_precision_at_1_diff1": 0.487875, + "nauc_precision_at_3_max": 0.391093, + "nauc_precision_at_3_std": 0.014533, + "nauc_precision_at_3_diff1": 0.262718, + "nauc_precision_at_5_max": 0.346393, + "nauc_precision_at_5_std": 0.028347, + "nauc_precision_at_5_diff1": 0.182454, + "nauc_precision_at_10_max": 0.312046, + "nauc_precision_at_10_std": 0.052469, + "nauc_precision_at_10_diff1": 0.115251, + "nauc_precision_at_20_max": 0.249726, + "nauc_precision_at_20_std": 0.060368, + "nauc_precision_at_20_diff1": 0.049952, + "nauc_precision_at_100_max": 0.076875, + "nauc_precision_at_100_std": 0.068812, + "nauc_precision_at_100_diff1": -0.080161, + "nauc_precision_at_1000_max": -0.054508, + "nauc_precision_at_1000_std": 0.018531, + "nauc_precision_at_1000_diff1": -0.163702, + "nauc_mrr_at_1_max": 0.421641, + "nauc_mrr_at_1_std": -0.045422, + "nauc_mrr_at_1_diff1": 0.487875, + "nauc_mrr_at_3_max": 0.427155, + "nauc_mrr_at_3_std": -0.019125, + "nauc_mrr_at_3_diff1": 0.438535, + "nauc_mrr_at_5_max": 0.42142, + "nauc_mrr_at_5_std": -0.016321, + "nauc_mrr_at_5_diff1": 0.432664, + "nauc_mrr_at_10_max": 0.424076, + "nauc_mrr_at_10_std": -0.013508, + "nauc_mrr_at_10_diff1": 0.432442, + "nauc_mrr_at_20_max": 0.42241, + "nauc_mrr_at_20_std": -0.013284, + "nauc_mrr_at_20_diff1": 0.432795, + "nauc_mrr_at_100_max": 0.422675, + "nauc_mrr_at_100_std": -0.013217, + "nauc_mrr_at_100_diff1": 0.433597, + "nauc_mrr_at_1000_max": 0.422852, + "nauc_mrr_at_1000_std": -0.013242, + "nauc_mrr_at_1000_diff1": 0.433731, + "main_score": 0.51134, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 36.18085241317749, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ClimateFEVER.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ClimateFEVER.json new file mode 100644 index 0000000000..3a82cdae26 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ClimateFEVER.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380", + "task_name": "ClimateFEVER", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.30554, + "ndcg_at_3": 0.26134, + "ndcg_at_5": 0.27962, + "ndcg_at_10": 0.31557, + "ndcg_at_20": 0.34339, + "ndcg_at_100": 0.38373, + "ndcg_at_1000": 0.41881, + "map_at_1": 0.136, + "map_at_3": 0.19065, + "map_at_5": 0.20872, + "map_at_10": 0.22645, + "map_at_20": 0.23659, + "map_at_100": 0.24469, + "map_at_1000": 0.24665, + "recall_at_1": 0.136, + "recall_at_3": 0.23837, + "recall_at_5": 0.29308, + "recall_at_10": 0.37277, + "recall_at_20": 0.45126, + "recall_at_100": 0.60613, + "recall_at_1000": 0.80316, + "precision_at_1": 0.30554, + "precision_at_3": 0.19283, + "precision_at_5": 0.14684, + "precision_at_10": 0.09733, + "precision_at_20": 0.06075, + "precision_at_100": 0.01702, + "precision_at_1000": 0.00236, + "mrr_at_1": 0.305537, + "mrr_at_3": 0.393702, + "mrr_at_5": 0.412432, + "mrr_at_10": 0.425944, + "mrr_at_20": 0.431046, + "mrr_at_100": 0.433628, + "mrr_at_1000": 0.434017, + "nauc_ndcg_at_1_max": 0.346221, + "nauc_ndcg_at_1_std": 0.166254, + "nauc_ndcg_at_1_diff1": 0.279232, + "nauc_ndcg_at_3_max": 0.374692, + "nauc_ndcg_at_3_std": 0.169178, + "nauc_ndcg_at_3_diff1": 0.240668, + "nauc_ndcg_at_5_max": 0.395978, + "nauc_ndcg_at_5_std": 0.197707, + "nauc_ndcg_at_5_diff1": 0.233957, + "nauc_ndcg_at_10_max": 0.401645, + "nauc_ndcg_at_10_std": 0.226743, + "nauc_ndcg_at_10_diff1": 0.21813, + "nauc_ndcg_at_20_max": 0.407321, + "nauc_ndcg_at_20_std": 0.238535, + "nauc_ndcg_at_20_diff1": 0.218055, + "nauc_ndcg_at_100_max": 0.408484, + "nauc_ndcg_at_100_std": 0.256218, + "nauc_ndcg_at_100_diff1": 0.210727, + "nauc_ndcg_at_1000_max": 0.409889, + "nauc_ndcg_at_1000_std": 0.258318, + "nauc_ndcg_at_1000_diff1": 0.217926, + "nauc_map_at_1_max": 0.401298, + "nauc_map_at_1_std": 0.139453, + "nauc_map_at_1_diff1": 0.339618, + "nauc_map_at_3_max": 0.386942, + "nauc_map_at_3_std": 0.143665, + "nauc_map_at_3_diff1": 0.26919, + "nauc_map_at_5_max": 0.395658, + "nauc_map_at_5_std": 0.168043, + "nauc_map_at_5_diff1": 0.257238, + "nauc_map_at_10_max": 0.397044, + "nauc_map_at_10_std": 0.186281, + "nauc_map_at_10_diff1": 0.243041, + "nauc_map_at_20_max": 0.400293, + "nauc_map_at_20_std": 0.192902, + "nauc_map_at_20_diff1": 0.242403, + "nauc_map_at_100_max": 0.401632, + "nauc_map_at_100_std": 0.198219, + "nauc_map_at_100_diff1": 0.240672, + "nauc_map_at_1000_max": 0.401812, + "nauc_map_at_1000_std": 0.198807, + "nauc_map_at_1000_diff1": 0.241117, + "nauc_recall_at_1_max": 0.401298, + "nauc_recall_at_1_std": 0.139453, + "nauc_recall_at_1_diff1": 0.339618, + "nauc_recall_at_3_max": 0.362432, + "nauc_recall_at_3_std": 0.153455, + "nauc_recall_at_3_diff1": 0.215589, + "nauc_recall_at_5_max": 0.373746, + "nauc_recall_at_5_std": 0.200884, + "nauc_recall_at_5_diff1": 0.183423, + "nauc_recall_at_10_max": 0.366813, + "nauc_recall_at_10_std": 0.247509, + "nauc_recall_at_10_diff1": 0.145858, + "nauc_recall_at_20_max": 0.358951, + "nauc_recall_at_20_std": 0.261103, + "nauc_recall_at_20_diff1": 0.133872, + "nauc_recall_at_100_max": 0.341472, + "nauc_recall_at_100_std": 0.311823, + "nauc_recall_at_100_diff1": 0.095465, + "nauc_recall_at_1000_max": 0.346908, + "nauc_recall_at_1000_std": 0.369776, + "nauc_recall_at_1000_diff1": 0.10488, + "nauc_precision_at_1_max": 0.346221, + "nauc_precision_at_1_std": 0.166254, + "nauc_precision_at_1_diff1": 0.279232, + "nauc_precision_at_3_max": 0.320378, + "nauc_precision_at_3_std": 0.192949, + "nauc_precision_at_3_diff1": 0.133217, + "nauc_precision_at_5_max": 0.318872, + "nauc_precision_at_5_std": 0.242223, + "nauc_precision_at_5_diff1": 0.096956, + "nauc_precision_at_10_max": 0.287361, + "nauc_precision_at_10_std": 0.28154, + "nauc_precision_at_10_diff1": 0.040429, + "nauc_precision_at_20_max": 0.256129, + "nauc_precision_at_20_std": 0.273047, + "nauc_precision_at_20_diff1": 0.034396, + "nauc_precision_at_100_max": 0.175244, + "nauc_precision_at_100_std": 0.272286, + "nauc_precision_at_100_diff1": -0.023567, + "nauc_precision_at_1000_max": 0.056622, + "nauc_precision_at_1000_std": 0.194828, + "nauc_precision_at_1000_diff1": -0.056242, + "nauc_mrr_at_1_max": 0.346221, + "nauc_mrr_at_1_std": 0.166254, + "nauc_mrr_at_1_diff1": 0.279232, + "nauc_mrr_at_3_max": 0.363959, + "nauc_mrr_at_3_std": 0.195216, + "nauc_mrr_at_3_diff1": 0.2409, + "nauc_mrr_at_5_max": 0.3741, + "nauc_mrr_at_5_std": 0.206158, + "nauc_mrr_at_5_diff1": 0.238697, + "nauc_mrr_at_10_max": 0.375779, + "nauc_mrr_at_10_std": 0.21268, + "nauc_mrr_at_10_diff1": 0.23932, + "nauc_mrr_at_20_max": 0.375841, + "nauc_mrr_at_20_std": 0.213451, + "nauc_mrr_at_20_diff1": 0.24001, + "nauc_mrr_at_100_max": 0.375249, + "nauc_mrr_at_100_std": 0.212732, + "nauc_mrr_at_100_diff1": 0.239741, + "nauc_mrr_at_1000_max": 0.375119, + "nauc_mrr_at_1000_std": 0.212463, + "nauc_mrr_at_1000_diff1": 0.239858, + "main_score": 0.31557, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3389.7072932720184, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ClimateFEVERHardNegatives.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ClimateFEVERHardNegatives.json new file mode 100644 index 0000000000..e00969b35e --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ClimateFEVERHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "3a309e201f3c2c4b13bd4a367a8f37eee2ec1d21", + "task_name": "ClimateFEVERHardNegatives", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.302, + "ndcg_at_3": 0.25808, + "ndcg_at_5": 0.27852, + "ndcg_at_10": 0.31688, + "ndcg_at_20": 0.3484, + "ndcg_at_100": 0.39255, + "ndcg_at_1000": 0.43152, + "map_at_1": 0.13777, + "map_at_3": 0.19038, + "map_at_5": 0.2095, + "map_at_10": 0.22827, + "map_at_20": 0.23949, + "map_at_100": 0.2485, + "map_at_1000": 0.25077, + "recall_at_1": 0.13777, + "recall_at_3": 0.23587, + "recall_at_5": 0.29312, + "recall_at_10": 0.37763, + "recall_at_20": 0.46765, + "recall_at_100": 0.6369, + "recall_at_1000": 0.85217, + "precision_at_1": 0.302, + "precision_at_3": 0.18867, + "precision_at_5": 0.1458, + "precision_at_10": 0.0986, + "precision_at_20": 0.0627, + "precision_at_100": 0.01787, + "precision_at_1000": 0.00253, + "mrr_at_1": 0.302, + "mrr_at_3": 0.386333, + "mrr_at_5": 0.405233, + "mrr_at_10": 0.42, + "mrr_at_20": 0.426315, + "mrr_at_100": 0.42893, + "mrr_at_1000": 0.429311, + "nauc_ndcg_at_1_max": 0.321183, + "nauc_ndcg_at_1_std": 0.113036, + "nauc_ndcg_at_1_diff1": 0.249704, + "nauc_ndcg_at_3_max": 0.36871, + "nauc_ndcg_at_3_std": 0.131328, + "nauc_ndcg_at_3_diff1": 0.225102, + "nauc_ndcg_at_5_max": 0.385086, + "nauc_ndcg_at_5_std": 0.147506, + "nauc_ndcg_at_5_diff1": 0.218208, + "nauc_ndcg_at_10_max": 0.392165, + "nauc_ndcg_at_10_std": 0.168916, + "nauc_ndcg_at_10_diff1": 0.20046, + "nauc_ndcg_at_20_max": 0.400084, + "nauc_ndcg_at_20_std": 0.173684, + "nauc_ndcg_at_20_diff1": 0.198403, + "nauc_ndcg_at_100_max": 0.39762, + "nauc_ndcg_at_100_std": 0.177175, + "nauc_ndcg_at_100_diff1": 0.18984, + "nauc_ndcg_at_1000_max": 0.397554, + "nauc_ndcg_at_1000_std": 0.17115, + "nauc_ndcg_at_1000_diff1": 0.19968, + "nauc_map_at_1_max": 0.393815, + "nauc_map_at_1_std": 0.084003, + "nauc_map_at_1_diff1": 0.32727, + "nauc_map_at_3_max": 0.375718, + "nauc_map_at_3_std": 0.102787, + "nauc_map_at_3_diff1": 0.25318, + "nauc_map_at_5_max": 0.385513, + "nauc_map_at_5_std": 0.120846, + "nauc_map_at_5_diff1": 0.242787, + "nauc_map_at_10_max": 0.388242, + "nauc_map_at_10_std": 0.133999, + "nauc_map_at_10_diff1": 0.227356, + "nauc_map_at_20_max": 0.391979, + "nauc_map_at_20_std": 0.138045, + "nauc_map_at_20_diff1": 0.224618, + "nauc_map_at_100_max": 0.393317, + "nauc_map_at_100_std": 0.140371, + "nauc_map_at_100_diff1": 0.222964, + "nauc_map_at_1000_max": 0.393164, + "nauc_map_at_1000_std": 0.139924, + "nauc_map_at_1000_diff1": 0.223728, + "nauc_recall_at_1_max": 0.393815, + "nauc_recall_at_1_std": 0.084003, + "nauc_recall_at_1_diff1": 0.32727, + "nauc_recall_at_3_max": 0.363412, + "nauc_recall_at_3_std": 0.126906, + "nauc_recall_at_3_diff1": 0.2019, + "nauc_recall_at_5_max": 0.368847, + "nauc_recall_at_5_std": 0.157553, + "nauc_recall_at_5_diff1": 0.171689, + "nauc_recall_at_10_max": 0.365084, + "nauc_recall_at_10_std": 0.191876, + "nauc_recall_at_10_diff1": 0.12704, + "nauc_recall_at_20_max": 0.361457, + "nauc_recall_at_20_std": 0.19269, + "nauc_recall_at_20_diff1": 0.113566, + "nauc_recall_at_100_max": 0.326609, + "nauc_recall_at_100_std": 0.197306, + "nauc_recall_at_100_diff1": 0.064856, + "nauc_recall_at_1000_max": 0.335686, + "nauc_recall_at_1000_std": 0.190615, + "nauc_recall_at_1000_diff1": 0.071827, + "nauc_precision_at_1_max": 0.321183, + "nauc_precision_at_1_std": 0.113036, + "nauc_precision_at_1_diff1": 0.249704, + "nauc_precision_at_3_max": 0.315856, + "nauc_precision_at_3_std": 0.167539, + "nauc_precision_at_3_diff1": 0.113458, + "nauc_precision_at_5_max": 0.30808, + "nauc_precision_at_5_std": 0.179023, + "nauc_precision_at_5_diff1": 0.08675, + "nauc_precision_at_10_max": 0.28045, + "nauc_precision_at_10_std": 0.206676, + "nauc_precision_at_10_diff1": 0.033775, + "nauc_precision_at_20_max": 0.257233, + "nauc_precision_at_20_std": 0.190482, + "nauc_precision_at_20_diff1": 0.016421, + "nauc_precision_at_100_max": 0.159986, + "nauc_precision_at_100_std": 0.152132, + "nauc_precision_at_100_diff1": -0.049059, + "nauc_precision_at_1000_max": 0.013909, + "nauc_precision_at_1000_std": 0.059046, + "nauc_precision_at_1000_diff1": -0.078125, + "nauc_mrr_at_1_max": 0.321183, + "nauc_mrr_at_1_std": 0.113036, + "nauc_mrr_at_1_diff1": 0.249704, + "nauc_mrr_at_3_max": 0.347533, + "nauc_mrr_at_3_std": 0.152187, + "nauc_mrr_at_3_diff1": 0.213731, + "nauc_mrr_at_5_max": 0.354539, + "nauc_mrr_at_5_std": 0.157112, + "nauc_mrr_at_5_diff1": 0.214883, + "nauc_mrr_at_10_max": 0.357085, + "nauc_mrr_at_10_std": 0.161864, + "nauc_mrr_at_10_diff1": 0.215641, + "nauc_mrr_at_20_max": 0.357273, + "nauc_mrr_at_20_std": 0.160676, + "nauc_mrr_at_20_diff1": 0.216895, + "nauc_mrr_at_100_max": 0.356414, + "nauc_mrr_at_100_std": 0.160447, + "nauc_mrr_at_100_diff1": 0.216867, + "nauc_mrr_at_1000_max": 0.356383, + "nauc_mrr_at_1000_std": 0.160323, + "nauc_mrr_at_1000_diff1": 0.21695, + "main_score": 0.31688, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 36.81579566001892, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeFeedbackMT.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeFeedbackMT.json new file mode 100644 index 0000000000..bee06fa6db --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeFeedbackMT.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b0f12fa0c0dd67f59c95a5c33d02aeeb4c398c5f", + "task_name": "CodeFeedbackMT", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.39768, + "ndcg_at_3": 0.47817, + "ndcg_at_5": 0.5004, + "ndcg_at_10": 0.52192, + "ndcg_at_20": 0.53698, + "ndcg_at_100": 0.55832, + "ndcg_at_1000": 0.57211, + "map_at_1": 0.39768, + "map_at_3": 0.45863, + "map_at_5": 0.47097, + "map_at_10": 0.47997, + "map_at_20": 0.48412, + "map_at_100": 0.48698, + "map_at_1000": 0.48746, + "recall_at_1": 0.39768, + "recall_at_3": 0.53461, + "recall_at_5": 0.58854, + "recall_at_10": 0.65452, + "recall_at_20": 0.71394, + "recall_at_100": 0.83031, + "recall_at_1000": 0.94095, + "precision_at_1": 0.39768, + "precision_at_3": 0.1782, + "precision_at_5": 0.11771, + "precision_at_10": 0.06545, + "precision_at_20": 0.0357, + "precision_at_100": 0.0083, + "precision_at_1000": 0.00094, + "mrr_at_1": 0.39768, + "mrr_at_3": 0.458625, + "mrr_at_5": 0.470974, + "mrr_at_10": 0.479967, + "mrr_at_20": 0.484118, + "mrr_at_100": 0.486977, + "mrr_at_1000": 0.487458, + "nauc_ndcg_at_1_max": 0.19407, + "nauc_ndcg_at_1_std": -0.217262, + "nauc_ndcg_at_1_diff1": 0.594624, + "nauc_ndcg_at_3_max": 0.218802, + "nauc_ndcg_at_3_std": -0.225638, + "nauc_ndcg_at_3_diff1": 0.553779, + "nauc_ndcg_at_5_max": 0.22128, + "nauc_ndcg_at_5_std": -0.222038, + "nauc_ndcg_at_5_diff1": 0.544866, + "nauc_ndcg_at_10_max": 0.224247, + "nauc_ndcg_at_10_std": -0.213667, + "nauc_ndcg_at_10_diff1": 0.540659, + "nauc_ndcg_at_20_max": 0.226388, + "nauc_ndcg_at_20_std": -0.210279, + "nauc_ndcg_at_20_diff1": 0.537077, + "nauc_ndcg_at_100_max": 0.225012, + "nauc_ndcg_at_100_std": -0.200868, + "nauc_ndcg_at_100_diff1": 0.538454, + "nauc_ndcg_at_1000_max": 0.223194, + "nauc_ndcg_at_1000_std": -0.201892, + "nauc_ndcg_at_1000_diff1": 0.542947, + "nauc_map_at_1_max": 0.19407, + "nauc_map_at_1_std": -0.217262, + "nauc_map_at_1_diff1": 0.594624, + "nauc_map_at_3_max": 0.212854, + "nauc_map_at_3_std": -0.223817, + "nauc_map_at_3_diff1": 0.563625, + "nauc_map_at_5_max": 0.214196, + "nauc_map_at_5_std": -0.221898, + "nauc_map_at_5_diff1": 0.558904, + "nauc_map_at_10_max": 0.215233, + "nauc_map_at_10_std": -0.21849, + "nauc_map_at_10_diff1": 0.557436, + "nauc_map_at_20_max": 0.215768, + "nauc_map_at_20_std": -0.217654, + "nauc_map_at_20_diff1": 0.556642, + "nauc_map_at_100_max": 0.215491, + "nauc_map_at_100_std": -0.216582, + "nauc_map_at_100_diff1": 0.556873, + "nauc_map_at_1000_max": 0.215437, + "nauc_map_at_1000_std": -0.216585, + "nauc_map_at_1000_diff1": 0.557018, + "nauc_recall_at_1_max": 0.19407, + "nauc_recall_at_1_std": -0.217262, + "nauc_recall_at_1_diff1": 0.594624, + "nauc_recall_at_3_max": 0.236617, + "nauc_recall_at_3_std": -0.231009, + "nauc_recall_at_3_diff1": 0.524274, + "nauc_recall_at_5_max": 0.243815, + "nauc_recall_at_5_std": -0.221711, + "nauc_recall_at_5_diff1": 0.499355, + "nauc_recall_at_10_max": 0.257437, + "nauc_recall_at_10_std": -0.192216, + "nauc_recall_at_10_diff1": 0.478712, + "nauc_recall_at_20_max": 0.272564, + "nauc_recall_at_20_std": -0.171892, + "nauc_recall_at_20_diff1": 0.451596, + "nauc_recall_at_100_max": 0.284924, + "nauc_recall_at_100_std": -0.053328, + "nauc_recall_at_100_diff1": 0.42072, + "nauc_recall_at_1000_max": 0.311533, + "nauc_recall_at_1000_std": 0.142382, + "nauc_recall_at_1000_diff1": 0.400505, + "nauc_precision_at_1_max": 0.19407, + "nauc_precision_at_1_std": -0.217262, + "nauc_precision_at_1_diff1": 0.594624, + "nauc_precision_at_3_max": 0.236617, + "nauc_precision_at_3_std": -0.231009, + "nauc_precision_at_3_diff1": 0.524274, + "nauc_precision_at_5_max": 0.243815, + "nauc_precision_at_5_std": -0.221711, + "nauc_precision_at_5_diff1": 0.499355, + "nauc_precision_at_10_max": 0.257437, + "nauc_precision_at_10_std": -0.192216, + "nauc_precision_at_10_diff1": 0.478712, + "nauc_precision_at_20_max": 0.272564, + "nauc_precision_at_20_std": -0.171892, + "nauc_precision_at_20_diff1": 0.451596, + "nauc_precision_at_100_max": 0.284924, + "nauc_precision_at_100_std": -0.053328, + "nauc_precision_at_100_diff1": 0.42072, + "nauc_precision_at_1000_max": 0.311533, + "nauc_precision_at_1000_std": 0.142382, + "nauc_precision_at_1000_diff1": 0.400505, + "nauc_mrr_at_1_max": 0.19407, + "nauc_mrr_at_1_std": -0.217262, + "nauc_mrr_at_1_diff1": 0.594624, + "nauc_mrr_at_3_max": 0.212854, + "nauc_mrr_at_3_std": -0.223817, + "nauc_mrr_at_3_diff1": 0.563625, + "nauc_mrr_at_5_max": 0.214196, + "nauc_mrr_at_5_std": -0.221898, + "nauc_mrr_at_5_diff1": 0.558904, + "nauc_mrr_at_10_max": 0.215233, + "nauc_mrr_at_10_std": -0.21849, + "nauc_mrr_at_10_diff1": 0.557436, + "nauc_mrr_at_20_max": 0.215768, + "nauc_mrr_at_20_std": -0.217654, + "nauc_mrr_at_20_diff1": 0.556642, + "nauc_mrr_at_100_max": 0.215491, + "nauc_mrr_at_100_std": -0.216582, + "nauc_mrr_at_100_diff1": 0.556873, + "nauc_mrr_at_1000_max": 0.215437, + "nauc_mrr_at_1000_std": -0.216585, + "nauc_mrr_at_1000_diff1": 0.557018, + "main_score": 0.52192, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 115.37248706817627, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeFeedbackST.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeFeedbackST.json new file mode 100644 index 0000000000..142bd626b4 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeFeedbackST.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "d213819e87aab9010628da8b73ab4eb337c89340", + "task_name": "CodeFeedbackST", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.61365, + "ndcg_at_3": 0.73113, + "ndcg_at_5": 0.75422, + "ndcg_at_10": 0.76852, + "ndcg_at_20": 0.77603, + "ndcg_at_100": 0.78307, + "ndcg_at_1000": 0.78522, + "map_at_1": 0.61365, + "map_at_3": 0.70279, + "map_at_5": 0.71568, + "map_at_10": 0.72167, + "map_at_20": 0.72375, + "map_at_100": 0.72476, + "map_at_1000": 0.72486, + "recall_at_1": 0.61365, + "recall_at_3": 0.81291, + "recall_at_5": 0.86865, + "recall_at_10": 0.91244, + "recall_at_20": 0.94196, + "recall_at_100": 0.97933, + "recall_at_1000": 0.99572, + "precision_at_1": 0.61365, + "precision_at_3": 0.27097, + "precision_at_5": 0.17373, + "precision_at_10": 0.09124, + "precision_at_20": 0.0471, + "precision_at_100": 0.00979, + "precision_at_1000": 0.001, + "mrr_at_1": 0.613141, + "mrr_at_3": 0.702411, + "mrr_at_5": 0.715418, + "mrr_at_10": 0.721394, + "mrr_at_20": 0.723487, + "mrr_at_100": 0.7245, + "mrr_at_1000": 0.72459, + "nauc_ndcg_at_1_max": 0.034951, + "nauc_ndcg_at_1_std": -0.268516, + "nauc_ndcg_at_1_diff1": 0.749819, + "nauc_ndcg_at_3_max": 0.085098, + "nauc_ndcg_at_3_std": -0.319908, + "nauc_ndcg_at_3_diff1": 0.700095, + "nauc_ndcg_at_5_max": 0.090283, + "nauc_ndcg_at_5_std": -0.321307, + "nauc_ndcg_at_5_diff1": 0.698093, + "nauc_ndcg_at_10_max": 0.084737, + "nauc_ndcg_at_10_std": -0.312965, + "nauc_ndcg_at_10_diff1": 0.700896, + "nauc_ndcg_at_20_max": 0.081619, + "nauc_ndcg_at_20_std": -0.306477, + "nauc_ndcg_at_20_diff1": 0.705384, + "nauc_ndcg_at_100_max": 0.077953, + "nauc_ndcg_at_100_std": -0.300432, + "nauc_ndcg_at_100_diff1": 0.70868, + "nauc_ndcg_at_1000_max": 0.076448, + "nauc_ndcg_at_1000_std": -0.300924, + "nauc_ndcg_at_1000_diff1": 0.710013, + "nauc_map_at_1_max": 0.034951, + "nauc_map_at_1_std": -0.268516, + "nauc_map_at_1_diff1": 0.749819, + "nauc_map_at_3_max": 0.070551, + "nauc_map_at_3_std": -0.305101, + "nauc_map_at_3_diff1": 0.713661, + "nauc_map_at_5_max": 0.072535, + "nauc_map_at_5_std": -0.305096, + "nauc_map_at_5_diff1": 0.713276, + "nauc_map_at_10_max": 0.070278, + "nauc_map_at_10_std": -0.301801, + "nauc_map_at_10_diff1": 0.714503, + "nauc_map_at_20_max": 0.069493, + "nauc_map_at_20_std": -0.300175, + "nauc_map_at_20_diff1": 0.715659, + "nauc_map_at_100_max": 0.06898, + "nauc_map_at_100_std": -0.299443, + "nauc_map_at_100_diff1": 0.716092, + "nauc_map_at_1000_max": 0.068937, + "nauc_map_at_1000_std": -0.299432, + "nauc_map_at_1000_diff1": 0.71614, + "nauc_recall_at_1_max": 0.034951, + "nauc_recall_at_1_std": -0.268516, + "nauc_recall_at_1_diff1": 0.749819, + "nauc_recall_at_3_max": 0.144442, + "nauc_recall_at_3_std": -0.380257, + "nauc_recall_at_3_diff1": 0.645267, + "nauc_recall_at_5_max": 0.186862, + "nauc_recall_at_5_std": -0.408801, + "nauc_recall_at_5_diff1": 0.616681, + "nauc_recall_at_10_max": 0.190655, + "nauc_recall_at_10_std": -0.390331, + "nauc_recall_at_10_diff1": 0.600787, + "nauc_recall_at_20_max": 0.20258, + "nauc_recall_at_20_std": -0.347528, + "nauc_recall_at_20_diff1": 0.607426, + "nauc_recall_at_100_max": 0.25492, + "nauc_recall_at_100_std": -0.139224, + "nauc_recall_at_100_diff1": 0.583678, + "nauc_recall_at_1000_max": 0.417273, + "nauc_recall_at_1000_std": 0.238259, + "nauc_recall_at_1000_diff1": 0.534128, + "nauc_precision_at_1_max": 0.034951, + "nauc_precision_at_1_std": -0.268516, + "nauc_precision_at_1_diff1": 0.749819, + "nauc_precision_at_3_max": 0.144442, + "nauc_precision_at_3_std": -0.380257, + "nauc_precision_at_3_diff1": 0.645267, + "nauc_precision_at_5_max": 0.186862, + "nauc_precision_at_5_std": -0.408801, + "nauc_precision_at_5_diff1": 0.616681, + "nauc_precision_at_10_max": 0.190655, + "nauc_precision_at_10_std": -0.390331, + "nauc_precision_at_10_diff1": 0.600787, + "nauc_precision_at_20_max": 0.20258, + "nauc_precision_at_20_std": -0.347528, + "nauc_precision_at_20_diff1": 0.607426, + "nauc_precision_at_100_max": 0.25492, + "nauc_precision_at_100_std": -0.139224, + "nauc_precision_at_100_diff1": 0.583678, + "nauc_precision_at_1000_max": 0.417273, + "nauc_precision_at_1000_std": 0.238259, + "nauc_precision_at_1000_diff1": 0.534128, + "nauc_mrr_at_1_max": 0.032917, + "nauc_mrr_at_1_std": -0.269459, + "nauc_mrr_at_1_diff1": 0.750479, + "nauc_mrr_at_3_max": 0.069216, + "nauc_mrr_at_3_std": -0.305261, + "nauc_mrr_at_3_diff1": 0.714295, + "nauc_mrr_at_5_max": 0.071255, + "nauc_mrr_at_5_std": -0.305637, + "nauc_mrr_at_5_diff1": 0.713854, + "nauc_mrr_at_10_max": 0.069031, + "nauc_mrr_at_10_std": -0.302176, + "nauc_mrr_at_10_diff1": 0.715064, + "nauc_mrr_at_20_max": 0.068251, + "nauc_mrr_at_20_std": -0.300588, + "nauc_mrr_at_20_diff1": 0.716222, + "nauc_mrr_at_100_max": 0.067724, + "nauc_mrr_at_100_std": -0.299864, + "nauc_mrr_at_100_diff1": 0.716657, + "nauc_mrr_at_1000_max": 0.067681, + "nauc_mrr_at_1000_std": -0.299853, + "nauc_mrr_at_1000_diff1": 0.716705, + "main_score": 0.76852, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 275.53679633140564, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeSearchNetCCRetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeSearchNetCCRetrieval.json new file mode 100644 index 0000000000..b70a1dceca --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeSearchNetCCRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "6e1effa2c03723c5fde48ee912b5ee08d4f211e8", + "task_name": "CodeSearchNetCCRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.38692, + "ndcg_at_3": 0.48062, + "ndcg_at_5": 0.50553, + "ndcg_at_10": 0.52769, + "ndcg_at_20": 0.54337, + "ndcg_at_100": 0.56304, + "ndcg_at_1000": 0.57537, + "map_at_1": 0.38692, + "map_at_3": 0.45796, + "map_at_5": 0.4718, + "map_at_10": 0.48101, + "map_at_20": 0.48532, + "map_at_100": 0.48801, + "map_at_1000": 0.48845, + "recall_at_1": 0.38692, + "recall_at_3": 0.54605, + "recall_at_5": 0.60645, + "recall_at_10": 0.67462, + "recall_at_20": 0.73656, + "recall_at_100": 0.84314, + "recall_at_1000": 0.94135, + "precision_at_1": 0.38692, + "precision_at_3": 0.18202, + "precision_at_5": 0.12129, + "precision_at_10": 0.06746, + "precision_at_20": 0.03683, + "precision_at_100": 0.00843, + "precision_at_1000": 0.00094, + "mrr_at_1": 0.387049, + "mrr_at_3": 0.458026, + "mrr_at_5": 0.471868, + "mrr_at_10": 0.481073, + "mrr_at_20": 0.485394, + "mrr_at_100": 0.488076, + "mrr_at_1000": 0.488523, + "nauc_ndcg_at_1_max": 0.291464, + "nauc_ndcg_at_1_std": -0.043576, + "nauc_ndcg_at_1_diff1": 0.545952, + "nauc_ndcg_at_3_max": 0.30535, + "nauc_ndcg_at_3_std": -0.028025, + "nauc_ndcg_at_3_diff1": 0.511003, + "nauc_ndcg_at_5_max": 0.306368, + "nauc_ndcg_at_5_std": -0.019815, + "nauc_ndcg_at_5_diff1": 0.5045, + "nauc_ndcg_at_10_max": 0.304778, + "nauc_ndcg_at_10_std": -0.014997, + "nauc_ndcg_at_10_diff1": 0.502947, + "nauc_ndcg_at_20_max": 0.301856, + "nauc_ndcg_at_20_std": -0.010165, + "nauc_ndcg_at_20_diff1": 0.501663, + "nauc_ndcg_at_100_max": 0.301512, + "nauc_ndcg_at_100_std": -0.006667, + "nauc_ndcg_at_100_diff1": 0.502216, + "nauc_ndcg_at_1000_max": 0.301326, + "nauc_ndcg_at_1000_std": -0.009884, + "nauc_ndcg_at_1000_diff1": 0.504694, + "nauc_map_at_1_max": 0.291464, + "nauc_map_at_1_std": -0.043576, + "nauc_map_at_1_diff1": 0.545952, + "nauc_map_at_3_max": 0.302005, + "nauc_map_at_3_std": -0.032243, + "nauc_map_at_3_diff1": 0.519296, + "nauc_map_at_5_max": 0.302429, + "nauc_map_at_5_std": -0.02787, + "nauc_map_at_5_diff1": 0.51586, + "nauc_map_at_10_max": 0.301752, + "nauc_map_at_10_std": -0.026156, + "nauc_map_at_10_diff1": 0.515513, + "nauc_map_at_20_max": 0.300963, + "nauc_map_at_20_std": -0.025025, + "nauc_map_at_20_diff1": 0.515235, + "nauc_map_at_100_max": 0.300886, + "nauc_map_at_100_std": -0.024639, + "nauc_map_at_100_diff1": 0.515351, + "nauc_map_at_1000_max": 0.300861, + "nauc_map_at_1000_std": -0.024735, + "nauc_map_at_1000_diff1": 0.515434, + "nauc_recall_at_1_max": 0.291464, + "nauc_recall_at_1_std": -0.043576, + "nauc_recall_at_1_diff1": 0.545952, + "nauc_recall_at_3_max": 0.315424, + "nauc_recall_at_3_std": -0.015113, + "nauc_recall_at_3_diff1": 0.486083, + "nauc_recall_at_5_max": 0.319195, + "nauc_recall_at_5_std": 0.007452, + "nauc_recall_at_5_diff1": 0.467346, + "nauc_recall_at_10_max": 0.315059, + "nauc_recall_at_10_std": 0.029304, + "nauc_recall_at_10_diff1": 0.455891, + "nauc_recall_at_20_max": 0.301451, + "nauc_recall_at_20_std": 0.062976, + "nauc_recall_at_20_diff1": 0.441416, + "nauc_recall_at_100_max": 0.298235, + "nauc_recall_at_100_std": 0.139795, + "nauc_recall_at_100_diff1": 0.414812, + "nauc_recall_at_1000_max": 0.292248, + "nauc_recall_at_1000_std": 0.232167, + "nauc_recall_at_1000_diff1": 0.38005, + "nauc_precision_at_1_max": 0.291464, + "nauc_precision_at_1_std": -0.043576, + "nauc_precision_at_1_diff1": 0.545952, + "nauc_precision_at_3_max": 0.315424, + "nauc_precision_at_3_std": -0.015113, + "nauc_precision_at_3_diff1": 0.486083, + "nauc_precision_at_5_max": 0.319195, + "nauc_precision_at_5_std": 0.007452, + "nauc_precision_at_5_diff1": 0.467346, + "nauc_precision_at_10_max": 0.315059, + "nauc_precision_at_10_std": 0.029304, + "nauc_precision_at_10_diff1": 0.455891, + "nauc_precision_at_20_max": 0.301451, + "nauc_precision_at_20_std": 0.062976, + "nauc_precision_at_20_diff1": 0.441416, + "nauc_precision_at_100_max": 0.298235, + "nauc_precision_at_100_std": 0.139795, + "nauc_precision_at_100_diff1": 0.414812, + "nauc_precision_at_1000_max": 0.292248, + "nauc_precision_at_1000_std": 0.232167, + "nauc_precision_at_1000_diff1": 0.38005, + "nauc_mrr_at_1_max": 0.291416, + "nauc_mrr_at_1_std": -0.043352, + "nauc_mrr_at_1_diff1": 0.545545, + "nauc_mrr_at_3_max": 0.30198, + "nauc_mrr_at_3_std": -0.032128, + "nauc_mrr_at_3_diff1": 0.519087, + "nauc_mrr_at_5_max": 0.302397, + "nauc_mrr_at_5_std": -0.027761, + "nauc_mrr_at_5_diff1": 0.515652, + "nauc_mrr_at_10_max": 0.301732, + "nauc_mrr_at_10_std": -0.026047, + "nauc_mrr_at_10_diff1": 0.5153, + "nauc_mrr_at_20_max": 0.300929, + "nauc_mrr_at_20_std": -0.024917, + "nauc_mrr_at_20_diff1": 0.51502, + "nauc_mrr_at_100_max": 0.300852, + "nauc_mrr_at_100_std": -0.02453, + "nauc_mrr_at_100_diff1": 0.515135, + "nauc_mrr_at_1000_max": 0.300827, + "nauc_mrr_at_1000_std": -0.024627, + "nauc_mrr_at_1000_diff1": 0.515217, + "main_score": 0.52769, + "hf_subset": "python", + "languages": [ + "python-Code" + ] + }, + { + "ndcg_at_1": 0.40413, + "ndcg_at_3": 0.49263, + "ndcg_at_5": 0.51668, + "ndcg_at_10": 0.5346, + "ndcg_at_20": 0.54883, + "ndcg_at_100": 0.56807, + "ndcg_at_1000": 0.58237, + "map_at_1": 0.40413, + "map_at_3": 0.47129, + "map_at_5": 0.48472, + "map_at_10": 0.49213, + "map_at_20": 0.49605, + "map_at_100": 0.49873, + "map_at_1000": 0.49922, + "recall_at_1": 0.40413, + "recall_at_3": 0.55424, + "recall_at_5": 0.61228, + "recall_at_10": 0.66758, + "recall_at_20": 0.72379, + "recall_at_100": 0.8271, + "recall_at_1000": 0.94196, + "precision_at_1": 0.40413, + "precision_at_3": 0.18475, + "precision_at_5": 0.12246, + "precision_at_10": 0.06676, + "precision_at_20": 0.03619, + "precision_at_100": 0.00827, + "precision_at_1000": 0.00094, + "mrr_at_1": 0.403525, + "mrr_at_3": 0.470981, + "mrr_at_5": 0.484412, + "mrr_at_10": 0.491828, + "mrr_at_20": 0.495741, + "mrr_at_100": 0.498424, + "mrr_at_1000": 0.49892, + "nauc_ndcg_at_1_max": 0.283225, + "nauc_ndcg_at_1_std": -0.073264, + "nauc_ndcg_at_1_diff1": 0.52513, + "nauc_ndcg_at_3_max": 0.316609, + "nauc_ndcg_at_3_std": -0.066902, + "nauc_ndcg_at_3_diff1": 0.492992, + "nauc_ndcg_at_5_max": 0.317058, + "nauc_ndcg_at_5_std": -0.063553, + "nauc_ndcg_at_5_diff1": 0.492294, + "nauc_ndcg_at_10_max": 0.311, + "nauc_ndcg_at_10_std": -0.064717, + "nauc_ndcg_at_10_diff1": 0.49158, + "nauc_ndcg_at_20_max": 0.305785, + "nauc_ndcg_at_20_std": -0.06197, + "nauc_ndcg_at_20_diff1": 0.48836, + "nauc_ndcg_at_100_max": 0.305391, + "nauc_ndcg_at_100_std": -0.057765, + "nauc_ndcg_at_100_diff1": 0.487385, + "nauc_ndcg_at_1000_max": 0.306858, + "nauc_ndcg_at_1000_std": -0.058424, + "nauc_ndcg_at_1000_diff1": 0.492756, + "nauc_map_at_1_max": 0.283225, + "nauc_map_at_1_std": -0.073264, + "nauc_map_at_1_diff1": 0.52513, + "nauc_map_at_3_max": 0.308567, + "nauc_map_at_3_std": -0.069241, + "nauc_map_at_3_diff1": 0.501134, + "nauc_map_at_5_max": 0.308432, + "nauc_map_at_5_std": -0.067536, + "nauc_map_at_5_diff1": 0.500922, + "nauc_map_at_10_max": 0.305865, + "nauc_map_at_10_std": -0.067971, + "nauc_map_at_10_diff1": 0.500523, + "nauc_map_at_20_max": 0.304347, + "nauc_map_at_20_std": -0.067372, + "nauc_map_at_20_diff1": 0.499702, + "nauc_map_at_100_max": 0.304434, + "nauc_map_at_100_std": -0.066701, + "nauc_map_at_100_diff1": 0.499628, + "nauc_map_at_1000_max": 0.304438, + "nauc_map_at_1000_std": -0.066731, + "nauc_map_at_1000_diff1": 0.499787, + "nauc_recall_at_1_max": 0.283225, + "nauc_recall_at_1_std": -0.073264, + "nauc_recall_at_1_diff1": 0.52513, + "nauc_recall_at_3_max": 0.340979, + "nauc_recall_at_3_std": -0.05945, + "nauc_recall_at_3_diff1": 0.468139, + "nauc_recall_at_5_max": 0.345179, + "nauc_recall_at_5_std": -0.049714, + "nauc_recall_at_5_diff1": 0.464239, + "nauc_recall_at_10_max": 0.326745, + "nauc_recall_at_10_std": -0.052891, + "nauc_recall_at_10_diff1": 0.459971, + "nauc_recall_at_20_max": 0.303512, + "nauc_recall_at_20_std": -0.036092, + "nauc_recall_at_20_diff1": 0.439354, + "nauc_recall_at_100_max": 0.293972, + "nauc_recall_at_100_std": 0.012682, + "nauc_recall_at_100_diff1": 0.406598, + "nauc_recall_at_1000_max": 0.343025, + "nauc_recall_at_1000_std": 0.111184, + "nauc_recall_at_1000_diff1": 0.479289, + "nauc_precision_at_1_max": 0.283225, + "nauc_precision_at_1_std": -0.073264, + "nauc_precision_at_1_diff1": 0.52513, + "nauc_precision_at_3_max": 0.340979, + "nauc_precision_at_3_std": -0.05945, + "nauc_precision_at_3_diff1": 0.468139, + "nauc_precision_at_5_max": 0.345179, + "nauc_precision_at_5_std": -0.049714, + "nauc_precision_at_5_diff1": 0.464239, + "nauc_precision_at_10_max": 0.326745, + "nauc_precision_at_10_std": -0.052891, + "nauc_precision_at_10_diff1": 0.459971, + "nauc_precision_at_20_max": 0.303512, + "nauc_precision_at_20_std": -0.036092, + "nauc_precision_at_20_diff1": 0.439354, + "nauc_precision_at_100_max": 0.293972, + "nauc_precision_at_100_std": 0.012682, + "nauc_precision_at_100_diff1": 0.406598, + "nauc_precision_at_1000_max": 0.343025, + "nauc_precision_at_1000_std": 0.111184, + "nauc_precision_at_1000_diff1": 0.479289, + "nauc_mrr_at_1_max": 0.283752, + "nauc_mrr_at_1_std": -0.075182, + "nauc_mrr_at_1_diff1": 0.5267, + "nauc_mrr_at_3_max": 0.308838, + "nauc_mrr_at_3_std": -0.070235, + "nauc_mrr_at_3_diff1": 0.501951, + "nauc_mrr_at_5_max": 0.308707, + "nauc_mrr_at_5_std": -0.068546, + "nauc_mrr_at_5_diff1": 0.50175, + "nauc_mrr_at_10_max": 0.306143, + "nauc_mrr_at_10_std": -0.068993, + "nauc_mrr_at_10_diff1": 0.501362, + "nauc_mrr_at_20_max": 0.304627, + "nauc_mrr_at_20_std": -0.068402, + "nauc_mrr_at_20_diff1": 0.500547, + "nauc_mrr_at_100_max": 0.304716, + "nauc_mrr_at_100_std": -0.067735, + "nauc_mrr_at_100_diff1": 0.500478, + "nauc_mrr_at_1000_max": 0.30472, + "nauc_mrr_at_1000_std": -0.067766, + "nauc_mrr_at_1000_diff1": 0.500638, + "main_score": 0.5346, + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ] + }, + { + "ndcg_at_1": 0.28885, + "ndcg_at_3": 0.36263, + "ndcg_at_5": 0.38472, + "ndcg_at_10": 0.40655, + "ndcg_at_20": 0.42247, + "ndcg_at_100": 0.44531, + "ndcg_at_1000": 0.46309, + "map_at_1": 0.28885, + "map_at_3": 0.34452, + "map_at_5": 0.35676, + "map_at_10": 0.36576, + "map_at_20": 0.37016, + "map_at_100": 0.37319, + "map_at_1000": 0.37379, + "recall_at_1": 0.28885, + "recall_at_3": 0.41505, + "recall_at_5": 0.46873, + "recall_at_10": 0.53632, + "recall_at_20": 0.59899, + "recall_at_100": 0.72408, + "recall_at_1000": 0.86826, + "precision_at_1": 0.28885, + "precision_at_3": 0.13835, + "precision_at_5": 0.09375, + "precision_at_10": 0.05363, + "precision_at_20": 0.02995, + "precision_at_100": 0.00724, + "precision_at_1000": 0.00087, + "mrr_at_1": 0.288845, + "mrr_at_3": 0.344537, + "mrr_at_5": 0.356807, + "mrr_at_10": 0.365785, + "mrr_at_20": 0.370178, + "mrr_at_100": 0.373216, + "mrr_at_1000": 0.373813, + "nauc_ndcg_at_1_max": 0.267744, + "nauc_ndcg_at_1_std": -0.019532, + "nauc_ndcg_at_1_diff1": 0.501107, + "nauc_ndcg_at_3_max": 0.255462, + "nauc_ndcg_at_3_std": -0.022104, + "nauc_ndcg_at_3_diff1": 0.453256, + "nauc_ndcg_at_5_max": 0.251967, + "nauc_ndcg_at_5_std": -0.019396, + "nauc_ndcg_at_5_diff1": 0.444159, + "nauc_ndcg_at_10_max": 0.2451, + "nauc_ndcg_at_10_std": -0.01541, + "nauc_ndcg_at_10_diff1": 0.435975, + "nauc_ndcg_at_20_max": 0.242176, + "nauc_ndcg_at_20_std": -0.010425, + "nauc_ndcg_at_20_diff1": 0.433472, + "nauc_ndcg_at_100_max": 0.242653, + "nauc_ndcg_at_100_std": -0.005279, + "nauc_ndcg_at_100_diff1": 0.434054, + "nauc_ndcg_at_1000_max": 0.245649, + "nauc_ndcg_at_1000_std": -0.00287, + "nauc_ndcg_at_1000_diff1": 0.436379, + "nauc_map_at_1_max": 0.267744, + "nauc_map_at_1_std": -0.019532, + "nauc_map_at_1_diff1": 0.501107, + "nauc_map_at_3_max": 0.257894, + "nauc_map_at_3_std": -0.021388, + "nauc_map_at_3_diff1": 0.464548, + "nauc_map_at_5_max": 0.256075, + "nauc_map_at_5_std": -0.019798, + "nauc_map_at_5_diff1": 0.459469, + "nauc_map_at_10_max": 0.253355, + "nauc_map_at_10_std": -0.018238, + "nauc_map_at_10_diff1": 0.45619, + "nauc_map_at_20_max": 0.252606, + "nauc_map_at_20_std": -0.016876, + "nauc_map_at_20_diff1": 0.455632, + "nauc_map_at_100_max": 0.252676, + "nauc_map_at_100_std": -0.016238, + "nauc_map_at_100_diff1": 0.455794, + "nauc_map_at_1000_max": 0.25281, + "nauc_map_at_1000_std": -0.016138, + "nauc_map_at_1000_diff1": 0.455918, + "nauc_recall_at_1_max": 0.267744, + "nauc_recall_at_1_std": -0.019532, + "nauc_recall_at_1_diff1": 0.501107, + "nauc_recall_at_3_max": 0.24884, + "nauc_recall_at_3_std": -0.024151, + "nauc_recall_at_3_diff1": 0.421657, + "nauc_recall_at_5_max": 0.239854, + "nauc_recall_at_5_std": -0.018008, + "nauc_recall_at_5_diff1": 0.399483, + "nauc_recall_at_10_max": 0.217129, + "nauc_recall_at_10_std": -0.00495, + "nauc_recall_at_10_diff1": 0.371442, + "nauc_recall_at_20_max": 0.202696, + "nauc_recall_at_20_std": 0.016778, + "nauc_recall_at_20_diff1": 0.355992, + "nauc_recall_at_100_max": 0.195117, + "nauc_recall_at_100_std": 0.062447, + "nauc_recall_at_100_diff1": 0.337311, + "nauc_recall_at_1000_max": 0.206332, + "nauc_recall_at_1000_std": 0.162987, + "nauc_recall_at_1000_diff1": 0.293336, + "nauc_precision_at_1_max": 0.267744, + "nauc_precision_at_1_std": -0.019532, + "nauc_precision_at_1_diff1": 0.501107, + "nauc_precision_at_3_max": 0.24884, + "nauc_precision_at_3_std": -0.024151, + "nauc_precision_at_3_diff1": 0.421657, + "nauc_precision_at_5_max": 0.239854, + "nauc_precision_at_5_std": -0.018008, + "nauc_precision_at_5_diff1": 0.399483, + "nauc_precision_at_10_max": 0.217129, + "nauc_precision_at_10_std": -0.00495, + "nauc_precision_at_10_diff1": 0.371442, + "nauc_precision_at_20_max": 0.202696, + "nauc_precision_at_20_std": 0.016778, + "nauc_precision_at_20_diff1": 0.355992, + "nauc_precision_at_100_max": 0.195117, + "nauc_precision_at_100_std": 0.062447, + "nauc_precision_at_100_diff1": 0.337311, + "nauc_precision_at_1000_max": 0.206332, + "nauc_precision_at_1000_std": 0.162987, + "nauc_precision_at_1000_diff1": 0.293336, + "nauc_mrr_at_1_max": 0.2675, + "nauc_mrr_at_1_std": -0.019696, + "nauc_mrr_at_1_diff1": 0.501107, + "nauc_mrr_at_3_max": 0.258041, + "nauc_mrr_at_3_std": -0.02089, + "nauc_mrr_at_3_diff1": 0.464557, + "nauc_mrr_at_5_max": 0.255969, + "nauc_mrr_at_5_std": -0.019634, + "nauc_mrr_at_5_diff1": 0.459405, + "nauc_mrr_at_10_max": 0.253304, + "nauc_mrr_at_10_std": -0.018047, + "nauc_mrr_at_10_diff1": 0.456173, + "nauc_mrr_at_20_max": 0.252583, + "nauc_mrr_at_20_std": -0.016671, + "nauc_mrr_at_20_diff1": 0.455609, + "nauc_mrr_at_100_max": 0.252636, + "nauc_mrr_at_100_std": -0.016039, + "nauc_mrr_at_100_diff1": 0.455773, + "nauc_mrr_at_1000_max": 0.252769, + "nauc_mrr_at_1000_std": -0.015939, + "nauc_mrr_at_1000_diff1": 0.455897, + "main_score": 0.40655, + "hf_subset": "go", + "languages": [ + "go-Code" + ] + }, + { + "ndcg_at_1": 0.40999, + "ndcg_at_3": 0.49401, + "ndcg_at_5": 0.51555, + "ndcg_at_10": 0.53328, + "ndcg_at_20": 0.54888, + "ndcg_at_100": 0.5686, + "ndcg_at_1000": 0.581, + "map_at_1": 0.40999, + "map_at_3": 0.47343, + "map_at_5": 0.48537, + "map_at_10": 0.49286, + "map_at_20": 0.49722, + "map_at_100": 0.4999, + "map_at_1000": 0.50032, + "recall_at_1": 0.40999, + "recall_at_3": 0.55353, + "recall_at_5": 0.60587, + "recall_at_10": 0.65979, + "recall_at_20": 0.72086, + "recall_at_100": 0.82791, + "recall_at_1000": 0.92863, + "precision_at_1": 0.40999, + "precision_at_3": 0.18451, + "precision_at_5": 0.12117, + "precision_at_10": 0.06598, + "precision_at_20": 0.03604, + "precision_at_100": 0.00828, + "precision_at_1000": 0.00093, + "mrr_at_1": 0.409992, + "mrr_at_3": 0.473434, + "mrr_at_5": 0.485369, + "mrr_at_10": 0.492846, + "mrr_at_20": 0.497209, + "mrr_at_100": 0.49989, + "mrr_at_1000": 0.500307, + "nauc_ndcg_at_1_max": 0.351862, + "nauc_ndcg_at_1_std": -0.105744, + "nauc_ndcg_at_1_diff1": 0.617568, + "nauc_ndcg_at_3_max": 0.370239, + "nauc_ndcg_at_3_std": -0.122137, + "nauc_ndcg_at_3_diff1": 0.576805, + "nauc_ndcg_at_5_max": 0.379846, + "nauc_ndcg_at_5_std": -0.117046, + "nauc_ndcg_at_5_diff1": 0.576362, + "nauc_ndcg_at_10_max": 0.377794, + "nauc_ndcg_at_10_std": -0.111989, + "nauc_ndcg_at_10_diff1": 0.568239, + "nauc_ndcg_at_20_max": 0.376536, + "nauc_ndcg_at_20_std": -0.101625, + "nauc_ndcg_at_20_diff1": 0.566607, + "nauc_ndcg_at_100_max": 0.368333, + "nauc_ndcg_at_100_std": -0.102939, + "nauc_ndcg_at_100_diff1": 0.564636, + "nauc_ndcg_at_1000_max": 0.371363, + "nauc_ndcg_at_1000_std": -0.101625, + "nauc_ndcg_at_1000_diff1": 0.567276, + "nauc_map_at_1_max": 0.351862, + "nauc_map_at_1_std": -0.105744, + "nauc_map_at_1_diff1": 0.617568, + "nauc_map_at_3_max": 0.365278, + "nauc_map_at_3_std": -0.118113, + "nauc_map_at_3_diff1": 0.586006, + "nauc_map_at_5_max": 0.370836, + "nauc_map_at_5_std": -0.115106, + "nauc_map_at_5_diff1": 0.58609, + "nauc_map_at_10_max": 0.370137, + "nauc_map_at_10_std": -0.113068, + "nauc_map_at_10_diff1": 0.582927, + "nauc_map_at_20_max": 0.369716, + "nauc_map_at_20_std": -0.110421, + "nauc_map_at_20_diff1": 0.582622, + "nauc_map_at_100_max": 0.368398, + "nauc_map_at_100_std": -0.110586, + "nauc_map_at_100_diff1": 0.582204, + "nauc_map_at_1000_max": 0.368472, + "nauc_map_at_1000_std": -0.110514, + "nauc_map_at_1000_diff1": 0.582263, + "nauc_recall_at_1_max": 0.351862, + "nauc_recall_at_1_std": -0.105744, + "nauc_recall_at_1_diff1": 0.617568, + "nauc_recall_at_3_max": 0.385501, + "nauc_recall_at_3_std": -0.134355, + "nauc_recall_at_3_diff1": 0.549226, + "nauc_recall_at_5_max": 0.409764, + "nauc_recall_at_5_std": -0.122678, + "nauc_recall_at_5_diff1": 0.545366, + "nauc_recall_at_10_max": 0.404102, + "nauc_recall_at_10_std": -0.105262, + "nauc_recall_at_10_diff1": 0.513322, + "nauc_recall_at_20_max": 0.402049, + "nauc_recall_at_20_std": -0.050891, + "nauc_recall_at_20_diff1": 0.49655, + "nauc_recall_at_100_max": 0.340374, + "nauc_recall_at_100_std": -0.040205, + "nauc_recall_at_100_diff1": 0.449399, + "nauc_recall_at_1000_max": 0.40926, + "nauc_recall_at_1000_std": 0.070537, + "nauc_recall_at_1000_diff1": 0.400614, + "nauc_precision_at_1_max": 0.351862, + "nauc_precision_at_1_std": -0.105744, + "nauc_precision_at_1_diff1": 0.617568, + "nauc_precision_at_3_max": 0.385501, + "nauc_precision_at_3_std": -0.134355, + "nauc_precision_at_3_diff1": 0.549226, + "nauc_precision_at_5_max": 0.409764, + "nauc_precision_at_5_std": -0.122678, + "nauc_precision_at_5_diff1": 0.545366, + "nauc_precision_at_10_max": 0.404102, + "nauc_precision_at_10_std": -0.105262, + "nauc_precision_at_10_diff1": 0.513322, + "nauc_precision_at_20_max": 0.402049, + "nauc_precision_at_20_std": -0.050891, + "nauc_precision_at_20_diff1": 0.49655, + "nauc_precision_at_100_max": 0.340374, + "nauc_precision_at_100_std": -0.040205, + "nauc_precision_at_100_diff1": 0.449399, + "nauc_precision_at_1000_max": 0.40926, + "nauc_precision_at_1000_std": 0.070537, + "nauc_precision_at_1000_diff1": 0.400614, + "nauc_mrr_at_1_max": 0.354447, + "nauc_mrr_at_1_std": -0.104452, + "nauc_mrr_at_1_diff1": 0.617568, + "nauc_mrr_at_3_max": 0.366615, + "nauc_mrr_at_3_std": -0.117445, + "nauc_mrr_at_3_diff1": 0.586006, + "nauc_mrr_at_5_max": 0.372191, + "nauc_mrr_at_5_std": -0.114429, + "nauc_mrr_at_5_diff1": 0.58609, + "nauc_mrr_at_10_max": 0.371519, + "nauc_mrr_at_10_std": -0.112363, + "nauc_mrr_at_10_diff1": 0.582911, + "nauc_mrr_at_20_max": 0.371109, + "nauc_mrr_at_20_std": -0.109711, + "nauc_mrr_at_20_diff1": 0.582606, + "nauc_mrr_at_100_max": 0.369799, + "nauc_mrr_at_100_std": -0.109874, + "nauc_mrr_at_100_diff1": 0.582189, + "nauc_mrr_at_1000_max": 0.369875, + "nauc_mrr_at_1000_std": -0.109801, + "nauc_mrr_at_1000_diff1": 0.582248, + "main_score": 0.53328, + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ] + }, + { + "ndcg_at_1": 0.37745, + "ndcg_at_3": 0.4665, + "ndcg_at_5": 0.48735, + "ndcg_at_10": 0.50803, + "ndcg_at_20": 0.52171, + "ndcg_at_100": 0.54151, + "ndcg_at_1000": 0.55538, + "map_at_1": 0.37745, + "map_at_3": 0.44503, + "map_at_5": 0.4566, + "map_at_10": 0.46519, + "map_at_20": 0.46898, + "map_at_100": 0.47167, + "map_at_1000": 0.47215, + "recall_at_1": 0.37745, + "recall_at_3": 0.52843, + "recall_at_5": 0.5791, + "recall_at_10": 0.64272, + "recall_at_20": 0.69658, + "recall_at_100": 0.80402, + "recall_at_1000": 0.91538, + "precision_at_1": 0.37745, + "precision_at_3": 0.17614, + "precision_at_5": 0.11582, + "precision_at_10": 0.06427, + "precision_at_20": 0.03483, + "precision_at_100": 0.00804, + "precision_at_1000": 0.00092, + "mrr_at_1": 0.377453, + "mrr_at_3": 0.444911, + "mrr_at_5": 0.456481, + "mrr_at_10": 0.465083, + "mrr_at_20": 0.468885, + "mrr_at_100": 0.471576, + "mrr_at_1000": 0.472057, + "nauc_ndcg_at_1_max": 0.297799, + "nauc_ndcg_at_1_std": -0.074458, + "nauc_ndcg_at_1_diff1": 0.571023, + "nauc_ndcg_at_3_max": 0.31825, + "nauc_ndcg_at_3_std": -0.068238, + "nauc_ndcg_at_3_diff1": 0.533845, + "nauc_ndcg_at_5_max": 0.318655, + "nauc_ndcg_at_5_std": -0.061885, + "nauc_ndcg_at_5_diff1": 0.526215, + "nauc_ndcg_at_10_max": 0.316128, + "nauc_ndcg_at_10_std": -0.054586, + "nauc_ndcg_at_10_diff1": 0.523433, + "nauc_ndcg_at_20_max": 0.314462, + "nauc_ndcg_at_20_std": -0.049862, + "nauc_ndcg_at_20_diff1": 0.519772, + "nauc_ndcg_at_100_max": 0.315183, + "nauc_ndcg_at_100_std": -0.044936, + "nauc_ndcg_at_100_diff1": 0.521493, + "nauc_ndcg_at_1000_max": 0.315796, + "nauc_ndcg_at_1000_std": -0.04678, + "nauc_ndcg_at_1000_diff1": 0.524545, + "nauc_map_at_1_max": 0.297799, + "nauc_map_at_1_std": -0.074458, + "nauc_map_at_1_diff1": 0.571023, + "nauc_map_at_3_max": 0.313082, + "nauc_map_at_3_std": -0.070641, + "nauc_map_at_3_diff1": 0.542397, + "nauc_map_at_5_max": 0.313332, + "nauc_map_at_5_std": -0.067121, + "nauc_map_at_5_diff1": 0.53841, + "nauc_map_at_10_max": 0.312247, + "nauc_map_at_10_std": -0.064174, + "nauc_map_at_10_diff1": 0.537376, + "nauc_map_at_20_max": 0.311769, + "nauc_map_at_20_std": -0.063008, + "nauc_map_at_20_diff1": 0.536494, + "nauc_map_at_100_max": 0.311776, + "nauc_map_at_100_std": -0.062359, + "nauc_map_at_100_diff1": 0.53672, + "nauc_map_at_1000_max": 0.311817, + "nauc_map_at_1000_std": -0.062409, + "nauc_map_at_1000_diff1": 0.536847, + "nauc_recall_at_1_max": 0.297799, + "nauc_recall_at_1_std": -0.074458, + "nauc_recall_at_1_diff1": 0.571023, + "nauc_recall_at_3_max": 0.333722, + "nauc_recall_at_3_std": -0.0607, + "nauc_recall_at_3_diff1": 0.508594, + "nauc_recall_at_5_max": 0.33523, + "nauc_recall_at_5_std": -0.044314, + "nauc_recall_at_5_diff1": 0.487234, + "nauc_recall_at_10_max": 0.328186, + "nauc_recall_at_10_std": -0.017369, + "nauc_recall_at_10_diff1": 0.473705, + "nauc_recall_at_20_max": 0.321605, + "nauc_recall_at_20_std": 0.010079, + "nauc_recall_at_20_diff1": 0.450203, + "nauc_recall_at_100_max": 0.332879, + "nauc_recall_at_100_std": 0.077146, + "nauc_recall_at_100_diff1": 0.438646, + "nauc_recall_at_1000_max": 0.365353, + "nauc_recall_at_1000_std": 0.162442, + "nauc_recall_at_1000_diff1": 0.424618, + "nauc_precision_at_1_max": 0.297799, + "nauc_precision_at_1_std": -0.074458, + "nauc_precision_at_1_diff1": 0.571023, + "nauc_precision_at_3_max": 0.333722, + "nauc_precision_at_3_std": -0.0607, + "nauc_precision_at_3_diff1": 0.508594, + "nauc_precision_at_5_max": 0.33523, + "nauc_precision_at_5_std": -0.044314, + "nauc_precision_at_5_diff1": 0.487234, + "nauc_precision_at_10_max": 0.328186, + "nauc_precision_at_10_std": -0.017369, + "nauc_precision_at_10_diff1": 0.473705, + "nauc_precision_at_20_max": 0.321605, + "nauc_precision_at_20_std": 0.010079, + "nauc_precision_at_20_diff1": 0.450203, + "nauc_precision_at_100_max": 0.332879, + "nauc_precision_at_100_std": 0.077146, + "nauc_precision_at_100_diff1": 0.438646, + "nauc_precision_at_1000_max": 0.365353, + "nauc_precision_at_1000_std": 0.162442, + "nauc_precision_at_1000_diff1": 0.424618, + "nauc_mrr_at_1_max": 0.298616, + "nauc_mrr_at_1_std": -0.074226, + "nauc_mrr_at_1_diff1": 0.571023, + "nauc_mrr_at_3_max": 0.313588, + "nauc_mrr_at_3_std": -0.070597, + "nauc_mrr_at_3_diff1": 0.542568, + "nauc_mrr_at_5_max": 0.313862, + "nauc_mrr_at_5_std": -0.06717, + "nauc_mrr_at_5_diff1": 0.538553, + "nauc_mrr_at_10_max": 0.312772, + "nauc_mrr_at_10_std": -0.064156, + "nauc_mrr_at_10_diff1": 0.537492, + "nauc_mrr_at_20_max": 0.312288, + "nauc_mrr_at_20_std": -0.063007, + "nauc_mrr_at_20_diff1": 0.536614, + "nauc_mrr_at_100_max": 0.3123, + "nauc_mrr_at_100_std": -0.062357, + "nauc_mrr_at_100_diff1": 0.53684, + "nauc_mrr_at_1000_max": 0.312341, + "nauc_mrr_at_1000_std": -0.062408, + "nauc_mrr_at_1000_diff1": 0.536967, + "main_score": 0.50803, + "hf_subset": "java", + "languages": [ + "java-Code" + ] + }, + { + "ndcg_at_1": 0.28036, + "ndcg_at_3": 0.35439, + "ndcg_at_5": 0.37426, + "ndcg_at_10": 0.39532, + "ndcg_at_20": 0.412, + "ndcg_at_100": 0.43674, + "ndcg_at_1000": 0.45618, + "map_at_1": 0.28036, + "map_at_3": 0.33628, + "map_at_5": 0.34731, + "map_at_10": 0.35601, + "map_at_20": 0.36059, + "map_at_100": 0.3639, + "map_at_1000": 0.36455, + "recall_at_1": 0.28036, + "recall_at_3": 0.40674, + "recall_at_5": 0.45497, + "recall_at_10": 0.52005, + "recall_at_20": 0.58599, + "recall_at_100": 0.72099, + "recall_at_1000": 0.87884, + "precision_at_1": 0.28036, + "precision_at_3": 0.13558, + "precision_at_5": 0.09099, + "precision_at_10": 0.05201, + "precision_at_20": 0.0293, + "precision_at_100": 0.00721, + "precision_at_1000": 0.00088, + "mrr_at_1": 0.28022, + "mrr_at_3": 0.336211, + "mrr_at_5": 0.347222, + "mrr_at_10": 0.355933, + "mrr_at_20": 0.36052, + "mrr_at_100": 0.36383, + "mrr_at_1000": 0.364479, + "nauc_ndcg_at_1_max": 0.233619, + "nauc_ndcg_at_1_std": -0.055831, + "nauc_ndcg_at_1_diff1": 0.520545, + "nauc_ndcg_at_3_max": 0.224626, + "nauc_ndcg_at_3_std": -0.048977, + "nauc_ndcg_at_3_diff1": 0.467511, + "nauc_ndcg_at_5_max": 0.220279, + "nauc_ndcg_at_5_std": -0.047317, + "nauc_ndcg_at_5_diff1": 0.462755, + "nauc_ndcg_at_10_max": 0.219324, + "nauc_ndcg_at_10_std": -0.039926, + "nauc_ndcg_at_10_diff1": 0.457309, + "nauc_ndcg_at_20_max": 0.214322, + "nauc_ndcg_at_20_std": -0.037124, + "nauc_ndcg_at_20_diff1": 0.451048, + "nauc_ndcg_at_100_max": 0.212409, + "nauc_ndcg_at_100_std": -0.031787, + "nauc_ndcg_at_100_diff1": 0.45043, + "nauc_ndcg_at_1000_max": 0.215952, + "nauc_ndcg_at_1000_std": -0.029877, + "nauc_ndcg_at_1000_diff1": 0.453763, + "nauc_map_at_1_max": 0.233619, + "nauc_map_at_1_std": -0.055831, + "nauc_map_at_1_diff1": 0.520545, + "nauc_map_at_3_max": 0.22691, + "nauc_map_at_3_std": -0.051074, + "nauc_map_at_3_diff1": 0.48005, + "nauc_map_at_5_max": 0.224466, + "nauc_map_at_5_std": -0.050263, + "nauc_map_at_5_diff1": 0.477504, + "nauc_map_at_10_max": 0.224149, + "nauc_map_at_10_std": -0.047252, + "nauc_map_at_10_diff1": 0.475183, + "nauc_map_at_20_max": 0.222823, + "nauc_map_at_20_std": -0.046566, + "nauc_map_at_20_diff1": 0.473568, + "nauc_map_at_100_max": 0.222571, + "nauc_map_at_100_std": -0.045903, + "nauc_map_at_100_diff1": 0.473565, + "nauc_map_at_1000_max": 0.222675, + "nauc_map_at_1000_std": -0.045825, + "nauc_map_at_1000_diff1": 0.473654, + "nauc_recall_at_1_max": 0.233619, + "nauc_recall_at_1_std": -0.055831, + "nauc_recall_at_1_diff1": 0.520545, + "nauc_recall_at_3_max": 0.218223, + "nauc_recall_at_3_std": -0.04297, + "nauc_recall_at_3_diff1": 0.4327, + "nauc_recall_at_5_max": 0.207867, + "nauc_recall_at_5_std": -0.038483, + "nauc_recall_at_5_diff1": 0.42055, + "nauc_recall_at_10_max": 0.204119, + "nauc_recall_at_10_std": -0.015082, + "nauc_recall_at_10_diff1": 0.402842, + "nauc_recall_at_20_max": 0.181964, + "nauc_recall_at_20_std": -0.001208, + "nauc_recall_at_20_diff1": 0.372602, + "nauc_recall_at_100_max": 0.15942, + "nauc_recall_at_100_std": 0.048116, + "nauc_recall_at_100_diff1": 0.344793, + "nauc_recall_at_1000_max": 0.181081, + "nauc_recall_at_1000_std": 0.16588, + "nauc_recall_at_1000_diff1": 0.310889, + "nauc_precision_at_1_max": 0.233619, + "nauc_precision_at_1_std": -0.055831, + "nauc_precision_at_1_diff1": 0.520545, + "nauc_precision_at_3_max": 0.218223, + "nauc_precision_at_3_std": -0.04297, + "nauc_precision_at_3_diff1": 0.4327, + "nauc_precision_at_5_max": 0.207867, + "nauc_precision_at_5_std": -0.038483, + "nauc_precision_at_5_diff1": 0.42055, + "nauc_precision_at_10_max": 0.204119, + "nauc_precision_at_10_std": -0.015082, + "nauc_precision_at_10_diff1": 0.402842, + "nauc_precision_at_20_max": 0.181964, + "nauc_precision_at_20_std": -0.001208, + "nauc_precision_at_20_diff1": 0.372602, + "nauc_precision_at_100_max": 0.15942, + "nauc_precision_at_100_std": 0.048116, + "nauc_precision_at_100_diff1": 0.344793, + "nauc_precision_at_1000_max": 0.181081, + "nauc_precision_at_1000_std": 0.16588, + "nauc_precision_at_1000_diff1": 0.310889, + "nauc_mrr_at_1_max": 0.233918, + "nauc_mrr_at_1_std": -0.054689, + "nauc_mrr_at_1_diff1": 0.521097, + "nauc_mrr_at_3_max": 0.22677, + "nauc_mrr_at_3_std": -0.050518, + "nauc_mrr_at_3_diff1": 0.480507, + "nauc_mrr_at_5_max": 0.224604, + "nauc_mrr_at_5_std": -0.049722, + "nauc_mrr_at_5_diff1": 0.477808, + "nauc_mrr_at_10_max": 0.224253, + "nauc_mrr_at_10_std": -0.04668, + "nauc_mrr_at_10_diff1": 0.475525, + "nauc_mrr_at_20_max": 0.222909, + "nauc_mrr_at_20_std": -0.04601, + "nauc_mrr_at_20_diff1": 0.473916, + "nauc_mrr_at_100_max": 0.22266, + "nauc_mrr_at_100_std": -0.045341, + "nauc_mrr_at_100_diff1": 0.473916, + "nauc_mrr_at_1000_max": 0.222765, + "nauc_mrr_at_1000_std": -0.045262, + "nauc_mrr_at_1000_diff1": 0.474006, + "main_score": 0.39532, + "hf_subset": "php", + "languages": [ + "php-Code" + ] + } + ] + }, + "evaluation_time": 482.4193916320801, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeTransOceanContest.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeTransOceanContest.json new file mode 100644 index 0000000000..854d5fc931 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeTransOceanContest.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "20da4eb20a4b17300c0986ee148c90867a7f2a4d", + "task_name": "CodeTransOceanContest", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.68326, + "ndcg_at_3": 0.75727, + "ndcg_at_5": 0.76272, + "ndcg_at_10": 0.77631, + "ndcg_at_20": 0.78582, + "ndcg_at_100": 0.79564, + "ndcg_at_1000": 0.8014, + "map_at_1": 0.68326, + "map_at_3": 0.73906, + "map_at_5": 0.74201, + "map_at_10": 0.74787, + "map_at_20": 0.75067, + "map_at_100": 0.75194, + "map_at_1000": 0.75214, + "recall_at_1": 0.68326, + "recall_at_3": 0.80995, + "recall_at_5": 0.82353, + "recall_at_10": 0.86425, + "recall_at_20": 0.90045, + "recall_at_100": 0.95475, + "recall_at_1000": 1.0, + "precision_at_1": 0.68326, + "precision_at_3": 0.26998, + "precision_at_5": 0.16471, + "precision_at_10": 0.08643, + "precision_at_20": 0.04502, + "precision_at_100": 0.00955, + "precision_at_1000": 0.001, + "mrr_at_1": 0.683258, + "mrr_at_3": 0.739065, + "mrr_at_5": 0.742006, + "mrr_at_10": 0.747867, + "mrr_at_20": 0.750671, + "mrr_at_100": 0.751936, + "mrr_at_1000": 0.752142, + "nauc_ndcg_at_1_max": 0.713711, + "nauc_ndcg_at_1_std": 0.078936, + "nauc_ndcg_at_1_diff1": 0.865522, + "nauc_ndcg_at_3_max": 0.733288, + "nauc_ndcg_at_3_std": 0.037285, + "nauc_ndcg_at_3_diff1": 0.839587, + "nauc_ndcg_at_5_max": 0.73323, + "nauc_ndcg_at_5_std": 0.041259, + "nauc_ndcg_at_5_diff1": 0.836306, + "nauc_ndcg_at_10_max": 0.730876, + "nauc_ndcg_at_10_std": 0.038437, + "nauc_ndcg_at_10_diff1": 0.830461, + "nauc_ndcg_at_20_max": 0.729448, + "nauc_ndcg_at_20_std": 0.038632, + "nauc_ndcg_at_20_diff1": 0.825122, + "nauc_ndcg_at_100_max": 0.728802, + "nauc_ndcg_at_100_std": 0.050167, + "nauc_ndcg_at_100_diff1": 0.829403, + "nauc_ndcg_at_1000_max": 0.728151, + "nauc_ndcg_at_1000_std": 0.048018, + "nauc_ndcg_at_1000_diff1": 0.834123, + "nauc_map_at_1_max": 0.713711, + "nauc_map_at_1_std": 0.078936, + "nauc_map_at_1_diff1": 0.865522, + "nauc_map_at_3_max": 0.727986, + "nauc_map_at_3_std": 0.049345, + "nauc_map_at_3_diff1": 0.844054, + "nauc_map_at_5_max": 0.72805, + "nauc_map_at_5_std": 0.051724, + "nauc_map_at_5_diff1": 0.842374, + "nauc_map_at_10_max": 0.727371, + "nauc_map_at_10_std": 0.05087, + "nauc_map_at_10_diff1": 0.840469, + "nauc_map_at_20_max": 0.726789, + "nauc_map_at_20_std": 0.05081, + "nauc_map_at_20_diff1": 0.839135, + "nauc_map_at_100_max": 0.726677, + "nauc_map_at_100_std": 0.052435, + "nauc_map_at_100_diff1": 0.839816, + "nauc_map_at_1000_max": 0.726653, + "nauc_map_at_1000_std": 0.052356, + "nauc_map_at_1000_diff1": 0.83997, + "nauc_recall_at_1_max": 0.713711, + "nauc_recall_at_1_std": 0.078936, + "nauc_recall_at_1_diff1": 0.865522, + "nauc_recall_at_3_max": 0.752809, + "nauc_recall_at_3_std": -0.007551, + "nauc_recall_at_3_diff1": 0.824555, + "nauc_recall_at_5_max": 0.753145, + "nauc_recall_at_5_std": 0.000462, + "nauc_recall_at_5_diff1": 0.813564, + "nauc_recall_at_10_max": 0.744645, + "nauc_recall_at_10_std": -0.023165, + "nauc_recall_at_10_diff1": 0.778693, + "nauc_recall_at_20_max": 0.741398, + "nauc_recall_at_20_std": -0.03729, + "nauc_recall_at_20_diff1": 0.724511, + "nauc_recall_at_100_max": 0.750007, + "nauc_recall_at_100_std": 0.113908, + "nauc_recall_at_100_diff1": 0.690789, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.713711, + "nauc_precision_at_1_std": 0.078936, + "nauc_precision_at_1_diff1": 0.865522, + "nauc_precision_at_3_max": 0.752809, + "nauc_precision_at_3_std": -0.007551, + "nauc_precision_at_3_diff1": 0.824555, + "nauc_precision_at_5_max": 0.753145, + "nauc_precision_at_5_std": 0.000462, + "nauc_precision_at_5_diff1": 0.813564, + "nauc_precision_at_10_max": 0.744645, + "nauc_precision_at_10_std": -0.023165, + "nauc_precision_at_10_diff1": 0.778693, + "nauc_precision_at_20_max": 0.741398, + "nauc_precision_at_20_std": -0.03729, + "nauc_precision_at_20_diff1": 0.724511, + "nauc_precision_at_100_max": 0.750007, + "nauc_precision_at_100_std": 0.113908, + "nauc_precision_at_100_diff1": 0.690789, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.713711, + "nauc_mrr_at_1_std": 0.078936, + "nauc_mrr_at_1_diff1": 0.865522, + "nauc_mrr_at_3_max": 0.727986, + "nauc_mrr_at_3_std": 0.049345, + "nauc_mrr_at_3_diff1": 0.844054, + "nauc_mrr_at_5_max": 0.72805, + "nauc_mrr_at_5_std": 0.051724, + "nauc_mrr_at_5_diff1": 0.842374, + "nauc_mrr_at_10_max": 0.727371, + "nauc_mrr_at_10_std": 0.05087, + "nauc_mrr_at_10_diff1": 0.840469, + "nauc_mrr_at_20_max": 0.726789, + "nauc_mrr_at_20_std": 0.05081, + "nauc_mrr_at_20_diff1": 0.839135, + "nauc_mrr_at_100_max": 0.726677, + "nauc_mrr_at_100_std": 0.052435, + "nauc_mrr_at_100_diff1": 0.839816, + "nauc_mrr_at_1000_max": 0.726653, + "nauc_mrr_at_1000_std": 0.052356, + "nauc_mrr_at_1000_diff1": 0.83997, + "main_score": 0.77631, + "hf_subset": "default", + "languages": [ + "python-Code", + "c++-Code" + ] + } + ] + }, + "evaluation_time": 2.2210798263549805, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeTransOceanDL.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeTransOceanDL.json new file mode 100644 index 0000000000..2e167788f5 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CodeTransOceanDL.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "281562cb8a1265ab5c0824bfa6ddcd9b0a15618f", + "task_name": "CodeTransOceanDL", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.08333, + "ndcg_at_3": 0.12163, + "ndcg_at_5": 0.18882, + "ndcg_at_10": 0.33627, + "ndcg_at_20": 0.3924, + "ndcg_at_100": 0.39446, + "ndcg_at_1000": 0.39446, + "map_at_1": 0.08333, + "map_at_3": 0.11019, + "map_at_5": 0.14796, + "map_at_10": 0.20678, + "map_at_20": 0.22443, + "map_at_100": 0.22471, + "map_at_1000": 0.22471, + "recall_at_1": 0.08333, + "recall_at_3": 0.15556, + "recall_at_5": 0.31667, + "recall_at_10": 0.78333, + "recall_at_20": 0.98889, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.08333, + "precision_at_3": 0.05185, + "precision_at_5": 0.06333, + "precision_at_10": 0.07833, + "precision_at_20": 0.04944, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.05, + "mrr_at_3": 0.105556, + "mrr_at_5": 0.13, + "mrr_at_10": 0.196316, + "mrr_at_20": 0.212456, + "mrr_at_100": 0.212724, + "mrr_at_1000": 0.212724, + "nauc_ndcg_at_1_max": -0.402737, + "nauc_ndcg_at_1_std": -0.191468, + "nauc_ndcg_at_1_diff1": 0.744567, + "nauc_ndcg_at_3_max": -0.413848, + "nauc_ndcg_at_3_std": -0.224866, + "nauc_ndcg_at_3_diff1": 0.52874, + "nauc_ndcg_at_5_max": -0.316301, + "nauc_ndcg_at_5_std": -0.144361, + "nauc_ndcg_at_5_diff1": 0.527048, + "nauc_ndcg_at_10_max": -0.189478, + "nauc_ndcg_at_10_std": -0.123814, + "nauc_ndcg_at_10_diff1": 0.585237, + "nauc_ndcg_at_20_max": -0.310401, + "nauc_ndcg_at_20_std": -0.154188, + "nauc_ndcg_at_20_diff1": 0.56773, + "nauc_ndcg_at_100_max": -0.316583, + "nauc_ndcg_at_100_std": -0.158141, + "nauc_ndcg_at_100_diff1": 0.570899, + "nauc_ndcg_at_1000_max": -0.316583, + "nauc_ndcg_at_1000_std": -0.158141, + "nauc_ndcg_at_1000_diff1": 0.570899, + "nauc_map_at_1_max": -0.402737, + "nauc_map_at_1_std": -0.191468, + "nauc_map_at_1_diff1": 0.744567, + "nauc_map_at_3_max": -0.412249, + "nauc_map_at_3_std": -0.218987, + "nauc_map_at_3_diff1": 0.573946, + "nauc_map_at_5_max": -0.345124, + "nauc_map_at_5_std": -0.165524, + "nauc_map_at_5_diff1": 0.561165, + "nauc_map_at_10_max": -0.29097, + "nauc_map_at_10_std": -0.155516, + "nauc_map_at_10_diff1": 0.585678, + "nauc_map_at_20_max": -0.331754, + "nauc_map_at_20_std": -0.165734, + "nauc_map_at_20_diff1": 0.581874, + "nauc_map_at_100_max": -0.332458, + "nauc_map_at_100_std": -0.166181, + "nauc_map_at_100_diff1": 0.582254, + "nauc_map_at_1000_max": -0.332458, + "nauc_map_at_1000_std": -0.166181, + "nauc_map_at_1000_diff1": 0.582254, + "nauc_recall_at_1_max": -0.402737, + "nauc_recall_at_1_std": -0.191468, + "nauc_recall_at_1_diff1": 0.744567, + "nauc_recall_at_3_max": -0.417183, + "nauc_recall_at_3_std": -0.237576, + "nauc_recall_at_3_diff1": 0.428528, + "nauc_recall_at_5_max": -0.262003, + "nauc_recall_at_5_std": -0.103016, + "nauc_recall_at_5_diff1": 0.471988, + "nauc_recall_at_10_max": 0.155762, + "nauc_recall_at_10_std": -0.023986, + "nauc_recall_at_10_diff1": 0.623623, + "nauc_recall_at_20_max": 0.122782, + "nauc_recall_at_20_std": 0.122782, + "nauc_recall_at_20_diff1": 0.348973, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.402737, + "nauc_precision_at_1_std": -0.191468, + "nauc_precision_at_1_diff1": 0.744567, + "nauc_precision_at_3_max": -0.417183, + "nauc_precision_at_3_std": -0.237576, + "nauc_precision_at_3_diff1": 0.428528, + "nauc_precision_at_5_max": -0.262003, + "nauc_precision_at_5_std": -0.103016, + "nauc_precision_at_5_diff1": 0.471988, + "nauc_precision_at_10_max": 0.155762, + "nauc_precision_at_10_std": -0.023986, + "nauc_precision_at_10_diff1": 0.623623, + "nauc_precision_at_20_max": 0.122782, + "nauc_precision_at_20_std": 0.122782, + "nauc_precision_at_20_diff1": 0.348973, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": -0.400322, + "nauc_mrr_at_1_std": -0.18278, + "nauc_mrr_at_1_diff1": 0.304147, + "nauc_mrr_at_3_max": -0.329336, + "nauc_mrr_at_3_std": -0.17309, + "nauc_mrr_at_3_diff1": 0.205728, + "nauc_mrr_at_5_max": -0.303036, + "nauc_mrr_at_5_std": -0.157551, + "nauc_mrr_at_5_diff1": 0.21714, + "nauc_mrr_at_10_max": -0.263855, + "nauc_mrr_at_10_std": -0.132614, + "nauc_mrr_at_10_diff1": 0.244516, + "nauc_mrr_at_20_max": -0.292754, + "nauc_mrr_at_20_std": -0.143156, + "nauc_mrr_at_20_diff1": 0.220734, + "nauc_mrr_at_100_max": -0.293486, + "nauc_mrr_at_100_std": -0.143625, + "nauc_mrr_at_100_diff1": 0.220519, + "nauc_mrr_at_1000_max": -0.293486, + "nauc_mrr_at_1000_std": -0.143625, + "nauc_mrr_at_1000_diff1": 0.220519, + "main_score": 0.33627, + "hf_subset": "default", + "languages": [ + "python-Code" + ] + } + ] + }, + "evaluation_time": 2.089700937271118, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CosQA.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CosQA.json new file mode 100644 index 0000000000..e1b65ac9ba --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/CosQA.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "bc5efb7e9d437246ce393ed19d772e08e4a79535", + "task_name": "CosQA", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.158, + "ndcg_at_3": 0.24695, + "ndcg_at_5": 0.30444, + "ndcg_at_10": 0.35576, + "ndcg_at_20": 0.38581, + "ndcg_at_100": 0.42316, + "ndcg_at_1000": 0.43179, + "map_at_1": 0.158, + "map_at_3": 0.22333, + "map_at_5": 0.25513, + "map_at_10": 0.27684, + "map_at_20": 0.28519, + "map_at_100": 0.29062, + "map_at_1000": 0.29098, + "recall_at_1": 0.158, + "recall_at_3": 0.316, + "recall_at_5": 0.456, + "recall_at_10": 0.612, + "recall_at_20": 0.73, + "recall_at_100": 0.928, + "recall_at_1000": 0.994, + "precision_at_1": 0.158, + "precision_at_3": 0.10533, + "precision_at_5": 0.0912, + "precision_at_10": 0.0612, + "precision_at_20": 0.0365, + "precision_at_100": 0.00928, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.146, + "mrr_at_3": 0.202, + "mrr_at_5": 0.2256, + "mrr_at_10": 0.253474, + "mrr_at_20": 0.262861, + "mrr_at_100": 0.268703, + "mrr_at_1000": 0.269061, + "nauc_ndcg_at_1_max": 0.051033, + "nauc_ndcg_at_1_std": -0.159799, + "nauc_ndcg_at_1_diff1": 0.356632, + "nauc_ndcg_at_3_max": 0.104409, + "nauc_ndcg_at_3_std": -0.116185, + "nauc_ndcg_at_3_diff1": 0.234699, + "nauc_ndcg_at_5_max": 0.134221, + "nauc_ndcg_at_5_std": -0.082948, + "nauc_ndcg_at_5_diff1": 0.203911, + "nauc_ndcg_at_10_max": 0.128973, + "nauc_ndcg_at_10_std": -0.083369, + "nauc_ndcg_at_10_diff1": 0.191918, + "nauc_ndcg_at_20_max": 0.129867, + "nauc_ndcg_at_20_std": -0.076803, + "nauc_ndcg_at_20_diff1": 0.191381, + "nauc_ndcg_at_100_max": 0.134401, + "nauc_ndcg_at_100_std": -0.069783, + "nauc_ndcg_at_100_diff1": 0.213594, + "nauc_ndcg_at_1000_max": 0.121895, + "nauc_ndcg_at_1000_std": -0.082747, + "nauc_ndcg_at_1000_diff1": 0.219743, + "nauc_map_at_1_max": 0.051033, + "nauc_map_at_1_std": -0.159799, + "nauc_map_at_1_diff1": 0.356632, + "nauc_map_at_3_max": 0.093941, + "nauc_map_at_3_std": -0.123991, + "nauc_map_at_3_diff1": 0.262469, + "nauc_map_at_5_max": 0.110101, + "nauc_map_at_5_std": -0.104855, + "nauc_map_at_5_diff1": 0.243262, + "nauc_map_at_10_max": 0.107998, + "nauc_map_at_10_std": -0.105305, + "nauc_map_at_10_diff1": 0.237956, + "nauc_map_at_20_max": 0.107995, + "nauc_map_at_20_std": -0.103533, + "nauc_map_at_20_diff1": 0.238383, + "nauc_map_at_100_max": 0.10777, + "nauc_map_at_100_std": -0.102615, + "nauc_map_at_100_diff1": 0.241838, + "nauc_map_at_1000_max": 0.107305, + "nauc_map_at_1000_std": -0.103038, + "nauc_map_at_1000_diff1": 0.242114, + "nauc_recall_at_1_max": 0.051033, + "nauc_recall_at_1_std": -0.159799, + "nauc_recall_at_1_diff1": 0.356632, + "nauc_recall_at_3_max": 0.129278, + "nauc_recall_at_3_std": -0.097902, + "nauc_recall_at_3_diff1": 0.167371, + "nauc_recall_at_5_max": 0.195974, + "nauc_recall_at_5_std": -0.027223, + "nauc_recall_at_5_diff1": 0.107452, + "nauc_recall_at_10_max": 0.18708, + "nauc_recall_at_10_std": -0.020817, + "nauc_recall_at_10_diff1": 0.062636, + "nauc_recall_at_20_max": 0.207309, + "nauc_recall_at_20_std": 0.024085, + "nauc_recall_at_20_diff1": 0.023739, + "nauc_recall_at_100_max": 0.490611, + "nauc_recall_at_100_std": 0.356533, + "nauc_recall_at_100_diff1": 0.038891, + "nauc_recall_at_1000_max": 0.637411, + "nauc_recall_at_1000_std": 0.956427, + "nauc_recall_at_1000_diff1": -0.125584, + "nauc_precision_at_1_max": 0.051033, + "nauc_precision_at_1_std": -0.159799, + "nauc_precision_at_1_diff1": 0.356632, + "nauc_precision_at_3_max": 0.129278, + "nauc_precision_at_3_std": -0.097902, + "nauc_precision_at_3_diff1": 0.167371, + "nauc_precision_at_5_max": 0.195974, + "nauc_precision_at_5_std": -0.027223, + "nauc_precision_at_5_diff1": 0.107452, + "nauc_precision_at_10_max": 0.18708, + "nauc_precision_at_10_std": -0.020817, + "nauc_precision_at_10_diff1": 0.062636, + "nauc_precision_at_20_max": 0.207309, + "nauc_precision_at_20_std": 0.024085, + "nauc_precision_at_20_diff1": 0.023739, + "nauc_precision_at_100_max": 0.490611, + "nauc_precision_at_100_std": 0.356533, + "nauc_precision_at_100_diff1": 0.038891, + "nauc_precision_at_1000_max": 0.637411, + "nauc_precision_at_1000_std": 0.956427, + "nauc_precision_at_1000_diff1": -0.125584, + "nauc_mrr_at_1_max": 0.096408, + "nauc_mrr_at_1_std": -0.185252, + "nauc_mrr_at_1_diff1": 0.407769, + "nauc_mrr_at_3_max": 0.126519, + "nauc_mrr_at_3_std": -0.168647, + "nauc_mrr_at_3_diff1": 0.296716, + "nauc_mrr_at_5_max": 0.12703, + "nauc_mrr_at_5_std": -0.14841, + "nauc_mrr_at_5_diff1": 0.269786, + "nauc_mrr_at_10_max": 0.132738, + "nauc_mrr_at_10_std": -0.143788, + "nauc_mrr_at_10_diff1": 0.263927, + "nauc_mrr_at_20_max": 0.129089, + "nauc_mrr_at_20_std": -0.142899, + "nauc_mrr_at_20_diff1": 0.264212, + "nauc_mrr_at_100_max": 0.129373, + "nauc_mrr_at_100_std": -0.143092, + "nauc_mrr_at_100_diff1": 0.267493, + "nauc_mrr_at_1000_max": 0.128924, + "nauc_mrr_at_1000_std": -0.143584, + "nauc_mrr_at_1000_diff1": 0.26781, + "main_score": 0.35576, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 6.46893835067749, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/DBPedia.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/DBPedia.json new file mode 100644 index 0000000000..9f4ed9bc53 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/DBPedia.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659", + "task_name": "DBPedia", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.51125, + "ndcg_at_3": 0.41836, + "ndcg_at_5": 0.39499, + "ndcg_at_10": 0.37849, + "ndcg_at_20": 0.37344, + "ndcg_at_100": 0.42396, + "ndcg_at_1000": 0.50272, + "map_at_1": 0.08695, + "map_at_3": 0.13187, + "map_at_5": 0.15418, + "map_at_10": 0.18299, + "map_at_20": 0.20938, + "map_at_100": 0.25139, + "map_at_1000": 0.26646, + "recall_at_1": 0.08695, + "recall_at_3": 0.14517, + "recall_at_5": 0.18079, + "recall_at_10": 0.24158, + "recall_at_20": 0.30831, + "recall_at_100": 0.49277, + "recall_at_1000": 0.74211, + "precision_at_1": 0.64, + "precision_at_3": 0.45333, + "precision_at_5": 0.382, + "precision_at_10": 0.2935, + "precision_at_20": 0.2195, + "precision_at_100": 0.092, + "precision_at_1000": 0.01949, + "mrr_at_1": 0.64, + "mrr_at_3": 0.71125, + "mrr_at_5": 0.719375, + "mrr_at_10": 0.725935, + "mrr_at_20": 0.728776, + "mrr_at_100": 0.729898, + "mrr_at_1000": 0.730029, + "nauc_ndcg_at_1_max": 0.390663, + "nauc_ndcg_at_1_std": 0.165072, + "nauc_ndcg_at_1_diff1": 0.382202, + "nauc_ndcg_at_3_max": 0.320758, + "nauc_ndcg_at_3_std": 0.1565, + "nauc_ndcg_at_3_diff1": 0.239631, + "nauc_ndcg_at_5_max": 0.291878, + "nauc_ndcg_at_5_std": 0.163255, + "nauc_ndcg_at_5_diff1": 0.230407, + "nauc_ndcg_at_10_max": 0.268541, + "nauc_ndcg_at_10_std": 0.142995, + "nauc_ndcg_at_10_diff1": 0.245817, + "nauc_ndcg_at_20_max": 0.23362, + "nauc_ndcg_at_20_std": 0.122736, + "nauc_ndcg_at_20_diff1": 0.245651, + "nauc_ndcg_at_100_max": 0.223467, + "nauc_ndcg_at_100_std": 0.171694, + "nauc_ndcg_at_100_diff1": 0.211022, + "nauc_ndcg_at_1000_max": 0.284342, + "nauc_ndcg_at_1000_std": 0.255672, + "nauc_ndcg_at_1000_diff1": 0.191441, + "nauc_map_at_1_max": -0.002275, + "nauc_map_at_1_std": -0.244658, + "nauc_map_at_1_diff1": 0.372066, + "nauc_map_at_3_max": 0.009865, + "nauc_map_at_3_std": -0.213329, + "nauc_map_at_3_diff1": 0.283803, + "nauc_map_at_5_max": 0.022897, + "nauc_map_at_5_std": -0.17884, + "nauc_map_at_5_diff1": 0.267507, + "nauc_map_at_10_max": 0.05537, + "nauc_map_at_10_std": -0.115263, + "nauc_map_at_10_diff1": 0.245628, + "nauc_map_at_20_max": 0.098953, + "nauc_map_at_20_std": -0.028245, + "nauc_map_at_20_diff1": 0.231129, + "nauc_map_at_100_max": 0.159436, + "nauc_map_at_100_std": 0.119869, + "nauc_map_at_100_diff1": 0.207075, + "nauc_map_at_1000_max": 0.179545, + "nauc_map_at_1000_std": 0.150864, + "nauc_map_at_1000_diff1": 0.198752, + "nauc_recall_at_1_max": -0.002275, + "nauc_recall_at_1_std": -0.244658, + "nauc_recall_at_1_diff1": 0.372066, + "nauc_recall_at_3_max": -0.013847, + "nauc_recall_at_3_std": -0.22386, + "nauc_recall_at_3_diff1": 0.247261, + "nauc_recall_at_5_max": -0.002092, + "nauc_recall_at_5_std": -0.188853, + "nauc_recall_at_5_diff1": 0.245158, + "nauc_recall_at_10_max": 0.011786, + "nauc_recall_at_10_std": -0.135706, + "nauc_recall_at_10_diff1": 0.213407, + "nauc_recall_at_20_max": 0.037171, + "nauc_recall_at_20_std": -0.043011, + "nauc_recall_at_20_diff1": 0.178323, + "nauc_recall_at_100_max": 0.107113, + "nauc_recall_at_100_std": 0.150146, + "nauc_recall_at_100_diff1": 0.114323, + "nauc_recall_at_1000_max": 0.159743, + "nauc_recall_at_1000_std": 0.298374, + "nauc_recall_at_1000_diff1": 0.060799, + "nauc_precision_at_1_max": 0.459933, + "nauc_precision_at_1_std": 0.218307, + "nauc_precision_at_1_diff1": 0.471652, + "nauc_precision_at_3_max": 0.371215, + "nauc_precision_at_3_std": 0.286604, + "nauc_precision_at_3_diff1": 0.122186, + "nauc_precision_at_5_max": 0.347484, + "nauc_precision_at_5_std": 0.364811, + "nauc_precision_at_5_diff1": 0.075576, + "nauc_precision_at_10_max": 0.37264, + "nauc_precision_at_10_std": 0.449082, + "nauc_precision_at_10_diff1": 0.059623, + "nauc_precision_at_20_max": 0.360542, + "nauc_precision_at_20_std": 0.507707, + "nauc_precision_at_20_diff1": 0.018127, + "nauc_precision_at_100_max": 0.296461, + "nauc_precision_at_100_std": 0.524118, + "nauc_precision_at_100_diff1": -0.057802, + "nauc_precision_at_1000_max": 0.216193, + "nauc_precision_at_1000_std": 0.279162, + "nauc_precision_at_1000_diff1": -0.165145, + "nauc_mrr_at_1_max": 0.459933, + "nauc_mrr_at_1_std": 0.218307, + "nauc_mrr_at_1_diff1": 0.471652, + "nauc_mrr_at_3_max": 0.484793, + "nauc_mrr_at_3_std": 0.246178, + "nauc_mrr_at_3_diff1": 0.459131, + "nauc_mrr_at_5_max": 0.487806, + "nauc_mrr_at_5_std": 0.249894, + "nauc_mrr_at_5_diff1": 0.467842, + "nauc_mrr_at_10_max": 0.491583, + "nauc_mrr_at_10_std": 0.252052, + "nauc_mrr_at_10_diff1": 0.469571, + "nauc_mrr_at_20_max": 0.490331, + "nauc_mrr_at_20_std": 0.255339, + "nauc_mrr_at_20_diff1": 0.4661, + "nauc_mrr_at_100_max": 0.490169, + "nauc_mrr_at_100_std": 0.254498, + "nauc_mrr_at_100_diff1": 0.466542, + "nauc_mrr_at_1000_max": 0.489971, + "nauc_mrr_at_1000_std": 0.254136, + "nauc_mrr_at_1000_diff1": 0.466519, + "main_score": 0.37849, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2203.046250104904, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/EmotionClassification.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/EmotionClassification.json new file mode 100644 index 0000000000..feecf8b4a0 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/EmotionClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "4f58c6b202a23cf9a4da393831edf4f9183cad37", + "task_name": "EmotionClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.34565, + "f1": 0.312211, + "f1_weighted": 0.365642, + "scores_per_experiment": [ + { + "accuracy": 0.378, + "f1": 0.332837, + "f1_weighted": 0.403311 + }, + { + "accuracy": 0.328, + "f1": 0.296203, + "f1_weighted": 0.340391 + }, + { + "accuracy": 0.314, + "f1": 0.286295, + "f1_weighted": 0.333503 + }, + { + "accuracy": 0.3275, + "f1": 0.306377, + "f1_weighted": 0.346474 + }, + { + "accuracy": 0.347, + "f1": 0.322369, + "f1_weighted": 0.370082 + }, + { + "accuracy": 0.3425, + "f1": 0.31065, + "f1_weighted": 0.366656 + }, + { + "accuracy": 0.357, + "f1": 0.31641, + "f1_weighted": 0.373635 + }, + { + "accuracy": 0.3175, + "f1": 0.286401, + "f1_weighted": 0.334374 + }, + { + "accuracy": 0.3635, + "f1": 0.324409, + "f1_weighted": 0.386065 + }, + { + "accuracy": 0.3815, + "f1": 0.340155, + "f1_weighted": 0.401933 + } + ], + "main_score": 0.34565, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 21.231767892837524, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/FEVER.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/FEVER.json new file mode 100644 index 0000000000..9f6e5470b2 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/FEVER.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12", + "task_name": "FEVER", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.80813, + "ndcg_at_3": 0.84986, + "ndcg_at_5": 0.85957, + "ndcg_at_10": 0.86481, + "ndcg_at_20": 0.86738, + "ndcg_at_100": 0.8709, + "ndcg_at_1000": 0.8733, + "map_at_1": 0.7494, + "map_at_3": 0.81712, + "map_at_5": 0.82371, + "map_at_10": 0.82643, + "map_at_20": 0.82733, + "map_at_100": 0.82802, + "map_at_1000": 0.82816, + "recall_at_1": 0.7494, + "recall_at_3": 0.88852, + "recall_at_5": 0.9146, + "recall_at_10": 0.93074, + "recall_at_20": 0.93974, + "recall_at_100": 0.95576, + "recall_at_1000": 0.97083, + "precision_at_1": 0.80813, + "precision_at_3": 0.32198, + "precision_at_5": 0.19925, + "precision_at_10": 0.10174, + "precision_at_20": 0.05159, + "precision_at_100": 0.01063, + "precision_at_1000": 0.0011, + "mrr_at_1": 0.808131, + "mrr_at_3": 0.873537, + "mrr_at_5": 0.87908, + "mrr_at_10": 0.880918, + "mrr_at_20": 0.881353, + "mrr_at_100": 0.881494, + "mrr_at_1000": 0.881503, + "nauc_ndcg_at_1_max": 0.25646, + "nauc_ndcg_at_1_std": -0.152753, + "nauc_ndcg_at_1_diff1": 0.647606, + "nauc_ndcg_at_3_max": 0.211629, + "nauc_ndcg_at_3_std": -0.105993, + "nauc_ndcg_at_3_diff1": 0.463592, + "nauc_ndcg_at_5_max": 0.212345, + "nauc_ndcg_at_5_std": -0.077859, + "nauc_ndcg_at_5_diff1": 0.450617, + "nauc_ndcg_at_10_max": 0.214322, + "nauc_ndcg_at_10_std": -0.069822, + "nauc_ndcg_at_10_diff1": 0.453029, + "nauc_ndcg_at_20_max": 0.212179, + "nauc_ndcg_at_20_std": -0.066422, + "nauc_ndcg_at_20_diff1": 0.451957, + "nauc_ndcg_at_100_max": 0.209908, + "nauc_ndcg_at_100_std": -0.067199, + "nauc_ndcg_at_100_diff1": 0.455255, + "nauc_ndcg_at_1000_max": 0.213597, + "nauc_ndcg_at_1000_std": -0.069701, + "nauc_ndcg_at_1000_diff1": 0.461614, + "nauc_map_at_1_max": 0.210447, + "nauc_map_at_1_std": -0.097026, + "nauc_map_at_1_diff1": 0.515213, + "nauc_map_at_3_max": 0.19866, + "nauc_map_at_3_std": -0.090464, + "nauc_map_at_3_diff1": 0.455221, + "nauc_map_at_5_max": 0.202612, + "nauc_map_at_5_std": -0.077709, + "nauc_map_at_5_diff1": 0.451438, + "nauc_map_at_10_max": 0.204422, + "nauc_map_at_10_std": -0.075366, + "nauc_map_at_10_diff1": 0.453019, + "nauc_map_at_20_max": 0.203832, + "nauc_map_at_20_std": -0.074859, + "nauc_map_at_20_diff1": 0.452823, + "nauc_map_at_100_max": 0.203842, + "nauc_map_at_100_std": -0.074648, + "nauc_map_at_100_diff1": 0.453278, + "nauc_map_at_1000_max": 0.204084, + "nauc_map_at_1000_std": -0.074689, + "nauc_map_at_1000_diff1": 0.453526, + "nauc_recall_at_1_max": 0.210447, + "nauc_recall_at_1_std": -0.097026, + "nauc_recall_at_1_diff1": 0.515213, + "nauc_recall_at_3_max": 0.150889, + "nauc_recall_at_3_std": -0.07084, + "nauc_recall_at_3_diff1": 0.286383, + "nauc_recall_at_5_max": 0.156312, + "nauc_recall_at_5_std": 0.03743, + "nauc_recall_at_5_diff1": 0.198437, + "nauc_recall_at_10_max": 0.163106, + "nauc_recall_at_10_std": 0.10233, + "nauc_recall_at_10_diff1": 0.162379, + "nauc_recall_at_20_max": 0.136307, + "nauc_recall_at_20_std": 0.153902, + "nauc_recall_at_20_diff1": 0.112707, + "nauc_recall_at_100_max": 0.066986, + "nauc_recall_at_100_std": 0.203591, + "nauc_recall_at_100_diff1": 0.046943, + "nauc_recall_at_1000_max": 0.06872, + "nauc_recall_at_1000_std": 0.262034, + "nauc_recall_at_1000_diff1": 0.023907, + "nauc_precision_at_1_max": 0.25646, + "nauc_precision_at_1_std": -0.152753, + "nauc_precision_at_1_diff1": 0.647606, + "nauc_precision_at_3_max": 0.159174, + "nauc_precision_at_3_std": -0.163004, + "nauc_precision_at_3_diff1": 0.284738, + "nauc_precision_at_5_max": 0.137561, + "nauc_precision_at_5_std": -0.020887, + "nauc_precision_at_5_diff1": 0.108048, + "nauc_precision_at_10_max": 0.110575, + "nauc_precision_at_10_std": 0.038058, + "nauc_precision_at_10_diff1": 0.013149, + "nauc_precision_at_20_max": 0.069594, + "nauc_precision_at_20_std": 0.073169, + "nauc_precision_at_20_diff1": -0.071282, + "nauc_precision_at_100_max": 0.013828, + "nauc_precision_at_100_std": 0.073762, + "nauc_precision_at_100_diff1": -0.109073, + "nauc_precision_at_1000_max": 0.032618, + "nauc_precision_at_1000_std": 0.043861, + "nauc_precision_at_1000_diff1": -0.086236, + "nauc_mrr_at_1_max": 0.25646, + "nauc_mrr_at_1_std": -0.152753, + "nauc_mrr_at_1_diff1": 0.647606, + "nauc_mrr_at_3_max": 0.267516, + "nauc_mrr_at_3_std": -0.172916, + "nauc_mrr_at_3_diff1": 0.620198, + "nauc_mrr_at_5_max": 0.27325, + "nauc_mrr_at_5_std": -0.16097, + "nauc_mrr_at_5_diff1": 0.622021, + "nauc_mrr_at_10_max": 0.275085, + "nauc_mrr_at_10_std": -0.159191, + "nauc_mrr_at_10_diff1": 0.625978, + "nauc_mrr_at_20_max": 0.274257, + "nauc_mrr_at_20_std": -0.158668, + "nauc_mrr_at_20_diff1": 0.626314, + "nauc_mrr_at_100_max": 0.273616, + "nauc_mrr_at_100_std": -0.158833, + "nauc_mrr_at_100_diff1": 0.626257, + "nauc_mrr_at_1000_max": 0.273567, + "nauc_mrr_at_1000_std": -0.158905, + "nauc_mrr_at_1000_diff1": 0.626259, + "main_score": 0.86481, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3729.318694114685, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/FEVERHardNegatives.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/FEVERHardNegatives.json new file mode 100644 index 0000000000..f0ee068c44 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/FEVERHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "080c9ed6267b65029207906e815d44a9240bafca", + "task_name": "FEVERHardNegatives", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.811, + "ndcg_at_3": 0.86011, + "ndcg_at_5": 0.87041, + "ndcg_at_10": 0.87581, + "ndcg_at_20": 0.87836, + "ndcg_at_100": 0.8822, + "ndcg_at_1000": 0.8848, + "map_at_1": 0.76038, + "map_at_3": 0.82846, + "map_at_5": 0.83553, + "map_at_10": 0.83835, + "map_at_20": 0.83929, + "map_at_100": 0.84006, + "map_at_1000": 0.84023, + "recall_at_1": 0.76038, + "recall_at_3": 0.90171, + "recall_at_5": 0.92925, + "recall_at_10": 0.94541, + "recall_at_20": 0.95428, + "recall_at_100": 0.97144, + "recall_at_1000": 0.98635, + "precision_at_1": 0.811, + "precision_at_3": 0.321, + "precision_at_5": 0.1992, + "precision_at_10": 0.1017, + "precision_at_20": 0.05165, + "precision_at_100": 0.01073, + "precision_at_1000": 0.00112, + "mrr_at_1": 0.811, + "mrr_at_3": 0.878167, + "mrr_at_5": 0.883867, + "mrr_at_10": 0.885899, + "mrr_at_20": 0.886237, + "mrr_at_100": 0.886287, + "mrr_at_1000": 0.886287, + "nauc_ndcg_at_1_max": 0.250415, + "nauc_ndcg_at_1_std": -0.149232, + "nauc_ndcg_at_1_diff1": 0.673693, + "nauc_ndcg_at_3_max": 0.170874, + "nauc_ndcg_at_3_std": -0.075767, + "nauc_ndcg_at_3_diff1": 0.500029, + "nauc_ndcg_at_5_max": 0.185971, + "nauc_ndcg_at_5_std": -0.072578, + "nauc_ndcg_at_5_diff1": 0.512308, + "nauc_ndcg_at_10_max": 0.187459, + "nauc_ndcg_at_10_std": -0.049331, + "nauc_ndcg_at_10_diff1": 0.512304, + "nauc_ndcg_at_20_max": 0.182704, + "nauc_ndcg_at_20_std": -0.053719, + "nauc_ndcg_at_20_diff1": 0.516198, + "nauc_ndcg_at_100_max": 0.185133, + "nauc_ndcg_at_100_std": -0.055427, + "nauc_ndcg_at_100_diff1": 0.525201, + "nauc_ndcg_at_1000_max": 0.186191, + "nauc_ndcg_at_1000_std": -0.057653, + "nauc_ndcg_at_1000_diff1": 0.531418, + "nauc_map_at_1_max": 0.21234, + "nauc_map_at_1_std": -0.098475, + "nauc_map_at_1_diff1": 0.578789, + "nauc_map_at_3_max": 0.169191, + "nauc_map_at_3_std": -0.07338, + "nauc_map_at_3_diff1": 0.50473, + "nauc_map_at_5_max": 0.180675, + "nauc_map_at_5_std": -0.074422, + "nauc_map_at_5_diff1": 0.514172, + "nauc_map_at_10_max": 0.181897, + "nauc_map_at_10_std": -0.064883, + "nauc_map_at_10_diff1": 0.514632, + "nauc_map_at_20_max": 0.180832, + "nauc_map_at_20_std": -0.066189, + "nauc_map_at_20_diff1": 0.51593, + "nauc_map_at_100_max": 0.181253, + "nauc_map_at_100_std": -0.066796, + "nauc_map_at_100_diff1": 0.517582, + "nauc_map_at_1000_max": 0.181369, + "nauc_map_at_1000_std": -0.066712, + "nauc_map_at_1000_diff1": 0.517827, + "nauc_recall_at_1_max": 0.21234, + "nauc_recall_at_1_std": -0.098475, + "nauc_recall_at_1_diff1": 0.578789, + "nauc_recall_at_3_max": 0.073839, + "nauc_recall_at_3_std": -0.005552, + "nauc_recall_at_3_diff1": 0.295423, + "nauc_recall_at_5_max": 0.107304, + "nauc_recall_at_5_std": 0.026176, + "nauc_recall_at_5_diff1": 0.263806, + "nauc_recall_at_10_max": 0.087324, + "nauc_recall_at_10_std": 0.17817, + "nauc_recall_at_10_diff1": 0.191379, + "nauc_recall_at_20_max": 0.026562, + "nauc_recall_at_20_std": 0.175443, + "nauc_recall_at_20_diff1": 0.173885, + "nauc_recall_at_100_max": -0.034653, + "nauc_recall_at_100_std": 0.279161, + "nauc_recall_at_100_diff1": 0.146051, + "nauc_recall_at_1000_max": -0.240521, + "nauc_recall_at_1000_std": 0.502295, + "nauc_recall_at_1000_diff1": 0.093146, + "nauc_precision_at_1_max": 0.250415, + "nauc_precision_at_1_std": -0.149232, + "nauc_precision_at_1_diff1": 0.673693, + "nauc_precision_at_3_max": 0.089676, + "nauc_precision_at_3_std": -0.09512, + "nauc_precision_at_3_diff1": 0.271179, + "nauc_precision_at_5_max": 0.155823, + "nauc_precision_at_5_std": -0.078834, + "nauc_precision_at_5_diff1": 0.14698, + "nauc_precision_at_10_max": 0.084564, + "nauc_precision_at_10_std": 0.142357, + "nauc_precision_at_10_diff1": -0.056931, + "nauc_precision_at_20_max": -0.026082, + "nauc_precision_at_20_std": 0.101411, + "nauc_precision_at_20_diff1": -0.131991, + "nauc_precision_at_100_max": -0.009038, + "nauc_precision_at_100_std": 0.08622, + "nauc_precision_at_100_diff1": -0.121358, + "nauc_precision_at_1000_max": -0.02604, + "nauc_precision_at_1000_std": 0.035015, + "nauc_precision_at_1000_diff1": -0.107754, + "nauc_mrr_at_1_max": 0.250415, + "nauc_mrr_at_1_std": -0.149232, + "nauc_mrr_at_1_diff1": 0.673693, + "nauc_mrr_at_3_max": 0.24147, + "nauc_mrr_at_3_std": -0.13276, + "nauc_mrr_at_3_diff1": 0.636933, + "nauc_mrr_at_5_max": 0.250307, + "nauc_mrr_at_5_std": -0.13403, + "nauc_mrr_at_5_diff1": 0.647393, + "nauc_mrr_at_10_max": 0.250845, + "nauc_mrr_at_10_std": -0.130872, + "nauc_mrr_at_10_diff1": 0.649813, + "nauc_mrr_at_20_max": 0.249853, + "nauc_mrr_at_20_std": -0.131831, + "nauc_mrr_at_20_diff1": 0.650705, + "nauc_mrr_at_100_max": 0.249506, + "nauc_mrr_at_100_std": -0.132286, + "nauc_mrr_at_100_diff1": 0.650557, + "nauc_mrr_at_1000_max": 0.249506, + "nauc_mrr_at_1000_std": -0.132286, + "nauc_mrr_at_1000_diff1": 0.650557, + "main_score": 0.87581, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 79.82257318496704, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/FiQA2018.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/FiQA2018.json new file mode 100644 index 0000000000..93b4f26577 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/FiQA2018.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "27a168819829fe9bcd655c2df245fb19452e8e06", + "task_name": "FiQA2018", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.40278, + "ndcg_at_3": 0.37064, + "ndcg_at_5": 0.38308, + "ndcg_at_10": 0.40808, + "ndcg_at_20": 0.43483, + "ndcg_at_100": 0.47714, + "ndcg_at_1000": 0.50627, + "map_at_1": 0.20854, + "map_at_3": 0.28863, + "map_at_5": 0.31181, + "map_at_10": 0.33083, + "map_at_20": 0.34063, + "map_at_100": 0.34898, + "map_at_1000": 0.35075, + "recall_at_1": 0.20854, + "recall_at_3": 0.33538, + "recall_at_5": 0.39653, + "recall_at_10": 0.47369, + "recall_at_20": 0.55631, + "recall_at_100": 0.73502, + "recall_at_1000": 0.90973, + "precision_at_1": 0.40278, + "precision_at_3": 0.2464, + "precision_at_5": 0.18241, + "precision_at_10": 0.11343, + "precision_at_20": 0.06806, + "precision_at_100": 0.01833, + "precision_at_1000": 0.00235, + "mrr_at_1": 0.402778, + "mrr_at_3": 0.461934, + "mrr_at_5": 0.476672, + "mrr_at_10": 0.487202, + "mrr_at_20": 0.492312, + "mrr_at_100": 0.49534, + "mrr_at_1000": 0.495711, + "nauc_ndcg_at_1_max": 0.338508, + "nauc_ndcg_at_1_std": 0.033934, + "nauc_ndcg_at_1_diff1": 0.522563, + "nauc_ndcg_at_3_max": 0.32208, + "nauc_ndcg_at_3_std": 0.009439, + "nauc_ndcg_at_3_diff1": 0.457483, + "nauc_ndcg_at_5_max": 0.308697, + "nauc_ndcg_at_5_std": 0.010349, + "nauc_ndcg_at_5_diff1": 0.443975, + "nauc_ndcg_at_10_max": 0.306793, + "nauc_ndcg_at_10_std": 0.016655, + "nauc_ndcg_at_10_diff1": 0.441654, + "nauc_ndcg_at_20_max": 0.313821, + "nauc_ndcg_at_20_std": 0.037561, + "nauc_ndcg_at_20_diff1": 0.43879, + "nauc_ndcg_at_100_max": 0.332244, + "nauc_ndcg_at_100_std": 0.063922, + "nauc_ndcg_at_100_diff1": 0.445778, + "nauc_ndcg_at_1000_max": 0.339235, + "nauc_ndcg_at_1000_std": 0.0585, + "nauc_ndcg_at_1000_diff1": 0.449925, + "nauc_map_at_1_max": 0.171671, + "nauc_map_at_1_std": -0.019445, + "nauc_map_at_1_diff1": 0.450065, + "nauc_map_at_3_max": 0.251495, + "nauc_map_at_3_std": -0.013694, + "nauc_map_at_3_diff1": 0.440429, + "nauc_map_at_5_max": 0.264184, + "nauc_map_at_5_std": -0.007288, + "nauc_map_at_5_diff1": 0.433537, + "nauc_map_at_10_max": 0.276688, + "nauc_map_at_10_std": 0.001871, + "nauc_map_at_10_diff1": 0.43409, + "nauc_map_at_20_max": 0.28169, + "nauc_map_at_20_std": 0.009944, + "nauc_map_at_20_diff1": 0.433142, + "nauc_map_at_100_max": 0.285503, + "nauc_map_at_100_std": 0.01568, + "nauc_map_at_100_diff1": 0.433168, + "nauc_map_at_1000_max": 0.286578, + "nauc_map_at_1000_std": 0.015585, + "nauc_map_at_1000_diff1": 0.433675, + "nauc_recall_at_1_max": 0.171671, + "nauc_recall_at_1_std": -0.019445, + "nauc_recall_at_1_diff1": 0.450065, + "nauc_recall_at_3_max": 0.252111, + "nauc_recall_at_3_std": -0.005842, + "nauc_recall_at_3_diff1": 0.411054, + "nauc_recall_at_5_max": 0.249532, + "nauc_recall_at_5_std": -0.003393, + "nauc_recall_at_5_diff1": 0.38418, + "nauc_recall_at_10_max": 0.249619, + "nauc_recall_at_10_std": 0.015686, + "nauc_recall_at_10_diff1": 0.356259, + "nauc_recall_at_20_max": 0.250657, + "nauc_recall_at_20_std": 0.083584, + "nauc_recall_at_20_diff1": 0.322262, + "nauc_recall_at_100_max": 0.315312, + "nauc_recall_at_100_std": 0.233591, + "nauc_recall_at_100_diff1": 0.341128, + "nauc_recall_at_1000_max": 0.41225, + "nauc_recall_at_1000_std": 0.394633, + "nauc_recall_at_1000_diff1": 0.359077, + "nauc_precision_at_1_max": 0.338508, + "nauc_precision_at_1_std": 0.033934, + "nauc_precision_at_1_diff1": 0.522563, + "nauc_precision_at_3_max": 0.376779, + "nauc_precision_at_3_std": 0.037159, + "nauc_precision_at_3_diff1": 0.34615, + "nauc_precision_at_5_max": 0.361584, + "nauc_precision_at_5_std": 0.054327, + "nauc_precision_at_5_diff1": 0.276104, + "nauc_precision_at_10_max": 0.346355, + "nauc_precision_at_10_std": 0.082535, + "nauc_precision_at_10_diff1": 0.219635, + "nauc_precision_at_20_max": 0.346602, + "nauc_precision_at_20_std": 0.124329, + "nauc_precision_at_20_diff1": 0.182776, + "nauc_precision_at_100_max": 0.327245, + "nauc_precision_at_100_std": 0.179406, + "nauc_precision_at_100_diff1": 0.100783, + "nauc_precision_at_1000_max": 0.254107, + "nauc_precision_at_1000_std": 0.120643, + "nauc_precision_at_1000_diff1": 0.015372, + "nauc_mrr_at_1_max": 0.338508, + "nauc_mrr_at_1_std": 0.033934, + "nauc_mrr_at_1_diff1": 0.522563, + "nauc_mrr_at_3_max": 0.370037, + "nauc_mrr_at_3_std": 0.035405, + "nauc_mrr_at_3_diff1": 0.515849, + "nauc_mrr_at_5_max": 0.368677, + "nauc_mrr_at_5_std": 0.038112, + "nauc_mrr_at_5_diff1": 0.508871, + "nauc_mrr_at_10_max": 0.367723, + "nauc_mrr_at_10_std": 0.037467, + "nauc_mrr_at_10_diff1": 0.503691, + "nauc_mrr_at_20_max": 0.366392, + "nauc_mrr_at_20_std": 0.040232, + "nauc_mrr_at_20_diff1": 0.502562, + "nauc_mrr_at_100_max": 0.366763, + "nauc_mrr_at_100_std": 0.041614, + "nauc_mrr_at_100_diff1": 0.503476, + "nauc_mrr_at_1000_max": 0.366683, + "nauc_mrr_at_1000_std": 0.041473, + "nauc_mrr_at_1000_diff1": 0.503491, + "main_score": 0.40808, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 30.390542030334473, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/HotpotQA.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/HotpotQA.json new file mode 100644 index 0000000000..6ff732c772 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/HotpotQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ab518f4d6fcca38d87c25209f94beba119d02014", + "task_name": "HotpotQA", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.75165, + "ndcg_at_3": 0.60952, + "ndcg_at_5": 0.63654, + "ndcg_at_10": 0.65651, + "ndcg_at_20": 0.66947, + "ndcg_at_100": 0.68636, + "ndcg_at_1000": 0.6995, + "map_at_1": 0.37583, + "map_at_3": 0.53531, + "map_at_5": 0.55626, + "map_at_10": 0.56805, + "map_at_20": 0.57318, + "map_at_100": 0.57654, + "map_at_1000": 0.57723, + "recall_at_1": 0.37583, + "recall_at_3": 0.57907, + "recall_at_5": 0.63241, + "recall_at_10": 0.68244, + "recall_at_20": 0.72411, + "recall_at_100": 0.79885, + "recall_at_1000": 0.88562, + "precision_at_1": 0.75165, + "precision_at_3": 0.38605, + "precision_at_5": 0.25296, + "precision_at_10": 0.13649, + "precision_at_20": 0.07241, + "precision_at_100": 0.01598, + "precision_at_1000": 0.00177, + "mrr_at_1": 0.751654, + "mrr_at_3": 0.803421, + "mrr_at_5": 0.810599, + "mrr_at_10": 0.814163, + "mrr_at_20": 0.815581, + "mrr_at_100": 0.816325, + "mrr_at_1000": 0.816406, + "nauc_ndcg_at_1_max": 0.459195, + "nauc_ndcg_at_1_std": -0.076514, + "nauc_ndcg_at_1_diff1": 0.655772, + "nauc_ndcg_at_3_max": 0.283647, + "nauc_ndcg_at_3_std": 0.0184, + "nauc_ndcg_at_3_diff1": 0.234104, + "nauc_ndcg_at_5_max": 0.274157, + "nauc_ndcg_at_5_std": 0.039408, + "nauc_ndcg_at_5_diff1": 0.210131, + "nauc_ndcg_at_10_max": 0.274094, + "nauc_ndcg_at_10_std": 0.059102, + "nauc_ndcg_at_10_diff1": 0.200976, + "nauc_ndcg_at_20_max": 0.275196, + "nauc_ndcg_at_20_std": 0.071506, + "nauc_ndcg_at_20_diff1": 0.200157, + "nauc_ndcg_at_100_max": 0.277123, + "nauc_ndcg_at_100_std": 0.085719, + "nauc_ndcg_at_100_diff1": 0.198235, + "nauc_ndcg_at_1000_max": 0.281942, + "nauc_ndcg_at_1000_std": 0.084439, + "nauc_ndcg_at_1000_diff1": 0.207913, + "nauc_map_at_1_max": 0.459195, + "nauc_map_at_1_std": -0.076514, + "nauc_map_at_1_diff1": 0.655772, + "nauc_map_at_3_max": 0.242411, + "nauc_map_at_3_std": 0.022111, + "nauc_map_at_3_diff1": 0.172704, + "nauc_map_at_5_max": 0.238121, + "nauc_map_at_5_std": 0.036719, + "nauc_map_at_5_diff1": 0.158696, + "nauc_map_at_10_max": 0.238935, + "nauc_map_at_10_std": 0.046754, + "nauc_map_at_10_diff1": 0.154846, + "nauc_map_at_20_max": 0.239747, + "nauc_map_at_20_std": 0.051121, + "nauc_map_at_20_diff1": 0.155293, + "nauc_map_at_100_max": 0.240369, + "nauc_map_at_100_std": 0.053803, + "nauc_map_at_100_diff1": 0.155084, + "nauc_map_at_1000_max": 0.240555, + "nauc_map_at_1000_std": 0.053835, + "nauc_map_at_1000_diff1": 0.155435, + "nauc_recall_at_1_max": 0.459195, + "nauc_recall_at_1_std": -0.076514, + "nauc_recall_at_1_diff1": 0.655772, + "nauc_recall_at_3_max": 0.22579, + "nauc_recall_at_3_std": 0.053678, + "nauc_recall_at_3_diff1": 0.099876, + "nauc_recall_at_5_max": 0.197048, + "nauc_recall_at_5_std": 0.093992, + "nauc_recall_at_5_diff1": 0.043756, + "nauc_recall_at_10_max": 0.185671, + "nauc_recall_at_10_std": 0.144288, + "nauc_recall_at_10_diff1": 0.006922, + "nauc_recall_at_20_max": 0.175816, + "nauc_recall_at_20_std": 0.189231, + "nauc_recall_at_20_diff1": -0.015081, + "nauc_recall_at_100_max": 0.159277, + "nauc_recall_at_100_std": 0.289938, + "nauc_recall_at_100_diff1": -0.076811, + "nauc_recall_at_1000_max": 0.164059, + "nauc_recall_at_1000_std": 0.384705, + "nauc_recall_at_1000_diff1": -0.102558, + "nauc_precision_at_1_max": 0.459195, + "nauc_precision_at_1_std": -0.076514, + "nauc_precision_at_1_diff1": 0.655772, + "nauc_precision_at_3_max": 0.22579, + "nauc_precision_at_3_std": 0.053678, + "nauc_precision_at_3_diff1": 0.099876, + "nauc_precision_at_5_max": 0.197048, + "nauc_precision_at_5_std": 0.093992, + "nauc_precision_at_5_diff1": 0.043756, + "nauc_precision_at_10_max": 0.185671, + "nauc_precision_at_10_std": 0.144288, + "nauc_precision_at_10_diff1": 0.006922, + "nauc_precision_at_20_max": 0.175816, + "nauc_precision_at_20_std": 0.189231, + "nauc_precision_at_20_diff1": -0.015081, + "nauc_precision_at_100_max": 0.159277, + "nauc_precision_at_100_std": 0.289938, + "nauc_precision_at_100_diff1": -0.076811, + "nauc_precision_at_1000_max": 0.164059, + "nauc_precision_at_1000_std": 0.384705, + "nauc_precision_at_1000_diff1": -0.102558, + "nauc_mrr_at_1_max": 0.459195, + "nauc_mrr_at_1_std": -0.076514, + "nauc_mrr_at_1_diff1": 0.655772, + "nauc_mrr_at_3_max": 0.489235, + "nauc_mrr_at_3_std": -0.055625, + "nauc_mrr_at_3_diff1": 0.636898, + "nauc_mrr_at_5_max": 0.486582, + "nauc_mrr_at_5_std": -0.051448, + "nauc_mrr_at_5_diff1": 0.634208, + "nauc_mrr_at_10_max": 0.486974, + "nauc_mrr_at_10_std": -0.048244, + "nauc_mrr_at_10_diff1": 0.635742, + "nauc_mrr_at_20_max": 0.487238, + "nauc_mrr_at_20_std": -0.047924, + "nauc_mrr_at_20_diff1": 0.6365, + "nauc_mrr_at_100_max": 0.486571, + "nauc_mrr_at_100_std": -0.048605, + "nauc_mrr_at_100_diff1": 0.636569, + "nauc_mrr_at_1000_max": 0.486543, + "nauc_mrr_at_1000_std": -0.048694, + "nauc_mrr_at_1000_diff1": 0.636603, + "main_score": 0.65651, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2708.363296985626, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/HotpotQAHardNegatives.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/HotpotQAHardNegatives.json new file mode 100644 index 0000000000..b9b19c6e33 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/HotpotQAHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "617612fa63afcb60e3b134bed8b7216a99707c37", + "task_name": "HotpotQAHardNegatives", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.742, + "ndcg_at_3": 0.60713, + "ndcg_at_5": 0.63737, + "ndcg_at_10": 0.66231, + "ndcg_at_20": 0.67646, + "ndcg_at_100": 0.7029, + "ndcg_at_1000": 0.71754, + "map_at_1": 0.371, + "map_at_3": 0.53217, + "map_at_5": 0.55572, + "map_at_10": 0.57043, + "map_at_20": 0.57602, + "map_at_100": 0.5812, + "map_at_1000": 0.58204, + "recall_at_1": 0.371, + "recall_at_3": 0.582, + "recall_at_5": 0.642, + "recall_at_10": 0.7045, + "recall_at_20": 0.7495, + "recall_at_100": 0.8675, + "recall_at_1000": 0.9625, + "precision_at_1": 0.742, + "precision_at_3": 0.388, + "precision_at_5": 0.2568, + "precision_at_10": 0.1409, + "precision_at_20": 0.07495, + "precision_at_100": 0.01735, + "precision_at_1000": 0.00192, + "mrr_at_1": 0.742, + "mrr_at_3": 0.795667, + "mrr_at_5": 0.803167, + "mrr_at_10": 0.807615, + "mrr_at_20": 0.809455, + "mrr_at_100": 0.810556, + "mrr_at_1000": 0.810596, + "nauc_ndcg_at_1_max": 0.421496, + "nauc_ndcg_at_1_std": -0.122834, + "nauc_ndcg_at_1_diff1": 0.642778, + "nauc_ndcg_at_3_max": 0.268185, + "nauc_ndcg_at_3_std": -0.088422, + "nauc_ndcg_at_3_diff1": 0.224775, + "nauc_ndcg_at_5_max": 0.280809, + "nauc_ndcg_at_5_std": -0.079013, + "nauc_ndcg_at_5_diff1": 0.21341, + "nauc_ndcg_at_10_max": 0.276017, + "nauc_ndcg_at_10_std": -0.065818, + "nauc_ndcg_at_10_diff1": 0.204495, + "nauc_ndcg_at_20_max": 0.274102, + "nauc_ndcg_at_20_std": -0.058242, + "nauc_ndcg_at_20_diff1": 0.208724, + "nauc_ndcg_at_100_max": 0.280219, + "nauc_ndcg_at_100_std": -0.041342, + "nauc_ndcg_at_100_diff1": 0.220446, + "nauc_ndcg_at_1000_max": 0.285326, + "nauc_ndcg_at_1000_std": -0.044146, + "nauc_ndcg_at_1000_diff1": 0.233367, + "nauc_map_at_1_max": 0.421496, + "nauc_map_at_1_std": -0.122834, + "nauc_map_at_1_diff1": 0.642778, + "nauc_map_at_3_max": 0.232058, + "nauc_map_at_3_std": -0.074468, + "nauc_map_at_3_diff1": 0.164514, + "nauc_map_at_5_max": 0.242096, + "nauc_map_at_5_std": -0.069511, + "nauc_map_at_5_diff1": 0.159977, + "nauc_map_at_10_max": 0.239442, + "nauc_map_at_10_std": -0.062492, + "nauc_map_at_10_diff1": 0.15497, + "nauc_map_at_20_max": 0.239227, + "nauc_map_at_20_std": -0.060756, + "nauc_map_at_20_diff1": 0.157419, + "nauc_map_at_100_max": 0.240255, + "nauc_map_at_100_std": -0.058029, + "nauc_map_at_100_diff1": 0.159698, + "nauc_map_at_1000_max": 0.240509, + "nauc_map_at_1000_std": -0.057816, + "nauc_map_at_1000_diff1": 0.160094, + "nauc_recall_at_1_max": 0.421496, + "nauc_recall_at_1_std": -0.122834, + "nauc_recall_at_1_diff1": 0.642778, + "nauc_recall_at_3_max": 0.211979, + "nauc_recall_at_3_std": -0.068152, + "nauc_recall_at_3_diff1": 0.083723, + "nauc_recall_at_5_max": 0.22776, + "nauc_recall_at_5_std": -0.046146, + "nauc_recall_at_5_diff1": 0.053494, + "nauc_recall_at_10_max": 0.205195, + "nauc_recall_at_10_std": -0.008551, + "nauc_recall_at_10_diff1": 0.012169, + "nauc_recall_at_20_max": 0.186387, + "nauc_recall_at_20_std": 0.026837, + "nauc_recall_at_20_diff1": 0.000186, + "nauc_recall_at_100_max": 0.179614, + "nauc_recall_at_100_std": 0.195757, + "nauc_recall_at_100_diff1": -0.042785, + "nauc_recall_at_1000_max": 0.196334, + "nauc_recall_at_1000_std": 0.541506, + "nauc_recall_at_1000_diff1": -0.124631, + "nauc_precision_at_1_max": 0.421496, + "nauc_precision_at_1_std": -0.122834, + "nauc_precision_at_1_diff1": 0.642778, + "nauc_precision_at_3_max": 0.211979, + "nauc_precision_at_3_std": -0.068152, + "nauc_precision_at_3_diff1": 0.083723, + "nauc_precision_at_5_max": 0.22776, + "nauc_precision_at_5_std": -0.046146, + "nauc_precision_at_5_diff1": 0.053494, + "nauc_precision_at_10_max": 0.205195, + "nauc_precision_at_10_std": -0.008551, + "nauc_precision_at_10_diff1": 0.012169, + "nauc_precision_at_20_max": 0.186387, + "nauc_precision_at_20_std": 0.026837, + "nauc_precision_at_20_diff1": 0.000186, + "nauc_precision_at_100_max": 0.179614, + "nauc_precision_at_100_std": 0.195757, + "nauc_precision_at_100_diff1": -0.042785, + "nauc_precision_at_1000_max": 0.196334, + "nauc_precision_at_1000_std": 0.541506, + "nauc_precision_at_1000_diff1": -0.124631, + "nauc_mrr_at_1_max": 0.421496, + "nauc_mrr_at_1_std": -0.122834, + "nauc_mrr_at_1_diff1": 0.642778, + "nauc_mrr_at_3_max": 0.447013, + "nauc_mrr_at_3_std": -0.154376, + "nauc_mrr_at_3_diff1": 0.623139, + "nauc_mrr_at_5_max": 0.453095, + "nauc_mrr_at_5_std": -0.150741, + "nauc_mrr_at_5_diff1": 0.621632, + "nauc_mrr_at_10_max": 0.453343, + "nauc_mrr_at_10_std": -0.149241, + "nauc_mrr_at_10_diff1": 0.626513, + "nauc_mrr_at_20_max": 0.452039, + "nauc_mrr_at_20_std": -0.147605, + "nauc_mrr_at_20_diff1": 0.627463, + "nauc_mrr_at_100_max": 0.451599, + "nauc_mrr_at_100_std": -0.147008, + "nauc_mrr_at_100_diff1": 0.627471, + "nauc_mrr_at_1000_max": 0.451512, + "nauc_mrr_at_1000_std": -0.147037, + "nauc_mrr_at_1000_diff1": 0.627433, + "main_score": 0.66231, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 72.93178129196167, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ImdbClassification.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ImdbClassification.json new file mode 100644 index 0000000000..fcdb91c9a2 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ImdbClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7", + "task_name": "ImdbClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.6037, + "f1": 0.601809, + "f1_weighted": 0.601809, + "ap": 0.563309, + "ap_weighted": 0.563309, + "scores_per_experiment": [ + { + "accuracy": 0.63232, + "f1": 0.63202, + "f1_weighted": 0.63202, + "ap": 0.584729, + "ap_weighted": 0.584729 + }, + { + "accuracy": 0.61888, + "f1": 0.616623, + "f1_weighted": 0.616623, + "ap": 0.571692, + "ap_weighted": 0.571692 + }, + { + "accuracy": 0.5464, + "f1": 0.54569, + "f1_weighted": 0.54569, + "ap": 0.525195, + "ap_weighted": 0.525195 + }, + { + "accuracy": 0.61872, + "f1": 0.618321, + "f1_weighted": 0.618321, + "ap": 0.574428, + "ap_weighted": 0.574428 + }, + { + "accuracy": 0.6314, + "f1": 0.631337, + "f1_weighted": 0.631337, + "ap": 0.58343, + "ap_weighted": 0.58343 + }, + { + "accuracy": 0.59812, + "f1": 0.597941, + "f1_weighted": 0.597941, + "ap": 0.559111, + "ap_weighted": 0.559111 + }, + { + "accuracy": 0.57788, + "f1": 0.574969, + "f1_weighted": 0.574969, + "ap": 0.546208, + "ap_weighted": 0.546208 + }, + { + "accuracy": 0.6076, + "f1": 0.60599, + "f1_weighted": 0.60599, + "ap": 0.564065, + "ap_weighted": 0.564065 + }, + { + "accuracy": 0.58448, + "f1": 0.574568, + "f1_weighted": 0.574568, + "ap": 0.547708, + "ap_weighted": 0.547708 + }, + { + "accuracy": 0.6212, + "f1": 0.620634, + "f1_weighted": 0.620634, + "ap": 0.57652, + "ap_weighted": 0.57652 + } + ], + "main_score": 0.6037, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 30.81363844871521, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBNarrativeQARetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBNarrativeQARetrieval.json new file mode 100644 index 0000000000..b159e29dee --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBNarrativeQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBNarrativeQARetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.31209, + "ndcg_at_3": 0.37222, + "ndcg_at_5": 0.39119, + "ndcg_at_10": 0.41324, + "ndcg_at_20": 0.43291, + "ndcg_at_100": 0.47151, + "ndcg_at_1000": 0.49563, + "map_at_1": 0.31209, + "map_at_3": 0.35753, + "map_at_5": 0.36801, + "map_at_10": 0.37714, + "map_at_20": 0.38253, + "map_at_100": 0.38756, + "map_at_1000": 0.38861, + "recall_at_1": 0.31209, + "recall_at_3": 0.41468, + "recall_at_5": 0.46091, + "recall_at_10": 0.52895, + "recall_at_20": 0.60685, + "recall_at_100": 0.81941, + "recall_at_1000": 1.0, + "precision_at_1": 0.31209, + "precision_at_3": 0.13823, + "precision_at_5": 0.09218, + "precision_at_10": 0.0529, + "precision_at_20": 0.03034, + "precision_at_100": 0.00819, + "precision_at_1000": 0.001, + "mrr_at_1": 0.312087, + "mrr_at_3": 0.35753, + "mrr_at_5": 0.368014, + "mrr_at_10": 0.377145, + "mrr_at_20": 0.382529, + "mrr_at_100": 0.387564, + "mrr_at_1000": 0.388615, + "nauc_ndcg_at_1_max": 0.569192, + "nauc_ndcg_at_1_std": 0.167784, + "nauc_ndcg_at_1_diff1": 0.655705, + "nauc_ndcg_at_3_max": 0.540232, + "nauc_ndcg_at_3_std": 0.185179, + "nauc_ndcg_at_3_diff1": 0.59318, + "nauc_ndcg_at_5_max": 0.531747, + "nauc_ndcg_at_5_std": 0.187259, + "nauc_ndcg_at_5_diff1": 0.581963, + "nauc_ndcg_at_10_max": 0.517257, + "nauc_ndcg_at_10_std": 0.190042, + "nauc_ndcg_at_10_diff1": 0.567971, + "nauc_ndcg_at_20_max": 0.511495, + "nauc_ndcg_at_20_std": 0.193182, + "nauc_ndcg_at_20_diff1": 0.562248, + "nauc_ndcg_at_100_max": 0.512231, + "nauc_ndcg_at_100_std": 0.198184, + "nauc_ndcg_at_100_diff1": 0.565235, + "nauc_ndcg_at_1000_max": 0.523793, + "nauc_ndcg_at_1000_std": 0.190454, + "nauc_ndcg_at_1000_diff1": 0.579256, + "nauc_map_at_1_max": 0.569192, + "nauc_map_at_1_std": 0.167784, + "nauc_map_at_1_diff1": 0.655705, + "nauc_map_at_3_max": 0.547629, + "nauc_map_at_3_std": 0.181075, + "nauc_map_at_3_diff1": 0.608001, + "nauc_map_at_5_max": 0.542944, + "nauc_map_at_5_std": 0.18215, + "nauc_map_at_5_diff1": 0.601878, + "nauc_map_at_10_max": 0.536955, + "nauc_map_at_10_std": 0.183364, + "nauc_map_at_10_diff1": 0.596117, + "nauc_map_at_20_max": 0.535555, + "nauc_map_at_20_std": 0.184225, + "nauc_map_at_20_diff1": 0.594818, + "nauc_map_at_100_max": 0.535568, + "nauc_map_at_100_std": 0.184748, + "nauc_map_at_100_diff1": 0.595183, + "nauc_map_at_1000_max": 0.536036, + "nauc_map_at_1000_std": 0.184507, + "nauc_map_at_1000_diff1": 0.595781, + "nauc_recall_at_1_max": 0.569192, + "nauc_recall_at_1_std": 0.167784, + "nauc_recall_at_1_diff1": 0.655705, + "nauc_recall_at_3_max": 0.519075, + "nauc_recall_at_3_std": 0.196789, + "nauc_recall_at_3_diff1": 0.551226, + "nauc_recall_at_5_max": 0.498215, + "nauc_recall_at_5_std": 0.202276, + "nauc_recall_at_5_diff1": 0.523206, + "nauc_recall_at_10_max": 0.451349, + "nauc_recall_at_10_std": 0.211212, + "nauc_recall_at_10_diff1": 0.476907, + "nauc_recall_at_20_max": 0.419847, + "nauc_recall_at_20_std": 0.226412, + "nauc_recall_at_20_diff1": 0.442861, + "nauc_recall_at_100_max": 0.360501, + "nauc_recall_at_100_std": 0.305139, + "nauc_recall_at_100_diff1": 0.383526, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.569192, + "nauc_precision_at_1_std": 0.167784, + "nauc_precision_at_1_diff1": 0.655705, + "nauc_precision_at_3_max": 0.519075, + "nauc_precision_at_3_std": 0.196789, + "nauc_precision_at_3_diff1": 0.551226, + "nauc_precision_at_5_max": 0.498215, + "nauc_precision_at_5_std": 0.202276, + "nauc_precision_at_5_diff1": 0.523206, + "nauc_precision_at_10_max": 0.451349, + "nauc_precision_at_10_std": 0.211212, + "nauc_precision_at_10_diff1": 0.476907, + "nauc_precision_at_20_max": 0.419847, + "nauc_precision_at_20_std": 0.226412, + "nauc_precision_at_20_diff1": 0.442861, + "nauc_precision_at_100_max": 0.360501, + "nauc_precision_at_100_std": 0.305139, + "nauc_precision_at_100_diff1": 0.383526, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.569192, + "nauc_mrr_at_1_std": 0.167784, + "nauc_mrr_at_1_diff1": 0.655705, + "nauc_mrr_at_3_max": 0.547629, + "nauc_mrr_at_3_std": 0.181075, + "nauc_mrr_at_3_diff1": 0.608001, + "nauc_mrr_at_5_max": 0.542944, + "nauc_mrr_at_5_std": 0.18215, + "nauc_mrr_at_5_diff1": 0.601878, + "nauc_mrr_at_10_max": 0.536955, + "nauc_mrr_at_10_std": 0.183364, + "nauc_mrr_at_10_diff1": 0.596117, + "nauc_mrr_at_20_max": 0.535555, + "nauc_mrr_at_20_std": 0.184225, + "nauc_mrr_at_20_diff1": 0.594818, + "nauc_mrr_at_100_max": 0.535568, + "nauc_mrr_at_100_std": 0.184748, + "nauc_mrr_at_100_diff1": 0.595183, + "nauc_mrr_at_1000_max": 0.536036, + "nauc_mrr_at_1000_std": 0.184507, + "nauc_mrr_at_1000_diff1": 0.595781, + "main_score": 0.41324, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 76.51239919662476, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBNeedleRetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBNeedleRetrieval.json new file mode 100644 index 0000000000..4301ef1a58 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBNeedleRetrieval.json @@ -0,0 +1,1201 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBNeedleRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test_256": [ + { + "ndcg_at_1": 0.86, + "ndcg_at_3": 0.92047, + "ndcg_at_5": 0.93595, + "ndcg_at_10": 0.93595, + "ndcg_at_20": 0.93595, + "ndcg_at_100": 0.93595, + "ndcg_at_1000": 0.93595, + "map_at_1": 0.86, + "map_at_3": 0.90667, + "map_at_5": 0.91467, + "map_at_10": 0.91467, + "map_at_20": 0.91467, + "map_at_100": 0.91467, + "map_at_1000": 0.91467, + "recall_at_1": 0.86, + "recall_at_3": 0.96, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.86, + "precision_at_3": 0.32, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.86, + "mrr_at_3": 0.906667, + "mrr_at_5": 0.914667, + "mrr_at_10": 0.914667, + "mrr_at_20": 0.914667, + "mrr_at_100": 0.914667, + "mrr_at_1000": 0.914667, + "nauc_ndcg_at_1_max": 0.659623, + "nauc_ndcg_at_1_std": -0.111404, + "nauc_ndcg_at_1_diff1": 0.873216, + "nauc_ndcg_at_3_max": 0.710973, + "nauc_ndcg_at_3_std": -0.179255, + "nauc_ndcg_at_3_diff1": 0.876215, + "nauc_ndcg_at_5_max": 0.689855, + "nauc_ndcg_at_5_std": -0.115465, + "nauc_ndcg_at_5_diff1": 0.877942, + "nauc_ndcg_at_10_max": 0.689855, + "nauc_ndcg_at_10_std": -0.115465, + "nauc_ndcg_at_10_diff1": 0.877942, + "nauc_ndcg_at_20_max": 0.689855, + "nauc_ndcg_at_20_std": -0.115465, + "nauc_ndcg_at_20_diff1": 0.877942, + "nauc_ndcg_at_100_max": 0.689855, + "nauc_ndcg_at_100_std": -0.115465, + "nauc_ndcg_at_100_diff1": 0.877942, + "nauc_ndcg_at_1000_max": 0.689855, + "nauc_ndcg_at_1000_std": -0.115465, + "nauc_ndcg_at_1000_diff1": 0.877942, + "nauc_map_at_1_max": 0.659623, + "nauc_map_at_1_std": -0.111404, + "nauc_map_at_1_diff1": 0.873216, + "nauc_map_at_3_max": 0.697338, + "nauc_map_at_3_std": -0.141982, + "nauc_map_at_3_diff1": 0.877007, + "nauc_map_at_5_max": 0.68782, + "nauc_map_at_5_std": -0.113611, + "nauc_map_at_5_diff1": 0.877754, + "nauc_map_at_10_max": 0.68782, + "nauc_map_at_10_std": -0.113611, + "nauc_map_at_10_diff1": 0.877754, + "nauc_map_at_20_max": 0.68782, + "nauc_map_at_20_std": -0.113611, + "nauc_map_at_20_diff1": 0.877754, + "nauc_map_at_100_max": 0.68782, + "nauc_map_at_100_std": -0.113611, + "nauc_map_at_100_diff1": 0.877754, + "nauc_map_at_1000_max": 0.68782, + "nauc_map_at_1000_std": -0.113611, + "nauc_map_at_1000_diff1": 0.877754, + "nauc_recall_at_1_max": 0.659623, + "nauc_recall_at_1_std": -0.111404, + "nauc_recall_at_1_diff1": 0.873216, + "nauc_recall_at_3_max": 0.795752, + "nauc_recall_at_3_std": -0.435341, + "nauc_recall_at_3_diff1": 0.869281, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.659623, + "nauc_precision_at_1_std": -0.111404, + "nauc_precision_at_1_diff1": 0.873216, + "nauc_precision_at_3_max": 0.795752, + "nauc_precision_at_3_std": -0.435341, + "nauc_precision_at_3_diff1": 0.869281, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.659623, + "nauc_mrr_at_1_std": -0.111404, + "nauc_mrr_at_1_diff1": 0.873216, + "nauc_mrr_at_3_max": 0.697338, + "nauc_mrr_at_3_std": -0.141982, + "nauc_mrr_at_3_diff1": 0.877007, + "nauc_mrr_at_5_max": 0.68782, + "nauc_mrr_at_5_std": -0.113611, + "nauc_mrr_at_5_diff1": 0.877754, + "nauc_mrr_at_10_max": 0.68782, + "nauc_mrr_at_10_std": -0.113611, + "nauc_mrr_at_10_diff1": 0.877754, + "nauc_mrr_at_20_max": 0.68782, + "nauc_mrr_at_20_std": -0.113611, + "nauc_mrr_at_20_diff1": 0.877754, + "nauc_mrr_at_100_max": 0.68782, + "nauc_mrr_at_100_std": -0.113611, + "nauc_mrr_at_100_diff1": 0.877754, + "nauc_mrr_at_1000_max": 0.68782, + "nauc_mrr_at_1000_std": -0.113611, + "nauc_mrr_at_1000_diff1": 0.877754, + "main_score": 0.86, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_512": [ + { + "ndcg_at_1": 0.76, + "ndcg_at_3": 0.85833, + "ndcg_at_5": 0.86694, + "ndcg_at_10": 0.88119, + "ndcg_at_20": 0.8866, + "ndcg_at_100": 0.8866, + "ndcg_at_1000": 0.8866, + "map_at_1": 0.76, + "map_at_3": 0.83667, + "map_at_5": 0.84167, + "map_at_10": 0.84833, + "map_at_20": 0.85, + "map_at_100": 0.85, + "map_at_1000": 0.85, + "recall_at_1": 0.76, + "recall_at_3": 0.92, + "recall_at_5": 0.94, + "recall_at_10": 0.98, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.76, + "precision_at_3": 0.30667, + "precision_at_5": 0.188, + "precision_at_10": 0.098, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.76, + "mrr_at_3": 0.836667, + "mrr_at_5": 0.841667, + "mrr_at_10": 0.848333, + "mrr_at_20": 0.85, + "mrr_at_100": 0.85, + "mrr_at_1000": 0.85, + "nauc_ndcg_at_1_max": 0.519036, + "nauc_ndcg_at_1_std": -0.185299, + "nauc_ndcg_at_1_diff1": 0.785671, + "nauc_ndcg_at_3_max": 0.594578, + "nauc_ndcg_at_3_std": -0.079921, + "nauc_ndcg_at_3_diff1": 0.745137, + "nauc_ndcg_at_5_max": 0.566446, + "nauc_ndcg_at_5_std": -0.145787, + "nauc_ndcg_at_5_diff1": 0.746727, + "nauc_ndcg_at_10_max": 0.58617, + "nauc_ndcg_at_10_std": -0.059842, + "nauc_ndcg_at_10_diff1": 0.76398, + "nauc_ndcg_at_20_max": 0.564762, + "nauc_ndcg_at_20_std": -0.107907, + "nauc_ndcg_at_20_diff1": 0.76614, + "nauc_ndcg_at_100_max": 0.564762, + "nauc_ndcg_at_100_std": -0.107907, + "nauc_ndcg_at_100_diff1": 0.76614, + "nauc_ndcg_at_1000_max": 0.564762, + "nauc_ndcg_at_1000_std": -0.107907, + "nauc_ndcg_at_1000_diff1": 0.76614, + "nauc_map_at_1_max": 0.519036, + "nauc_map_at_1_std": -0.185299, + "nauc_map_at_1_diff1": 0.785671, + "nauc_map_at_3_max": 0.57327, + "nauc_map_at_3_std": -0.102142, + "nauc_map_at_3_diff1": 0.75926, + "nauc_map_at_5_max": 0.558681, + "nauc_map_at_5_std": -0.135355, + "nauc_map_at_5_diff1": 0.760527, + "nauc_map_at_10_max": 0.565575, + "nauc_map_at_10_std": -0.103199, + "nauc_map_at_10_diff1": 0.767541, + "nauc_map_at_20_max": 0.560325, + "nauc_map_at_20_std": -0.114951, + "nauc_map_at_20_diff1": 0.768088, + "nauc_map_at_100_max": 0.560325, + "nauc_map_at_100_std": -0.114951, + "nauc_map_at_100_diff1": 0.768088, + "nauc_map_at_1000_max": 0.560325, + "nauc_map_at_1000_std": -0.114951, + "nauc_map_at_1000_diff1": 0.768088, + "nauc_recall_at_1_max": 0.519036, + "nauc_recall_at_1_std": -0.185299, + "nauc_recall_at_1_diff1": 0.785671, + "nauc_recall_at_3_max": 0.707166, + "nauc_recall_at_3_std": 0.030345, + "nauc_recall_at_3_diff1": 0.667951, + "nauc_recall_at_5_max": 0.609555, + "nauc_recall_at_5_std": -0.2493, + "nauc_recall_at_5_diff1": 0.64986, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": 0.869281, + "nauc_recall_at_10_diff1": 0.722222, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.519036, + "nauc_precision_at_1_std": -0.185299, + "nauc_precision_at_1_diff1": 0.785671, + "nauc_precision_at_3_max": 0.707166, + "nauc_precision_at_3_std": 0.030345, + "nauc_precision_at_3_diff1": 0.667951, + "nauc_precision_at_5_max": 0.609555, + "nauc_precision_at_5_std": -0.2493, + "nauc_precision_at_5_diff1": 0.64986, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 0.869281, + "nauc_precision_at_10_diff1": 0.722222, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.519036, + "nauc_mrr_at_1_std": -0.185299, + "nauc_mrr_at_1_diff1": 0.785671, + "nauc_mrr_at_3_max": 0.57327, + "nauc_mrr_at_3_std": -0.102142, + "nauc_mrr_at_3_diff1": 0.75926, + "nauc_mrr_at_5_max": 0.558681, + "nauc_mrr_at_5_std": -0.135355, + "nauc_mrr_at_5_diff1": 0.760527, + "nauc_mrr_at_10_max": 0.565575, + "nauc_mrr_at_10_std": -0.103199, + "nauc_mrr_at_10_diff1": 0.767541, + "nauc_mrr_at_20_max": 0.560325, + "nauc_mrr_at_20_std": -0.114951, + "nauc_mrr_at_20_diff1": 0.768088, + "nauc_mrr_at_100_max": 0.560325, + "nauc_mrr_at_100_std": -0.114951, + "nauc_mrr_at_100_diff1": 0.768088, + "nauc_mrr_at_1000_max": 0.560325, + "nauc_mrr_at_1000_std": -0.114951, + "nauc_mrr_at_1000_diff1": 0.768088, + "main_score": 0.76, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_1024": [ + { + "ndcg_at_1": 0.66, + "ndcg_at_3": 0.74309, + "ndcg_at_5": 0.77579, + "ndcg_at_10": 0.80887, + "ndcg_at_20": 0.80887, + "ndcg_at_100": 0.81329, + "ndcg_at_1000": 0.81329, + "map_at_1": 0.66, + "map_at_3": 0.72333, + "map_at_5": 0.74133, + "map_at_10": 0.75538, + "map_at_20": 0.75538, + "map_at_100": 0.75629, + "map_at_1000": 0.75629, + "recall_at_1": 0.66, + "recall_at_3": 0.8, + "recall_at_5": 0.88, + "recall_at_10": 0.98, + "recall_at_20": 0.98, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.66, + "precision_at_3": 0.26667, + "precision_at_5": 0.176, + "precision_at_10": 0.098, + "precision_at_20": 0.049, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.66, + "mrr_at_3": 0.723333, + "mrr_at_5": 0.741333, + "mrr_at_10": 0.755381, + "mrr_at_20": 0.755381, + "mrr_at_100": 0.75629, + "mrr_at_1000": 0.75629, + "nauc_ndcg_at_1_max": 0.378373, + "nauc_ndcg_at_1_std": -0.090179, + "nauc_ndcg_at_1_diff1": 0.816489, + "nauc_ndcg_at_3_max": 0.374418, + "nauc_ndcg_at_3_std": -0.229005, + "nauc_ndcg_at_3_diff1": 0.749085, + "nauc_ndcg_at_5_max": 0.312374, + "nauc_ndcg_at_5_std": -0.22401, + "nauc_ndcg_at_5_diff1": 0.715491, + "nauc_ndcg_at_10_max": 0.317232, + "nauc_ndcg_at_10_std": -0.202842, + "nauc_ndcg_at_10_diff1": 0.690632, + "nauc_ndcg_at_20_max": 0.317232, + "nauc_ndcg_at_20_std": -0.202842, + "nauc_ndcg_at_20_diff1": 0.690632, + "nauc_ndcg_at_100_max": 0.341485, + "nauc_ndcg_at_100_std": -0.176712, + "nauc_ndcg_at_100_diff1": 0.741366, + "nauc_ndcg_at_1000_max": 0.341485, + "nauc_ndcg_at_1000_std": -0.176712, + "nauc_ndcg_at_1000_diff1": 0.741366, + "nauc_map_at_1_max": 0.378373, + "nauc_map_at_1_std": -0.090179, + "nauc_map_at_1_diff1": 0.816489, + "nauc_map_at_3_max": 0.371738, + "nauc_map_at_3_std": -0.186882, + "nauc_map_at_3_diff1": 0.767007, + "nauc_map_at_5_max": 0.341501, + "nauc_map_at_5_std": -0.182122, + "nauc_map_at_5_diff1": 0.752582, + "nauc_map_at_10_max": 0.344446, + "nauc_map_at_10_std": -0.170348, + "nauc_map_at_10_diff1": 0.746529, + "nauc_map_at_20_max": 0.344446, + "nauc_map_at_20_std": -0.170348, + "nauc_map_at_20_diff1": 0.746529, + "nauc_map_at_100_max": 0.348399, + "nauc_map_at_100_std": -0.166075, + "nauc_map_at_100_diff1": 0.754795, + "nauc_map_at_1000_max": 0.348399, + "nauc_map_at_1000_std": -0.166075, + "nauc_map_at_1000_diff1": 0.754795, + "nauc_recall_at_1_max": 0.378373, + "nauc_recall_at_1_std": -0.090179, + "nauc_recall_at_1_diff1": 0.816489, + "nauc_recall_at_3_max": 0.386314, + "nauc_recall_at_3_std": -0.385814, + "nauc_recall_at_3_diff1": 0.683616, + "nauc_recall_at_5_max": 0.137687, + "nauc_recall_at_5_std": -0.453309, + "nauc_recall_at_5_diff1": 0.50175, + "nauc_recall_at_10_max": -0.563492, + "nauc_recall_at_10_std": -1.151727, + "nauc_recall_at_10_diff1": -1.151727, + "nauc_recall_at_20_max": -0.563492, + "nauc_recall_at_20_std": -1.151727, + "nauc_recall_at_20_diff1": -1.151727, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.378373, + "nauc_precision_at_1_std": -0.090179, + "nauc_precision_at_1_diff1": 0.816489, + "nauc_precision_at_3_max": 0.386314, + "nauc_precision_at_3_std": -0.385814, + "nauc_precision_at_3_diff1": 0.683616, + "nauc_precision_at_5_max": 0.137687, + "nauc_precision_at_5_std": -0.453309, + "nauc_precision_at_5_diff1": 0.50175, + "nauc_precision_at_10_max": -0.563492, + "nauc_precision_at_10_std": -1.151727, + "nauc_precision_at_10_diff1": -1.151727, + "nauc_precision_at_20_max": -0.563492, + "nauc_precision_at_20_std": -1.151727, + "nauc_precision_at_20_diff1": -1.151727, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.378373, + "nauc_mrr_at_1_std": -0.090179, + "nauc_mrr_at_1_diff1": 0.816489, + "nauc_mrr_at_3_max": 0.371738, + "nauc_mrr_at_3_std": -0.186882, + "nauc_mrr_at_3_diff1": 0.767007, + "nauc_mrr_at_5_max": 0.341501, + "nauc_mrr_at_5_std": -0.182122, + "nauc_mrr_at_5_diff1": 0.752582, + "nauc_mrr_at_10_max": 0.344446, + "nauc_mrr_at_10_std": -0.170348, + "nauc_mrr_at_10_diff1": 0.746529, + "nauc_mrr_at_20_max": 0.344446, + "nauc_mrr_at_20_std": -0.170348, + "nauc_mrr_at_20_diff1": 0.746529, + "nauc_mrr_at_100_max": 0.348399, + "nauc_mrr_at_100_std": -0.166075, + "nauc_mrr_at_100_diff1": 0.754795, + "nauc_mrr_at_1000_max": 0.348399, + "nauc_mrr_at_1000_std": -0.166075, + "nauc_mrr_at_1000_diff1": 0.754795, + "main_score": 0.66, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_2048": [ + { + "ndcg_at_1": 0.6, + "ndcg_at_3": 0.69309, + "ndcg_at_5": 0.72492, + "ndcg_at_10": 0.75185, + "ndcg_at_20": 0.76644, + "ndcg_at_100": 0.76987, + "ndcg_at_1000": 0.76987, + "map_at_1": 0.6, + "map_at_3": 0.67, + "map_at_5": 0.687, + "map_at_10": 0.69875, + "map_at_20": 0.70245, + "map_at_100": 0.70281, + "map_at_1000": 0.70281, + "recall_at_1": 0.6, + "recall_at_3": 0.76, + "recall_at_5": 0.84, + "recall_at_10": 0.92, + "recall_at_20": 0.98, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.6, + "precision_at_3": 0.25333, + "precision_at_5": 0.168, + "precision_at_10": 0.092, + "precision_at_20": 0.049, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.6, + "mrr_at_3": 0.67, + "mrr_at_5": 0.687, + "mrr_at_10": 0.698746, + "mrr_at_20": 0.702448, + "mrr_at_100": 0.702805, + "mrr_at_1000": 0.702805, + "nauc_ndcg_at_1_max": 0.318789, + "nauc_ndcg_at_1_std": 0.040071, + "nauc_ndcg_at_1_diff1": 0.295013, + "nauc_ndcg_at_3_max": 0.330893, + "nauc_ndcg_at_3_std": 0.106981, + "nauc_ndcg_at_3_diff1": 0.377133, + "nauc_ndcg_at_5_max": 0.336413, + "nauc_ndcg_at_5_std": 0.181748, + "nauc_ndcg_at_5_diff1": 0.382962, + "nauc_ndcg_at_10_max": 0.254506, + "nauc_ndcg_at_10_std": 0.179352, + "nauc_ndcg_at_10_diff1": 0.330577, + "nauc_ndcg_at_20_max": 0.311501, + "nauc_ndcg_at_20_std": 0.130978, + "nauc_ndcg_at_20_diff1": 0.339111, + "nauc_ndcg_at_100_max": 0.299106, + "nauc_ndcg_at_100_std": 0.120334, + "nauc_ndcg_at_100_diff1": 0.327213, + "nauc_ndcg_at_1000_max": 0.299106, + "nauc_ndcg_at_1000_std": 0.120334, + "nauc_ndcg_at_1000_diff1": 0.327213, + "nauc_map_at_1_max": 0.318789, + "nauc_map_at_1_std": 0.040071, + "nauc_map_at_1_diff1": 0.295013, + "nauc_map_at_3_max": 0.319138, + "nauc_map_at_3_std": 0.090453, + "nauc_map_at_3_diff1": 0.34044, + "nauc_map_at_5_max": 0.321958, + "nauc_map_at_5_std": 0.122652, + "nauc_map_at_5_diff1": 0.34106, + "nauc_map_at_10_max": 0.291212, + "nauc_map_at_10_std": 0.117641, + "nauc_map_at_10_diff1": 0.320298, + "nauc_map_at_20_max": 0.302357, + "nauc_map_at_20_std": 0.10675, + "nauc_map_at_20_diff1": 0.320887, + "nauc_map_at_100_max": 0.301337, + "nauc_map_at_100_std": 0.10585, + "nauc_map_at_100_diff1": 0.319895, + "nauc_map_at_1000_max": 0.301337, + "nauc_map_at_1000_std": 0.10585, + "nauc_map_at_1000_diff1": 0.319895, + "nauc_recall_at_1_max": 0.318789, + "nauc_recall_at_1_std": 0.040071, + "nauc_recall_at_1_diff1": 0.295013, + "nauc_recall_at_3_max": 0.378083, + "nauc_recall_at_3_std": 0.16609, + "nauc_recall_at_3_diff1": 0.517998, + "nauc_recall_at_5_max": 0.413537, + "nauc_recall_at_5_std": 0.498159, + "nauc_recall_at_5_diff1": 0.601252, + "nauc_recall_at_10_max": -0.148109, + "nauc_recall_at_10_std": 0.791667, + "nauc_recall_at_10_diff1": 0.389823, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 0.722222, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.318789, + "nauc_precision_at_1_std": 0.040071, + "nauc_precision_at_1_diff1": 0.295013, + "nauc_precision_at_3_max": 0.378083, + "nauc_precision_at_3_std": 0.16609, + "nauc_precision_at_3_diff1": 0.517998, + "nauc_precision_at_5_max": 0.413537, + "nauc_precision_at_5_std": 0.498159, + "nauc_precision_at_5_diff1": 0.601252, + "nauc_precision_at_10_max": -0.148109, + "nauc_precision_at_10_std": 0.791667, + "nauc_precision_at_10_diff1": 0.389823, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 0.722222, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.318789, + "nauc_mrr_at_1_std": 0.040071, + "nauc_mrr_at_1_diff1": 0.295013, + "nauc_mrr_at_3_max": 0.319138, + "nauc_mrr_at_3_std": 0.090453, + "nauc_mrr_at_3_diff1": 0.34044, + "nauc_mrr_at_5_max": 0.321958, + "nauc_mrr_at_5_std": 0.122652, + "nauc_mrr_at_5_diff1": 0.34106, + "nauc_mrr_at_10_max": 0.291212, + "nauc_mrr_at_10_std": 0.117641, + "nauc_mrr_at_10_diff1": 0.320298, + "nauc_mrr_at_20_max": 0.302357, + "nauc_mrr_at_20_std": 0.10675, + "nauc_mrr_at_20_diff1": 0.320887, + "nauc_mrr_at_100_max": 0.301337, + "nauc_mrr_at_100_std": 0.10585, + "nauc_mrr_at_100_diff1": 0.319895, + "nauc_mrr_at_1000_max": 0.301337, + "nauc_mrr_at_1000_std": 0.10585, + "nauc_mrr_at_1000_diff1": 0.319895, + "main_score": 0.6, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_4096": [ + { + "ndcg_at_1": 0.54, + "ndcg_at_3": 0.73404, + "ndcg_at_5": 0.76674, + "ndcg_at_10": 0.77883, + "ndcg_at_20": 0.78424, + "ndcg_at_100": 0.78424, + "ndcg_at_1000": 0.78424, + "map_at_1": 0.54, + "map_at_3": 0.69, + "map_at_5": 0.708, + "map_at_10": 0.7125, + "map_at_20": 0.71417, + "map_at_100": 0.71417, + "map_at_1000": 0.71417, + "recall_at_1": 0.54, + "recall_at_3": 0.86, + "recall_at_5": 0.94, + "recall_at_10": 0.98, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.54, + "precision_at_3": 0.28667, + "precision_at_5": 0.188, + "precision_at_10": 0.098, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.54, + "mrr_at_3": 0.69, + "mrr_at_5": 0.708, + "mrr_at_10": 0.7125, + "mrr_at_20": 0.714167, + "mrr_at_100": 0.714167, + "mrr_at_1000": 0.714167, + "nauc_ndcg_at_1_max": 0.324526, + "nauc_ndcg_at_1_std": -0.380734, + "nauc_ndcg_at_1_diff1": 0.208024, + "nauc_ndcg_at_3_max": 0.392936, + "nauc_ndcg_at_3_std": -0.199539, + "nauc_ndcg_at_3_diff1": 0.334691, + "nauc_ndcg_at_5_max": 0.37202, + "nauc_ndcg_at_5_std": -0.264154, + "nauc_ndcg_at_5_diff1": 0.297528, + "nauc_ndcg_at_10_max": 0.370853, + "nauc_ndcg_at_10_std": -0.280437, + "nauc_ndcg_at_10_diff1": 0.284819, + "nauc_ndcg_at_20_max": 0.354813, + "nauc_ndcg_at_20_std": -0.271328, + "nauc_ndcg_at_20_diff1": 0.290034, + "nauc_ndcg_at_100_max": 0.354813, + "nauc_ndcg_at_100_std": -0.271328, + "nauc_ndcg_at_100_diff1": 0.290034, + "nauc_ndcg_at_1000_max": 0.354813, + "nauc_ndcg_at_1000_std": -0.271328, + "nauc_ndcg_at_1000_diff1": 0.290034, + "nauc_map_at_1_max": 0.324526, + "nauc_map_at_1_std": -0.380734, + "nauc_map_at_1_diff1": 0.208024, + "nauc_map_at_3_max": 0.365865, + "nauc_map_at_3_std": -0.244803, + "nauc_map_at_3_diff1": 0.303441, + "nauc_map_at_5_max": 0.354106, + "nauc_map_at_5_std": -0.275705, + "nauc_map_at_5_diff1": 0.28643, + "nauc_map_at_10_max": 0.354204, + "nauc_map_at_10_std": -0.280451, + "nauc_map_at_10_diff1": 0.283036, + "nauc_map_at_20_max": 0.350324, + "nauc_map_at_20_std": -0.278319, + "nauc_map_at_20_diff1": 0.284243, + "nauc_map_at_100_max": 0.350324, + "nauc_map_at_100_std": -0.278319, + "nauc_map_at_100_diff1": 0.284243, + "nauc_map_at_1000_max": 0.350324, + "nauc_map_at_1000_std": -0.278319, + "nauc_map_at_1000_diff1": 0.284243, + "nauc_recall_at_1_max": 0.324526, + "nauc_recall_at_1_std": -0.380734, + "nauc_recall_at_1_diff1": 0.208024, + "nauc_recall_at_3_max": 0.537689, + "nauc_recall_at_3_std": 0.025911, + "nauc_recall_at_3_diff1": 0.489954, + "nauc_recall_at_5_max": 0.565982, + "nauc_recall_at_5_std": -0.204015, + "nauc_recall_at_5_diff1": 0.371615, + "nauc_recall_at_10_max": 0.869281, + "nauc_recall_at_10_std": -0.563492, + "nauc_recall_at_10_diff1": 0.122782, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.324526, + "nauc_precision_at_1_std": -0.380734, + "nauc_precision_at_1_diff1": 0.208024, + "nauc_precision_at_3_max": 0.537689, + "nauc_precision_at_3_std": 0.025911, + "nauc_precision_at_3_diff1": 0.489954, + "nauc_precision_at_5_max": 0.565982, + "nauc_precision_at_5_std": -0.204015, + "nauc_precision_at_5_diff1": 0.371615, + "nauc_precision_at_10_max": 0.869281, + "nauc_precision_at_10_std": -0.563492, + "nauc_precision_at_10_diff1": 0.122782, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.324526, + "nauc_mrr_at_1_std": -0.380734, + "nauc_mrr_at_1_diff1": 0.208024, + "nauc_mrr_at_3_max": 0.365865, + "nauc_mrr_at_3_std": -0.244803, + "nauc_mrr_at_3_diff1": 0.303441, + "nauc_mrr_at_5_max": 0.354106, + "nauc_mrr_at_5_std": -0.275705, + "nauc_mrr_at_5_diff1": 0.28643, + "nauc_mrr_at_10_max": 0.354204, + "nauc_mrr_at_10_std": -0.280451, + "nauc_mrr_at_10_diff1": 0.283036, + "nauc_mrr_at_20_max": 0.350324, + "nauc_mrr_at_20_std": -0.278319, + "nauc_mrr_at_20_diff1": 0.284243, + "nauc_mrr_at_100_max": 0.350324, + "nauc_mrr_at_100_std": -0.278319, + "nauc_mrr_at_100_diff1": 0.284243, + "nauc_mrr_at_1000_max": 0.350324, + "nauc_mrr_at_1000_std": -0.278319, + "nauc_mrr_at_1000_diff1": 0.284243, + "main_score": 0.54, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_8192": [ + { + "ndcg_at_1": 0.5, + "ndcg_at_3": 0.63571, + "ndcg_at_5": 0.6779, + "ndcg_at_10": 0.71739, + "ndcg_at_20": 0.72822, + "ndcg_at_100": 0.72822, + "ndcg_at_1000": 0.72822, + "map_at_1": 0.5, + "map_at_3": 0.6, + "map_at_5": 0.624, + "map_at_10": 0.64072, + "map_at_20": 0.64408, + "map_at_100": 0.64408, + "map_at_1000": 0.64408, + "recall_at_1": 0.5, + "recall_at_3": 0.74, + "recall_at_5": 0.84, + "recall_at_10": 0.96, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.5, + "precision_at_3": 0.24667, + "precision_at_5": 0.168, + "precision_at_10": 0.096, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.5, + "mrr_at_3": 0.6, + "mrr_at_5": 0.624, + "mrr_at_10": 0.640722, + "mrr_at_20": 0.644079, + "mrr_at_100": 0.644079, + "mrr_at_1000": 0.644079, + "nauc_ndcg_at_1_max": 0.23547, + "nauc_ndcg_at_1_std": 0.104144, + "nauc_ndcg_at_1_diff1": 0.489807, + "nauc_ndcg_at_3_max": 0.190496, + "nauc_ndcg_at_3_std": 0.215207, + "nauc_ndcg_at_3_diff1": 0.439363, + "nauc_ndcg_at_5_max": 0.163662, + "nauc_ndcg_at_5_std": 0.218582, + "nauc_ndcg_at_5_diff1": 0.464294, + "nauc_ndcg_at_10_max": 0.185486, + "nauc_ndcg_at_10_std": 0.165591, + "nauc_ndcg_at_10_diff1": 0.463916, + "nauc_ndcg_at_20_max": 0.196487, + "nauc_ndcg_at_20_std": 0.163639, + "nauc_ndcg_at_20_diff1": 0.47024, + "nauc_ndcg_at_100_max": 0.196487, + "nauc_ndcg_at_100_std": 0.163639, + "nauc_ndcg_at_100_diff1": 0.47024, + "nauc_ndcg_at_1000_max": 0.196487, + "nauc_ndcg_at_1000_std": 0.163639, + "nauc_ndcg_at_1000_diff1": 0.47024, + "nauc_map_at_1_max": 0.23547, + "nauc_map_at_1_std": 0.104144, + "nauc_map_at_1_diff1": 0.489807, + "nauc_map_at_3_max": 0.196193, + "nauc_map_at_3_std": 0.179336, + "nauc_map_at_3_diff1": 0.455715, + "nauc_map_at_5_max": 0.182195, + "nauc_map_at_5_std": 0.179894, + "nauc_map_at_5_diff1": 0.470671, + "nauc_map_at_10_max": 0.194895, + "nauc_map_at_10_std": 0.1586, + "nauc_map_at_10_diff1": 0.470654, + "nauc_map_at_20_max": 0.198011, + "nauc_map_at_20_std": 0.157802, + "nauc_map_at_20_diff1": 0.4721, + "nauc_map_at_100_max": 0.198011, + "nauc_map_at_100_std": 0.157802, + "nauc_map_at_100_diff1": 0.4721, + "nauc_map_at_1000_max": 0.198011, + "nauc_map_at_1000_std": 0.157802, + "nauc_map_at_1000_diff1": 0.4721, + "nauc_recall_at_1_max": 0.23547, + "nauc_recall_at_1_std": 0.104144, + "nauc_recall_at_1_diff1": 0.489807, + "nauc_recall_at_3_max": 0.172621, + "nauc_recall_at_3_std": 0.353838, + "nauc_recall_at_3_diff1": 0.37614, + "nauc_recall_at_5_max": 0.065047, + "nauc_recall_at_5_std": 0.428817, + "nauc_recall_at_5_diff1": 0.431026, + "nauc_recall_at_10_max": -0.004669, + "nauc_recall_at_10_std": 0.19141, + "nauc_recall_at_10_diff1": 0.338469, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.23547, + "nauc_precision_at_1_std": 0.104144, + "nauc_precision_at_1_diff1": 0.489807, + "nauc_precision_at_3_max": 0.172621, + "nauc_precision_at_3_std": 0.353838, + "nauc_precision_at_3_diff1": 0.37614, + "nauc_precision_at_5_max": 0.065047, + "nauc_precision_at_5_std": 0.428817, + "nauc_precision_at_5_diff1": 0.431026, + "nauc_precision_at_10_max": -0.004669, + "nauc_precision_at_10_std": 0.19141, + "nauc_precision_at_10_diff1": 0.338469, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.23547, + "nauc_mrr_at_1_std": 0.104144, + "nauc_mrr_at_1_diff1": 0.489807, + "nauc_mrr_at_3_max": 0.196193, + "nauc_mrr_at_3_std": 0.179336, + "nauc_mrr_at_3_diff1": 0.455715, + "nauc_mrr_at_5_max": 0.182195, + "nauc_mrr_at_5_std": 0.179894, + "nauc_mrr_at_5_diff1": 0.470671, + "nauc_mrr_at_10_max": 0.194895, + "nauc_mrr_at_10_std": 0.1586, + "nauc_mrr_at_10_diff1": 0.470654, + "nauc_mrr_at_20_max": 0.198011, + "nauc_mrr_at_20_std": 0.157802, + "nauc_mrr_at_20_diff1": 0.4721, + "nauc_mrr_at_100_max": 0.198011, + "nauc_mrr_at_100_std": 0.157802, + "nauc_mrr_at_100_diff1": 0.4721, + "nauc_mrr_at_1000_max": 0.198011, + "nauc_mrr_at_1000_std": 0.157802, + "nauc_mrr_at_1000_diff1": 0.4721, + "main_score": 0.5, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_16384": [ + { + "ndcg_at_1": 0.34, + "ndcg_at_3": 0.43571, + "ndcg_at_5": 0.43571, + "ndcg_at_10": 0.44149, + "ndcg_at_20": 0.44707, + "ndcg_at_100": 0.52485, + "ndcg_at_1000": 0.52485, + "map_at_1": 0.34, + "map_at_3": 0.41333, + "map_at_5": 0.41333, + "map_at_10": 0.41533, + "map_at_20": 0.41715, + "map_at_100": 0.42509, + "map_at_1000": 0.42509, + "recall_at_1": 0.34, + "recall_at_3": 0.5, + "recall_at_5": 0.5, + "recall_at_10": 0.52, + "recall_at_20": 0.54, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.34, + "precision_at_3": 0.16667, + "precision_at_5": 0.1, + "precision_at_10": 0.052, + "precision_at_20": 0.027, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.34, + "mrr_at_3": 0.413333, + "mrr_at_5": 0.413333, + "mrr_at_10": 0.415333, + "mrr_at_20": 0.417152, + "mrr_at_100": 0.42492, + "mrr_at_1000": 0.42492, + "nauc_ndcg_at_1_max": 0.038051, + "nauc_ndcg_at_1_std": 0.468984, + "nauc_ndcg_at_1_diff1": 0.19876, + "nauc_ndcg_at_3_max": 0.002505, + "nauc_ndcg_at_3_std": 0.470481, + "nauc_ndcg_at_3_diff1": 0.104317, + "nauc_ndcg_at_5_max": 0.002505, + "nauc_ndcg_at_5_std": 0.470481, + "nauc_ndcg_at_5_diff1": 0.104317, + "nauc_ndcg_at_10_max": -0.014013, + "nauc_ndcg_at_10_std": 0.461325, + "nauc_ndcg_at_10_diff1": 0.090563, + "nauc_ndcg_at_20_max": -0.025682, + "nauc_ndcg_at_20_std": 0.452387, + "nauc_ndcg_at_20_diff1": 0.075038, + "nauc_ndcg_at_100_max": -0.009673, + "nauc_ndcg_at_100_std": 0.461995, + "nauc_ndcg_at_100_diff1": 0.106464, + "nauc_ndcg_at_1000_max": -0.009673, + "nauc_ndcg_at_1000_std": 0.461995, + "nauc_ndcg_at_1000_diff1": 0.106464, + "nauc_map_at_1_max": 0.038051, + "nauc_map_at_1_std": 0.468984, + "nauc_map_at_1_diff1": 0.19876, + "nauc_map_at_3_max": 0.003756, + "nauc_map_at_3_std": 0.471125, + "nauc_map_at_3_diff1": 0.120724, + "nauc_map_at_5_max": 0.003756, + "nauc_map_at_5_std": 0.471125, + "nauc_map_at_5_diff1": 0.120724, + "nauc_map_at_10_max": -0.001927, + "nauc_map_at_10_std": 0.467975, + "nauc_map_at_10_diff1": 0.116023, + "nauc_map_at_20_max": -0.005673, + "nauc_map_at_20_std": 0.4651, + "nauc_map_at_20_diff1": 0.111056, + "nauc_map_at_100_max": -0.003477, + "nauc_map_at_100_std": 0.466114, + "nauc_map_at_100_diff1": 0.116589, + "nauc_map_at_1000_max": -0.003477, + "nauc_map_at_1000_std": 0.466114, + "nauc_map_at_1000_diff1": 0.116589, + "nauc_recall_at_1_max": 0.038051, + "nauc_recall_at_1_std": 0.468984, + "nauc_recall_at_1_diff1": 0.19876, + "nauc_recall_at_3_max": 0.001919, + "nauc_recall_at_3_std": 0.468195, + "nauc_recall_at_3_diff1": 0.059374, + "nauc_recall_at_5_max": 0.001919, + "nauc_recall_at_5_std": 0.468195, + "nauc_recall_at_5_diff1": 0.059374, + "nauc_recall_at_10_max": -0.056314, + "nauc_recall_at_10_std": 0.435878, + "nauc_recall_at_10_diff1": 0.009957, + "nauc_recall_at_20_max": -0.100497, + "nauc_recall_at_20_std": 0.402184, + "nauc_recall_at_20_diff1": -0.049346, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.038051, + "nauc_precision_at_1_std": 0.468984, + "nauc_precision_at_1_diff1": 0.19876, + "nauc_precision_at_3_max": 0.001919, + "nauc_precision_at_3_std": 0.468195, + "nauc_precision_at_3_diff1": 0.059374, + "nauc_precision_at_5_max": 0.001919, + "nauc_precision_at_5_std": 0.468195, + "nauc_precision_at_5_diff1": 0.059374, + "nauc_precision_at_10_max": -0.056314, + "nauc_precision_at_10_std": 0.435878, + "nauc_precision_at_10_diff1": 0.009957, + "nauc_precision_at_20_max": -0.100497, + "nauc_precision_at_20_std": 0.402184, + "nauc_precision_at_20_diff1": -0.049346, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.038051, + "nauc_mrr_at_1_std": 0.468984, + "nauc_mrr_at_1_diff1": 0.19876, + "nauc_mrr_at_3_max": 0.003756, + "nauc_mrr_at_3_std": 0.471125, + "nauc_mrr_at_3_diff1": 0.120724, + "nauc_mrr_at_5_max": 0.003756, + "nauc_mrr_at_5_std": 0.471125, + "nauc_mrr_at_5_diff1": 0.120724, + "nauc_mrr_at_10_max": -0.001927, + "nauc_mrr_at_10_std": 0.467975, + "nauc_mrr_at_10_diff1": 0.116023, + "nauc_mrr_at_20_max": -0.005673, + "nauc_mrr_at_20_std": 0.4651, + "nauc_mrr_at_20_diff1": 0.111056, + "nauc_mrr_at_100_max": -0.004099, + "nauc_mrr_at_100_std": 0.46445, + "nauc_mrr_at_100_diff1": 0.115089, + "nauc_mrr_at_1000_max": -0.004099, + "nauc_mrr_at_1000_std": 0.46445, + "nauc_mrr_at_1000_diff1": 0.115089, + "main_score": 0.34, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_32768": [ + { + "ndcg_at_1": 0.14, + "ndcg_at_3": 0.20309, + "ndcg_at_5": 0.21944, + "ndcg_at_10": 0.21944, + "ndcg_at_20": 0.21944, + "ndcg_at_100": 0.34598, + "ndcg_at_1000": 0.34598, + "map_at_1": 0.14, + "map_at_3": 0.19, + "map_at_5": 0.199, + "map_at_10": 0.199, + "map_at_20": 0.199, + "map_at_100": 0.21392, + "map_at_1000": 0.21392, + "recall_at_1": 0.14, + "recall_at_3": 0.24, + "recall_at_5": 0.28, + "recall_at_10": 0.28, + "recall_at_20": 0.28, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.14, + "precision_at_3": 0.08, + "precision_at_5": 0.056, + "precision_at_10": 0.028, + "precision_at_20": 0.014, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.14, + "mrr_at_3": 0.19, + "mrr_at_5": 0.199, + "mrr_at_10": 0.199, + "mrr_at_20": 0.199, + "mrr_at_100": 0.213944, + "mrr_at_1000": 0.213944, + "nauc_ndcg_at_1_max": 0.234503, + "nauc_ndcg_at_1_std": 0.221265, + "nauc_ndcg_at_1_diff1": 0.349442, + "nauc_ndcg_at_3_max": 0.347482, + "nauc_ndcg_at_3_std": 0.425083, + "nauc_ndcg_at_3_diff1": 0.410841, + "nauc_ndcg_at_5_max": 0.361584, + "nauc_ndcg_at_5_std": 0.426532, + "nauc_ndcg_at_5_diff1": 0.412747, + "nauc_ndcg_at_10_max": 0.361584, + "nauc_ndcg_at_10_std": 0.426532, + "nauc_ndcg_at_10_diff1": 0.412747, + "nauc_ndcg_at_20_max": 0.361584, + "nauc_ndcg_at_20_std": 0.426532, + "nauc_ndcg_at_20_diff1": 0.412747, + "nauc_ndcg_at_100_max": 0.339529, + "nauc_ndcg_at_100_std": 0.404739, + "nauc_ndcg_at_100_diff1": 0.396797, + "nauc_ndcg_at_1000_max": 0.339529, + "nauc_ndcg_at_1000_std": 0.404739, + "nauc_ndcg_at_1000_diff1": 0.396797, + "nauc_map_at_1_max": 0.234503, + "nauc_map_at_1_std": 0.221265, + "nauc_map_at_1_diff1": 0.349442, + "nauc_map_at_3_max": 0.328078, + "nauc_map_at_3_std": 0.390076, + "nauc_map_at_3_diff1": 0.400295, + "nauc_map_at_5_max": 0.339282, + "nauc_map_at_5_std": 0.393366, + "nauc_map_at_5_diff1": 0.403394, + "nauc_map_at_10_max": 0.339282, + "nauc_map_at_10_std": 0.393366, + "nauc_map_at_10_diff1": 0.403394, + "nauc_map_at_20_max": 0.339282, + "nauc_map_at_20_std": 0.393366, + "nauc_map_at_20_diff1": 0.403394, + "nauc_map_at_100_max": 0.333488, + "nauc_map_at_100_std": 0.389318, + "nauc_map_at_100_diff1": 0.397802, + "nauc_map_at_1000_max": 0.333488, + "nauc_map_at_1000_std": 0.389318, + "nauc_map_at_1000_diff1": 0.397802, + "nauc_recall_at_1_max": 0.234503, + "nauc_recall_at_1_std": 0.221265, + "nauc_recall_at_1_diff1": 0.349442, + "nauc_recall_at_3_max": 0.394141, + "nauc_recall_at_3_std": 0.509256, + "nauc_recall_at_3_diff1": 0.436197, + "nauc_recall_at_5_max": 0.411839, + "nauc_recall_at_5_std": 0.500615, + "nauc_recall_at_5_diff1": 0.432482, + "nauc_recall_at_10_max": 0.411839, + "nauc_recall_at_10_std": 0.500615, + "nauc_recall_at_10_diff1": 0.432482, + "nauc_recall_at_20_max": 0.411839, + "nauc_recall_at_20_std": 0.500615, + "nauc_recall_at_20_diff1": 0.432482, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.234503, + "nauc_precision_at_1_std": 0.221265, + "nauc_precision_at_1_diff1": 0.349442, + "nauc_precision_at_3_max": 0.394141, + "nauc_precision_at_3_std": 0.509256, + "nauc_precision_at_3_diff1": 0.436197, + "nauc_precision_at_5_max": 0.411839, + "nauc_precision_at_5_std": 0.500615, + "nauc_precision_at_5_diff1": 0.432482, + "nauc_precision_at_10_max": 0.411839, + "nauc_precision_at_10_std": 0.500615, + "nauc_precision_at_10_diff1": 0.432482, + "nauc_precision_at_20_max": 0.411839, + "nauc_precision_at_20_std": 0.500615, + "nauc_precision_at_20_diff1": 0.432482, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.234503, + "nauc_mrr_at_1_std": 0.221265, + "nauc_mrr_at_1_diff1": 0.349442, + "nauc_mrr_at_3_max": 0.328078, + "nauc_mrr_at_3_std": 0.390076, + "nauc_mrr_at_3_diff1": 0.400295, + "nauc_mrr_at_5_max": 0.339282, + "nauc_mrr_at_5_std": 0.393366, + "nauc_mrr_at_5_diff1": 0.403394, + "nauc_mrr_at_10_max": 0.339282, + "nauc_mrr_at_10_std": 0.393366, + "nauc_mrr_at_10_diff1": 0.403394, + "nauc_mrr_at_20_max": 0.339282, + "nauc_mrr_at_20_std": 0.393366, + "nauc_mrr_at_20_diff1": 0.403394, + "nauc_mrr_at_100_max": 0.334974, + "nauc_mrr_at_100_std": 0.384154, + "nauc_mrr_at_100_diff1": 0.399405, + "nauc_mrr_at_1000_max": 0.334974, + "nauc_mrr_at_1000_std": 0.384154, + "nauc_mrr_at_1000_diff1": 0.399405, + "main_score": 0.14, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 15.47933316230774, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBPasskeyRetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBPasskeyRetrieval.json new file mode 100644 index 0000000000..7c91722ef0 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBPasskeyRetrieval.json @@ -0,0 +1,1201 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBPasskeyRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test_256": [ + { + "ndcg_at_1": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "map_at_1": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "map_at_10": 1.0, + "map_at_20": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "recall_at_1": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 1.0, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "mrr_at_10": 1.0, + "mrr_at_20": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_1000_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "main_score": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_512": [ + { + "ndcg_at_1": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "map_at_1": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "map_at_10": 1.0, + "map_at_20": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "recall_at_1": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 1.0, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "mrr_at_10": 1.0, + "mrr_at_20": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_1000_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "main_score": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_1024": [ + { + "ndcg_at_1": 0.98, + "ndcg_at_3": 0.99262, + "ndcg_at_5": 0.99262, + "ndcg_at_10": 0.99262, + "ndcg_at_20": 0.99262, + "ndcg_at_100": 0.99262, + "ndcg_at_1000": 0.99262, + "map_at_1": 0.98, + "map_at_3": 0.99, + "map_at_5": 0.99, + "map_at_10": 0.99, + "map_at_20": 0.99, + "map_at_100": 0.99, + "map_at_1000": 0.99, + "recall_at_1": 0.98, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.98, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.98, + "mrr_at_3": 0.99, + "mrr_at_5": 0.99, + "mrr_at_10": 0.99, + "mrr_at_20": 0.99, + "mrr_at_100": 0.99, + "mrr_at_1000": 0.99, + "nauc_ndcg_at_1_max": 1.0, + "nauc_ndcg_at_1_std": 1.0, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 1.0, + "nauc_ndcg_at_3_std": 1.0, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 1.0, + "nauc_ndcg_at_5_std": 1.0, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 1.0, + "nauc_ndcg_at_10_std": 1.0, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 1.0, + "nauc_ndcg_at_20_std": 1.0, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 1.0, + "nauc_ndcg_at_100_std": 1.0, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 1.0, + "nauc_ndcg_at_1000_std": 1.0, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 1.0, + "nauc_map_at_1_std": 1.0, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 1.0, + "nauc_map_at_3_std": 1.0, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 1.0, + "nauc_map_at_5_std": 1.0, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 1.0, + "nauc_map_at_10_std": 1.0, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 1.0, + "nauc_map_at_20_std": 1.0, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 1.0, + "nauc_map_at_100_std": 1.0, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 1.0, + "nauc_map_at_1000_std": 1.0, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 1.0, + "nauc_recall_at_1_std": 1.0, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 1.0, + "nauc_precision_at_1_std": 1.0, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 1.0, + "nauc_mrr_at_1_std": 1.0, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 1.0, + "nauc_mrr_at_3_std": 1.0, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 1.0, + "nauc_mrr_at_5_std": 1.0, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 1.0, + "nauc_mrr_at_10_std": 1.0, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 1.0, + "nauc_mrr_at_20_std": 1.0, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 1.0, + "nauc_mrr_at_100_std": 1.0, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 1.0, + "nauc_mrr_at_1000_std": 1.0, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.98, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_2048": [ + { + "ndcg_at_1": 0.98, + "ndcg_at_3": 0.99262, + "ndcg_at_5": 0.99262, + "ndcg_at_10": 0.99262, + "ndcg_at_20": 0.99262, + "ndcg_at_100": 0.99262, + "ndcg_at_1000": 0.99262, + "map_at_1": 0.98, + "map_at_3": 0.99, + "map_at_5": 0.99, + "map_at_10": 0.99, + "map_at_20": 0.99, + "map_at_100": 0.99, + "map_at_1000": 0.99, + "recall_at_1": 0.98, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.98, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.98, + "mrr_at_3": 0.99, + "mrr_at_5": 0.99, + "mrr_at_10": 0.99, + "mrr_at_20": 0.99, + "mrr_at_100": 0.99, + "mrr_at_1000": 0.99, + "nauc_ndcg_at_1_max": 0.869281, + "nauc_ndcg_at_1_std": 0.554155, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.869281, + "nauc_ndcg_at_3_std": 0.554155, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.869281, + "nauc_ndcg_at_5_std": 0.554155, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.869281, + "nauc_ndcg_at_10_std": 0.554155, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.869281, + "nauc_ndcg_at_20_std": 0.554155, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.869281, + "nauc_ndcg_at_100_std": 0.554155, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.869281, + "nauc_ndcg_at_1000_std": 0.554155, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.869281, + "nauc_map_at_1_std": 0.554155, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.869281, + "nauc_map_at_3_std": 0.554155, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.869281, + "nauc_map_at_5_std": 0.554155, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.869281, + "nauc_map_at_10_std": 0.554155, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.869281, + "nauc_map_at_20_std": 0.554155, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.869281, + "nauc_map_at_100_std": 0.554155, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.869281, + "nauc_map_at_1000_std": 0.554155, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.869281, + "nauc_recall_at_1_std": 0.554155, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.869281, + "nauc_precision_at_1_std": 0.554155, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.869281, + "nauc_mrr_at_1_std": 0.554155, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.869281, + "nauc_mrr_at_3_std": 0.554155, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.869281, + "nauc_mrr_at_5_std": 0.554155, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.869281, + "nauc_mrr_at_10_std": 0.554155, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.869281, + "nauc_mrr_at_20_std": 0.554155, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.869281, + "nauc_mrr_at_100_std": 0.554155, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.869281, + "nauc_mrr_at_1000_std": 0.554155, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.98, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_4096": [ + { + "ndcg_at_1": 0.78, + "ndcg_at_3": 0.87571, + "ndcg_at_5": 0.89119, + "ndcg_at_10": 0.89721, + "ndcg_at_20": 0.89721, + "ndcg_at_100": 0.89721, + "ndcg_at_1000": 0.89721, + "map_at_1": 0.78, + "map_at_3": 0.85333, + "map_at_5": 0.86133, + "map_at_10": 0.86356, + "map_at_20": 0.86356, + "map_at_100": 0.86356, + "map_at_1000": 0.86356, + "recall_at_1": 0.78, + "recall_at_3": 0.94, + "recall_at_5": 0.98, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.78, + "precision_at_3": 0.31333, + "precision_at_5": 0.196, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.78, + "mrr_at_3": 0.853333, + "mrr_at_5": 0.861333, + "mrr_at_10": 0.863556, + "mrr_at_20": 0.863556, + "mrr_at_100": 0.863556, + "mrr_at_1000": 0.863556, + "nauc_ndcg_at_1_max": 0.649094, + "nauc_ndcg_at_1_std": 0.632401, + "nauc_ndcg_at_1_diff1": 0.744517, + "nauc_ndcg_at_3_max": 0.631486, + "nauc_ndcg_at_3_std": 0.695413, + "nauc_ndcg_at_3_diff1": 0.761346, + "nauc_ndcg_at_5_max": 0.66445, + "nauc_ndcg_at_5_std": 0.649178, + "nauc_ndcg_at_5_diff1": 0.746202, + "nauc_ndcg_at_10_max": 0.643388, + "nauc_ndcg_at_10_std": 0.66745, + "nauc_ndcg_at_10_diff1": 0.747707, + "nauc_ndcg_at_20_max": 0.643388, + "nauc_ndcg_at_20_std": 0.66745, + "nauc_ndcg_at_20_diff1": 0.747707, + "nauc_ndcg_at_100_max": 0.643388, + "nauc_ndcg_at_100_std": 0.66745, + "nauc_ndcg_at_100_diff1": 0.747707, + "nauc_ndcg_at_1000_max": 0.643388, + "nauc_ndcg_at_1000_std": 0.66745, + "nauc_ndcg_at_1000_diff1": 0.747707, + "nauc_map_at_1_max": 0.649094, + "nauc_map_at_1_std": 0.632401, + "nauc_map_at_1_diff1": 0.744517, + "nauc_map_at_3_max": 0.63517, + "nauc_map_at_3_std": 0.680625, + "nauc_map_at_3_diff1": 0.753521, + "nauc_map_at_5_max": 0.648828, + "nauc_map_at_5_std": 0.660873, + "nauc_map_at_5_diff1": 0.746867, + "nauc_map_at_10_max": 0.642689, + "nauc_map_at_10_std": 0.666166, + "nauc_map_at_10_diff1": 0.747297, + "nauc_map_at_20_max": 0.642689, + "nauc_map_at_20_std": 0.666166, + "nauc_map_at_20_diff1": 0.747297, + "nauc_map_at_100_max": 0.642689, + "nauc_map_at_100_std": 0.666166, + "nauc_map_at_100_diff1": 0.747297, + "nauc_map_at_1000_max": 0.642689, + "nauc_map_at_1000_std": 0.666166, + "nauc_map_at_1000_diff1": 0.747297, + "nauc_recall_at_1_max": 0.649094, + "nauc_recall_at_1_std": 0.632401, + "nauc_recall_at_1_diff1": 0.744517, + "nauc_recall_at_3_max": 0.609555, + "nauc_recall_at_3_std": 0.786026, + "nauc_recall_at_3_diff1": 0.814815, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": 0.358077, + "nauc_recall_at_5_diff1": 0.722222, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.649094, + "nauc_precision_at_1_std": 0.632401, + "nauc_precision_at_1_diff1": 0.744517, + "nauc_precision_at_3_max": 0.609555, + "nauc_precision_at_3_std": 0.786026, + "nauc_precision_at_3_diff1": 0.814815, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 0.358077, + "nauc_precision_at_5_diff1": 0.722222, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.649094, + "nauc_mrr_at_1_std": 0.632401, + "nauc_mrr_at_1_diff1": 0.744517, + "nauc_mrr_at_3_max": 0.63517, + "nauc_mrr_at_3_std": 0.680625, + "nauc_mrr_at_3_diff1": 0.753521, + "nauc_mrr_at_5_max": 0.648828, + "nauc_mrr_at_5_std": 0.660873, + "nauc_mrr_at_5_diff1": 0.746867, + "nauc_mrr_at_10_max": 0.642689, + "nauc_mrr_at_10_std": 0.666166, + "nauc_mrr_at_10_diff1": 0.747297, + "nauc_mrr_at_20_max": 0.642689, + "nauc_mrr_at_20_std": 0.666166, + "nauc_mrr_at_20_diff1": 0.747297, + "nauc_mrr_at_100_max": 0.642689, + "nauc_mrr_at_100_std": 0.666166, + "nauc_mrr_at_100_diff1": 0.747297, + "nauc_mrr_at_1000_max": 0.642689, + "nauc_mrr_at_1000_std": 0.666166, + "nauc_mrr_at_1000_diff1": 0.747297, + "main_score": 0.78, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_8192": [ + { + "ndcg_at_1": 0.9, + "ndcg_at_3": 0.92, + "ndcg_at_5": 0.93635, + "ndcg_at_10": 0.94266, + "ndcg_at_20": 0.94266, + "ndcg_at_100": 0.94266, + "ndcg_at_1000": 0.94266, + "map_at_1": 0.9, + "map_at_3": 0.91333, + "map_at_5": 0.92233, + "map_at_10": 0.92483, + "map_at_20": 0.92483, + "map_at_100": 0.92483, + "map_at_1000": 0.92483, + "recall_at_1": 0.9, + "recall_at_3": 0.94, + "recall_at_5": 0.98, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.9, + "precision_at_3": 0.31333, + "precision_at_5": 0.196, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.9, + "mrr_at_3": 0.913333, + "mrr_at_5": 0.922333, + "mrr_at_10": 0.924833, + "mrr_at_20": 0.924833, + "mrr_at_100": 0.924833, + "mrr_at_1000": 0.924833, + "nauc_ndcg_at_1_max": 0.782446, + "nauc_ndcg_at_1_std": 0.700747, + "nauc_ndcg_at_1_diff1": 0.884687, + "nauc_ndcg_at_3_max": 0.808298, + "nauc_ndcg_at_3_std": 0.660656, + "nauc_ndcg_at_3_diff1": 0.872199, + "nauc_ndcg_at_5_max": 0.813249, + "nauc_ndcg_at_5_std": 0.714549, + "nauc_ndcg_at_5_diff1": 0.893564, + "nauc_ndcg_at_10_max": 0.7927, + "nauc_ndcg_at_10_std": 0.697524, + "nauc_ndcg_at_10_diff1": 0.881853, + "nauc_ndcg_at_20_max": 0.7927, + "nauc_ndcg_at_20_std": 0.697524, + "nauc_ndcg_at_20_diff1": 0.881853, + "nauc_ndcg_at_100_max": 0.7927, + "nauc_ndcg_at_100_std": 0.697524, + "nauc_ndcg_at_100_diff1": 0.881853, + "nauc_ndcg_at_1000_max": 0.7927, + "nauc_ndcg_at_1000_std": 0.697524, + "nauc_ndcg_at_1000_diff1": 0.881853, + "nauc_map_at_1_max": 0.782446, + "nauc_map_at_1_std": 0.700747, + "nauc_map_at_1_diff1": 0.884687, + "nauc_map_at_3_max": 0.798355, + "nauc_map_at_3_std": 0.676076, + "nauc_map_at_3_diff1": 0.877002, + "nauc_map_at_5_max": 0.797951, + "nauc_map_at_5_std": 0.702827, + "nauc_map_at_5_diff1": 0.885711, + "nauc_map_at_10_max": 0.791231, + "nauc_map_at_10_std": 0.697291, + "nauc_map_at_10_diff1": 0.88191, + "nauc_map_at_20_max": 0.791231, + "nauc_map_at_20_std": 0.697291, + "nauc_map_at_20_diff1": 0.88191, + "nauc_map_at_100_max": 0.791231, + "nauc_map_at_100_std": 0.697291, + "nauc_map_at_100_diff1": 0.88191, + "nauc_map_at_1000_max": 0.791231, + "nauc_map_at_1000_std": 0.697291, + "nauc_map_at_1000_diff1": 0.88191, + "nauc_recall_at_1_max": 0.782446, + "nauc_recall_at_1_std": 0.700747, + "nauc_recall_at_1_diff1": 0.884687, + "nauc_recall_at_3_max": 0.851385, + "nauc_recall_at_3_std": 0.593838, + "nauc_recall_at_3_diff1": 0.851385, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": 0.869281, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.782446, + "nauc_precision_at_1_std": 0.700747, + "nauc_precision_at_1_diff1": 0.884687, + "nauc_precision_at_3_max": 0.851385, + "nauc_precision_at_3_std": 0.593838, + "nauc_precision_at_3_diff1": 0.851385, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 0.869281, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.782446, + "nauc_mrr_at_1_std": 0.700747, + "nauc_mrr_at_1_diff1": 0.884687, + "nauc_mrr_at_3_max": 0.798355, + "nauc_mrr_at_3_std": 0.676076, + "nauc_mrr_at_3_diff1": 0.877002, + "nauc_mrr_at_5_max": 0.797951, + "nauc_mrr_at_5_std": 0.702827, + "nauc_mrr_at_5_diff1": 0.885711, + "nauc_mrr_at_10_max": 0.791231, + "nauc_mrr_at_10_std": 0.697291, + "nauc_mrr_at_10_diff1": 0.88191, + "nauc_mrr_at_20_max": 0.791231, + "nauc_mrr_at_20_std": 0.697291, + "nauc_mrr_at_20_diff1": 0.88191, + "nauc_mrr_at_100_max": 0.791231, + "nauc_mrr_at_100_std": 0.697291, + "nauc_mrr_at_100_diff1": 0.88191, + "nauc_mrr_at_1000_max": 0.791231, + "nauc_mrr_at_1000_std": 0.697291, + "nauc_mrr_at_1000_diff1": 0.88191, + "main_score": 0.9, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_16384": [ + { + "ndcg_at_1": 0.48, + "ndcg_at_3": 0.51524, + "ndcg_at_5": 0.51524, + "ndcg_at_10": 0.51524, + "ndcg_at_20": 0.51524, + "ndcg_at_100": 0.58937, + "ndcg_at_1000": 0.58937, + "map_at_1": 0.48, + "map_at_3": 0.50667, + "map_at_5": 0.50667, + "map_at_10": 0.50667, + "map_at_20": 0.50667, + "map_at_100": 0.51305, + "map_at_1000": 0.51305, + "recall_at_1": 0.48, + "recall_at_3": 0.54, + "recall_at_5": 0.54, + "recall_at_10": 0.54, + "recall_at_20": 0.54, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.48, + "precision_at_3": 0.18, + "precision_at_5": 0.108, + "precision_at_10": 0.054, + "precision_at_20": 0.027, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.48, + "mrr_at_3": 0.506667, + "mrr_at_5": 0.506667, + "mrr_at_10": 0.506667, + "mrr_at_20": 0.506667, + "mrr_at_100": 0.512732, + "mrr_at_1000": 0.512732, + "nauc_ndcg_at_1_max": 0.618159, + "nauc_ndcg_at_1_std": 0.271885, + "nauc_ndcg_at_1_diff1": 0.756801, + "nauc_ndcg_at_3_max": 0.55787, + "nauc_ndcg_at_3_std": 0.276172, + "nauc_ndcg_at_3_diff1": 0.722162, + "nauc_ndcg_at_5_max": 0.55787, + "nauc_ndcg_at_5_std": 0.276172, + "nauc_ndcg_at_5_diff1": 0.722162, + "nauc_ndcg_at_10_max": 0.55787, + "nauc_ndcg_at_10_std": 0.276172, + "nauc_ndcg_at_10_diff1": 0.722162, + "nauc_ndcg_at_20_max": 0.55787, + "nauc_ndcg_at_20_std": 0.276172, + "nauc_ndcg_at_20_diff1": 0.722162, + "nauc_ndcg_at_100_max": 0.56557, + "nauc_ndcg_at_100_std": 0.279913, + "nauc_ndcg_at_100_diff1": 0.723528, + "nauc_ndcg_at_1000_max": 0.56557, + "nauc_ndcg_at_1000_std": 0.279913, + "nauc_ndcg_at_1000_diff1": 0.723528, + "nauc_map_at_1_max": 0.618159, + "nauc_map_at_1_std": 0.271885, + "nauc_map_at_1_diff1": 0.756801, + "nauc_map_at_3_max": 0.573078, + "nauc_map_at_3_std": 0.278068, + "nauc_map_at_3_diff1": 0.729005, + "nauc_map_at_5_max": 0.573078, + "nauc_map_at_5_std": 0.278068, + "nauc_map_at_5_diff1": 0.729005, + "nauc_map_at_10_max": 0.573078, + "nauc_map_at_10_std": 0.278068, + "nauc_map_at_10_diff1": 0.729005, + "nauc_map_at_20_max": 0.573078, + "nauc_map_at_20_std": 0.278068, + "nauc_map_at_20_diff1": 0.729005, + "nauc_map_at_100_max": 0.573646, + "nauc_map_at_100_std": 0.279004, + "nauc_map_at_100_diff1": 0.728616, + "nauc_map_at_1000_max": 0.573646, + "nauc_map_at_1000_std": 0.279004, + "nauc_map_at_1000_diff1": 0.728616, + "nauc_recall_at_1_max": 0.618159, + "nauc_recall_at_1_std": 0.271885, + "nauc_recall_at_1_diff1": 0.756801, + "nauc_recall_at_3_max": 0.512573, + "nauc_recall_at_3_std": 0.26929, + "nauc_recall_at_3_diff1": 0.702565, + "nauc_recall_at_5_max": 0.512573, + "nauc_recall_at_5_std": 0.26929, + "nauc_recall_at_5_diff1": 0.702565, + "nauc_recall_at_10_max": 0.512573, + "nauc_recall_at_10_std": 0.26929, + "nauc_recall_at_10_diff1": 0.702565, + "nauc_recall_at_20_max": 0.512573, + "nauc_recall_at_20_std": 0.26929, + "nauc_recall_at_20_diff1": 0.702565, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.618159, + "nauc_precision_at_1_std": 0.271885, + "nauc_precision_at_1_diff1": 0.756801, + "nauc_precision_at_3_max": 0.512573, + "nauc_precision_at_3_std": 0.26929, + "nauc_precision_at_3_diff1": 0.702565, + "nauc_precision_at_5_max": 0.512573, + "nauc_precision_at_5_std": 0.26929, + "nauc_precision_at_5_diff1": 0.702565, + "nauc_precision_at_10_max": 0.512573, + "nauc_precision_at_10_std": 0.26929, + "nauc_precision_at_10_diff1": 0.702565, + "nauc_precision_at_20_max": 0.512573, + "nauc_precision_at_20_std": 0.26929, + "nauc_precision_at_20_diff1": 0.702565, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.618159, + "nauc_mrr_at_1_std": 0.271885, + "nauc_mrr_at_1_diff1": 0.756801, + "nauc_mrr_at_3_max": 0.573078, + "nauc_mrr_at_3_std": 0.278068, + "nauc_mrr_at_3_diff1": 0.729005, + "nauc_mrr_at_5_max": 0.573078, + "nauc_mrr_at_5_std": 0.278068, + "nauc_mrr_at_5_diff1": 0.729005, + "nauc_mrr_at_10_max": 0.573078, + "nauc_mrr_at_10_std": 0.278068, + "nauc_mrr_at_10_diff1": 0.729005, + "nauc_mrr_at_20_max": 0.573078, + "nauc_mrr_at_20_std": 0.278068, + "nauc_mrr_at_20_diff1": 0.729005, + "nauc_mrr_at_100_max": 0.573154, + "nauc_mrr_at_100_std": 0.277863, + "nauc_mrr_at_100_diff1": 0.728334, + "nauc_mrr_at_1000_max": 0.573154, + "nauc_mrr_at_1000_std": 0.277863, + "nauc_mrr_at_1000_diff1": 0.728334, + "main_score": 0.48, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_32768": [ + { + "ndcg_at_1": 0.26, + "ndcg_at_3": 0.26, + "ndcg_at_5": 0.26, + "ndcg_at_10": 0.26, + "ndcg_at_20": 0.26, + "ndcg_at_100": 0.38854, + "ndcg_at_1000": 0.38854, + "map_at_1": 0.26, + "map_at_3": 0.26, + "map_at_5": 0.26, + "map_at_10": 0.26, + "map_at_20": 0.26, + "map_at_100": 0.27459, + "map_at_1000": 0.27459, + "recall_at_1": 0.26, + "recall_at_3": 0.26, + "recall_at_5": 0.26, + "recall_at_10": 0.26, + "recall_at_20": 0.26, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.26, + "precision_at_3": 0.08667, + "precision_at_5": 0.052, + "precision_at_10": 0.026, + "precision_at_20": 0.013, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.26, + "mrr_at_3": 0.26, + "mrr_at_5": 0.26, + "mrr_at_10": 0.26, + "mrr_at_20": 0.26, + "mrr_at_100": 0.27323, + "mrr_at_1000": 0.27323, + "nauc_ndcg_at_1_max": 0.566882, + "nauc_ndcg_at_1_std": 0.608337, + "nauc_ndcg_at_1_diff1": 0.821514, + "nauc_ndcg_at_3_max": 0.566882, + "nauc_ndcg_at_3_std": 0.608337, + "nauc_ndcg_at_3_diff1": 0.821514, + "nauc_ndcg_at_5_max": 0.566882, + "nauc_ndcg_at_5_std": 0.608337, + "nauc_ndcg_at_5_diff1": 0.821514, + "nauc_ndcg_at_10_max": 0.566882, + "nauc_ndcg_at_10_std": 0.608337, + "nauc_ndcg_at_10_diff1": 0.821514, + "nauc_ndcg_at_20_max": 0.566882, + "nauc_ndcg_at_20_std": 0.608337, + "nauc_ndcg_at_20_diff1": 0.821514, + "nauc_ndcg_at_100_max": 0.568169, + "nauc_ndcg_at_100_std": 0.602938, + "nauc_ndcg_at_100_diff1": 0.809097, + "nauc_ndcg_at_1000_max": 0.568169, + "nauc_ndcg_at_1000_std": 0.602938, + "nauc_ndcg_at_1000_diff1": 0.809097, + "nauc_map_at_1_max": 0.566882, + "nauc_map_at_1_std": 0.608337, + "nauc_map_at_1_diff1": 0.821514, + "nauc_map_at_3_max": 0.566882, + "nauc_map_at_3_std": 0.608337, + "nauc_map_at_3_diff1": 0.821514, + "nauc_map_at_5_max": 0.566882, + "nauc_map_at_5_std": 0.608337, + "nauc_map_at_5_diff1": 0.821514, + "nauc_map_at_10_max": 0.566882, + "nauc_map_at_10_std": 0.608337, + "nauc_map_at_10_diff1": 0.821514, + "nauc_map_at_20_max": 0.566882, + "nauc_map_at_20_std": 0.608337, + "nauc_map_at_20_diff1": 0.821514, + "nauc_map_at_100_max": 0.567036, + "nauc_map_at_100_std": 0.606194, + "nauc_map_at_100_diff1": 0.816362, + "nauc_map_at_1000_max": 0.567036, + "nauc_map_at_1000_std": 0.606194, + "nauc_map_at_1000_diff1": 0.816362, + "nauc_recall_at_1_max": 0.566882, + "nauc_recall_at_1_std": 0.608337, + "nauc_recall_at_1_diff1": 0.821514, + "nauc_recall_at_3_max": 0.566882, + "nauc_recall_at_3_std": 0.608337, + "nauc_recall_at_3_diff1": 0.821514, + "nauc_recall_at_5_max": 0.566882, + "nauc_recall_at_5_std": 0.608337, + "nauc_recall_at_5_diff1": 0.821514, + "nauc_recall_at_10_max": 0.566882, + "nauc_recall_at_10_std": 0.608337, + "nauc_recall_at_10_diff1": 0.821514, + "nauc_recall_at_20_max": 0.566882, + "nauc_recall_at_20_std": 0.608337, + "nauc_recall_at_20_diff1": 0.821514, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.566882, + "nauc_precision_at_1_std": 0.608337, + "nauc_precision_at_1_diff1": 0.821514, + "nauc_precision_at_3_max": 0.566882, + "nauc_precision_at_3_std": 0.608337, + "nauc_precision_at_3_diff1": 0.821514, + "nauc_precision_at_5_max": 0.566882, + "nauc_precision_at_5_std": 0.608337, + "nauc_precision_at_5_diff1": 0.821514, + "nauc_precision_at_10_max": 0.566882, + "nauc_precision_at_10_std": 0.608337, + "nauc_precision_at_10_diff1": 0.821514, + "nauc_precision_at_20_max": 0.566882, + "nauc_precision_at_20_std": 0.608337, + "nauc_precision_at_20_diff1": 0.821514, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.566882, + "nauc_mrr_at_1_std": 0.608337, + "nauc_mrr_at_1_diff1": 0.821514, + "nauc_mrr_at_3_max": 0.566882, + "nauc_mrr_at_3_std": 0.608337, + "nauc_mrr_at_3_diff1": 0.821514, + "nauc_mrr_at_5_max": 0.566882, + "nauc_mrr_at_5_std": 0.608337, + "nauc_mrr_at_5_diff1": 0.821514, + "nauc_mrr_at_10_max": 0.566882, + "nauc_mrr_at_10_std": 0.608337, + "nauc_mrr_at_10_diff1": 0.821514, + "nauc_mrr_at_20_max": 0.566882, + "nauc_mrr_at_20_std": 0.608337, + "nauc_mrr_at_20_diff1": 0.821514, + "nauc_mrr_at_100_max": 0.560309, + "nauc_mrr_at_100_std": 0.602163, + "nauc_mrr_at_100_diff1": 0.813608, + "nauc_mrr_at_1000_max": 0.560309, + "nauc_mrr_at_1000_std": 0.602163, + "nauc_mrr_at_1000_diff1": 0.813608, + "main_score": 0.26, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 13.261255025863647, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBQMSumRetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBQMSumRetrieval.json new file mode 100644 index 0000000000..c5e7bff1a0 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBQMSumRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBQMSumRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.22593, + "ndcg_at_3": 0.29837, + "ndcg_at_5": 0.32897, + "ndcg_at_10": 0.36482, + "ndcg_at_20": 0.3869, + "ndcg_at_100": 0.447, + "ndcg_at_1000": 0.45343, + "map_at_1": 0.22593, + "map_at_3": 0.28062, + "map_at_5": 0.29751, + "map_at_10": 0.31222, + "map_at_20": 0.31831, + "map_at_100": 0.326, + "map_at_1000": 0.32639, + "recall_at_1": 0.22593, + "recall_at_3": 0.34971, + "recall_at_5": 0.42436, + "recall_at_10": 0.53569, + "recall_at_20": 0.62279, + "recall_at_100": 0.95612, + "recall_at_1000": 1.0, + "precision_at_1": 0.22593, + "precision_at_3": 0.11657, + "precision_at_5": 0.08487, + "precision_at_10": 0.05357, + "precision_at_20": 0.03114, + "precision_at_100": 0.00956, + "precision_at_1000": 0.001, + "mrr_at_1": 0.225933, + "mrr_at_3": 0.280616, + "mrr_at_5": 0.297511, + "mrr_at_10": 0.312217, + "mrr_at_20": 0.318311, + "mrr_at_100": 0.325996, + "mrr_at_1000": 0.326387, + "nauc_ndcg_at_1_max": 0.080246, + "nauc_ndcg_at_1_std": -0.35208, + "nauc_ndcg_at_1_diff1": 0.522022, + "nauc_ndcg_at_3_max": 0.055686, + "nauc_ndcg_at_3_std": -0.383488, + "nauc_ndcg_at_3_diff1": 0.473616, + "nauc_ndcg_at_5_max": 0.030008, + "nauc_ndcg_at_5_std": -0.405115, + "nauc_ndcg_at_5_diff1": 0.471013, + "nauc_ndcg_at_10_max": 0.00418, + "nauc_ndcg_at_10_std": -0.42911, + "nauc_ndcg_at_10_diff1": 0.467478, + "nauc_ndcg_at_20_max": 0.002325, + "nauc_ndcg_at_20_std": -0.429008, + "nauc_ndcg_at_20_diff1": 0.462072, + "nauc_ndcg_at_100_max": 0.025699, + "nauc_ndcg_at_100_std": -0.403059, + "nauc_ndcg_at_100_diff1": 0.46515, + "nauc_ndcg_at_1000_max": 0.024884, + "nauc_ndcg_at_1000_std": -0.406645, + "nauc_ndcg_at_1000_diff1": 0.470101, + "nauc_map_at_1_max": 0.080246, + "nauc_map_at_1_std": -0.35208, + "nauc_map_at_1_diff1": 0.522022, + "nauc_map_at_3_max": 0.060571, + "nauc_map_at_3_std": -0.375558, + "nauc_map_at_3_diff1": 0.483263, + "nauc_map_at_5_max": 0.045611, + "nauc_map_at_5_std": -0.38828, + "nauc_map_at_5_diff1": 0.481771, + "nauc_map_at_10_max": 0.034818, + "nauc_map_at_10_std": -0.397965, + "nauc_map_at_10_diff1": 0.47992, + "nauc_map_at_20_max": 0.034749, + "nauc_map_at_20_std": -0.397433, + "nauc_map_at_20_diff1": 0.478382, + "nauc_map_at_100_max": 0.037333, + "nauc_map_at_100_std": -0.3946, + "nauc_map_at_100_diff1": 0.478349, + "nauc_map_at_1000_max": 0.037341, + "nauc_map_at_1000_std": -0.394731, + "nauc_map_at_1000_diff1": 0.478638, + "nauc_recall_at_1_max": 0.080246, + "nauc_recall_at_1_std": -0.35208, + "nauc_recall_at_1_diff1": 0.522022, + "nauc_recall_at_3_max": 0.042869, + "nauc_recall_at_3_std": -0.405017, + "nauc_recall_at_3_diff1": 0.448298, + "nauc_recall_at_5_max": -0.014015, + "nauc_recall_at_5_std": -0.45236, + "nauc_recall_at_5_diff1": 0.442665, + "nauc_recall_at_10_max": -0.094302, + "nauc_recall_at_10_std": -0.529032, + "nauc_recall_at_10_diff1": 0.432503, + "nauc_recall_at_20_max": -0.11528, + "nauc_recall_at_20_std": -0.542444, + "nauc_recall_at_20_diff1": 0.406872, + "nauc_recall_at_100_max": 0.0683, + "nauc_recall_at_100_std": -0.242651, + "nauc_recall_at_100_diff1": 0.262057, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.080246, + "nauc_precision_at_1_std": -0.35208, + "nauc_precision_at_1_diff1": 0.522022, + "nauc_precision_at_3_max": 0.042869, + "nauc_precision_at_3_std": -0.405017, + "nauc_precision_at_3_diff1": 0.448298, + "nauc_precision_at_5_max": -0.014015, + "nauc_precision_at_5_std": -0.45236, + "nauc_precision_at_5_diff1": 0.442665, + "nauc_precision_at_10_max": -0.094302, + "nauc_precision_at_10_std": -0.529032, + "nauc_precision_at_10_diff1": 0.432503, + "nauc_precision_at_20_max": -0.11528, + "nauc_precision_at_20_std": -0.542444, + "nauc_precision_at_20_diff1": 0.406872, + "nauc_precision_at_100_max": 0.0683, + "nauc_precision_at_100_std": -0.242651, + "nauc_precision_at_100_diff1": 0.262057, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.080246, + "nauc_mrr_at_1_std": -0.35208, + "nauc_mrr_at_1_diff1": 0.522022, + "nauc_mrr_at_3_max": 0.060571, + "nauc_mrr_at_3_std": -0.375558, + "nauc_mrr_at_3_diff1": 0.483263, + "nauc_mrr_at_5_max": 0.045611, + "nauc_mrr_at_5_std": -0.38828, + "nauc_mrr_at_5_diff1": 0.481771, + "nauc_mrr_at_10_max": 0.034818, + "nauc_mrr_at_10_std": -0.397965, + "nauc_mrr_at_10_diff1": 0.47992, + "nauc_mrr_at_20_max": 0.034749, + "nauc_mrr_at_20_std": -0.397433, + "nauc_mrr_at_20_diff1": 0.478382, + "nauc_mrr_at_100_max": 0.037333, + "nauc_mrr_at_100_std": -0.3946, + "nauc_mrr_at_100_diff1": 0.478349, + "nauc_mrr_at_1000_max": 0.037341, + "nauc_mrr_at_1000_std": -0.394731, + "nauc_mrr_at_1000_diff1": 0.478638, + "main_score": 0.36482, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.1698994636535645, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBSummScreenFDRetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBSummScreenFDRetrieval.json new file mode 100644 index 0000000000..e9f4352f6e --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBSummScreenFDRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBSummScreenFDRetrieval", + "mteb_version": "1.36.8", + "scores": { + "validation": [ + { + "ndcg_at_1": 0.80655, + "ndcg_at_3": 0.87103, + "ndcg_at_5": 0.88704, + "ndcg_at_10": 0.89911, + "ndcg_at_20": 0.90138, + "ndcg_at_100": 0.90198, + "ndcg_at_1000": 0.90198, + "map_at_1": 0.80655, + "map_at_3": 0.85615, + "map_at_5": 0.86508, + "map_at_10": 0.87036, + "map_at_20": 0.871, + "map_at_100": 0.8711, + "map_at_1000": 0.8711, + "recall_at_1": 0.80655, + "recall_at_3": 0.91369, + "recall_at_5": 0.95238, + "recall_at_10": 0.9881, + "recall_at_20": 0.99702, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.80655, + "precision_at_3": 0.30456, + "precision_at_5": 0.19048, + "precision_at_10": 0.09881, + "precision_at_20": 0.04985, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.806548, + "mrr_at_3": 0.856151, + "mrr_at_5": 0.865079, + "mrr_at_10": 0.870364, + "mrr_at_20": 0.870996, + "mrr_at_100": 0.871099, + "mrr_at_1000": 0.871099, + "nauc_ndcg_at_1_max": 0.446869, + "nauc_ndcg_at_1_std": -0.050075, + "nauc_ndcg_at_1_diff1": 0.82984, + "nauc_ndcg_at_3_max": 0.443094, + "nauc_ndcg_at_3_std": -0.092722, + "nauc_ndcg_at_3_diff1": 0.819499, + "nauc_ndcg_at_5_max": 0.480819, + "nauc_ndcg_at_5_std": -0.026166, + "nauc_ndcg_at_5_diff1": 0.821524, + "nauc_ndcg_at_10_max": 0.463888, + "nauc_ndcg_at_10_std": -0.057547, + "nauc_ndcg_at_10_diff1": 0.818677, + "nauc_ndcg_at_20_max": 0.456594, + "nauc_ndcg_at_20_std": -0.054888, + "nauc_ndcg_at_20_diff1": 0.82049, + "nauc_ndcg_at_100_max": 0.453915, + "nauc_ndcg_at_100_std": -0.059934, + "nauc_ndcg_at_100_diff1": 0.820174, + "nauc_ndcg_at_1000_max": 0.453915, + "nauc_ndcg_at_1000_std": -0.059934, + "nauc_ndcg_at_1000_diff1": 0.820174, + "nauc_map_at_1_max": 0.446869, + "nauc_map_at_1_std": -0.050075, + "nauc_map_at_1_diff1": 0.82984, + "nauc_map_at_3_max": 0.442283, + "nauc_map_at_3_std": -0.079518, + "nauc_map_at_3_diff1": 0.820802, + "nauc_map_at_5_max": 0.460125, + "nauc_map_at_5_std": -0.048124, + "nauc_map_at_5_diff1": 0.821275, + "nauc_map_at_10_max": 0.453228, + "nauc_map_at_10_std": -0.0595, + "nauc_map_at_10_diff1": 0.820204, + "nauc_map_at_20_max": 0.451665, + "nauc_map_at_20_std": -0.059636, + "nauc_map_at_20_diff1": 0.820743, + "nauc_map_at_100_max": 0.451316, + "nauc_map_at_100_std": -0.06029, + "nauc_map_at_100_diff1": 0.820702, + "nauc_map_at_1000_max": 0.451316, + "nauc_map_at_1000_std": -0.06029, + "nauc_map_at_1000_diff1": 0.820702, + "nauc_recall_at_1_max": 0.446869, + "nauc_recall_at_1_std": -0.050075, + "nauc_recall_at_1_diff1": 0.82984, + "nauc_recall_at_3_max": 0.448149, + "nauc_recall_at_3_std": -0.153243, + "nauc_recall_at_3_diff1": 0.814807, + "nauc_recall_at_5_max": 0.657604, + "nauc_recall_at_5_std": 0.17216, + "nauc_recall_at_5_diff1": 0.828501, + "nauc_recall_at_10_max": 0.795633, + "nauc_recall_at_10_std": 0.001336, + "nauc_recall_at_10_diff1": 0.786304, + "nauc_recall_at_20_max": 0.869108, + "nauc_recall_at_20_std": 0.722158, + "nauc_recall_at_20_diff1": 0.869108, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.446869, + "nauc_precision_at_1_std": -0.050075, + "nauc_precision_at_1_diff1": 0.82984, + "nauc_precision_at_3_max": 0.448149, + "nauc_precision_at_3_std": -0.153243, + "nauc_precision_at_3_diff1": 0.814807, + "nauc_precision_at_5_max": 0.657604, + "nauc_precision_at_5_std": 0.17216, + "nauc_precision_at_5_diff1": 0.828501, + "nauc_precision_at_10_max": 0.795633, + "nauc_precision_at_10_std": 0.001336, + "nauc_precision_at_10_diff1": 0.786304, + "nauc_precision_at_20_max": 0.869108, + "nauc_precision_at_20_std": 0.722158, + "nauc_precision_at_20_diff1": 0.869108, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.446869, + "nauc_mrr_at_1_std": -0.050075, + "nauc_mrr_at_1_diff1": 0.82984, + "nauc_mrr_at_3_max": 0.442283, + "nauc_mrr_at_3_std": -0.079518, + "nauc_mrr_at_3_diff1": 0.820802, + "nauc_mrr_at_5_max": 0.460125, + "nauc_mrr_at_5_std": -0.048124, + "nauc_mrr_at_5_diff1": 0.821275, + "nauc_mrr_at_10_max": 0.453228, + "nauc_mrr_at_10_std": -0.0595, + "nauc_mrr_at_10_diff1": 0.820204, + "nauc_mrr_at_20_max": 0.451665, + "nauc_mrr_at_20_std": -0.059636, + "nauc_mrr_at_20_diff1": 0.820743, + "nauc_mrr_at_100_max": 0.451316, + "nauc_mrr_at_100_std": -0.06029, + "nauc_mrr_at_100_diff1": 0.820702, + "nauc_mrr_at_1000_max": 0.451316, + "nauc_mrr_at_1000_std": -0.06029, + "nauc_mrr_at_1000_diff1": 0.820702, + "main_score": 0.89911, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.449986219406128, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBWikimQARetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBWikimQARetrieval.json new file mode 100644 index 0000000000..2e5b60a93b --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/LEMBWikimQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBWikimQARetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.72, + "ndcg_at_3": 0.77873, + "ndcg_at_5": 0.78849, + "ndcg_at_10": 0.79948, + "ndcg_at_20": 0.8091, + "ndcg_at_100": 0.82185, + "ndcg_at_1000": 0.82454, + "map_at_1": 0.72, + "map_at_3": 0.76444, + "map_at_5": 0.76994, + "map_at_10": 0.77461, + "map_at_20": 0.77744, + "map_at_100": 0.77932, + "map_at_1000": 0.77944, + "recall_at_1": 0.72, + "recall_at_3": 0.82, + "recall_at_5": 0.84333, + "recall_at_10": 0.87667, + "recall_at_20": 0.91333, + "recall_at_100": 0.98, + "recall_at_1000": 1.0, + "precision_at_1": 0.72, + "precision_at_3": 0.27333, + "precision_at_5": 0.16867, + "precision_at_10": 0.08767, + "precision_at_20": 0.04567, + "precision_at_100": 0.0098, + "precision_at_1000": 0.001, + "mrr_at_1": 0.72, + "mrr_at_3": 0.764444, + "mrr_at_5": 0.769944, + "mrr_at_10": 0.77461, + "mrr_at_20": 0.777438, + "mrr_at_100": 0.779318, + "mrr_at_1000": 0.779437, + "nauc_ndcg_at_1_max": 0.827514, + "nauc_ndcg_at_1_std": 0.113451, + "nauc_ndcg_at_1_diff1": 0.837702, + "nauc_ndcg_at_3_max": 0.822408, + "nauc_ndcg_at_3_std": 0.106863, + "nauc_ndcg_at_3_diff1": 0.813087, + "nauc_ndcg_at_5_max": 0.828451, + "nauc_ndcg_at_5_std": 0.095333, + "nauc_ndcg_at_5_diff1": 0.808455, + "nauc_ndcg_at_10_max": 0.822355, + "nauc_ndcg_at_10_std": 0.099167, + "nauc_ndcg_at_10_diff1": 0.8039, + "nauc_ndcg_at_20_max": 0.821378, + "nauc_ndcg_at_20_std": 0.087523, + "nauc_ndcg_at_20_diff1": 0.805638, + "nauc_ndcg_at_100_max": 0.822299, + "nauc_ndcg_at_100_std": 0.10261, + "nauc_ndcg_at_100_diff1": 0.808192, + "nauc_ndcg_at_1000_max": 0.824203, + "nauc_ndcg_at_1000_std": 0.098495, + "nauc_ndcg_at_1000_diff1": 0.811916, + "nauc_map_at_1_max": 0.827514, + "nauc_map_at_1_std": 0.113451, + "nauc_map_at_1_diff1": 0.837702, + "nauc_map_at_3_max": 0.824327, + "nauc_map_at_3_std": 0.107477, + "nauc_map_at_3_diff1": 0.819359, + "nauc_map_at_5_max": 0.827495, + "nauc_map_at_5_std": 0.100787, + "nauc_map_at_5_diff1": 0.817008, + "nauc_map_at_10_max": 0.825488, + "nauc_map_at_10_std": 0.102781, + "nauc_map_at_10_diff1": 0.815674, + "nauc_map_at_20_max": 0.825345, + "nauc_map_at_20_std": 0.099576, + "nauc_map_at_20_diff1": 0.816313, + "nauc_map_at_100_max": 0.825522, + "nauc_map_at_100_std": 0.101235, + "nauc_map_at_100_diff1": 0.816737, + "nauc_map_at_1000_max": 0.825584, + "nauc_map_at_1000_std": 0.101063, + "nauc_map_at_1000_diff1": 0.816867, + "nauc_recall_at_1_max": 0.827514, + "nauc_recall_at_1_std": 0.113451, + "nauc_recall_at_1_diff1": 0.837702, + "nauc_recall_at_3_max": 0.815242, + "nauc_recall_at_3_std": 0.10528, + "nauc_recall_at_3_diff1": 0.790799, + "nauc_recall_at_5_max": 0.833142, + "nauc_recall_at_5_std": 0.073326, + "nauc_recall_at_5_diff1": 0.773891, + "nauc_recall_at_10_max": 0.803167, + "nauc_recall_at_10_std": 0.081805, + "nauc_recall_at_10_diff1": 0.742503, + "nauc_recall_at_20_max": 0.787133, + "nauc_recall_at_20_std": -0.009319, + "nauc_recall_at_20_diff1": 0.731326, + "nauc_recall_at_100_max": 0.707049, + "nauc_recall_at_100_std": 0.334578, + "nauc_recall_at_100_diff1": 0.586134, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.827514, + "nauc_precision_at_1_std": 0.113451, + "nauc_precision_at_1_diff1": 0.837702, + "nauc_precision_at_3_max": 0.815242, + "nauc_precision_at_3_std": 0.10528, + "nauc_precision_at_3_diff1": 0.790799, + "nauc_precision_at_5_max": 0.833142, + "nauc_precision_at_5_std": 0.073326, + "nauc_precision_at_5_diff1": 0.773891, + "nauc_precision_at_10_max": 0.803167, + "nauc_precision_at_10_std": 0.081805, + "nauc_precision_at_10_diff1": 0.742503, + "nauc_precision_at_20_max": 0.787133, + "nauc_precision_at_20_std": -0.009319, + "nauc_precision_at_20_diff1": 0.731326, + "nauc_precision_at_100_max": 0.707049, + "nauc_precision_at_100_std": 0.334578, + "nauc_precision_at_100_diff1": 0.586134, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.827514, + "nauc_mrr_at_1_std": 0.113451, + "nauc_mrr_at_1_diff1": 0.837702, + "nauc_mrr_at_3_max": 0.824327, + "nauc_mrr_at_3_std": 0.107477, + "nauc_mrr_at_3_diff1": 0.819359, + "nauc_mrr_at_5_max": 0.827495, + "nauc_mrr_at_5_std": 0.100787, + "nauc_mrr_at_5_diff1": 0.817008, + "nauc_mrr_at_10_max": 0.825488, + "nauc_mrr_at_10_std": 0.102781, + "nauc_mrr_at_10_diff1": 0.815674, + "nauc_mrr_at_20_max": 0.825345, + "nauc_mrr_at_20_std": 0.099576, + "nauc_mrr_at_20_diff1": 0.816313, + "nauc_mrr_at_100_max": 0.825522, + "nauc_mrr_at_100_std": 0.101235, + "nauc_mrr_at_100_diff1": 0.816737, + "nauc_mrr_at_1000_max": 0.825584, + "nauc_mrr_at_1000_std": 0.101063, + "nauc_mrr_at_1000_diff1": 0.816867, + "main_score": 0.79948, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.313003778457642, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MSMARCO.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MSMARCO.json new file mode 100644 index 0000000000..0e8bc5b7f5 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MSMARCO.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c5a29a104738b98a9e76336939199e264163d4a0", + "task_name": "MSMARCO", + "mteb_version": "1.36.8", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.14928, + "ndcg_at_3": 0.22721, + "ndcg_at_5": 0.26362, + "ndcg_at_10": 0.30133, + "ndcg_at_20": 0.33019, + "ndcg_at_100": 0.36923, + "ndcg_at_1000": 0.3877, + "map_at_1": 0.14468, + "map_at_3": 0.2055, + "map_at_5": 0.22577, + "map_at_10": 0.2414, + "map_at_20": 0.24944, + "map_at_100": 0.25489, + "map_at_1000": 0.25562, + "recall_at_1": 0.14468, + "recall_at_3": 0.28526, + "recall_at_5": 0.37293, + "recall_at_10": 0.48879, + "recall_at_20": 0.60117, + "recall_at_100": 0.80918, + "recall_at_1000": 0.95098, + "precision_at_1": 0.14928, + "precision_at_3": 0.09842, + "precision_at_5": 0.07745, + "precision_at_10": 0.05089, + "precision_at_20": 0.03145, + "precision_at_100": 0.00853, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.149284, + "mrr_at_3": 0.210697, + "mrr_at_5": 0.230898, + "mrr_at_10": 0.246298, + "mrr_at_20": 0.254182, + "mrr_at_100": 0.259361, + "mrr_at_1000": 0.260035, + "nauc_ndcg_at_1_max": 0.008113, + "nauc_ndcg_at_1_std": -0.145364, + "nauc_ndcg_at_1_diff1": 0.290862, + "nauc_ndcg_at_3_max": 0.017905, + "nauc_ndcg_at_3_std": -0.151302, + "nauc_ndcg_at_3_diff1": 0.263629, + "nauc_ndcg_at_5_max": 0.023476, + "nauc_ndcg_at_5_std": -0.152505, + "nauc_ndcg_at_5_diff1": 0.260592, + "nauc_ndcg_at_10_max": 0.023065, + "nauc_ndcg_at_10_std": -0.1462, + "nauc_ndcg_at_10_diff1": 0.260568, + "nauc_ndcg_at_20_max": 0.029536, + "nauc_ndcg_at_20_std": -0.128712, + "nauc_ndcg_at_20_diff1": 0.258493, + "nauc_ndcg_at_100_max": 0.037737, + "nauc_ndcg_at_100_std": -0.099451, + "nauc_ndcg_at_100_diff1": 0.253926, + "nauc_ndcg_at_1000_max": 0.038952, + "nauc_ndcg_at_1000_std": -0.107251, + "nauc_ndcg_at_1000_diff1": 0.258398, + "nauc_map_at_1_max": 0.00871, + "nauc_map_at_1_std": -0.148305, + "nauc_map_at_1_diff1": 0.2969, + "nauc_map_at_3_max": 0.015661, + "nauc_map_at_3_std": -0.151173, + "nauc_map_at_3_diff1": 0.270153, + "nauc_map_at_5_max": 0.018674, + "nauc_map_at_5_std": -0.152444, + "nauc_map_at_5_diff1": 0.267834, + "nauc_map_at_10_max": 0.018489, + "nauc_map_at_10_std": -0.149654, + "nauc_map_at_10_diff1": 0.268022, + "nauc_map_at_20_max": 0.020368, + "nauc_map_at_20_std": -0.144664, + "nauc_map_at_20_diff1": 0.267604, + "nauc_map_at_100_max": 0.021505, + "nauc_map_at_100_std": -0.140592, + "nauc_map_at_100_diff1": 0.267041, + "nauc_map_at_1000_max": 0.021615, + "nauc_map_at_1000_std": -0.140727, + "nauc_map_at_1000_diff1": 0.267214, + "nauc_recall_at_1_max": 0.00871, + "nauc_recall_at_1_std": -0.148305, + "nauc_recall_at_1_diff1": 0.2969, + "nauc_recall_at_3_max": 0.021814, + "nauc_recall_at_3_std": -0.15397, + "nauc_recall_at_3_diff1": 0.250483, + "nauc_recall_at_5_max": 0.033002, + "nauc_recall_at_5_std": -0.15518, + "nauc_recall_at_5_diff1": 0.244865, + "nauc_recall_at_10_max": 0.031561, + "nauc_recall_at_10_std": -0.138918, + "nauc_recall_at_10_diff1": 0.24325, + "nauc_recall_at_20_max": 0.054112, + "nauc_recall_at_20_std": -0.07597, + "nauc_recall_at_20_diff1": 0.232733, + "nauc_recall_at_100_max": 0.13036, + "nauc_recall_at_100_std": 0.186657, + "nauc_recall_at_100_diff1": 0.178637, + "nauc_recall_at_1000_max": 0.378127, + "nauc_recall_at_1000_std": 0.569032, + "nauc_recall_at_1000_diff1": 0.173737, + "nauc_precision_at_1_max": 0.008113, + "nauc_precision_at_1_std": -0.145364, + "nauc_precision_at_1_diff1": 0.290862, + "nauc_precision_at_3_max": 0.020905, + "nauc_precision_at_3_std": -0.152719, + "nauc_precision_at_3_diff1": 0.247023, + "nauc_precision_at_5_max": 0.034243, + "nauc_precision_at_5_std": -0.151609, + "nauc_precision_at_5_diff1": 0.240205, + "nauc_precision_at_10_max": 0.033903, + "nauc_precision_at_10_std": -0.132923, + "nauc_precision_at_10_diff1": 0.23463, + "nauc_precision_at_20_max": 0.063159, + "nauc_precision_at_20_std": -0.062733, + "nauc_precision_at_20_diff1": 0.216764, + "nauc_precision_at_100_max": 0.144448, + "nauc_precision_at_100_std": 0.179173, + "nauc_precision_at_100_diff1": 0.132987, + "nauc_precision_at_1000_max": 0.241942, + "nauc_precision_at_1000_std": 0.263013, + "nauc_precision_at_1000_diff1": 0.029854, + "nauc_mrr_at_1_max": 0.008113, + "nauc_mrr_at_1_std": -0.145364, + "nauc_mrr_at_1_diff1": 0.290862, + "nauc_mrr_at_3_max": 0.017138, + "nauc_mrr_at_3_std": -0.147344, + "nauc_mrr_at_3_diff1": 0.265581, + "nauc_mrr_at_5_max": 0.0207, + "nauc_mrr_at_5_std": -0.147469, + "nauc_mrr_at_5_diff1": 0.263991, + "nauc_mrr_at_10_max": 0.020629, + "nauc_mrr_at_10_std": -0.144685, + "nauc_mrr_at_10_diff1": 0.264362, + "nauc_mrr_at_20_max": 0.022566, + "nauc_mrr_at_20_std": -0.139665, + "nauc_mrr_at_20_diff1": 0.264071, + "nauc_mrr_at_100_max": 0.023416, + "nauc_mrr_at_100_std": -0.136211, + "nauc_mrr_at_100_diff1": 0.263502, + "nauc_mrr_at_1000_max": 0.023448, + "nauc_mrr_at_1000_std": -0.136409, + "nauc_mrr_at_1000_diff1": 0.26367, + "main_score": 0.30133, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 4944.7892072200775, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MTOPDomainClassification.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MTOPDomainClassification.json new file mode 100644 index 0000000000..125ea2da43 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MTOPDomainClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf", + "task_name": "MTOPDomainClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.901505, + "f1": 0.89471, + "f1_weighted": 0.902081, + "scores_per_experiment": [ + { + "accuracy": 0.899453, + "f1": 0.891496, + "f1_weighted": 0.899171 + }, + { + "accuracy": 0.898541, + "f1": 0.892939, + "f1_weighted": 0.898977 + }, + { + "accuracy": 0.897629, + "f1": 0.891607, + "f1_weighted": 0.89745 + }, + { + "accuracy": 0.911537, + "f1": 0.903942, + "f1_weighted": 0.913002 + }, + { + "accuracy": 0.893981, + "f1": 0.88568, + "f1_weighted": 0.895361 + }, + { + "accuracy": 0.904241, + "f1": 0.901478, + "f1_weighted": 0.904107 + }, + { + "accuracy": 0.894665, + "f1": 0.886591, + "f1_weighted": 0.8947 + }, + { + "accuracy": 0.903557, + "f1": 0.894211, + "f1_weighted": 0.904534 + }, + { + "accuracy": 0.903329, + "f1": 0.900107, + "f1_weighted": 0.904472 + }, + { + "accuracy": 0.908117, + "f1": 0.899047, + "f1_weighted": 0.909037 + } + ], + "main_score": 0.901505, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.79361891746521, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MTOPIntentClassification.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MTOPIntentClassification.json new file mode 100644 index 0000000000..54190ed75f --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MTOPIntentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba", + "task_name": "MTOPIntentClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.66881, + "f1": 0.487132, + "f1_weighted": 0.705166, + "scores_per_experiment": [ + { + "accuracy": 0.651619, + "f1": 0.468773, + "f1_weighted": 0.692139 + }, + { + "accuracy": 0.683539, + "f1": 0.482602, + "f1_weighted": 0.719215 + }, + { + "accuracy": 0.687415, + "f1": 0.485862, + "f1_weighted": 0.723183 + }, + { + "accuracy": 0.685591, + "f1": 0.520053, + "f1_weighted": 0.720524 + }, + { + "accuracy": 0.648427, + "f1": 0.489967, + "f1_weighted": 0.684228 + }, + { + "accuracy": 0.649339, + "f1": 0.479603, + "f1_weighted": 0.683869 + }, + { + "accuracy": 0.674647, + "f1": 0.4878, + "f1_weighted": 0.714311 + }, + { + "accuracy": 0.667351, + "f1": 0.487977, + "f1_weighted": 0.699928 + }, + { + "accuracy": 0.676015, + "f1": 0.491938, + "f1_weighted": 0.712803 + }, + { + "accuracy": 0.664159, + "f1": 0.476748, + "f1_weighted": 0.701464 + } + ], + "main_score": 0.66881, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 33.72898197174072, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MassiveIntentClassification.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MassiveIntentClassification.json new file mode 100644 index 0000000000..f9af67668b --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MassiveIntentClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "task_name": "MassiveIntentClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.67078, + "f1": 0.646556, + "f1_weighted": 0.673098, + "scores_per_experiment": [ + { + "accuracy": 0.667115, + "f1": 0.648123, + "f1_weighted": 0.670533 + }, + { + "accuracy": 0.69536, + "f1": 0.662063, + "f1_weighted": 0.69845 + }, + { + "accuracy": 0.662408, + "f1": 0.641398, + "f1_weighted": 0.662367 + }, + { + "accuracy": 0.686281, + "f1": 0.658689, + "f1_weighted": 0.68825 + }, + { + "accuracy": 0.670814, + "f1": 0.645885, + "f1_weighted": 0.670414 + }, + { + "accuracy": 0.650303, + "f1": 0.633803, + "f1_weighted": 0.655691 + }, + { + "accuracy": 0.661735, + "f1": 0.643435, + "f1_weighted": 0.662825 + }, + { + "accuracy": 0.677875, + "f1": 0.651194, + "f1_weighted": 0.68261 + }, + { + "accuracy": 0.661063, + "f1": 0.630538, + "f1_weighted": 0.66264 + }, + { + "accuracy": 0.674849, + "f1": 0.650426, + "f1_weighted": 0.677202 + } + ], + "main_score": 0.67078, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.338367938995361, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MassiveScenarioClassification.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MassiveScenarioClassification.json new file mode 100644 index 0000000000..c9080da94f --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MassiveScenarioClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "task_name": "MassiveScenarioClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.727875, + "f1": 0.722514, + "f1_weighted": 0.72896, + "scores_per_experiment": [ + { + "accuracy": 0.746133, + "f1": 0.743685, + "f1_weighted": 0.746076 + }, + { + "accuracy": 0.738736, + "f1": 0.730862, + "f1_weighted": 0.737903 + }, + { + "accuracy": 0.705447, + "f1": 0.697797, + "f1_weighted": 0.704739 + }, + { + "accuracy": 0.7115, + "f1": 0.701932, + "f1_weighted": 0.713173 + }, + { + "accuracy": 0.752522, + "f1": 0.744942, + "f1_weighted": 0.75203 + }, + { + "accuracy": 0.720578, + "f1": 0.709205, + "f1_weighted": 0.723093 + }, + { + "accuracy": 0.720242, + "f1": 0.719237, + "f1_weighted": 0.72475 + }, + { + "accuracy": 0.715535, + "f1": 0.715999, + "f1_weighted": 0.718423 + }, + { + "accuracy": 0.729321, + "f1": 0.727966, + "f1_weighted": 0.730098 + }, + { + "accuracy": 0.738736, + "f1": 0.733516, + "f1_weighted": 0.739316 + } + ], + "main_score": 0.727875, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.182679653167725, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MedrxivClusteringP2P.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MedrxivClusteringP2P.json new file mode 100644 index 0000000000..029a0a55b3 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MedrxivClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "task_name": "MedrxivClusteringP2P", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.329031, + "v_measure_std": 0.015608, + "v_measures": [ + 0.316199, + 0.319192, + 0.307649, + 0.311978, + 0.321524, + 0.351735, + 0.333597, + 0.353624, + 0.344067, + 0.330749 + ], + "main_score": 0.329031, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 83.0384874343872, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MedrxivClusteringP2P.v2.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MedrxivClusteringP2P.v2.json new file mode 100644 index 0000000000..29aec59e10 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MedrxivClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "task_name": "MedrxivClusteringP2P.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.383775, + 0.359468, + 0.347567, + 0.359938, + 0.370181, + 0.352278, + 0.355466, + 0.366653, + 0.368376, + 0.382581 + ] + }, + "v_measure": 0.364628, + "v_measure_std": 0.011463, + "main_score": 0.364628, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 43.240434408187866, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MedrxivClusteringS2S.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MedrxivClusteringS2S.json new file mode 100644 index 0000000000..85b6c720fb --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MedrxivClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "task_name": "MedrxivClusteringS2S", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.326103, + "v_measure_std": 0.016933, + "v_measures": [ + 0.301654, + 0.317968, + 0.30788, + 0.310842, + 0.317383, + 0.34608, + 0.332956, + 0.353694, + 0.344877, + 0.327692 + ], + "main_score": 0.326103, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 32.04099941253662, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MedrxivClusteringS2S.v2.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MedrxivClusteringS2S.v2.json new file mode 100644 index 0000000000..0dd907bb05 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MedrxivClusteringS2S.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "task_name": "MedrxivClusteringS2S.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.34764, + 0.367447, + 0.364155, + 0.358587, + 0.358843, + 0.357281, + 0.346298, + 0.369759, + 0.360473, + 0.369339 + ] + }, + "v_measure": 0.359982, + "v_measure_std": 0.007789, + "main_score": 0.359982, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 44.00247311592102, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MindSmallReranking.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MindSmallReranking.json new file mode 100644 index 0000000000..b57e1a998a --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MindSmallReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "59042f120c80e8afa9cdbb224f67076cec0fc9a7", + "task_name": "MindSmallReranking", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "map": 0.304159, + "mrr": 0.313432, + "nAUC_map_max": -0.2254, + "nAUC_map_std": -0.066124, + "nAUC_map_diff1": 0.113923, + "nAUC_mrr_max": -0.168304, + "nAUC_mrr_std": -0.042285, + "nAUC_mrr_diff1": 0.110738, + "main_score": 0.304159, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 653.072781085968, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MultiLongDocRetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MultiLongDocRetrieval.json new file mode 100644 index 0000000000..17ac1547b2 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/MultiLongDocRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", + "task_name": "MultiLongDocRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.30625, + "ndcg_at_3": 0.36966, + "ndcg_at_5": 0.38617, + "ndcg_at_10": 0.40068, + "ndcg_at_20": 0.41597, + "ndcg_at_100": 0.43267, + "ndcg_at_1000": 0.45005, + "map_at_1": 0.30625, + "map_at_3": 0.35438, + "map_at_5": 0.36356, + "map_at_10": 0.36953, + "map_at_20": 0.37362, + "map_at_100": 0.37577, + "map_at_1000": 0.37635, + "recall_at_1": 0.30625, + "recall_at_3": 0.41375, + "recall_at_5": 0.45375, + "recall_at_10": 0.49875, + "recall_at_20": 0.56, + "recall_at_100": 0.6525, + "recall_at_1000": 0.79375, + "precision_at_1": 0.30625, + "precision_at_3": 0.13792, + "precision_at_5": 0.09075, + "precision_at_10": 0.04988, + "precision_at_20": 0.028, + "precision_at_100": 0.00652, + "precision_at_1000": 0.00079, + "mrr_at_1": 0.30625, + "mrr_at_3": 0.354375, + "mrr_at_5": 0.363562, + "mrr_at_10": 0.369531, + "mrr_at_20": 0.373622, + "mrr_at_100": 0.375771, + "mrr_at_1000": 0.376355, + "nauc_ndcg_at_1_max": 0.451155, + "nauc_ndcg_at_1_std": 0.015329, + "nauc_ndcg_at_1_diff1": 0.55712, + "nauc_ndcg_at_3_max": 0.455453, + "nauc_ndcg_at_3_std": 0.032054, + "nauc_ndcg_at_3_diff1": 0.509316, + "nauc_ndcg_at_5_max": 0.456824, + "nauc_ndcg_at_5_std": 0.041532, + "nauc_ndcg_at_5_diff1": 0.503774, + "nauc_ndcg_at_10_max": 0.464918, + "nauc_ndcg_at_10_std": 0.068796, + "nauc_ndcg_at_10_diff1": 0.499872, + "nauc_ndcg_at_20_max": 0.465204, + "nauc_ndcg_at_20_std": 0.077813, + "nauc_ndcg_at_20_diff1": 0.496788, + "nauc_ndcg_at_100_max": 0.464901, + "nauc_ndcg_at_100_std": 0.085679, + "nauc_ndcg_at_100_diff1": 0.493751, + "nauc_ndcg_at_1000_max": 0.463991, + "nauc_ndcg_at_1000_std": 0.086082, + "nauc_ndcg_at_1000_diff1": 0.493072, + "nauc_map_at_1_max": 0.451155, + "nauc_map_at_1_std": 0.015329, + "nauc_map_at_1_diff1": 0.55712, + "nauc_map_at_3_max": 0.45549, + "nauc_map_at_3_std": 0.026595, + "nauc_map_at_3_diff1": 0.521645, + "nauc_map_at_5_max": 0.456498, + "nauc_map_at_5_std": 0.032247, + "nauc_map_at_5_diff1": 0.518697, + "nauc_map_at_10_max": 0.459508, + "nauc_map_at_10_std": 0.04332, + "nauc_map_at_10_diff1": 0.516675, + "nauc_map_at_20_max": 0.45943, + "nauc_map_at_20_std": 0.045285, + "nauc_map_at_20_diff1": 0.516074, + "nauc_map_at_100_max": 0.458965, + "nauc_map_at_100_std": 0.046038, + "nauc_map_at_100_diff1": 0.515409, + "nauc_map_at_1000_max": 0.458917, + "nauc_map_at_1000_std": 0.046049, + "nauc_map_at_1000_diff1": 0.515417, + "nauc_recall_at_1_max": 0.451155, + "nauc_recall_at_1_std": 0.015329, + "nauc_recall_at_1_diff1": 0.55712, + "nauc_recall_at_3_max": 0.454924, + "nauc_recall_at_3_std": 0.048087, + "nauc_recall_at_3_diff1": 0.474059, + "nauc_recall_at_5_max": 0.457213, + "nauc_recall_at_5_std": 0.069533, + "nauc_recall_at_5_diff1": 0.459821, + "nauc_recall_at_10_max": 0.48416, + "nauc_recall_at_10_std": 0.155477, + "nauc_recall_at_10_diff1": 0.449415, + "nauc_recall_at_20_max": 0.487617, + "nauc_recall_at_20_std": 0.201729, + "nauc_recall_at_20_diff1": 0.43187, + "nauc_recall_at_100_max": 0.496049, + "nauc_recall_at_100_std": 0.275, + "nauc_recall_at_100_diff1": 0.40726, + "nauc_recall_at_1000_max": 0.501855, + "nauc_recall_at_1000_std": 0.371682, + "nauc_recall_at_1000_diff1": 0.355114, + "nauc_precision_at_1_max": 0.451155, + "nauc_precision_at_1_std": 0.015329, + "nauc_precision_at_1_diff1": 0.55712, + "nauc_precision_at_3_max": 0.454924, + "nauc_precision_at_3_std": 0.048087, + "nauc_precision_at_3_diff1": 0.474059, + "nauc_precision_at_5_max": 0.457213, + "nauc_precision_at_5_std": 0.069533, + "nauc_precision_at_5_diff1": 0.459821, + "nauc_precision_at_10_max": 0.48416, + "nauc_precision_at_10_std": 0.155477, + "nauc_precision_at_10_diff1": 0.449415, + "nauc_precision_at_20_max": 0.487617, + "nauc_precision_at_20_std": 0.201729, + "nauc_precision_at_20_diff1": 0.43187, + "nauc_precision_at_100_max": 0.496049, + "nauc_precision_at_100_std": 0.275, + "nauc_precision_at_100_diff1": 0.40726, + "nauc_precision_at_1000_max": 0.501855, + "nauc_precision_at_1000_std": 0.371682, + "nauc_precision_at_1000_diff1": 0.355114, + "nauc_mrr_at_1_max": 0.451155, + "nauc_mrr_at_1_std": 0.015329, + "nauc_mrr_at_1_diff1": 0.55712, + "nauc_mrr_at_3_max": 0.45549, + "nauc_mrr_at_3_std": 0.026595, + "nauc_mrr_at_3_diff1": 0.521645, + "nauc_mrr_at_5_max": 0.456498, + "nauc_mrr_at_5_std": 0.032247, + "nauc_mrr_at_5_diff1": 0.518697, + "nauc_mrr_at_10_max": 0.459508, + "nauc_mrr_at_10_std": 0.04332, + "nauc_mrr_at_10_diff1": 0.516675, + "nauc_mrr_at_20_max": 0.45943, + "nauc_mrr_at_20_std": 0.045285, + "nauc_mrr_at_20_diff1": 0.516074, + "nauc_mrr_at_100_max": 0.458965, + "nauc_mrr_at_100_std": 0.046038, + "nauc_mrr_at_100_diff1": 0.515409, + "nauc_mrr_at_1000_max": 0.458917, + "nauc_mrr_at_1000_std": 0.046049, + "nauc_mrr_at_1000_diff1": 0.515417, + "main_score": 0.40068, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1626.0403215885162, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/NFCorpus.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/NFCorpus.json new file mode 100644 index 0000000000..2e08b6722d --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/NFCorpus.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814", + "task_name": "NFCorpus", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.43498, + "ndcg_at_3": 0.42022, + "ndcg_at_5": 0.39829, + "ndcg_at_10": 0.37144, + "ndcg_at_20": 0.34511, + "ndcg_at_100": 0.34287, + "ndcg_at_1000": 0.43022, + "map_at_1": 0.0616, + "map_at_3": 0.10384, + "map_at_5": 0.12142, + "map_at_10": 0.14318, + "map_at_20": 0.15929, + "map_at_100": 0.18417, + "map_at_1000": 0.1997, + "recall_at_1": 0.0616, + "recall_at_3": 0.1149, + "recall_at_5": 0.14264, + "recall_at_10": 0.18261, + "recall_at_20": 0.22139, + "recall_at_100": 0.34696, + "recall_at_1000": 0.65764, + "precision_at_1": 0.44892, + "precision_at_3": 0.40248, + "precision_at_5": 0.34923, + "precision_at_10": 0.28173, + "precision_at_20": 0.20526, + "precision_at_100": 0.08879, + "precision_at_1000": 0.02185, + "mrr_at_1": 0.44582, + "mrr_at_3": 0.527864, + "mrr_at_5": 0.538854, + "mrr_at_10": 0.545857, + "mrr_at_20": 0.548054, + "mrr_at_100": 0.550273, + "mrr_at_1000": 0.550635, + "nauc_ndcg_at_1_max": 0.382859, + "nauc_ndcg_at_1_std": 0.273927, + "nauc_ndcg_at_1_diff1": 0.370387, + "nauc_ndcg_at_3_max": 0.452432, + "nauc_ndcg_at_3_std": 0.336465, + "nauc_ndcg_at_3_diff1": 0.272404, + "nauc_ndcg_at_5_max": 0.449715, + "nauc_ndcg_at_5_std": 0.355513, + "nauc_ndcg_at_5_diff1": 0.243128, + "nauc_ndcg_at_10_max": 0.438667, + "nauc_ndcg_at_10_std": 0.360792, + "nauc_ndcg_at_10_diff1": 0.22433, + "nauc_ndcg_at_20_max": 0.4237, + "nauc_ndcg_at_20_std": 0.353787, + "nauc_ndcg_at_20_diff1": 0.223147, + "nauc_ndcg_at_100_max": 0.450609, + "nauc_ndcg_at_100_std": 0.366569, + "nauc_ndcg_at_100_diff1": 0.245027, + "nauc_ndcg_at_1000_max": 0.49536, + "nauc_ndcg_at_1000_std": 0.4134, + "nauc_ndcg_at_1000_diff1": 0.260126, + "nauc_map_at_1_max": 0.117957, + "nauc_map_at_1_std": -0.088796, + "nauc_map_at_1_diff1": 0.412522, + "nauc_map_at_3_max": 0.189957, + "nauc_map_at_3_std": -0.017529, + "nauc_map_at_3_diff1": 0.324108, + "nauc_map_at_5_max": 0.222213, + "nauc_map_at_5_std": 0.018758, + "nauc_map_at_5_diff1": 0.297637, + "nauc_map_at_10_max": 0.270464, + "nauc_map_at_10_std": 0.07229, + "nauc_map_at_10_diff1": 0.272553, + "nauc_map_at_20_max": 0.296645, + "nauc_map_at_20_std": 0.116879, + "nauc_map_at_20_diff1": 0.260061, + "nauc_map_at_100_max": 0.335141, + "nauc_map_at_100_std": 0.191462, + "nauc_map_at_100_diff1": 0.243276, + "nauc_map_at_1000_max": 0.343864, + "nauc_map_at_1000_std": 0.223521, + "nauc_map_at_1000_diff1": 0.238326, + "nauc_recall_at_1_max": 0.117957, + "nauc_recall_at_1_std": -0.088796, + "nauc_recall_at_1_diff1": 0.412522, + "nauc_recall_at_3_max": 0.200562, + "nauc_recall_at_3_std": -0.001257, + "nauc_recall_at_3_diff1": 0.295069, + "nauc_recall_at_5_max": 0.219303, + "nauc_recall_at_5_std": 0.035282, + "nauc_recall_at_5_diff1": 0.249688, + "nauc_recall_at_10_max": 0.260906, + "nauc_recall_at_10_std": 0.082724, + "nauc_recall_at_10_diff1": 0.217859, + "nauc_recall_at_20_max": 0.287804, + "nauc_recall_at_20_std": 0.144226, + "nauc_recall_at_20_diff1": 0.213548, + "nauc_recall_at_100_max": 0.347643, + "nauc_recall_at_100_std": 0.264689, + "nauc_recall_at_100_diff1": 0.214243, + "nauc_recall_at_1000_max": 0.321988, + "nauc_recall_at_1000_std": 0.312295, + "nauc_recall_at_1000_diff1": 0.121254, + "nauc_precision_at_1_max": 0.391275, + "nauc_precision_at_1_std": 0.284887, + "nauc_precision_at_1_diff1": 0.368498, + "nauc_precision_at_3_max": 0.47322, + "nauc_precision_at_3_std": 0.399756, + "nauc_precision_at_3_diff1": 0.162279, + "nauc_precision_at_5_max": 0.447441, + "nauc_precision_at_5_std": 0.429854, + "nauc_precision_at_5_diff1": 0.092557, + "nauc_precision_at_10_max": 0.414019, + "nauc_precision_at_10_std": 0.458515, + "nauc_precision_at_10_diff1": 0.041599, + "nauc_precision_at_20_max": 0.357026, + "nauc_precision_at_20_std": 0.466783, + "nauc_precision_at_20_diff1": 0.014195, + "nauc_precision_at_100_max": 0.249512, + "nauc_precision_at_100_std": 0.462886, + "nauc_precision_at_100_diff1": -0.046182, + "nauc_precision_at_1000_max": 0.086444, + "nauc_precision_at_1000_std": 0.304731, + "nauc_precision_at_1000_diff1": -0.062589, + "nauc_mrr_at_1_max": 0.385973, + "nauc_mrr_at_1_std": 0.269488, + "nauc_mrr_at_1_diff1": 0.376773, + "nauc_mrr_at_3_max": 0.469972, + "nauc_mrr_at_3_std": 0.346883, + "nauc_mrr_at_3_diff1": 0.366006, + "nauc_mrr_at_5_max": 0.47315, + "nauc_mrr_at_5_std": 0.358286, + "nauc_mrr_at_5_diff1": 0.363282, + "nauc_mrr_at_10_max": 0.475392, + "nauc_mrr_at_10_std": 0.361043, + "nauc_mrr_at_10_diff1": 0.35706, + "nauc_mrr_at_20_max": 0.475476, + "nauc_mrr_at_20_std": 0.362862, + "nauc_mrr_at_20_diff1": 0.360532, + "nauc_mrr_at_100_max": 0.475012, + "nauc_mrr_at_100_std": 0.362285, + "nauc_mrr_at_100_diff1": 0.359783, + "nauc_mrr_at_1000_max": 0.4747, + "nauc_mrr_at_1000_std": 0.362006, + "nauc_mrr_at_1000_diff1": 0.359846, + "main_score": 0.37144, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.217962265014648, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/NQ.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/NQ.json new file mode 100644 index 0000000000..46c05079af --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/NQ.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31", + "task_name": "NQ", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.35255, + "ndcg_at_3": 0.46354, + "ndcg_at_5": 0.51281, + "ndcg_at_10": 0.5537, + "ndcg_at_20": 0.57365, + "ndcg_at_100": 0.59306, + "ndcg_at_1000": 0.59854, + "map_at_1": 0.31069, + "map_at_3": 0.42297, + "map_at_5": 0.45291, + "map_at_10": 0.47216, + "map_at_20": 0.47846, + "map_at_100": 0.48163, + "map_at_1000": 0.48188, + "recall_at_1": 0.31069, + "recall_at_3": 0.54601, + "recall_at_5": 0.65957, + "recall_at_10": 0.77692, + "recall_at_20": 0.85074, + "recall_at_100": 0.94752, + "recall_at_1000": 0.98793, + "precision_at_1": 0.35255, + "precision_at_3": 0.21292, + "precision_at_5": 0.15608, + "precision_at_10": 0.09334, + "precision_at_20": 0.05145, + "precision_at_100": 0.01152, + "precision_at_1000": 0.0012, + "mrr_at_1": 0.352839, + "mrr_at_3": 0.457947, + "mrr_at_5": 0.48231, + "mrr_at_10": 0.49688, + "mrr_at_20": 0.50135, + "mrr_at_100": 0.503727, + "mrr_at_1000": 0.503906, + "nauc_ndcg_at_1_max": 0.223749, + "nauc_ndcg_at_1_std": 0.020285, + "nauc_ndcg_at_1_diff1": 0.297809, + "nauc_ndcg_at_3_max": 0.274427, + "nauc_ndcg_at_3_std": 0.020708, + "nauc_ndcg_at_3_diff1": 0.267501, + "nauc_ndcg_at_5_max": 0.295777, + "nauc_ndcg_at_5_std": 0.02801, + "nauc_ndcg_at_5_diff1": 0.268275, + "nauc_ndcg_at_10_max": 0.316843, + "nauc_ndcg_at_10_std": 0.053395, + "nauc_ndcg_at_10_diff1": 0.266206, + "nauc_ndcg_at_20_max": 0.315083, + "nauc_ndcg_at_20_std": 0.066367, + "nauc_ndcg_at_20_diff1": 0.265303, + "nauc_ndcg_at_100_max": 0.304041, + "nauc_ndcg_at_100_std": 0.063879, + "nauc_ndcg_at_100_diff1": 0.268631, + "nauc_ndcg_at_1000_max": 0.296795, + "nauc_ndcg_at_1000_std": 0.0563, + "nauc_ndcg_at_1000_diff1": 0.270178, + "nauc_map_at_1_max": 0.21075, + "nauc_map_at_1_std": -0.001451, + "nauc_map_at_1_diff1": 0.306372, + "nauc_map_at_3_max": 0.259868, + "nauc_map_at_3_std": 0.014907, + "nauc_map_at_3_diff1": 0.275291, + "nauc_map_at_5_max": 0.272937, + "nauc_map_at_5_std": 0.020212, + "nauc_map_at_5_diff1": 0.275912, + "nauc_map_at_10_max": 0.281664, + "nauc_map_at_10_std": 0.031258, + "nauc_map_at_10_diff1": 0.275441, + "nauc_map_at_20_max": 0.281005, + "nauc_map_at_20_std": 0.035306, + "nauc_map_at_20_diff1": 0.27545, + "nauc_map_at_100_max": 0.279284, + "nauc_map_at_100_std": 0.035095, + "nauc_map_at_100_diff1": 0.275687, + "nauc_map_at_1000_max": 0.279036, + "nauc_map_at_1000_std": 0.034848, + "nauc_map_at_1000_diff1": 0.275737, + "nauc_recall_at_1_max": 0.21075, + "nauc_recall_at_1_std": -0.001451, + "nauc_recall_at_1_diff1": 0.306372, + "nauc_recall_at_3_max": 0.304566, + "nauc_recall_at_3_std": 0.019385, + "nauc_recall_at_3_diff1": 0.238896, + "nauc_recall_at_5_max": 0.362886, + "nauc_recall_at_5_std": 0.034556, + "nauc_recall_at_5_diff1": 0.234704, + "nauc_recall_at_10_max": 0.478422, + "nauc_recall_at_10_std": 0.136168, + "nauc_recall_at_10_diff1": 0.219344, + "nauc_recall_at_20_max": 0.538696, + "nauc_recall_at_20_std": 0.257889, + "nauc_recall_at_20_diff1": 0.199972, + "nauc_recall_at_100_max": 0.667167, + "nauc_recall_at_100_std": 0.50111, + "nauc_recall_at_100_diff1": 0.199422, + "nauc_recall_at_1000_max": 0.755415, + "nauc_recall_at_1000_std": 0.747457, + "nauc_recall_at_1000_diff1": 0.227066, + "nauc_precision_at_1_max": 0.223749, + "nauc_precision_at_1_std": 0.020285, + "nauc_precision_at_1_diff1": 0.297809, + "nauc_precision_at_3_max": 0.293226, + "nauc_precision_at_3_std": 0.061994, + "nauc_precision_at_3_diff1": 0.188042, + "nauc_precision_at_5_max": 0.298898, + "nauc_precision_at_5_std": 0.083449, + "nauc_precision_at_5_diff1": 0.15004, + "nauc_precision_at_10_max": 0.291691, + "nauc_precision_at_10_std": 0.154882, + "nauc_precision_at_10_diff1": 0.087128, + "nauc_precision_at_20_max": 0.239689, + "nauc_precision_at_20_std": 0.197311, + "nauc_precision_at_20_diff1": 0.038646, + "nauc_precision_at_100_max": 0.113179, + "nauc_precision_at_100_std": 0.191007, + "nauc_precision_at_100_diff1": -0.028752, + "nauc_precision_at_1000_max": 0.024361, + "nauc_precision_at_1000_std": 0.138194, + "nauc_precision_at_1000_diff1": -0.056081, + "nauc_mrr_at_1_max": 0.224178, + "nauc_mrr_at_1_std": 0.019572, + "nauc_mrr_at_1_diff1": 0.29695, + "nauc_mrr_at_3_max": 0.262214, + "nauc_mrr_at_3_std": 0.031489, + "nauc_mrr_at_3_diff1": 0.272521, + "nauc_mrr_at_5_max": 0.270507, + "nauc_mrr_at_5_std": 0.033505, + "nauc_mrr_at_5_diff1": 0.273267, + "nauc_mrr_at_10_max": 0.276657, + "nauc_mrr_at_10_std": 0.041924, + "nauc_mrr_at_10_diff1": 0.272286, + "nauc_mrr_at_20_max": 0.275691, + "nauc_mrr_at_20_std": 0.043796, + "nauc_mrr_at_20_diff1": 0.271995, + "nauc_mrr_at_100_max": 0.274526, + "nauc_mrr_at_100_std": 0.043277, + "nauc_mrr_at_100_diff1": 0.272679, + "nauc_mrr_at_1000_max": 0.274316, + "nauc_mrr_at_1000_std": 0.043079, + "nauc_mrr_at_1000_diff1": 0.272725, + "main_score": 0.5537, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1696.8312747478485, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/QuoraRetrieval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/QuoraRetrieval.json new file mode 100644 index 0000000000..f6e615877e --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/QuoraRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e4e08e0b7dbe3c8700f0daef558ff32256715259", + "task_name": "QuoraRetrieval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.804, + "ndcg_at_3": 0.84446, + "ndcg_at_5": 0.85988, + "ndcg_at_10": 0.87362, + "ndcg_at_20": 0.88085, + "ndcg_at_100": 0.88708, + "ndcg_at_1000": 0.88848, + "map_at_1": 0.69873, + "map_at_3": 0.80548, + "map_at_5": 0.8234, + "map_at_10": 0.83469, + "map_at_20": 0.839, + "map_at_100": 0.8413, + "map_at_1000": 0.84149, + "recall_at_1": 0.69873, + "recall_at_3": 0.86252, + "recall_at_5": 0.90688, + "recall_at_10": 0.9466, + "recall_at_20": 0.96991, + "recall_at_100": 0.99279, + "recall_at_1000": 0.9993, + "precision_at_1": 0.804, + "precision_at_3": 0.36793, + "precision_at_5": 0.24144, + "precision_at_10": 0.13206, + "precision_at_20": 0.07018, + "precision_at_100": 0.01515, + "precision_at_1000": 0.00156, + "mrr_at_1": 0.8036, + "mrr_at_3": 0.856983, + "mrr_at_5": 0.863903, + "mrr_at_10": 0.867264, + "mrr_at_20": 0.868175, + "mrr_at_100": 0.868464, + "mrr_at_1000": 0.868479, + "nauc_ndcg_at_1_max": 0.390225, + "nauc_ndcg_at_1_std": -0.300088, + "nauc_ndcg_at_1_diff1": 0.761111, + "nauc_ndcg_at_3_max": 0.378953, + "nauc_ndcg_at_3_std": -0.34937, + "nauc_ndcg_at_3_diff1": 0.743518, + "nauc_ndcg_at_5_max": 0.377013, + "nauc_ndcg_at_5_std": -0.362558, + "nauc_ndcg_at_5_diff1": 0.747791, + "nauc_ndcg_at_10_max": 0.385919, + "nauc_ndcg_at_10_std": -0.346077, + "nauc_ndcg_at_10_diff1": 0.749659, + "nauc_ndcg_at_20_max": 0.387767, + "nauc_ndcg_at_20_std": -0.341171, + "nauc_ndcg_at_20_diff1": 0.749728, + "nauc_ndcg_at_100_max": 0.391858, + "nauc_ndcg_at_100_std": -0.3218, + "nauc_ndcg_at_100_diff1": 0.749017, + "nauc_ndcg_at_1000_max": 0.391835, + "nauc_ndcg_at_1000_std": -0.320719, + "nauc_ndcg_at_1000_diff1": 0.748809, + "nauc_map_at_1_max": 0.293514, + "nauc_map_at_1_std": -0.328817, + "nauc_map_at_1_diff1": 0.783187, + "nauc_map_at_3_max": 0.354874, + "nauc_map_at_3_std": -0.374032, + "nauc_map_at_3_diff1": 0.75708, + "nauc_map_at_5_max": 0.364446, + "nauc_map_at_5_std": -0.374343, + "nauc_map_at_5_diff1": 0.755042, + "nauc_map_at_10_max": 0.373925, + "nauc_map_at_10_std": -0.359355, + "nauc_map_at_10_diff1": 0.753443, + "nauc_map_at_20_max": 0.376489, + "nauc_map_at_20_std": -0.352068, + "nauc_map_at_20_diff1": 0.752059, + "nauc_map_at_100_max": 0.377679, + "nauc_map_at_100_std": -0.344935, + "nauc_map_at_100_diff1": 0.75116, + "nauc_map_at_1000_max": 0.377751, + "nauc_map_at_1000_std": -0.344491, + "nauc_map_at_1000_diff1": 0.751117, + "nauc_recall_at_1_max": 0.293514, + "nauc_recall_at_1_std": -0.328817, + "nauc_recall_at_1_diff1": 0.783187, + "nauc_recall_at_3_max": 0.334874, + "nauc_recall_at_3_std": -0.414839, + "nauc_recall_at_3_diff1": 0.717966, + "nauc_recall_at_5_max": 0.328683, + "nauc_recall_at_5_std": -0.464735, + "nauc_recall_at_5_diff1": 0.706934, + "nauc_recall_at_10_max": 0.353121, + "nauc_recall_at_10_std": -0.427204, + "nauc_recall_at_10_diff1": 0.701432, + "nauc_recall_at_20_max": 0.350195, + "nauc_recall_at_20_std": -0.46354, + "nauc_recall_at_20_diff1": 0.703883, + "nauc_recall_at_100_max": 0.432742, + "nauc_recall_at_100_std": -0.101072, + "nauc_recall_at_100_diff1": 0.707138, + "nauc_recall_at_1000_max": 0.602277, + "nauc_recall_at_1000_std": 0.568063, + "nauc_recall_at_1000_diff1": 0.722231, + "nauc_precision_at_1_max": 0.390225, + "nauc_precision_at_1_std": -0.300088, + "nauc_precision_at_1_diff1": 0.761111, + "nauc_precision_at_3_max": 0.100371, + "nauc_precision_at_3_std": 0.047789, + "nauc_precision_at_3_diff1": -0.135148, + "nauc_precision_at_5_max": 0.032834, + "nauc_precision_at_5_std": 0.13219, + "nauc_precision_at_5_diff1": -0.268133, + "nauc_precision_at_10_max": -0.010163, + "nauc_precision_at_10_std": 0.227438, + "nauc_precision_at_10_diff1": -0.354064, + "nauc_precision_at_20_max": -0.041509, + "nauc_precision_at_20_std": 0.274193, + "nauc_precision_at_20_diff1": -0.39462, + "nauc_precision_at_100_max": -0.068951, + "nauc_precision_at_100_std": 0.330266, + "nauc_precision_at_100_diff1": -0.422539, + "nauc_precision_at_1000_max": -0.078863, + "nauc_precision_at_1000_std": 0.341681, + "nauc_precision_at_1000_diff1": -0.427565, + "nauc_mrr_at_1_max": 0.388557, + "nauc_mrr_at_1_std": -0.300554, + "nauc_mrr_at_1_diff1": 0.761847, + "nauc_mrr_at_3_max": 0.400854, + "nauc_mrr_at_3_std": -0.310362, + "nauc_mrr_at_3_diff1": 0.749949, + "nauc_mrr_at_5_max": 0.399615, + "nauc_mrr_at_5_std": -0.311687, + "nauc_mrr_at_5_diff1": 0.752656, + "nauc_mrr_at_10_max": 0.39972, + "nauc_mrr_at_10_std": -0.307074, + "nauc_mrr_at_10_diff1": 0.753105, + "nauc_mrr_at_20_max": 0.399635, + "nauc_mrr_at_20_std": -0.307534, + "nauc_mrr_at_20_diff1": 0.753208, + "nauc_mrr_at_100_max": 0.399625, + "nauc_mrr_at_100_std": -0.306996, + "nauc_mrr_at_100_diff1": 0.753207, + "nauc_mrr_at_1000_max": 0.399585, + "nauc_mrr_at_1000_std": -0.307067, + "nauc_mrr_at_1000_diff1": 0.753216, + "main_score": 0.87362, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 217.4433081150055, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/RedditClustering.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/RedditClustering.json new file mode 100644 index 0000000000..2f60978add --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/RedditClustering.json @@ -0,0 +1,47 @@ +{ + "dataset_revision": "24640382cdbf8abc73003fb0fa6d111a705499eb", + "task_name": "RedditClustering", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.502394, + "v_measure_std": 0.046239, + "v_measures": [ + 0.505014, + 0.575565, + 0.463331, + 0.468542, + 0.476707, + 0.483059, + 0.525108, + 0.466673, + 0.464279, + 0.480471, + 0.457882, + 0.565264, + 0.512915, + 0.518638, + 0.591314, + 0.47158, + 0.50431, + 0.559056, + 0.458695, + 0.47946, + 0.478197, + 0.484176, + 0.62897, + 0.490923, + 0.449731 + ], + "main_score": 0.502394, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 247.14496231079102, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/RedditClusteringP2P.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/RedditClusteringP2P.json new file mode 100644 index 0000000000..b420508ccd --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/RedditClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "385e3cb46b4cfa89021f56c4380204149d0efe33", + "task_name": "RedditClusteringP2P", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.549229, + "v_measure_std": 0.12686, + "v_measures": [ + 0.601133, + 0.590766, + 0.616661, + 0.335681, + 0.589794, + 0.524433, + 0.282014, + 0.653469, + 0.618676, + 0.67966 + ], + "main_score": 0.549229, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 488.66356468200684, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SCIDOCS.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SCIDOCS.json new file mode 100644 index 0000000000..912f1b64eb --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SCIDOCS.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88", + "task_name": "SCIDOCS", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.277, + "ndcg_at_3": 0.2277, + "ndcg_at_5": 0.20271, + "ndcg_at_10": 0.24059, + "ndcg_at_20": 0.27495, + "ndcg_at_100": 0.33716, + "ndcg_at_1000": 0.39541, + "map_at_1": 0.05628, + "map_at_3": 0.10432, + "map_at_5": 0.12781, + "map_at_10": 0.14876, + "map_at_20": 0.16216, + "map_at_100": 0.17574, + "map_at_1000": 0.17955, + "recall_at_1": 0.05628, + "recall_at_3": 0.12967, + "recall_at_5": 0.18282, + "recall_at_10": 0.25225, + "recall_at_20": 0.33273, + "recall_at_100": 0.53395, + "recall_at_1000": 0.81503, + "precision_at_1": 0.277, + "precision_at_3": 0.21333, + "precision_at_5": 0.1802, + "precision_at_10": 0.1245, + "precision_at_20": 0.0821, + "precision_at_100": 0.02632, + "precision_at_1000": 0.00402, + "mrr_at_1": 0.277, + "mrr_at_3": 0.358333, + "mrr_at_5": 0.381933, + "mrr_at_10": 0.39534, + "mrr_at_20": 0.401922, + "mrr_at_100": 0.405945, + "mrr_at_1000": 0.406365, + "nauc_ndcg_at_1_max": 0.21981, + "nauc_ndcg_at_1_std": 0.125088, + "nauc_ndcg_at_1_diff1": 0.186307, + "nauc_ndcg_at_3_max": 0.267503, + "nauc_ndcg_at_3_std": 0.148274, + "nauc_ndcg_at_3_diff1": 0.16122, + "nauc_ndcg_at_5_max": 0.28449, + "nauc_ndcg_at_5_std": 0.169323, + "nauc_ndcg_at_5_diff1": 0.155514, + "nauc_ndcg_at_10_max": 0.307979, + "nauc_ndcg_at_10_std": 0.214414, + "nauc_ndcg_at_10_diff1": 0.144282, + "nauc_ndcg_at_20_max": 0.315324, + "nauc_ndcg_at_20_std": 0.248, + "nauc_ndcg_at_20_diff1": 0.141686, + "nauc_ndcg_at_100_max": 0.316806, + "nauc_ndcg_at_100_std": 0.298303, + "nauc_ndcg_at_100_diff1": 0.134599, + "nauc_ndcg_at_1000_max": 0.316761, + "nauc_ndcg_at_1000_std": 0.295287, + "nauc_ndcg_at_1000_diff1": 0.136419, + "nauc_map_at_1_max": 0.214957, + "nauc_map_at_1_std": 0.121279, + "nauc_map_at_1_diff1": 0.183206, + "nauc_map_at_3_max": 0.258202, + "nauc_map_at_3_std": 0.135759, + "nauc_map_at_3_diff1": 0.16546, + "nauc_map_at_5_max": 0.271434, + "nauc_map_at_5_std": 0.155176, + "nauc_map_at_5_diff1": 0.155973, + "nauc_map_at_10_max": 0.295155, + "nauc_map_at_10_std": 0.194214, + "nauc_map_at_10_diff1": 0.142803, + "nauc_map_at_20_max": 0.300529, + "nauc_map_at_20_std": 0.215665, + "nauc_map_at_20_diff1": 0.137706, + "nauc_map_at_100_max": 0.305721, + "nauc_map_at_100_std": 0.237276, + "nauc_map_at_100_diff1": 0.13631, + "nauc_map_at_1000_max": 0.305561, + "nauc_map_at_1000_std": 0.238398, + "nauc_map_at_1000_diff1": 0.136375, + "nauc_recall_at_1_max": 0.214957, + "nauc_recall_at_1_std": 0.121279, + "nauc_recall_at_1_diff1": 0.183206, + "nauc_recall_at_3_max": 0.278854, + "nauc_recall_at_3_std": 0.156181, + "nauc_recall_at_3_diff1": 0.144963, + "nauc_recall_at_5_max": 0.294754, + "nauc_recall_at_5_std": 0.185605, + "nauc_recall_at_5_diff1": 0.134108, + "nauc_recall_at_10_max": 0.323135, + "nauc_recall_at_10_std": 0.259473, + "nauc_recall_at_10_diff1": 0.10907, + "nauc_recall_at_20_max": 0.316692, + "nauc_recall_at_20_std": 0.313252, + "nauc_recall_at_20_diff1": 0.097837, + "nauc_recall_at_100_max": 0.276311, + "nauc_recall_at_100_std": 0.411162, + "nauc_recall_at_100_diff1": 0.063697, + "nauc_recall_at_1000_max": 0.244821, + "nauc_recall_at_1000_std": 0.45149, + "nauc_recall_at_1000_diff1": 0.027752, + "nauc_precision_at_1_max": 0.21981, + "nauc_precision_at_1_std": 0.125088, + "nauc_precision_at_1_diff1": 0.186307, + "nauc_precision_at_3_max": 0.283504, + "nauc_precision_at_3_std": 0.159033, + "nauc_precision_at_3_diff1": 0.149158, + "nauc_precision_at_5_max": 0.301035, + "nauc_precision_at_5_std": 0.18822, + "nauc_precision_at_5_diff1": 0.140202, + "nauc_precision_at_10_max": 0.327746, + "nauc_precision_at_10_std": 0.260935, + "nauc_precision_at_10_diff1": 0.114938, + "nauc_precision_at_20_max": 0.32226, + "nauc_precision_at_20_std": 0.313962, + "nauc_precision_at_20_diff1": 0.102939, + "nauc_precision_at_100_max": 0.27941, + "nauc_precision_at_100_std": 0.403559, + "nauc_precision_at_100_diff1": 0.071595, + "nauc_precision_at_1000_max": 0.252201, + "nauc_precision_at_1000_std": 0.423311, + "nauc_precision_at_1000_diff1": 0.04365, + "nauc_mrr_at_1_max": 0.21981, + "nauc_mrr_at_1_std": 0.125088, + "nauc_mrr_at_1_diff1": 0.186307, + "nauc_mrr_at_3_max": 0.252575, + "nauc_mrr_at_3_std": 0.147397, + "nauc_mrr_at_3_diff1": 0.16259, + "nauc_mrr_at_5_max": 0.262746, + "nauc_mrr_at_5_std": 0.155352, + "nauc_mrr_at_5_diff1": 0.16804, + "nauc_mrr_at_10_max": 0.260389, + "nauc_mrr_at_10_std": 0.161157, + "nauc_mrr_at_10_diff1": 0.168882, + "nauc_mrr_at_20_max": 0.259443, + "nauc_mrr_at_20_std": 0.161487, + "nauc_mrr_at_20_diff1": 0.171441, + "nauc_mrr_at_100_max": 0.258283, + "nauc_mrr_at_100_std": 0.160651, + "nauc_mrr_at_100_diff1": 0.170454, + "nauc_mrr_at_1000_max": 0.258236, + "nauc_mrr_at_1000_std": 0.160436, + "nauc_mrr_at_1000_diff1": 0.170417, + "main_score": 0.24059, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 20.15979242324829, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SICK-R.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SICK-R.json new file mode 100644 index 0000000000..86364b48a1 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SICK-R.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "20a6d6f312dd54037fe07a32d58e5e168867909d", + "task_name": "SICK-R", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.773724, + "spearman": 0.688621, + "cosine_pearson": 0.773724, + "cosine_spearman": 0.688621, + "manhattan_pearson": 0.731188, + "manhattan_spearman": 0.688466, + "euclidean_pearson": 0.731642, + "euclidean_spearman": 0.688479, + "main_score": 0.688621, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.4615602493286133, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS12.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS12.json new file mode 100644 index 0000000000..1bbefeb71d --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS12.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384", + "task_name": "STS12", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.727902, + "spearman": 0.674093, + "cosine_pearson": 0.727902, + "cosine_spearman": 0.674098, + "manhattan_pearson": 0.690258, + "manhattan_spearman": 0.674288, + "euclidean_pearson": 0.691227, + "euclidean_spearman": 0.674511, + "main_score": 0.674098, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.8719387054443359, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS13.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS13.json new file mode 100644 index 0000000000..1c4566e510 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS13.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca", + "task_name": "STS13", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.802984, + "spearman": 0.805685, + "cosine_pearson": 0.802984, + "cosine_spearman": 0.805687, + "manhattan_pearson": 0.797037, + "manhattan_spearman": 0.8064, + "euclidean_pearson": 0.798173, + "euclidean_spearman": 0.807777, + "main_score": 0.805687, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.402846097946167, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS14.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS14.json new file mode 100644 index 0000000000..faf95867c0 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS14.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375", + "task_name": "STS14", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.785584, + "spearman": 0.729385, + "cosine_pearson": 0.785584, + "cosine_spearman": 0.729385, + "manhattan_pearson": 0.764729, + "manhattan_spearman": 0.729288, + "euclidean_pearson": 0.765175, + "euclidean_spearman": 0.729534, + "main_score": 0.729385, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.0606813430786133, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS15.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS15.json new file mode 100644 index 0000000000..0164527e12 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS15.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3", + "task_name": "STS15", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.828357, + "spearman": 0.838624, + "cosine_pearson": 0.828357, + "cosine_spearman": 0.838624, + "manhattan_pearson": 0.830701, + "manhattan_spearman": 0.837865, + "euclidean_pearson": 0.830894, + "euclidean_spearman": 0.838192, + "main_score": 0.838624, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.8056259155273438, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS16.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS16.json new file mode 100644 index 0000000000..309230fa5c --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS16.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "4d8694f8f0e0100860b497b999b3dbed754a0513", + "task_name": "STS16", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.772578, + "spearman": 0.779888, + "cosine_pearson": 0.772578, + "cosine_spearman": 0.779889, + "manhattan_pearson": 0.775839, + "manhattan_spearman": 0.780423, + "euclidean_pearson": 0.775886, + "euclidean_spearman": 0.780414, + "main_score": 0.779889, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.9737656116485596, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS17.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS17.json new file mode 100644 index 0000000000..b0e0d2d274 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS17.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "faeb762787bd10488a50c8b5be4a3b82e411949c", + "task_name": "STS17", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.847098, + "spearman": 0.851797, + "cosine_pearson": 0.847098, + "cosine_spearman": 0.851797, + "manhattan_pearson": 0.847506, + "manhattan_spearman": 0.85216, + "euclidean_pearson": 0.847236, + "euclidean_spearman": 0.852259, + "main_score": 0.851797, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.08304786682128906, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS22.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS22.json new file mode 100644 index 0000000000..cb816ef010 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS22.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3", + "task_name": "STS22", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.668075, + "spearman": 0.668389, + "cosine_pearson": 0.668075, + "cosine_spearman": 0.668389, + "manhattan_pearson": 0.681247, + "manhattan_spearman": 0.667374, + "euclidean_pearson": 0.682078, + "euclidean_spearman": 0.667897, + "main_score": 0.668389, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.4133682250976562, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS22.v2.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS22.v2.json new file mode 100644 index 0000000000..53c8aa898d --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STS22.v2.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d31f33a128469b20e357535c39b82fb3c3f6f2bd", + "task_name": "STS22.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.668062, + "spearman": 0.668478, + "cosine_pearson": 0.668062, + "cosine_spearman": 0.668478, + "manhattan_pearson": 0.681259, + "manhattan_spearman": 0.667374, + "euclidean_pearson": 0.682077, + "euclidean_spearman": 0.667991, + "main_score": 0.668478, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.6459414958953857, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STSBenchmark.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STSBenchmark.json new file mode 100644 index 0000000000..0b399ceb71 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/STSBenchmark.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831", + "task_name": "STSBenchmark", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.781817, + "spearman": 0.771027, + "cosine_pearson": 0.781817, + "cosine_spearman": 0.77103, + "manhattan_pearson": 0.777917, + "manhattan_spearman": 0.772853, + "euclidean_pearson": 0.777412, + "euclidean_spearman": 0.772525, + "main_score": 0.77103, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.36873507499694824, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SciDocsRR.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SciDocsRR.json new file mode 100644 index 0000000000..3cb2d3826e --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SciDocsRR.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d3c5e1fc0b855ab6097bf1cda04dd73947d7caab", + "task_name": "SciDocsRR", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "map": 0.875441, + "mrr": 0.965804, + "nAUC_map_max": 0.538023, + "nAUC_map_std": 0.679005, + "nAUC_map_diff1": 0.010903, + "nAUC_mrr_max": 0.869758, + "nAUC_mrr_std": 0.795696, + "nAUC_mrr_diff1": 0.450022, + "main_score": 0.875441, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 28.715125799179077, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SciFact.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SciFact.json new file mode 100644 index 0000000000..27d0f37fef --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SciFact.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "0228b52cf27578f30900b9e5271d331663a030d7", + "task_name": "SciFact", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.64333, + "ndcg_at_3": 0.70718, + "ndcg_at_5": 0.73592, + "ndcg_at_10": 0.75494, + "ndcg_at_20": 0.76649, + "ndcg_at_100": 0.77725, + "ndcg_at_1000": 0.77963, + "map_at_1": 0.60928, + "map_at_3": 0.68017, + "map_at_5": 0.69964, + "map_at_10": 0.70981, + "map_at_20": 0.71339, + "map_at_100": 0.71505, + "map_at_1000": 0.71512, + "recall_at_1": 0.60928, + "recall_at_3": 0.75672, + "recall_at_5": 0.82778, + "recall_at_10": 0.881, + "recall_at_20": 0.92433, + "recall_at_100": 0.98, + "recall_at_1000": 1.0, + "precision_at_1": 0.64333, + "precision_at_3": 0.27556, + "precision_at_5": 0.18467, + "precision_at_10": 0.10033, + "precision_at_20": 0.0525, + "precision_at_100": 0.0111, + "precision_at_1000": 0.00113, + "mrr_at_1": 0.643333, + "mrr_at_3": 0.701111, + "mrr_at_5": 0.715944, + "mrr_at_10": 0.721327, + "mrr_at_20": 0.724414, + "mrr_at_100": 0.725877, + "mrr_at_1000": 0.725941, + "nauc_ndcg_at_1_max": 0.587883, + "nauc_ndcg_at_1_std": 0.033641, + "nauc_ndcg_at_1_diff1": 0.738358, + "nauc_ndcg_at_3_max": 0.609525, + "nauc_ndcg_at_3_std": -0.00045, + "nauc_ndcg_at_3_diff1": 0.722731, + "nauc_ndcg_at_5_max": 0.623804, + "nauc_ndcg_at_5_std": 0.014322, + "nauc_ndcg_at_5_diff1": 0.716652, + "nauc_ndcg_at_10_max": 0.639094, + "nauc_ndcg_at_10_std": 0.037789, + "nauc_ndcg_at_10_diff1": 0.705555, + "nauc_ndcg_at_20_max": 0.634712, + "nauc_ndcg_at_20_std": 0.033587, + "nauc_ndcg_at_20_diff1": 0.705545, + "nauc_ndcg_at_100_max": 0.629579, + "nauc_ndcg_at_100_std": 0.033205, + "nauc_ndcg_at_100_diff1": 0.715621, + "nauc_ndcg_at_1000_max": 0.626647, + "nauc_ndcg_at_1000_std": 0.032662, + "nauc_ndcg_at_1000_diff1": 0.716175, + "nauc_map_at_1_max": 0.522977, + "nauc_map_at_1_std": -0.067575, + "nauc_map_at_1_diff1": 0.76237, + "nauc_map_at_3_max": 0.579052, + "nauc_map_at_3_std": -0.032632, + "nauc_map_at_3_diff1": 0.735922, + "nauc_map_at_5_max": 0.603133, + "nauc_map_at_5_std": -0.007378, + "nauc_map_at_5_diff1": 0.731028, + "nauc_map_at_10_max": 0.613253, + "nauc_map_at_10_std": 0.010766, + "nauc_map_at_10_diff1": 0.726085, + "nauc_map_at_20_max": 0.612468, + "nauc_map_at_20_std": 0.009667, + "nauc_map_at_20_diff1": 0.726127, + "nauc_map_at_100_max": 0.61151, + "nauc_map_at_100_std": 0.009876, + "nauc_map_at_100_diff1": 0.726933, + "nauc_map_at_1000_max": 0.611442, + "nauc_map_at_1000_std": 0.009907, + "nauc_map_at_1000_diff1": 0.726959, + "nauc_recall_at_1_max": 0.522977, + "nauc_recall_at_1_std": -0.067575, + "nauc_recall_at_1_diff1": 0.76237, + "nauc_recall_at_3_max": 0.603739, + "nauc_recall_at_3_std": -0.037794, + "nauc_recall_at_3_diff1": 0.692171, + "nauc_recall_at_5_max": 0.653842, + "nauc_recall_at_5_std": 0.010882, + "nauc_recall_at_5_diff1": 0.657669, + "nauc_recall_at_10_max": 0.740956, + "nauc_recall_at_10_std": 0.085492, + "nauc_recall_at_10_diff1": 0.596045, + "nauc_recall_at_20_max": 0.747046, + "nauc_recall_at_20_std": 0.073403, + "nauc_recall_at_20_diff1": 0.541455, + "nauc_recall_at_100_max": 0.85761, + "nauc_recall_at_100_std": 0.115079, + "nauc_recall_at_100_diff1": 0.68254, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.587883, + "nauc_precision_at_1_std": 0.033641, + "nauc_precision_at_1_diff1": 0.738358, + "nauc_precision_at_3_max": 0.561841, + "nauc_precision_at_3_std": 0.210568, + "nauc_precision_at_3_diff1": 0.391717, + "nauc_precision_at_5_max": 0.538585, + "nauc_precision_at_5_std": 0.3565, + "nauc_precision_at_5_diff1": 0.180589, + "nauc_precision_at_10_max": 0.410625, + "nauc_precision_at_10_std": 0.449308, + "nauc_precision_at_10_diff1": -0.011719, + "nauc_precision_at_20_max": 0.348578, + "nauc_precision_at_20_std": 0.488934, + "nauc_precision_at_20_diff1": -0.121682, + "nauc_precision_at_100_max": 0.235024, + "nauc_precision_at_100_std": 0.573699, + "nauc_precision_at_100_diff1": -0.265162, + "nauc_precision_at_1000_max": 0.179976, + "nauc_precision_at_1000_std": 0.614249, + "nauc_precision_at_1000_diff1": -0.348957, + "nauc_mrr_at_1_max": 0.587883, + "nauc_mrr_at_1_std": 0.033641, + "nauc_mrr_at_1_diff1": 0.738358, + "nauc_mrr_at_3_max": 0.62171, + "nauc_mrr_at_3_std": 0.04879, + "nauc_mrr_at_3_diff1": 0.720348, + "nauc_mrr_at_5_max": 0.626733, + "nauc_mrr_at_5_std": 0.054388, + "nauc_mrr_at_5_diff1": 0.716576, + "nauc_mrr_at_10_max": 0.62902, + "nauc_mrr_at_10_std": 0.051662, + "nauc_mrr_at_10_diff1": 0.713798, + "nauc_mrr_at_20_max": 0.626601, + "nauc_mrr_at_20_std": 0.048461, + "nauc_mrr_at_20_diff1": 0.714327, + "nauc_mrr_at_100_max": 0.62601, + "nauc_mrr_at_100_std": 0.048652, + "nauc_mrr_at_100_diff1": 0.715347, + "nauc_mrr_at_1000_max": 0.625944, + "nauc_mrr_at_1000_std": 0.048695, + "nauc_mrr_at_1000_diff1": 0.715371, + "main_score": 0.75494, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 8.252904891967773, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SprintDuplicateQuestions.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SprintDuplicateQuestions.json new file mode 100644 index 0000000000..f3648408ea --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SprintDuplicateQuestions.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", + "task_name": "SprintDuplicateQuestions", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "similarity_accuracy": 0.997782, + "similarity_accuracy_threshold": 0.926957, + "similarity_f1": 0.887226, + "similarity_f1_threshold": 0.925032, + "similarity_precision": 0.885458, + "similarity_recall": 0.889, + "similarity_ap": 0.948919, + "cosine_accuracy": 0.997782, + "cosine_accuracy_threshold": 0.926957, + "cosine_f1": 0.887226, + "cosine_f1_threshold": 0.925032, + "cosine_precision": 0.885458, + "cosine_recall": 0.889, + "cosine_ap": 0.948917, + "manhattan_accuracy": 0.997812, + "manhattan_accuracy_threshold": 104.1763, + "manhattan_f1": 0.888327, + "manhattan_f1_threshold": 104.1763, + "manhattan_precision": 0.897855, + "manhattan_recall": 0.879, + "manhattan_ap": 0.948718, + "euclidean_accuracy": 0.997792, + "euclidean_accuracy_threshold": 6.685794, + "euclidean_f1": 0.887544, + "euclidean_f1_threshold": 6.685794, + "euclidean_precision": 0.895219, + "euclidean_recall": 0.88, + "euclidean_ap": 0.949257, + "dot_accuracy": 0.997584, + "dot_accuracy_threshold": 282.234772, + "dot_f1": 0.878578, + "dot_f1_threshold": 281.367523, + "dot_precision": 0.867446, + "dot_recall": 0.89, + "dot_ap": 0.940745, + "max_accuracy": 0.997812, + "max_f1": 0.888327, + "max_precision": 0.897855, + "max_recall": 0.89, + "max_ap": 0.949257, + "main_score": 0.949257, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.3820717334747314, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackExchangeClustering.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackExchangeClustering.json new file mode 100644 index 0000000000..19c3306957 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackExchangeClustering.json @@ -0,0 +1,47 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "task_name": "StackExchangeClustering", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.660337, + "v_measure_std": 0.044478, + "v_measures": [ + 0.725818, + 0.66401, + 0.593318, + 0.674461, + 0.700635, + 0.631147, + 0.640965, + 0.719508, + 0.651621, + 0.658779, + 0.69204, + 0.725003, + 0.769922, + 0.716773, + 0.594657, + 0.665988, + 0.622276, + 0.644908, + 0.618644, + 0.600792, + 0.634937, + 0.64332, + 0.642113, + 0.645701, + 0.631081 + ], + "main_score": 0.660337, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 223.81509518623352, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackExchangeClustering.v2.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackExchangeClustering.v2.json new file mode 100644 index 0000000000..e69098690b --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackExchangeClustering.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "task_name": "StackExchangeClustering.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.584523, + 0.57748, + 0.575937, + 0.577753, + 0.590318, + 0.585003, + 0.580976, + 0.578572, + 0.577301, + 0.600377 + ] + }, + "v_measure": 0.582824, + "v_measure_std": 0.00725, + "main_score": 0.582824, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 47.873615741729736, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackExchangeClusteringP2P.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackExchangeClusteringP2P.json new file mode 100644 index 0000000000..a817a100c6 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackExchangeClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "task_name": "StackExchangeClusteringP2P", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.350937, + "v_measure_std": 0.016275, + "v_measures": [ + 0.331977, + 0.344658, + 0.334956, + 0.332391, + 0.334893, + 0.379993, + 0.363007, + 0.362034, + 0.36178, + 0.363681 + ], + "main_score": 0.350937, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 130.58435726165771, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackExchangeClusteringP2P.v2.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackExchangeClusteringP2P.v2.json new file mode 100644 index 0000000000..3bf2f2075b --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackExchangeClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "task_name": "StackExchangeClusteringP2P.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.396832, + 0.40863, + 0.406614, + 0.402877, + 0.404996, + 0.417857, + 0.409814, + 0.411541, + 0.406247, + 0.402206 + ] + }, + "v_measure": 0.406761, + "v_measure_std": 0.005446, + "main_score": 0.406761, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 49.298858642578125, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackOverflowDupQuestions.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackOverflowDupQuestions.json new file mode 100644 index 0000000000..3e27e11b18 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackOverflowDupQuestions.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "e185fbe320c72810689fc5848eb6114e1ef5ec69", + "task_name": "StackOverflowDupQuestions", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "map": 0.540481, + "mrr": 0.549945, + "nAUC_map_max": 0.110479, + "nAUC_map_std": 0.090058, + "nAUC_map_diff1": 0.40054, + "nAUC_mrr_max": 0.117672, + "nAUC_mrr_std": 0.0962, + "nAUC_mrr_diff1": 0.400062, + "main_score": 0.540481, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 24.778701782226562, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackOverflowQA.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackOverflowQA.json new file mode 100644 index 0000000000..b4a5779798 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/StackOverflowQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "db8f169f3894c14a00251061f957b2063eef2bd5", + "task_name": "StackOverflowQA", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.83751, + "ndcg_at_3": 0.8852, + "ndcg_at_5": 0.89418, + "ndcg_at_10": 0.90043, + "ndcg_at_20": 0.90374, + "ndcg_at_100": 0.90703, + "ndcg_at_1000": 0.90836, + "map_at_1": 0.83751, + "map_at_3": 0.87404, + "map_at_5": 0.87908, + "map_at_10": 0.88172, + "map_at_20": 0.88263, + "map_at_100": 0.88313, + "map_at_1000": 0.88318, + "recall_at_1": 0.83751, + "recall_at_3": 0.91725, + "recall_at_5": 0.93882, + "recall_at_10": 0.95787, + "recall_at_20": 0.97091, + "recall_at_100": 0.98796, + "recall_at_1000": 0.9985, + "precision_at_1": 0.83751, + "precision_at_3": 0.30575, + "precision_at_5": 0.18776, + "precision_at_10": 0.09579, + "precision_at_20": 0.04855, + "precision_at_100": 0.00988, + "precision_at_1000": 0.001, + "mrr_at_1": 0.837513, + "mrr_at_3": 0.874039, + "mrr_at_5": 0.879079, + "mrr_at_10": 0.881717, + "mrr_at_20": 0.882629, + "mrr_at_100": 0.883134, + "mrr_at_1000": 0.88318, + "nauc_ndcg_at_1_max": 0.81318, + "nauc_ndcg_at_1_std": -0.034808, + "nauc_ndcg_at_1_diff1": 0.903444, + "nauc_ndcg_at_3_max": 0.823859, + "nauc_ndcg_at_3_std": 0.009028, + "nauc_ndcg_at_3_diff1": 0.87878, + "nauc_ndcg_at_5_max": 0.826373, + "nauc_ndcg_at_5_std": 0.002398, + "nauc_ndcg_at_5_diff1": 0.883761, + "nauc_ndcg_at_10_max": 0.827247, + "nauc_ndcg_at_10_std": -0.002675, + "nauc_ndcg_at_10_diff1": 0.885765, + "nauc_ndcg_at_20_max": 0.827344, + "nauc_ndcg_at_20_std": 0.001372, + "nauc_ndcg_at_20_diff1": 0.888643, + "nauc_ndcg_at_100_max": 0.826345, + "nauc_ndcg_at_100_std": 0.002521, + "nauc_ndcg_at_100_diff1": 0.888245, + "nauc_ndcg_at_1000_max": 0.824935, + "nauc_ndcg_at_1000_std": -0.001878, + "nauc_ndcg_at_1000_diff1": 0.888525, + "nauc_map_at_1_max": 0.81318, + "nauc_map_at_1_std": -0.034808, + "nauc_map_at_1_diff1": 0.903444, + "nauc_map_at_3_max": 0.820897, + "nauc_map_at_3_std": -0.002848, + "nauc_map_at_3_diff1": 0.8854, + "nauc_map_at_5_max": 0.822125, + "nauc_map_at_5_std": -0.006674, + "nauc_map_at_5_diff1": 0.88822, + "nauc_map_at_10_max": 0.822424, + "nauc_map_at_10_std": -0.008554, + "nauc_map_at_10_diff1": 0.88913, + "nauc_map_at_20_max": 0.822336, + "nauc_map_at_20_std": -0.007961, + "nauc_map_at_20_diff1": 0.889809, + "nauc_map_at_100_max": 0.822239, + "nauc_map_at_100_std": -0.007872, + "nauc_map_at_100_diff1": 0.889806, + "nauc_map_at_1000_max": 0.822203, + "nauc_map_at_1000_std": -0.007992, + "nauc_map_at_1000_diff1": 0.889827, + "nauc_recall_at_1_max": 0.81318, + "nauc_recall_at_1_std": -0.034808, + "nauc_recall_at_1_diff1": 0.903444, + "nauc_recall_at_3_max": 0.836259, + "nauc_recall_at_3_std": 0.058547, + "nauc_recall_at_3_diff1": 0.851215, + "nauc_recall_at_5_max": 0.849864, + "nauc_recall_at_5_std": 0.049576, + "nauc_recall_at_5_diff1": 0.861011, + "nauc_recall_at_10_max": 0.865297, + "nauc_recall_at_10_std": 0.033621, + "nauc_recall_at_10_diff1": 0.863322, + "nauc_recall_at_20_max": 0.885713, + "nauc_recall_at_20_std": 0.109303, + "nauc_recall_at_20_diff1": 0.889735, + "nauc_recall_at_100_max": 0.922399, + "nauc_recall_at_100_std": 0.300151, + "nauc_recall_at_100_diff1": 0.870264, + "nauc_recall_at_1000_max": 0.912873, + "nauc_recall_at_1000_std": 0.279614, + "nauc_recall_at_1000_diff1": 0.820284, + "nauc_precision_at_1_max": 0.81318, + "nauc_precision_at_1_std": -0.034808, + "nauc_precision_at_1_diff1": 0.903444, + "nauc_precision_at_3_max": 0.836259, + "nauc_precision_at_3_std": 0.058547, + "nauc_precision_at_3_diff1": 0.851215, + "nauc_precision_at_5_max": 0.849864, + "nauc_precision_at_5_std": 0.049576, + "nauc_precision_at_5_diff1": 0.861011, + "nauc_precision_at_10_max": 0.865297, + "nauc_precision_at_10_std": 0.033621, + "nauc_precision_at_10_diff1": 0.863322, + "nauc_precision_at_20_max": 0.885713, + "nauc_precision_at_20_std": 0.109303, + "nauc_precision_at_20_diff1": 0.889735, + "nauc_precision_at_100_max": 0.922399, + "nauc_precision_at_100_std": 0.300151, + "nauc_precision_at_100_diff1": 0.870264, + "nauc_precision_at_1000_max": 0.912873, + "nauc_precision_at_1000_std": 0.279614, + "nauc_precision_at_1000_diff1": 0.820284, + "nauc_mrr_at_1_max": 0.81318, + "nauc_mrr_at_1_std": -0.034808, + "nauc_mrr_at_1_diff1": 0.903444, + "nauc_mrr_at_3_max": 0.820897, + "nauc_mrr_at_3_std": -0.002848, + "nauc_mrr_at_3_diff1": 0.8854, + "nauc_mrr_at_5_max": 0.822125, + "nauc_mrr_at_5_std": -0.006674, + "nauc_mrr_at_5_diff1": 0.88822, + "nauc_mrr_at_10_max": 0.822424, + "nauc_mrr_at_10_std": -0.008554, + "nauc_mrr_at_10_diff1": 0.88913, + "nauc_mrr_at_20_max": 0.822336, + "nauc_mrr_at_20_std": -0.007961, + "nauc_mrr_at_20_diff1": 0.889809, + "nauc_mrr_at_100_max": 0.822239, + "nauc_mrr_at_100_std": -0.007872, + "nauc_mrr_at_100_diff1": 0.889806, + "nauc_mrr_at_1000_max": 0.822203, + "nauc_mrr_at_1000_std": -0.007992, + "nauc_mrr_at_1000_diff1": 0.889827, + "main_score": 0.90043, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 22.38176989555359, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SummEval.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SummEval.json new file mode 100644 index 0000000000..fb6caebb43 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SummEval.json @@ -0,0 +1,24 @@ +{ + "dataset_revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c", + "task_name": "SummEval", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.282361, + "spearman": 0.286997, + "cosine_spearman": 0.286997, + "cosine_pearson": 0.282361, + "dot_spearman": 0.286874, + "dot_pearson": 0.285702, + "main_score": 0.286997, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.3015658855438232, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SummEvalSummarization.v2.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SummEvalSummarization.v2.json new file mode 100644 index 0000000000..8e35fb2140 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SummEvalSummarization.v2.json @@ -0,0 +1,24 @@ +{ + "dataset_revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c", + "task_name": "SummEvalSummarization.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "pearson": 0.294013, + "spearman": 0.26738, + "cosine_spearman": 0.26738, + "cosine_pearson": 0.294013, + "dot_spearman": 0.273153, + "dot_pearson": 0.293884, + "main_score": 0.26738, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.614858865737915, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SyntheticText2SQL.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SyntheticText2SQL.json new file mode 100644 index 0000000000..66711e16b8 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/SyntheticText2SQL.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "686b87296c3a0191b5d9415a00526c62db9fce09", + "task_name": "SyntheticText2SQL", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.07554, + "ndcg_at_3": 0.40085, + "ndcg_at_5": 0.43501, + "ndcg_at_10": 0.46329, + "ndcg_at_20": 0.48066, + "ndcg_at_100": 0.49913, + "ndcg_at_1000": 0.50527, + "map_at_1": 0.07554, + "map_at_3": 0.32758, + "map_at_5": 0.34658, + "map_at_10": 0.35838, + "map_at_20": 0.36323, + "map_at_100": 0.36584, + "map_at_1000": 0.36607, + "recall_at_1": 0.07554, + "recall_at_3": 0.61015, + "recall_at_5": 0.69287, + "recall_at_10": 0.7797, + "recall_at_20": 0.84772, + "recall_at_100": 0.94668, + "recall_at_1000": 0.99453, + "precision_at_1": 0.07554, + "precision_at_3": 0.20338, + "precision_at_5": 0.13857, + "precision_at_10": 0.07797, + "precision_at_20": 0.04239, + "precision_at_100": 0.00947, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.276534, + "mrr_at_3": 0.440779, + "mrr_at_5": 0.4577, + "mrr_at_10": 0.468677, + "mrr_at_20": 0.473501, + "mrr_at_100": 0.476082, + "mrr_at_1000": 0.476316, + "nauc_ndcg_at_1_max": 0.076626, + "nauc_ndcg_at_1_std": -0.132716, + "nauc_ndcg_at_1_diff1": 0.202184, + "nauc_ndcg_at_3_max": 0.315275, + "nauc_ndcg_at_3_std": -0.202293, + "nauc_ndcg_at_3_diff1": -0.543524, + "nauc_ndcg_at_5_max": 0.313919, + "nauc_ndcg_at_5_std": -0.209716, + "nauc_ndcg_at_5_diff1": -0.505223, + "nauc_ndcg_at_10_max": 0.310163, + "nauc_ndcg_at_10_std": -0.197989, + "nauc_ndcg_at_10_diff1": -0.475662, + "nauc_ndcg_at_20_max": 0.30651, + "nauc_ndcg_at_20_std": -0.193168, + "nauc_ndcg_at_20_diff1": -0.465263, + "nauc_ndcg_at_100_max": 0.299882, + "nauc_ndcg_at_100_std": -0.189972, + "nauc_ndcg_at_100_diff1": -0.452061, + "nauc_ndcg_at_1000_max": 0.294872, + "nauc_ndcg_at_1000_std": -0.189964, + "nauc_ndcg_at_1000_diff1": -0.447025, + "nauc_map_at_1_max": 0.076626, + "nauc_map_at_1_std": -0.132716, + "nauc_map_at_1_diff1": 0.202184, + "nauc_map_at_3_max": 0.285874, + "nauc_map_at_3_std": -0.193335, + "nauc_map_at_3_diff1": -0.464708, + "nauc_map_at_5_max": 0.283443, + "nauc_map_at_5_std": -0.197695, + "nauc_map_at_5_diff1": -0.438496, + "nauc_map_at_10_max": 0.280797, + "nauc_map_at_10_std": -0.19212, + "nauc_map_at_10_diff1": -0.423436, + "nauc_map_at_20_max": 0.279234, + "nauc_map_at_20_std": -0.190842, + "nauc_map_at_20_diff1": -0.419614, + "nauc_map_at_100_max": 0.278136, + "nauc_map_at_100_std": -0.190356, + "nauc_map_at_100_diff1": -0.417522, + "nauc_map_at_1000_max": 0.277982, + "nauc_map_at_1000_std": -0.190307, + "nauc_map_at_1000_diff1": -0.41733, + "nauc_recall_at_1_max": 0.076626, + "nauc_recall_at_1_std": -0.132716, + "nauc_recall_at_1_diff1": 0.202184, + "nauc_recall_at_3_max": 0.376111, + "nauc_recall_at_3_std": -0.220962, + "nauc_recall_at_3_diff1": -0.702118, + "nauc_recall_at_5_max": 0.384502, + "nauc_recall_at_5_std": -0.238484, + "nauc_recall_at_5_diff1": -0.64799, + "nauc_recall_at_10_max": 0.393245, + "nauc_recall_at_10_std": -0.210757, + "nauc_recall_at_10_diff1": -0.597098, + "nauc_recall_at_20_max": 0.406283, + "nauc_recall_at_20_std": -0.188964, + "nauc_recall_at_20_diff1": -0.589498, + "nauc_recall_at_100_max": 0.457246, + "nauc_recall_at_100_std": -0.128916, + "nauc_recall_at_100_diff1": -0.564103, + "nauc_recall_at_1000_max": 0.526974, + "nauc_recall_at_1000_std": 0.2754, + "nauc_recall_at_1000_diff1": -0.288769, + "nauc_precision_at_1_max": 0.076626, + "nauc_precision_at_1_std": -0.132716, + "nauc_precision_at_1_diff1": 0.202184, + "nauc_precision_at_3_max": 0.376111, + "nauc_precision_at_3_std": -0.220962, + "nauc_precision_at_3_diff1": -0.702118, + "nauc_precision_at_5_max": 0.384502, + "nauc_precision_at_5_std": -0.238484, + "nauc_precision_at_5_diff1": -0.64799, + "nauc_precision_at_10_max": 0.393245, + "nauc_precision_at_10_std": -0.210757, + "nauc_precision_at_10_diff1": -0.597098, + "nauc_precision_at_20_max": 0.406283, + "nauc_precision_at_20_std": -0.188964, + "nauc_precision_at_20_diff1": -0.589498, + "nauc_precision_at_100_max": 0.457246, + "nauc_precision_at_100_std": -0.128916, + "nauc_precision_at_100_diff1": -0.564103, + "nauc_precision_at_1000_max": 0.526974, + "nauc_precision_at_1000_std": 0.2754, + "nauc_precision_at_1000_diff1": -0.288769, + "nauc_mrr_at_1_max": 0.194394, + "nauc_mrr_at_1_std": -0.137126, + "nauc_mrr_at_1_diff1": -0.389071, + "nauc_mrr_at_3_max": 0.282502, + "nauc_mrr_at_3_std": -0.176625, + "nauc_mrr_at_3_diff1": -0.545867, + "nauc_mrr_at_5_max": 0.279183, + "nauc_mrr_at_5_std": -0.177667, + "nauc_mrr_at_5_diff1": -0.532487, + "nauc_mrr_at_10_max": 0.278008, + "nauc_mrr_at_10_std": -0.173777, + "nauc_mrr_at_10_diff1": -0.525508, + "nauc_mrr_at_20_max": 0.276713, + "nauc_mrr_at_20_std": -0.172889, + "nauc_mrr_at_20_diff1": -0.524418, + "nauc_mrr_at_100_max": 0.275991, + "nauc_mrr_at_100_std": -0.172347, + "nauc_mrr_at_100_diff1": -0.523834, + "nauc_mrr_at_1000_max": 0.275878, + "nauc_mrr_at_1000_std": -0.172294, + "nauc_mrr_at_1000_diff1": -0.523792, + "main_score": 0.46329, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "sql-Code" + ] + } + ] + }, + "evaluation_time": 31.795472621917725, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TRECCOVID.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TRECCOVID.json new file mode 100644 index 0000000000..cd7ed42a98 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TRECCOVID.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bb9466bac8153a0349341eb1b22e06409e78ef4e", + "task_name": "TRECCOVID", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.74, + "ndcg_at_3": 0.69458, + "ndcg_at_5": 0.67462, + "ndcg_at_10": 0.64666, + "ndcg_at_20": 0.62613, + "ndcg_at_100": 0.51672, + "ndcg_at_1000": 0.49504, + "map_at_1": 0.00217, + "map_at_3": 0.00574, + "map_at_5": 0.00866, + "map_at_10": 0.01555, + "map_at_20": 0.02818, + "map_at_100": 0.09653, + "map_at_1000": 0.24663, + "recall_at_1": 0.00217, + "recall_at_3": 0.00607, + "recall_at_5": 0.00956, + "recall_at_10": 0.01801, + "recall_at_20": 0.034, + "recall_at_100": 0.13203, + "recall_at_1000": 0.47482, + "precision_at_1": 0.8, + "precision_at_3": 0.72667, + "precision_at_5": 0.712, + "precision_at_10": 0.69, + "precision_at_20": 0.661, + "precision_at_100": 0.5336, + "precision_at_1000": 0.21992, + "mrr_at_1": 0.8, + "mrr_at_3": 0.863333, + "mrr_at_5": 0.871333, + "mrr_at_10": 0.873833, + "mrr_at_20": 0.873833, + "mrr_at_100": 0.873833, + "mrr_at_1000": 0.873833, + "nauc_ndcg_at_1_max": 0.203525, + "nauc_ndcg_at_1_std": 0.18351, + "nauc_ndcg_at_1_diff1": 0.232466, + "nauc_ndcg_at_3_max": 0.409188, + "nauc_ndcg_at_3_std": 0.370261, + "nauc_ndcg_at_3_diff1": 0.266199, + "nauc_ndcg_at_5_max": 0.441362, + "nauc_ndcg_at_5_std": 0.435711, + "nauc_ndcg_at_5_diff1": 0.189988, + "nauc_ndcg_at_10_max": 0.423127, + "nauc_ndcg_at_10_std": 0.510638, + "nauc_ndcg_at_10_diff1": 0.075489, + "nauc_ndcg_at_20_max": 0.445718, + "nauc_ndcg_at_20_std": 0.619758, + "nauc_ndcg_at_20_diff1": -0.015555, + "nauc_ndcg_at_100_max": 0.482419, + "nauc_ndcg_at_100_std": 0.780726, + "nauc_ndcg_at_100_diff1": -0.143878, + "nauc_ndcg_at_1000_max": 0.529928, + "nauc_ndcg_at_1000_std": 0.790005, + "nauc_ndcg_at_1000_diff1": -0.117751, + "nauc_map_at_1_max": 0.098188, + "nauc_map_at_1_std": -0.016614, + "nauc_map_at_1_diff1": 0.309291, + "nauc_map_at_3_max": 0.142299, + "nauc_map_at_3_std": 0.040655, + "nauc_map_at_3_diff1": 0.276221, + "nauc_map_at_5_max": 0.254099, + "nauc_map_at_5_std": 0.154945, + "nauc_map_at_5_diff1": 0.316153, + "nauc_map_at_10_max": 0.315796, + "nauc_map_at_10_std": 0.27087, + "nauc_map_at_10_diff1": 0.259202, + "nauc_map_at_20_max": 0.349581, + "nauc_map_at_20_std": 0.367847, + "nauc_map_at_20_diff1": 0.206922, + "nauc_map_at_100_max": 0.434384, + "nauc_map_at_100_std": 0.608506, + "nauc_map_at_100_diff1": 0.049022, + "nauc_map_at_1000_max": 0.567462, + "nauc_map_at_1000_std": 0.824652, + "nauc_map_at_1000_diff1": -0.121838, + "nauc_recall_at_1_max": 0.098188, + "nauc_recall_at_1_std": -0.016614, + "nauc_recall_at_1_diff1": 0.309291, + "nauc_recall_at_3_max": 0.131102, + "nauc_recall_at_3_std": 0.033149, + "nauc_recall_at_3_diff1": 0.24404, + "nauc_recall_at_5_max": 0.219445, + "nauc_recall_at_5_std": 0.1282, + "nauc_recall_at_5_diff1": 0.272628, + "nauc_recall_at_10_max": 0.255562, + "nauc_recall_at_10_std": 0.215714, + "nauc_recall_at_10_diff1": 0.203584, + "nauc_recall_at_20_max": 0.281813, + "nauc_recall_at_20_std": 0.297993, + "nauc_recall_at_20_diff1": 0.174821, + "nauc_recall_at_100_max": 0.368559, + "nauc_recall_at_100_std": 0.507743, + "nauc_recall_at_100_diff1": 0.086497, + "nauc_recall_at_1000_max": 0.510776, + "nauc_recall_at_1000_std": 0.73079, + "nauc_recall_at_1000_diff1": -0.052855, + "nauc_precision_at_1_max": 0.450649, + "nauc_precision_at_1_std": 0.194206, + "nauc_precision_at_1_diff1": 0.525125, + "nauc_precision_at_3_max": 0.485752, + "nauc_precision_at_3_std": 0.384713, + "nauc_precision_at_3_diff1": 0.334005, + "nauc_precision_at_5_max": 0.561393, + "nauc_precision_at_5_std": 0.49271, + "nauc_precision_at_5_diff1": 0.193453, + "nauc_precision_at_10_max": 0.523088, + "nauc_precision_at_10_std": 0.558344, + "nauc_precision_at_10_diff1": 0.075465, + "nauc_precision_at_20_max": 0.500305, + "nauc_precision_at_20_std": 0.665076, + "nauc_precision_at_20_diff1": -0.018414, + "nauc_precision_at_100_max": 0.513466, + "nauc_precision_at_100_std": 0.815081, + "nauc_precision_at_100_diff1": -0.140202, + "nauc_precision_at_1000_max": 0.430057, + "nauc_precision_at_1000_std": 0.610152, + "nauc_precision_at_1000_diff1": -0.230339, + "nauc_mrr_at_1_max": 0.450649, + "nauc_mrr_at_1_std": 0.194206, + "nauc_mrr_at_1_diff1": 0.525125, + "nauc_mrr_at_3_max": 0.536438, + "nauc_mrr_at_3_std": 0.28117, + "nauc_mrr_at_3_diff1": 0.531726, + "nauc_mrr_at_5_max": 0.529773, + "nauc_mrr_at_5_std": 0.25939, + "nauc_mrr_at_5_diff1": 0.522077, + "nauc_mrr_at_10_max": 0.522678, + "nauc_mrr_at_10_std": 0.25323, + "nauc_mrr_at_10_diff1": 0.530421, + "nauc_mrr_at_20_max": 0.522678, + "nauc_mrr_at_20_std": 0.25323, + "nauc_mrr_at_20_diff1": 0.530421, + "nauc_mrr_at_100_max": 0.522678, + "nauc_mrr_at_100_std": 0.25323, + "nauc_mrr_at_100_diff1": 0.530421, + "nauc_mrr_at_1000_max": 0.522678, + "nauc_mrr_at_1000_std": 0.25323, + "nauc_mrr_at_1000_diff1": 0.530421, + "main_score": 0.64666, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 121.1473822593689, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/Touche2020.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/Touche2020.json new file mode 100644 index 0000000000..e74b090f42 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/Touche2020.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f", + "task_name": "Touche2020", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.2449, + "ndcg_at_3": 0.23983, + "ndcg_at_5": 0.23247, + "ndcg_at_10": 0.24174, + "ndcg_at_20": 0.25419, + "ndcg_at_100": 0.35806, + "ndcg_at_1000": 0.48482, + "map_at_1": 0.01921, + "map_at_3": 0.04502, + "map_at_5": 0.06198, + "map_at_10": 0.09527, + "map_at_20": 0.12, + "map_at_100": 0.15497, + "map_at_1000": 0.17361, + "recall_at_1": 0.01921, + "recall_at_3": 0.06064, + "recall_at_5": 0.09652, + "recall_at_10": 0.16759, + "recall_at_20": 0.24, + "recall_at_100": 0.48321, + "recall_at_1000": 0.86811, + "precision_at_1": 0.26531, + "precision_at_3": 0.26531, + "precision_at_5": 0.2449, + "precision_at_10": 0.23061, + "precision_at_20": 0.17653, + "precision_at_100": 0.07571, + "precision_at_1000": 0.01598, + "mrr_at_1": 0.265306, + "mrr_at_3": 0.380952, + "mrr_at_5": 0.410544, + "mrr_at_10": 0.425729, + "mrr_at_20": 0.42886, + "mrr_at_100": 0.434129, + "mrr_at_1000": 0.434129, + "nauc_ndcg_at_1_max": -0.257389, + "nauc_ndcg_at_1_std": 0.117323, + "nauc_ndcg_at_1_diff1": 0.095402, + "nauc_ndcg_at_3_max": -0.17096, + "nauc_ndcg_at_3_std": 0.031128, + "nauc_ndcg_at_3_diff1": -0.001741, + "nauc_ndcg_at_5_max": -0.164988, + "nauc_ndcg_at_5_std": 0.099874, + "nauc_ndcg_at_5_diff1": -0.081971, + "nauc_ndcg_at_10_max": -0.169988, + "nauc_ndcg_at_10_std": 0.087897, + "nauc_ndcg_at_10_diff1": -0.037142, + "nauc_ndcg_at_20_max": -0.196104, + "nauc_ndcg_at_20_std": 0.104772, + "nauc_ndcg_at_20_diff1": -0.077501, + "nauc_ndcg_at_100_max": -0.162869, + "nauc_ndcg_at_100_std": 0.29864, + "nauc_ndcg_at_100_diff1": -0.07459, + "nauc_ndcg_at_1000_max": -0.171677, + "nauc_ndcg_at_1000_std": 0.378947, + "nauc_ndcg_at_1000_diff1": -0.089988, + "nauc_map_at_1_max": -0.273775, + "nauc_map_at_1_std": 0.010533, + "nauc_map_at_1_diff1": 0.104094, + "nauc_map_at_3_max": -0.265399, + "nauc_map_at_3_std": -0.077665, + "nauc_map_at_3_diff1": -0.079334, + "nauc_map_at_5_max": -0.213081, + "nauc_map_at_5_std": -0.048554, + "nauc_map_at_5_diff1": -0.096587, + "nauc_map_at_10_max": -0.138897, + "nauc_map_at_10_std": 0.011327, + "nauc_map_at_10_diff1": -0.051847, + "nauc_map_at_20_max": -0.12039, + "nauc_map_at_20_std": 0.065866, + "nauc_map_at_20_diff1": -0.087957, + "nauc_map_at_100_max": -0.113609, + "nauc_map_at_100_std": 0.141516, + "nauc_map_at_100_diff1": -0.080843, + "nauc_map_at_1000_max": -0.112997, + "nauc_map_at_1000_std": 0.190131, + "nauc_map_at_1000_diff1": -0.08179, + "nauc_recall_at_1_max": -0.273775, + "nauc_recall_at_1_std": 0.010533, + "nauc_recall_at_1_diff1": 0.104094, + "nauc_recall_at_3_max": -0.248834, + "nauc_recall_at_3_std": -0.1349, + "nauc_recall_at_3_diff1": -0.129659, + "nauc_recall_at_5_max": -0.197863, + "nauc_recall_at_5_std": -0.057649, + "nauc_recall_at_5_diff1": -0.147228, + "nauc_recall_at_10_max": -0.175287, + "nauc_recall_at_10_std": 0.009452, + "nauc_recall_at_10_diff1": -0.067548, + "nauc_recall_at_20_max": -0.178237, + "nauc_recall_at_20_std": 0.086459, + "nauc_recall_at_20_diff1": -0.114912, + "nauc_recall_at_100_max": -0.163452, + "nauc_recall_at_100_std": 0.3708, + "nauc_recall_at_100_diff1": -0.155522, + "nauc_recall_at_1000_max": -0.221479, + "nauc_recall_at_1000_std": 0.728637, + "nauc_recall_at_1000_diff1": -0.319628, + "nauc_precision_at_1_max": -0.291233, + "nauc_precision_at_1_std": 0.058241, + "nauc_precision_at_1_diff1": 0.114746, + "nauc_precision_at_3_max": -0.123708, + "nauc_precision_at_3_std": -0.036962, + "nauc_precision_at_3_diff1": -0.006932, + "nauc_precision_at_5_max": -0.112697, + "nauc_precision_at_5_std": 0.087854, + "nauc_precision_at_5_diff1": -0.11157, + "nauc_precision_at_10_max": -0.096707, + "nauc_precision_at_10_std": 0.141638, + "nauc_precision_at_10_diff1": -0.008426, + "nauc_precision_at_20_max": -0.093255, + "nauc_precision_at_20_std": 0.277106, + "nauc_precision_at_20_diff1": -0.053662, + "nauc_precision_at_100_max": 0.016253, + "nauc_precision_at_100_std": 0.658258, + "nauc_precision_at_100_diff1": -0.030023, + "nauc_precision_at_1000_max": 0.184955, + "nauc_precision_at_1000_std": 0.20845, + "nauc_precision_at_1000_diff1": 0.052535, + "nauc_mrr_at_1_max": -0.291233, + "nauc_mrr_at_1_std": 0.058241, + "nauc_mrr_at_1_diff1": 0.114746, + "nauc_mrr_at_3_max": -0.211674, + "nauc_mrr_at_3_std": -0.021579, + "nauc_mrr_at_3_diff1": 0.122871, + "nauc_mrr_at_5_max": -0.217307, + "nauc_mrr_at_5_std": 0.023035, + "nauc_mrr_at_5_diff1": 0.110249, + "nauc_mrr_at_10_max": -0.240811, + "nauc_mrr_at_10_std": 0.035908, + "nauc_mrr_at_10_diff1": 0.106902, + "nauc_mrr_at_20_max": -0.241123, + "nauc_mrr_at_20_std": 0.041887, + "nauc_mrr_at_20_diff1": 0.108899, + "nauc_mrr_at_100_max": -0.240583, + "nauc_mrr_at_100_std": 0.035461, + "nauc_mrr_at_100_diff1": 0.109962, + "nauc_mrr_at_1000_max": -0.240583, + "nauc_mrr_at_1000_std": 0.035461, + "nauc_mrr_at_1000_diff1": 0.109962, + "main_score": 0.24174, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 595.8074662685394, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/Touche2020Retrieval.v3.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/Touche2020Retrieval.v3.json new file mode 100644 index 0000000000..7145957a2a --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/Touche2020Retrieval.v3.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "431886eaecc48f067a3975b70d0949ea2862463c", + "task_name": "Touche2020Retrieval.v3", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "ndcg_at_1": 0.64286, + "ndcg_at_3": 0.62498, + "ndcg_at_5": 0.61126, + "ndcg_at_10": 0.56253, + "ndcg_at_20": 0.51507, + "ndcg_at_100": 0.58639, + "ndcg_at_1000": 0.71063, + "map_at_1": 0.02655, + "map_at_3": 0.07199, + "map_at_5": 0.11188, + "map_at_10": 0.18542, + "map_at_20": 0.2683, + "map_at_100": 0.36883, + "map_at_1000": 0.40279, + "recall_at_1": 0.02655, + "recall_at_3": 0.07728, + "recall_at_5": 0.1272, + "recall_at_10": 0.22226, + "recall_at_20": 0.35009, + "recall_at_100": 0.6255, + "recall_at_1000": 0.92967, + "precision_at_1": 0.73469, + "precision_at_3": 0.72789, + "precision_at_5": 0.70204, + "precision_at_10": 0.61837, + "precision_at_20": 0.50612, + "precision_at_100": 0.19755, + "precision_at_1000": 0.03206, + "mrr_at_1": 0.734694, + "mrr_at_3": 0.836735, + "mrr_at_5": 0.845918, + "mrr_at_10": 0.845918, + "mrr_at_20": 0.845918, + "mrr_at_100": 0.845918, + "mrr_at_1000": 0.845918, + "nauc_ndcg_at_1_max": -0.075779, + "nauc_ndcg_at_1_std": 0.184916, + "nauc_ndcg_at_1_diff1": -0.057327, + "nauc_ndcg_at_3_max": -0.043998, + "nauc_ndcg_at_3_std": 0.084628, + "nauc_ndcg_at_3_diff1": -0.060011, + "nauc_ndcg_at_5_max": 0.089581, + "nauc_ndcg_at_5_std": 0.094817, + "nauc_ndcg_at_5_diff1": 0.003617, + "nauc_ndcg_at_10_max": 0.108444, + "nauc_ndcg_at_10_std": 0.166303, + "nauc_ndcg_at_10_diff1": 0.002537, + "nauc_ndcg_at_20_max": 0.143453, + "nauc_ndcg_at_20_std": 0.126158, + "nauc_ndcg_at_20_diff1": 0.00588, + "nauc_ndcg_at_100_max": 0.123688, + "nauc_ndcg_at_100_std": 0.244896, + "nauc_ndcg_at_100_diff1": 0.056983, + "nauc_ndcg_at_1000_max": 0.195859, + "nauc_ndcg_at_1000_std": 0.390337, + "nauc_ndcg_at_1000_diff1": 0.030607, + "nauc_map_at_1_max": -0.102829, + "nauc_map_at_1_std": -0.075624, + "nauc_map_at_1_diff1": -0.000402, + "nauc_map_at_3_max": -0.013374, + "nauc_map_at_3_std": -0.095292, + "nauc_map_at_3_diff1": 0.006344, + "nauc_map_at_5_max": 0.057605, + "nauc_map_at_5_std": -0.099339, + "nauc_map_at_5_diff1": 0.042888, + "nauc_map_at_10_max": 0.062115, + "nauc_map_at_10_std": -0.089739, + "nauc_map_at_10_diff1": 0.012094, + "nauc_map_at_20_max": 0.101831, + "nauc_map_at_20_std": -0.041266, + "nauc_map_at_20_diff1": 0.025944, + "nauc_map_at_100_max": 0.131313, + "nauc_map_at_100_std": 0.130075, + "nauc_map_at_100_diff1": 0.017219, + "nauc_map_at_1000_max": 0.142607, + "nauc_map_at_1000_std": 0.174298, + "nauc_map_at_1000_diff1": 0.013461, + "nauc_recall_at_1_max": -0.102829, + "nauc_recall_at_1_std": -0.075624, + "nauc_recall_at_1_diff1": -0.000402, + "nauc_recall_at_3_max": -0.015781, + "nauc_recall_at_3_std": -0.109655, + "nauc_recall_at_3_diff1": 0.015901, + "nauc_recall_at_5_max": 0.027938, + "nauc_recall_at_5_std": -0.153873, + "nauc_recall_at_5_diff1": 0.116292, + "nauc_recall_at_10_max": 0.02387, + "nauc_recall_at_10_std": -0.120133, + "nauc_recall_at_10_diff1": 0.082405, + "nauc_recall_at_20_max": 0.066596, + "nauc_recall_at_20_std": -0.068635, + "nauc_recall_at_20_diff1": 0.097501, + "nauc_recall_at_100_max": 0.078576, + "nauc_recall_at_100_std": 0.251726, + "nauc_recall_at_100_diff1": 0.126271, + "nauc_recall_at_1000_max": 0.248725, + "nauc_recall_at_1000_std": 0.695074, + "nauc_recall_at_1000_diff1": 0.329801, + "nauc_precision_at_1_max": -0.226613, + "nauc_precision_at_1_std": 0.089778, + "nauc_precision_at_1_diff1": -0.074389, + "nauc_precision_at_3_max": -0.07693, + "nauc_precision_at_3_std": -0.020793, + "nauc_precision_at_3_diff1": -0.061114, + "nauc_precision_at_5_max": 0.17285, + "nauc_precision_at_5_std": 0.035861, + "nauc_precision_at_5_diff1": 0.066669, + "nauc_precision_at_10_max": 0.127061, + "nauc_precision_at_10_std": 0.129311, + "nauc_precision_at_10_diff1": -0.009838, + "nauc_precision_at_20_max": 0.184653, + "nauc_precision_at_20_std": 0.240544, + "nauc_precision_at_20_diff1": -0.047972, + "nauc_precision_at_100_max": 0.152019, + "nauc_precision_at_100_std": 0.633027, + "nauc_precision_at_100_diff1": -0.196971, + "nauc_precision_at_1000_max": 0.144883, + "nauc_precision_at_1000_std": 0.343892, + "nauc_precision_at_1000_diff1": -0.185675, + "nauc_mrr_at_1_max": -0.226613, + "nauc_mrr_at_1_std": 0.089778, + "nauc_mrr_at_1_diff1": -0.074389, + "nauc_mrr_at_3_max": -0.212142, + "nauc_mrr_at_3_std": 0.07496, + "nauc_mrr_at_3_diff1": -0.120611, + "nauc_mrr_at_5_max": -0.214166, + "nauc_mrr_at_5_std": 0.113105, + "nauc_mrr_at_5_diff1": -0.115736, + "nauc_mrr_at_10_max": -0.214166, + "nauc_mrr_at_10_std": 0.113105, + "nauc_mrr_at_10_diff1": -0.115736, + "nauc_mrr_at_20_max": -0.214166, + "nauc_mrr_at_20_std": 0.113105, + "nauc_mrr_at_20_diff1": -0.115736, + "nauc_mrr_at_100_max": -0.214166, + "nauc_mrr_at_100_std": 0.113105, + "nauc_mrr_at_100_diff1": -0.115736, + "nauc_mrr_at_1000_max": -0.214166, + "nauc_mrr_at_1000_std": 0.113105, + "nauc_mrr_at_1000_diff1": -0.115736, + "main_score": 0.56253, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 378.3850872516632, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ToxicConversationsClassification.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ToxicConversationsClassification.json new file mode 100644 index 0000000000..b99ad3f253 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/ToxicConversationsClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "edfaf9da55d3dd50d43143d90c1ac476895ae6de", + "task_name": "ToxicConversationsClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.593701, + "f1": 0.455573, + "f1_weighted": 0.684044, + "ap": 0.096798, + "ap_weighted": 0.096798, + "scores_per_experiment": [ + { + "accuracy": 0.632324, + "f1": 0.484387, + "f1_weighted": 0.716879, + "ap": 0.107441, + "ap_weighted": 0.107441 + }, + { + "accuracy": 0.645508, + "f1": 0.483087, + "f1_weighted": 0.727001, + "ap": 0.09993, + "ap_weighted": 0.09993 + }, + { + "accuracy": 0.674805, + "f1": 0.495929, + "f1_weighted": 0.748701, + "ap": 0.100362, + "ap_weighted": 0.100362 + }, + { + "accuracy": 0.640625, + "f1": 0.480913, + "f1_weighted": 0.723293, + "ap": 0.099869, + "ap_weighted": 0.099869 + }, + { + "accuracy": 0.472656, + "f1": 0.386364, + "f1_weighted": 0.580073, + "ap": 0.085487, + "ap_weighted": 0.085487 + }, + { + "accuracy": 0.523926, + "f1": 0.411543, + "f1_weighted": 0.628021, + "ap": 0.084984, + "ap_weighted": 0.084984 + }, + { + "accuracy": 0.652832, + "f1": 0.478989, + "f1_weighted": 0.732332, + "ap": 0.093565, + "ap_weighted": 0.093565 + }, + { + "accuracy": 0.526855, + "f1": 0.42061, + "f1_weighted": 0.629467, + "ap": 0.092193, + "ap_weighted": 0.092193 + }, + { + "accuracy": 0.553223, + "f1": 0.439345, + "f1_weighted": 0.652048, + "ap": 0.098502, + "ap_weighted": 0.098502 + }, + { + "accuracy": 0.614258, + "f1": 0.474559, + "f1_weighted": 0.702628, + "ap": 0.105652, + "ap_weighted": 0.105652 + } + ], + "main_score": 0.593701, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 13.333395004272461, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TweetSentimentExtractionClassification.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TweetSentimentExtractionClassification.json new file mode 100644 index 0000000000..09a8c5b7d9 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TweetSentimentExtractionClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a", + "task_name": "TweetSentimentExtractionClassification", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "accuracy": 0.500481, + "f1": 0.501904, + "f1_weighted": 0.498331, + "scores_per_experiment": [ + { + "accuracy": 0.494058, + "f1": 0.492552, + "f1_weighted": 0.485314 + }, + { + "accuracy": 0.505376, + "f1": 0.507375, + "f1_weighted": 0.50518 + }, + { + "accuracy": 0.494907, + "f1": 0.495124, + "f1_weighted": 0.494142 + }, + { + "accuracy": 0.505942, + "f1": 0.508962, + "f1_weighted": 0.503951 + }, + { + "accuracy": 0.488398, + "f1": 0.487901, + "f1_weighted": 0.482863 + }, + { + "accuracy": 0.508772, + "f1": 0.511098, + "f1_weighted": 0.508689 + }, + { + "accuracy": 0.494907, + "f1": 0.496336, + "f1_weighted": 0.493017 + }, + { + "accuracy": 0.512168, + "f1": 0.514878, + "f1_weighted": 0.511096 + }, + { + "accuracy": 0.494341, + "f1": 0.497462, + "f1_weighted": 0.494509 + }, + { + "accuracy": 0.505942, + "f1": 0.507349, + "f1_weighted": 0.504549 + } + ], + "main_score": 0.500481, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.734870195388794, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TwentyNewsgroupsClustering.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TwentyNewsgroupsClustering.json new file mode 100644 index 0000000000..2d458381d8 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TwentyNewsgroupsClustering.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "task_name": "TwentyNewsgroupsClustering", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measure": 0.46035, + "v_measure_std": 0.012818, + "v_measures": [ + 0.459674, + 0.47561, + 0.460004, + 0.480946, + 0.46763, + 0.460389, + 0.45876, + 0.445909, + 0.433505, + 0.461071 + ], + "main_score": 0.46035, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 38.048712968826294, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TwentyNewsgroupsClustering.v2.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TwentyNewsgroupsClustering.v2.json new file mode 100644 index 0000000000..9f7f0f0257 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TwentyNewsgroupsClustering.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "task_name": "TwentyNewsgroupsClustering.v2", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.466454, + 0.460749, + 0.456455, + 0.44876, + 0.435224, + 0.445242, + 0.436034, + 0.442794, + 0.444031, + 0.441427 + ] + }, + "v_measure": 0.447717, + "v_measure_std": 0.009871, + "main_score": 0.447717, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 36.36772608757019, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TwitterSemEval2015.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TwitterSemEval2015.json new file mode 100644 index 0000000000..446a72d04d --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TwitterSemEval2015.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", + "task_name": "TwitterSemEval2015", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "similarity_accuracy": 0.816296, + "similarity_accuracy_threshold": 0.922274, + "similarity_f1": 0.561499, + "similarity_f1_threshold": 0.884554, + "similarity_precision": 0.492921, + "similarity_recall": 0.652243, + "similarity_ap": 0.581538, + "cosine_accuracy": 0.816296, + "cosine_accuracy_threshold": 0.922274, + "cosine_f1": 0.561499, + "cosine_f1_threshold": 0.884554, + "cosine_precision": 0.492921, + "cosine_recall": 0.652243, + "cosine_ap": 0.581539, + "manhattan_accuracy": 0.815462, + "manhattan_accuracy_threshold": 111.983948, + "manhattan_f1": 0.563982, + "manhattan_f1_threshold": 131.863083, + "manhattan_precision": 0.489515, + "manhattan_recall": 0.665172, + "manhattan_ap": 0.580747, + "euclidean_accuracy": 0.815998, + "euclidean_accuracy_threshold": 6.88906, + "euclidean_f1": 0.562204, + "euclidean_f1_threshold": 8.412651, + "euclidean_precision": 0.491318, + "euclidean_recall": 0.656992, + "euclidean_ap": 0.581238, + "dot_accuracy": 0.817727, + "dot_accuracy_threshold": 280.642517, + "dot_f1": 0.558875, + "dot_f1_threshold": 272.533936, + "dot_precision": 0.533046, + "dot_recall": 0.587335, + "dot_ap": 0.580233, + "max_accuracy": 0.817727, + "max_f1": 0.563982, + "max_precision": 0.533046, + "max_recall": 0.665172, + "max_ap": 0.581539, + "main_score": 0.581539, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.909769296646118, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TwitterURLCorpus.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TwitterURLCorpus.json new file mode 100644 index 0000000000..8a9f138851 --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/TwitterURLCorpus.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", + "task_name": "TwitterURLCorpus", + "mteb_version": "1.36.8", + "scores": { + "test": [ + { + "similarity_accuracy": 0.874821, + "similarity_accuracy_threshold": 0.90733, + "similarity_f1": 0.753411, + "similarity_f1_threshold": 0.89242, + "similarity_precision": 0.714246, + "similarity_recall": 0.79712, + "similarity_ap": 0.827706, + "cosine_accuracy": 0.874821, + "cosine_accuracy_threshold": 0.90733, + "cosine_f1": 0.753411, + "cosine_f1_threshold": 0.89242, + "cosine_precision": 0.714246, + "cosine_recall": 0.79712, + "cosine_ap": 0.827706, + "manhattan_accuracy": 0.874452, + "manhattan_accuracy_threshold": 117.073288, + "manhattan_f1": 0.7517, + "manhattan_f1_threshold": 126.252991, + "manhattan_precision": 0.715648, + "manhattan_recall": 0.791577, + "manhattan_ap": 0.826601, + "euclidean_accuracy": 0.874626, + "euclidean_accuracy_threshold": 7.556748, + "euclidean_f1": 0.753153, + "euclidean_f1_threshold": 8.089161, + "euclidean_precision": 0.715146, + "euclidean_recall": 0.795427, + "euclidean_ap": 0.827203, + "dot_accuracy": 0.874704, + "dot_accuracy_threshold": 275.302856, + "dot_f1": 0.752801, + "dot_f1_threshold": 271.106659, + "dot_precision": 0.697687, + "dot_recall": 0.81737, + "dot_ap": 0.826179, + "max_accuracy": 0.874821, + "max_f1": 0.753411, + "max_precision": 0.715648, + "max_recall": 0.81737, + "max_ap": 0.827706, + "main_score": 0.827706, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 8.636502265930176, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/model_meta.json b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/model_meta.json new file mode 100644 index 0000000000..d52807cacb --- /dev/null +++ b/results/ibm-granite__granite-embedding-small-english-r2/54a8d2616a0844355a5164432d3f6dafb37b17a3/model_meta.json @@ -0,0 +1 @@ +{"name": "ibm-granite/granite-embedding-small-english-r2", "revision": "54a8d2616a0844355a5164432d3f6dafb37b17a3", "release_date": "2025-08-15", "languages": ["eng-Latn"], "n_parameters": 47000000, "memory_usage_mb": 91.0, "max_tokens": 8192.0, "embed_dim": 384, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/ibm-granite/granite-embedding-small-english-r2", "similarity_fn_name": "cosine", "use_instructions": false, "training_datasets": {"WikipediaRetrievalMultilingual": [], "WikipediaRerankingMultilingual": [], "StackOverflowDupQuestions": [], "AskUbuntuDupQuestions": [], "StackExchangeClusteringP2P": [], "StackExchangeClusteringP2P.v2": [], "StackExchangeClustering": [], "StackExchangeClustering.v2": [], "NQ": ["test"], "NQ-NL": ["test"], "NQHardNegatives": ["test"], "HotPotQA": ["test"], "HotPotQAHardNegatives": ["test"], "HotPotQA-PL": ["test"], "HotpotQA-NL": ["test"], "FEVER": ["test"], "FEVERHardNegatives": ["test"], "FEVER-NL": ["test"], "MIRACLRetrieval": ["train"], "MIRACLRetrievalHardNegatives": ["train"], "MIRACLReranking": ["train"], "MrTidyRetrieval": ["train"], "DBPedia": ["train"], "DBPedia-NL": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["text"], "loader": "sentence_transformers_loader"} \ No newline at end of file