diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/AILACasedocs.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/AILACasedocs.json new file mode 100644 index 0000000000..1ba067fa70 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/AILACasedocs.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4106e6bcc72e0698d714ea8b101355e3e238431a", + "task_name": "AILACasedocs", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.26, + "ndcg_at_3": 0.26018, + "ndcg_at_5": 0.27016, + "ndcg_at_10": 0.29981, + "ndcg_at_20": 0.34041, + "ndcg_at_100": 0.44641, + "ndcg_at_1000": 0.47277, + "map_at_1": 0.09504, + "map_at_3": 0.16173, + "map_at_5": 0.18449, + "map_at_10": 0.20796, + "map_at_20": 0.22519, + "map_at_100": 0.2525, + "map_at_1000": 0.25625, + "recall_at_1": 0.09504, + "recall_at_3": 0.20641, + "recall_at_5": 0.27234, + "recall_at_10": 0.38082, + "recall_at_20": 0.4952, + "recall_at_100": 0.87364, + "recall_at_1000": 1.0, + "precision_at_1": 0.26, + "precision_at_3": 0.20667, + "precision_at_5": 0.172, + "precision_at_10": 0.12, + "precision_at_20": 0.081, + "precision_at_100": 0.0334, + "precision_at_1000": 0.0039, + "mrr_at_1": 0.26, + "mrr_at_3": 0.323333, + "mrr_at_5": 0.350333, + "mrr_at_10": 0.372952, + "mrr_at_20": 0.380013, + "mrr_at_100": 0.384189, + "mrr_at_1000": 0.384638, + "nauc_ndcg_at_1_max": 0.634529, + "nauc_ndcg_at_1_std": -0.151774, + "nauc_ndcg_at_1_diff1": 0.519933, + "nauc_ndcg_at_3_max": 0.541066, + "nauc_ndcg_at_3_std": -0.12, + "nauc_ndcg_at_3_diff1": 0.48106, + "nauc_ndcg_at_5_max": 0.520596, + "nauc_ndcg_at_5_std": -0.101539, + "nauc_ndcg_at_5_diff1": 0.450842, + "nauc_ndcg_at_10_max": 0.550251, + "nauc_ndcg_at_10_std": -0.09872, + "nauc_ndcg_at_10_diff1": 0.467025, + "nauc_ndcg_at_20_max": 0.487271, + "nauc_ndcg_at_20_std": -0.136847, + "nauc_ndcg_at_20_diff1": 0.465239, + "nauc_ndcg_at_100_max": 0.55967, + "nauc_ndcg_at_100_std": -0.050633, + "nauc_ndcg_at_100_diff1": 0.492625, + "nauc_ndcg_at_1000_max": 0.564518, + "nauc_ndcg_at_1000_std": -0.092988, + "nauc_ndcg_at_1000_diff1": 0.523239, + "nauc_map_at_1_max": 0.546606, + "nauc_map_at_1_std": -0.044766, + "nauc_map_at_1_diff1": 0.425279, + "nauc_map_at_3_max": 0.538254, + "nauc_map_at_3_std": -0.105903, + "nauc_map_at_3_diff1": 0.477036, + "nauc_map_at_5_max": 0.538385, + "nauc_map_at_5_std": -0.101066, + "nauc_map_at_5_diff1": 0.490156, + "nauc_map_at_10_max": 0.54262, + "nauc_map_at_10_std": -0.085109, + "nauc_map_at_10_diff1": 0.490242, + "nauc_map_at_20_max": 0.507275, + "nauc_map_at_20_std": -0.108962, + "nauc_map_at_20_diff1": 0.481491, + "nauc_map_at_100_max": 0.528365, + "nauc_map_at_100_std": -0.090259, + "nauc_map_at_100_diff1": 0.499263, + "nauc_map_at_1000_max": 0.533206, + "nauc_map_at_1000_std": -0.09222, + "nauc_map_at_1000_diff1": 0.506243, + "nauc_recall_at_1_max": 0.546606, + "nauc_recall_at_1_std": -0.044766, + "nauc_recall_at_1_diff1": 0.425279, + "nauc_recall_at_3_max": 0.418008, + "nauc_recall_at_3_std": -0.123651, + "nauc_recall_at_3_diff1": 0.378729, + "nauc_recall_at_5_max": 0.403916, + "nauc_recall_at_5_std": -0.139452, + "nauc_recall_at_5_diff1": 0.354635, + "nauc_recall_at_10_max": 0.38616, + "nauc_recall_at_10_std": -0.076434, + "nauc_recall_at_10_diff1": 0.264934, + "nauc_recall_at_20_max": 0.213452, + "nauc_recall_at_20_std": -0.154048, + "nauc_recall_at_20_diff1": 0.222938, + "nauc_recall_at_100_max": 0.395936, + "nauc_recall_at_100_std": 0.413017, + "nauc_recall_at_100_diff1": 0.146421, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.634529, + "nauc_precision_at_1_std": -0.151774, + "nauc_precision_at_1_diff1": 0.519933, + "nauc_precision_at_3_max": 0.51219, + "nauc_precision_at_3_std": -0.133251, + "nauc_precision_at_3_diff1": 0.510109, + "nauc_precision_at_5_max": 0.464116, + "nauc_precision_at_5_std": -0.088129, + "nauc_precision_at_5_diff1": 0.448881, + "nauc_precision_at_10_max": 0.47723, + "nauc_precision_at_10_std": -0.001242, + "nauc_precision_at_10_diff1": 0.508931, + "nauc_precision_at_20_max": 0.299703, + "nauc_precision_at_20_std": -0.071361, + "nauc_precision_at_20_diff1": 0.43304, + "nauc_precision_at_100_max": 0.302324, + "nauc_precision_at_100_std": 0.108069, + "nauc_precision_at_100_diff1": 0.27284, + "nauc_precision_at_1000_max": 0.282376, + "nauc_precision_at_1000_std": 0.05804, + "nauc_precision_at_1000_diff1": 0.301468, + "nauc_mrr_at_1_max": 0.634529, + "nauc_mrr_at_1_std": -0.151774, + "nauc_mrr_at_1_diff1": 0.519933, + "nauc_mrr_at_3_max": 0.552366, + "nauc_mrr_at_3_std": -0.15416, + "nauc_mrr_at_3_diff1": 0.471404, + "nauc_mrr_at_5_max": 0.554163, + "nauc_mrr_at_5_std": -0.156668, + "nauc_mrr_at_5_diff1": 0.448356, + "nauc_mrr_at_10_max": 0.578667, + "nauc_mrr_at_10_std": -0.130699, + "nauc_mrr_at_10_diff1": 0.464797, + "nauc_mrr_at_20_max": 0.58007, + "nauc_mrr_at_20_std": -0.137417, + "nauc_mrr_at_20_diff1": 0.475354, + "nauc_mrr_at_100_max": 0.582718, + "nauc_mrr_at_100_std": -0.13447, + "nauc_mrr_at_100_diff1": 0.476302, + "nauc_mrr_at_1000_max": 0.582309, + "nauc_mrr_at_1000_std": -0.135577, + "nauc_mrr_at_1000_diff1": 0.475961, + "main_score": 0.29981, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.647683620452881, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/AILAStatutes.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/AILAStatutes.json new file mode 100644 index 0000000000..2c48eb0c6b --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/AILAStatutes.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e", + "task_name": "AILAStatutes", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.14, + "ndcg_at_3": 0.16965, + "ndcg_at_5": 0.18069, + "ndcg_at_10": 0.23612, + "ndcg_at_20": 0.301, + "ndcg_at_100": 0.46693, + "ndcg_at_1000": 0.46693, + "map_at_1": 0.044, + "map_at_3": 0.09183, + "map_at_5": 0.11337, + "map_at_10": 0.14182, + "map_at_20": 0.16327, + "map_at_100": 0.21205, + "map_at_1000": 0.21205, + "recall_at_1": 0.044, + "recall_at_3": 0.13167, + "recall_at_5": 0.19233, + "recall_at_10": 0.30033, + "recall_at_20": 0.457, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.14, + "precision_at_3": 0.16667, + "precision_at_5": 0.152, + "precision_at_10": 0.124, + "precision_at_20": 0.098, + "precision_at_100": 0.0434, + "precision_at_1000": 0.00434, + "mrr_at_1": 0.14, + "mrr_at_3": 0.246667, + "mrr_at_5": 0.283667, + "mrr_at_10": 0.299714, + "mrr_at_20": 0.316448, + "mrr_at_100": 0.320584, + "mrr_at_1000": 0.320584, + "nauc_ndcg_at_1_max": 0.52777, + "nauc_ndcg_at_1_std": 0.196529, + "nauc_ndcg_at_1_diff1": 0.244431, + "nauc_ndcg_at_3_max": 0.455446, + "nauc_ndcg_at_3_std": 0.211664, + "nauc_ndcg_at_3_diff1": 0.212615, + "nauc_ndcg_at_5_max": 0.505357, + "nauc_ndcg_at_5_std": 0.259802, + "nauc_ndcg_at_5_diff1": 0.212096, + "nauc_ndcg_at_10_max": 0.530473, + "nauc_ndcg_at_10_std": 0.259089, + "nauc_ndcg_at_10_diff1": 0.167999, + "nauc_ndcg_at_20_max": 0.507583, + "nauc_ndcg_at_20_std": 0.256566, + "nauc_ndcg_at_20_diff1": 0.149752, + "nauc_ndcg_at_100_max": 0.485475, + "nauc_ndcg_at_100_std": 0.2592, + "nauc_ndcg_at_100_diff1": 0.191278, + "nauc_ndcg_at_1000_max": 0.485475, + "nauc_ndcg_at_1000_std": 0.2592, + "nauc_ndcg_at_1000_diff1": 0.191278, + "nauc_map_at_1_max": 0.430454, + "nauc_map_at_1_std": 0.173949, + "nauc_map_at_1_diff1": 0.279126, + "nauc_map_at_3_max": 0.460972, + "nauc_map_at_3_std": 0.173354, + "nauc_map_at_3_diff1": 0.217193, + "nauc_map_at_5_max": 0.505919, + "nauc_map_at_5_std": 0.247825, + "nauc_map_at_5_diff1": 0.19226, + "nauc_map_at_10_max": 0.55143, + "nauc_map_at_10_std": 0.256796, + "nauc_map_at_10_diff1": 0.182143, + "nauc_map_at_20_max": 0.540592, + "nauc_map_at_20_std": 0.265593, + "nauc_map_at_20_diff1": 0.18117, + "nauc_map_at_100_max": 0.526805, + "nauc_map_at_100_std": 0.27223, + "nauc_map_at_100_diff1": 0.182015, + "nauc_map_at_1000_max": 0.526805, + "nauc_map_at_1000_std": 0.27223, + "nauc_map_at_1000_diff1": 0.182015, + "nauc_recall_at_1_max": 0.430454, + "nauc_recall_at_1_std": 0.173949, + "nauc_recall_at_1_diff1": 0.279126, + "nauc_recall_at_3_max": 0.44384, + "nauc_recall_at_3_std": 0.183181, + "nauc_recall_at_3_diff1": 0.214745, + "nauc_recall_at_5_max": 0.533037, + "nauc_recall_at_5_std": 0.290115, + "nauc_recall_at_5_diff1": 0.187234, + "nauc_recall_at_10_max": 0.545921, + "nauc_recall_at_10_std": 0.265722, + "nauc_recall_at_10_diff1": 0.105344, + "nauc_recall_at_20_max": 0.436662, + "nauc_recall_at_20_std": 0.233972, + "nauc_recall_at_20_diff1": 0.010741, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.52777, + "nauc_precision_at_1_std": 0.196529, + "nauc_precision_at_1_diff1": 0.244431, + "nauc_precision_at_3_max": 0.445254, + "nauc_precision_at_3_std": 0.225785, + "nauc_precision_at_3_diff1": 0.204402, + "nauc_precision_at_5_max": 0.535413, + "nauc_precision_at_5_std": 0.350745, + "nauc_precision_at_5_diff1": 0.210793, + "nauc_precision_at_10_max": 0.488651, + "nauc_precision_at_10_std": 0.287712, + "nauc_precision_at_10_diff1": 0.098968, + "nauc_precision_at_20_max": 0.29879, + "nauc_precision_at_20_std": 0.228019, + "nauc_precision_at_20_diff1": 0.011992, + "nauc_precision_at_100_max": -0.373118, + "nauc_precision_at_100_std": 0.047255, + "nauc_precision_at_100_diff1": 0.000604, + "nauc_precision_at_1000_max": -0.373118, + "nauc_precision_at_1000_std": 0.047255, + "nauc_precision_at_1000_diff1": 0.000604, + "nauc_mrr_at_1_max": 0.52777, + "nauc_mrr_at_1_std": 0.196529, + "nauc_mrr_at_1_diff1": 0.244431, + "nauc_mrr_at_3_max": 0.425046, + "nauc_mrr_at_3_std": 0.196051, + "nauc_mrr_at_3_diff1": 0.215857, + "nauc_mrr_at_5_max": 0.436256, + "nauc_mrr_at_5_std": 0.202878, + "nauc_mrr_at_5_diff1": 0.240142, + "nauc_mrr_at_10_max": 0.43631, + "nauc_mrr_at_10_std": 0.213423, + "nauc_mrr_at_10_diff1": 0.222824, + "nauc_mrr_at_20_max": 0.43365, + "nauc_mrr_at_20_std": 0.196137, + "nauc_mrr_at_20_diff1": 0.220054, + "nauc_mrr_at_100_max": 0.433228, + "nauc_mrr_at_100_std": 0.196124, + "nauc_mrr_at_100_diff1": 0.222027, + "nauc_mrr_at_1000_max": 0.433228, + "nauc_mrr_at_1000_std": 0.196124, + "nauc_mrr_at_1000_diff1": 0.222027, + "main_score": 0.23612, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.4910318851470947, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ARCChallenge.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ARCChallenge.json new file mode 100644 index 0000000000..d24c4f2fe0 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ARCChallenge.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c481e0da3dcbbad8bce7721dea9085b74320a0a3", + "task_name": "ARCChallenge", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.03925, + "ndcg_at_3": 0.08529, + "ndcg_at_5": 0.10332, + "ndcg_at_10": 0.11605, + "ndcg_at_20": 0.13071, + "ndcg_at_100": 0.16499, + "ndcg_at_1000": 0.2084, + "map_at_1": 0.03925, + "map_at_3": 0.07381, + "map_at_5": 0.08387, + "map_at_10": 0.08914, + "map_at_20": 0.09316, + "map_at_100": 0.09759, + "map_at_1000": 0.099, + "recall_at_1": 0.03925, + "recall_at_3": 0.1186, + "recall_at_5": 0.16212, + "recall_at_10": 0.20137, + "recall_at_20": 0.25939, + "recall_at_100": 0.44881, + "recall_at_1000": 0.80375, + "precision_at_1": 0.03925, + "precision_at_3": 0.03953, + "precision_at_5": 0.03242, + "precision_at_10": 0.02014, + "precision_at_20": 0.01297, + "precision_at_100": 0.00449, + "precision_at_1000": 0.0008, + "mrr_at_1": 0.039249, + "mrr_at_3": 0.073805, + "mrr_at_5": 0.083874, + "mrr_at_10": 0.089143, + "mrr_at_20": 0.093157, + "mrr_at_100": 0.097589, + "mrr_at_1000": 0.099002, + "nauc_ndcg_at_1_max": 0.353748, + "nauc_ndcg_at_1_std": -0.202273, + "nauc_ndcg_at_1_diff1": -0.024802, + "nauc_ndcg_at_3_max": 0.386037, + "nauc_ndcg_at_3_std": -0.151854, + "nauc_ndcg_at_3_diff1": -0.021888, + "nauc_ndcg_at_5_max": 0.39342, + "nauc_ndcg_at_5_std": -0.141953, + "nauc_ndcg_at_5_diff1": -0.024193, + "nauc_ndcg_at_10_max": 0.37043, + "nauc_ndcg_at_10_std": -0.144946, + "nauc_ndcg_at_10_diff1": 0.002884, + "nauc_ndcg_at_20_max": 0.364688, + "nauc_ndcg_at_20_std": -0.142594, + "nauc_ndcg_at_20_diff1": 0.006663, + "nauc_ndcg_at_100_max": 0.338096, + "nauc_ndcg_at_100_std": -0.13121, + "nauc_ndcg_at_100_diff1": 0.004364, + "nauc_ndcg_at_1000_max": 0.323608, + "nauc_ndcg_at_1000_std": -0.116335, + "nauc_ndcg_at_1000_diff1": -0.003829, + "nauc_map_at_1_max": 0.353748, + "nauc_map_at_1_std": -0.202273, + "nauc_map_at_1_diff1": -0.024802, + "nauc_map_at_3_max": 0.378308, + "nauc_map_at_3_std": -0.15763, + "nauc_map_at_3_diff1": -0.023815, + "nauc_map_at_5_max": 0.38505, + "nauc_map_at_5_std": -0.150842, + "nauc_map_at_5_diff1": -0.026262, + "nauc_map_at_10_max": 0.373422, + "nauc_map_at_10_std": -0.151576, + "nauc_map_at_10_diff1": -0.01174, + "nauc_map_at_20_max": 0.371332, + "nauc_map_at_20_std": -0.150259, + "nauc_map_at_20_diff1": -0.010462, + "nauc_map_at_100_max": 0.365719, + "nauc_map_at_100_std": -0.147523, + "nauc_map_at_100_diff1": -0.010302, + "nauc_map_at_1000_max": 0.365117, + "nauc_map_at_1000_std": -0.146693, + "nauc_map_at_1000_diff1": -0.010354, + "nauc_recall_at_1_max": 0.353748, + "nauc_recall_at_1_std": -0.202273, + "nauc_recall_at_1_diff1": -0.024802, + "nauc_recall_at_3_max": 0.401197, + "nauc_recall_at_3_std": -0.14144, + "nauc_recall_at_3_diff1": -0.017931, + "nauc_recall_at_5_max": 0.407368, + "nauc_recall_at_5_std": -0.126852, + "nauc_recall_at_5_diff1": -0.019993, + "nauc_recall_at_10_max": 0.361856, + "nauc_recall_at_10_std": -0.136001, + "nauc_recall_at_10_diff1": 0.029671, + "nauc_recall_at_20_max": 0.349574, + "nauc_recall_at_20_std": -0.1328, + "nauc_recall_at_20_diff1": 0.036595, + "nauc_recall_at_100_max": 0.277469, + "nauc_recall_at_100_std": -0.10458, + "nauc_recall_at_100_diff1": 0.024265, + "nauc_recall_at_1000_max": 0.117684, + "nauc_recall_at_1000_std": 0.021399, + "nauc_recall_at_1000_diff1": -0.031764, + "nauc_precision_at_1_max": 0.353748, + "nauc_precision_at_1_std": -0.202273, + "nauc_precision_at_1_diff1": -0.024802, + "nauc_precision_at_3_max": 0.401197, + "nauc_precision_at_3_std": -0.14144, + "nauc_precision_at_3_diff1": -0.017931, + "nauc_precision_at_5_max": 0.407368, + "nauc_precision_at_5_std": -0.126852, + "nauc_precision_at_5_diff1": -0.019993, + "nauc_precision_at_10_max": 0.361856, + "nauc_precision_at_10_std": -0.136001, + "nauc_precision_at_10_diff1": 0.029671, + "nauc_precision_at_20_max": 0.349574, + "nauc_precision_at_20_std": -0.1328, + "nauc_precision_at_20_diff1": 0.036595, + "nauc_precision_at_100_max": 0.277469, + "nauc_precision_at_100_std": -0.10458, + "nauc_precision_at_100_diff1": 0.024265, + "nauc_precision_at_1000_max": 0.117684, + "nauc_precision_at_1000_std": 0.021399, + "nauc_precision_at_1000_diff1": -0.031764, + "nauc_mrr_at_1_max": 0.353748, + "nauc_mrr_at_1_std": -0.202273, + "nauc_mrr_at_1_diff1": -0.024802, + "nauc_mrr_at_3_max": 0.378308, + "nauc_mrr_at_3_std": -0.15763, + "nauc_mrr_at_3_diff1": -0.023815, + "nauc_mrr_at_5_max": 0.38505, + "nauc_mrr_at_5_std": -0.150842, + "nauc_mrr_at_5_diff1": -0.026262, + "nauc_mrr_at_10_max": 0.373422, + "nauc_mrr_at_10_std": -0.151576, + "nauc_mrr_at_10_diff1": -0.01174, + "nauc_mrr_at_20_max": 0.371332, + "nauc_mrr_at_20_std": -0.150259, + "nauc_mrr_at_20_diff1": -0.010462, + "nauc_mrr_at_100_max": 0.365719, + "nauc_mrr_at_100_std": -0.147523, + "nauc_mrr_at_100_diff1": -0.010302, + "nauc_mrr_at_1000_max": 0.365117, + "nauc_mrr_at_1000_std": -0.146692, + "nauc_mrr_at_1000_diff1": -0.010354, + "main_score": 0.11605, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7.7207465171813965, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/AlphaNLI.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/AlphaNLI.json new file mode 100644 index 0000000000..b191f1de93 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/AlphaNLI.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "303f40ef3d50918d3dc43577d33f2f7344ad72c1", + "task_name": "AlphaNLI", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.21736, + "ndcg_at_3": 0.28926, + "ndcg_at_5": 0.30862, + "ndcg_at_10": 0.32338, + "ndcg_at_20": 0.33767, + "ndcg_at_100": 0.36233, + "ndcg_at_1000": 0.38354, + "map_at_1": 0.21736, + "map_at_3": 0.27208, + "map_at_5": 0.28282, + "map_at_10": 0.28891, + "map_at_20": 0.2928, + "map_at_100": 0.29606, + "map_at_1000": 0.29674, + "recall_at_1": 0.21736, + "recall_at_3": 0.33877, + "recall_at_5": 0.38577, + "recall_at_10": 0.43146, + "recall_at_20": 0.48825, + "recall_at_100": 0.62337, + "recall_at_1000": 0.797, + "precision_at_1": 0.21736, + "precision_at_3": 0.11292, + "precision_at_5": 0.07715, + "precision_at_10": 0.04315, + "precision_at_20": 0.02441, + "precision_at_100": 0.00623, + "precision_at_1000": 0.0008, + "mrr_at_1": 0.217363, + "mrr_at_3": 0.272084, + "mrr_at_5": 0.282822, + "mrr_at_10": 0.288905, + "mrr_at_20": 0.292796, + "mrr_at_100": 0.296059, + "mrr_at_1000": 0.29674, + "nauc_ndcg_at_1_max": 0.275119, + "nauc_ndcg_at_1_std": -0.056175, + "nauc_ndcg_at_1_diff1": 0.389988, + "nauc_ndcg_at_3_max": 0.289285, + "nauc_ndcg_at_3_std": -0.055313, + "nauc_ndcg_at_3_diff1": 0.355594, + "nauc_ndcg_at_5_max": 0.273527, + "nauc_ndcg_at_5_std": -0.046119, + "nauc_ndcg_at_5_diff1": 0.341888, + "nauc_ndcg_at_10_max": 0.264966, + "nauc_ndcg_at_10_std": -0.037576, + "nauc_ndcg_at_10_diff1": 0.331942, + "nauc_ndcg_at_20_max": 0.263137, + "nauc_ndcg_at_20_std": -0.036078, + "nauc_ndcg_at_20_diff1": 0.33252, + "nauc_ndcg_at_100_max": 0.262916, + "nauc_ndcg_at_100_std": -0.031886, + "nauc_ndcg_at_100_diff1": 0.326118, + "nauc_ndcg_at_1000_max": 0.267189, + "nauc_ndcg_at_1000_std": -0.030426, + "nauc_ndcg_at_1000_diff1": 0.327817, + "nauc_map_at_1_max": 0.275119, + "nauc_map_at_1_std": -0.056175, + "nauc_map_at_1_diff1": 0.389988, + "nauc_map_at_3_max": 0.286847, + "nauc_map_at_3_std": -0.055732, + "nauc_map_at_3_diff1": 0.362875, + "nauc_map_at_5_max": 0.277955, + "nauc_map_at_5_std": -0.050036, + "nauc_map_at_5_diff1": 0.35525, + "nauc_map_at_10_max": 0.274246, + "nauc_map_at_10_std": -0.046336, + "nauc_map_at_10_diff1": 0.350885, + "nauc_map_at_20_max": 0.273801, + "nauc_map_at_20_std": -0.046173, + "nauc_map_at_20_diff1": 0.35102, + "nauc_map_at_100_max": 0.27406, + "nauc_map_at_100_std": -0.045552, + "nauc_map_at_100_diff1": 0.350343, + "nauc_map_at_1000_max": 0.274252, + "nauc_map_at_1000_std": -0.045377, + "nauc_map_at_1000_diff1": 0.350408, + "nauc_recall_at_1_max": 0.275119, + "nauc_recall_at_1_std": -0.056175, + "nauc_recall_at_1_diff1": 0.389988, + "nauc_recall_at_3_max": 0.295496, + "nauc_recall_at_3_std": -0.054114, + "nauc_recall_at_3_diff1": 0.336371, + "nauc_recall_at_5_max": 0.259511, + "nauc_recall_at_5_std": -0.03509, + "nauc_recall_at_5_diff1": 0.305014, + "nauc_recall_at_10_max": 0.235122, + "nauc_recall_at_10_std": -0.010826, + "nauc_recall_at_10_diff1": 0.276911, + "nauc_recall_at_20_max": 0.227251, + "nauc_recall_at_20_std": -0.003011, + "nauc_recall_at_20_diff1": 0.278373, + "nauc_recall_at_100_max": 0.216217, + "nauc_recall_at_100_std": 0.025549, + "nauc_recall_at_100_diff1": 0.229727, + "nauc_recall_at_1000_max": 0.240066, + "nauc_recall_at_1000_std": 0.062576, + "nauc_recall_at_1000_diff1": 0.194722, + "nauc_precision_at_1_max": 0.275119, + "nauc_precision_at_1_std": -0.056175, + "nauc_precision_at_1_diff1": 0.389988, + "nauc_precision_at_3_max": 0.295496, + "nauc_precision_at_3_std": -0.054114, + "nauc_precision_at_3_diff1": 0.336371, + "nauc_precision_at_5_max": 0.259511, + "nauc_precision_at_5_std": -0.03509, + "nauc_precision_at_5_diff1": 0.305014, + "nauc_precision_at_10_max": 0.235122, + "nauc_precision_at_10_std": -0.010826, + "nauc_precision_at_10_diff1": 0.276911, + "nauc_precision_at_20_max": 0.227251, + "nauc_precision_at_20_std": -0.003011, + "nauc_precision_at_20_diff1": 0.278373, + "nauc_precision_at_100_max": 0.216217, + "nauc_precision_at_100_std": 0.025549, + "nauc_precision_at_100_diff1": 0.229727, + "nauc_precision_at_1000_max": 0.240066, + "nauc_precision_at_1000_std": 0.062576, + "nauc_precision_at_1000_diff1": 0.194722, + "nauc_mrr_at_1_max": 0.275119, + "nauc_mrr_at_1_std": -0.056175, + "nauc_mrr_at_1_diff1": 0.389988, + "nauc_mrr_at_3_max": 0.286847, + "nauc_mrr_at_3_std": -0.055732, + "nauc_mrr_at_3_diff1": 0.362875, + "nauc_mrr_at_5_max": 0.277955, + "nauc_mrr_at_5_std": -0.050036, + "nauc_mrr_at_5_diff1": 0.35525, + "nauc_mrr_at_10_max": 0.274246, + "nauc_mrr_at_10_std": -0.046336, + "nauc_mrr_at_10_diff1": 0.350885, + "nauc_mrr_at_20_max": 0.273801, + "nauc_mrr_at_20_std": -0.046173, + "nauc_mrr_at_20_diff1": 0.35102, + "nauc_mrr_at_100_max": 0.27406, + "nauc_mrr_at_100_std": -0.045552, + "nauc_mrr_at_100_diff1": 0.350343, + "nauc_mrr_at_1000_max": 0.274252, + "nauc_mrr_at_1000_std": -0.045377, + "nauc_mrr_at_1000_diff1": 0.350408, + "main_score": 0.32338, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 84.96897578239441, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/AppsRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/AppsRetrieval.json new file mode 100644 index 0000000000..28251c2b1b --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/AppsRetrieval.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "f22508f96b7a36c2415181ed8bb76f76e04ae2d5", + "task_name": "AppsRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.05525, + "ndcg_at_3": 0.07181, + "ndcg_at_5": 0.08054, + "ndcg_at_10": 0.08909, + "ndcg_at_20": 0.09633, + "ndcg_at_100": 0.11866, + "ndcg_at_1000": 0.1599, + "map_at_1": 0.05525, + "map_at_3": 0.06764, + "map_at_5": 0.07253, + "map_at_10": 0.07604, + "map_at_20": 0.07802, + "map_at_100": 0.08079, + "map_at_1000": 0.08197, + "recall_at_1": 0.05525, + "recall_at_3": 0.08393, + "recall_at_5": 0.10491, + "recall_at_10": 0.13147, + "recall_at_20": 0.16016, + "recall_at_100": 0.28526, + "recall_at_1000": 0.63187, + "precision_at_1": 0.05525, + "precision_at_3": 0.02798, + "precision_at_5": 0.02098, + "precision_at_10": 0.01315, + "precision_at_20": 0.00801, + "precision_at_100": 0.00285, + "precision_at_1000": 0.00063, + "mrr_at_1": 0.055246, + "mrr_at_3": 0.067641, + "mrr_at_5": 0.072475, + "mrr_at_10": 0.076034, + "mrr_at_20": 0.078015, + "mrr_at_100": 0.080781, + "mrr_at_1000": 0.081962, + "nauc_ndcg_at_1_max": 0.406122, + "nauc_ndcg_at_1_std": 0.06953, + "nauc_ndcg_at_1_diff1": 0.466237, + "nauc_ndcg_at_3_max": 0.336371, + "nauc_ndcg_at_3_std": 0.062366, + "nauc_ndcg_at_3_diff1": 0.345895, + "nauc_ndcg_at_5_max": 0.314153, + "nauc_ndcg_at_5_std": 0.068649, + "nauc_ndcg_at_5_diff1": 0.307742, + "nauc_ndcg_at_10_max": 0.297173, + "nauc_ndcg_at_10_std": 0.076191, + "nauc_ndcg_at_10_diff1": 0.283142, + "nauc_ndcg_at_20_max": 0.277158, + "nauc_ndcg_at_20_std": 0.075972, + "nauc_ndcg_at_20_diff1": 0.261273, + "nauc_ndcg_at_100_max": 0.244899, + "nauc_ndcg_at_100_std": 0.079308, + "nauc_ndcg_at_100_diff1": 0.219698, + "nauc_ndcg_at_1000_max": 0.248712, + "nauc_ndcg_at_1000_std": 0.078194, + "nauc_ndcg_at_1000_diff1": 0.212881, + "nauc_map_at_1_max": 0.406122, + "nauc_map_at_1_std": 0.06953, + "nauc_map_at_1_diff1": 0.466237, + "nauc_map_at_3_max": 0.351753, + "nauc_map_at_3_std": 0.063857, + "nauc_map_at_3_diff1": 0.370432, + "nauc_map_at_5_max": 0.336925, + "nauc_map_at_5_std": 0.067512, + "nauc_map_at_5_diff1": 0.34494, + "nauc_map_at_10_max": 0.328283, + "nauc_map_at_10_std": 0.070773, + "nauc_map_at_10_diff1": 0.332077, + "nauc_map_at_20_max": 0.321198, + "nauc_map_at_20_std": 0.070578, + "nauc_map_at_20_diff1": 0.324002, + "nauc_map_at_100_max": 0.315095, + "nauc_map_at_100_std": 0.070612, + "nauc_map_at_100_diff1": 0.316031, + "nauc_map_at_1000_max": 0.314959, + "nauc_map_at_1000_std": 0.070561, + "nauc_map_at_1000_diff1": 0.315432, + "nauc_recall_at_1_max": 0.406122, + "nauc_recall_at_1_std": 0.06953, + "nauc_recall_at_1_diff1": 0.466237, + "nauc_recall_at_3_max": 0.300052, + "nauc_recall_at_3_std": 0.058872, + "nauc_recall_at_3_diff1": 0.288552, + "nauc_recall_at_5_max": 0.26574, + "nauc_recall_at_5_std": 0.071644, + "nauc_recall_at_5_diff1": 0.22912, + "nauc_recall_at_10_max": 0.235234, + "nauc_recall_at_10_std": 0.088898, + "nauc_recall_at_10_diff1": 0.186843, + "nauc_recall_at_20_max": 0.192497, + "nauc_recall_at_20_std": 0.087452, + "nauc_recall_at_20_diff1": 0.144348, + "nauc_recall_at_100_max": 0.123237, + "nauc_recall_at_100_std": 0.096905, + "nauc_recall_at_100_diff1": 0.057062, + "nauc_recall_at_1000_max": 0.131607, + "nauc_recall_at_1000_std": 0.093003, + "nauc_recall_at_1000_diff1": 0.011422, + "nauc_precision_at_1_max": 0.406122, + "nauc_precision_at_1_std": 0.06953, + "nauc_precision_at_1_diff1": 0.466237, + "nauc_precision_at_3_max": 0.300052, + "nauc_precision_at_3_std": 0.058872, + "nauc_precision_at_3_diff1": 0.288552, + "nauc_precision_at_5_max": 0.26574, + "nauc_precision_at_5_std": 0.071644, + "nauc_precision_at_5_diff1": 0.22912, + "nauc_precision_at_10_max": 0.235234, + "nauc_precision_at_10_std": 0.088898, + "nauc_precision_at_10_diff1": 0.186843, + "nauc_precision_at_20_max": 0.192497, + "nauc_precision_at_20_std": 0.087452, + "nauc_precision_at_20_diff1": 0.144348, + "nauc_precision_at_100_max": 0.123237, + "nauc_precision_at_100_std": 0.096905, + "nauc_precision_at_100_diff1": 0.057062, + "nauc_precision_at_1000_max": 0.131607, + "nauc_precision_at_1000_std": 0.093003, + "nauc_precision_at_1000_diff1": 0.011422, + "nauc_mrr_at_1_max": 0.406122, + "nauc_mrr_at_1_std": 0.06953, + "nauc_mrr_at_1_diff1": 0.466237, + "nauc_mrr_at_3_max": 0.351753, + "nauc_mrr_at_3_std": 0.063857, + "nauc_mrr_at_3_diff1": 0.370432, + "nauc_mrr_at_5_max": 0.337241, + "nauc_mrr_at_5_std": 0.06785, + "nauc_mrr_at_5_diff1": 0.345476, + "nauc_mrr_at_10_max": 0.328333, + "nauc_mrr_at_10_std": 0.070828, + "nauc_mrr_at_10_diff1": 0.332162, + "nauc_mrr_at_20_max": 0.321235, + "nauc_mrr_at_20_std": 0.07062, + "nauc_mrr_at_20_diff1": 0.324075, + "nauc_mrr_at_100_max": 0.315132, + "nauc_mrr_at_100_std": 0.070653, + "nauc_mrr_at_100_diff1": 0.316102, + "nauc_mrr_at_1000_max": 0.314996, + "nauc_mrr_at_1000_std": 0.070602, + "nauc_mrr_at_1000_diff1": 0.315503, + "main_score": 0.08909, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 63.48354768753052, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ArguAna.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ArguAna.json new file mode 100644 index 0000000000..b20fc08fc9 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ArguAna.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a", + "task_name": "ArguAna", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.29516, + "ndcg_at_3": 0.4569, + "ndcg_at_5": 0.50991, + "ndcg_at_10": 0.5616, + "ndcg_at_20": 0.5823, + "ndcg_at_100": 0.59084, + "ndcg_at_1000": 0.59194, + "map_at_1": 0.29516, + "map_at_3": 0.41631, + "map_at_5": 0.44572, + "map_at_10": 0.46689, + "map_at_20": 0.47267, + "map_at_100": 0.47404, + "map_at_1000": 0.4741, + "recall_at_1": 0.29516, + "recall_at_3": 0.57468, + "recall_at_5": 0.70341, + "recall_at_10": 0.86415, + "recall_at_20": 0.94523, + "recall_at_100": 0.98862, + "recall_at_1000": 0.99644, + "precision_at_1": 0.29516, + "precision_at_3": 0.19156, + "precision_at_5": 0.14068, + "precision_at_10": 0.08642, + "precision_at_20": 0.04726, + "precision_at_100": 0.00989, + "precision_at_1000": 0.001, + "mrr_at_1": 0.298009, + "mrr_at_3": 0.418445, + "mrr_at_5": 0.446965, + "mrr_at_10": 0.468279, + "mrr_at_20": 0.474054, + "mrr_at_100": 0.475425, + "mrr_at_1000": 0.475483, + "nauc_ndcg_at_1_max": -0.089609, + "nauc_ndcg_at_1_std": -0.154713, + "nauc_ndcg_at_1_diff1": 0.105282, + "nauc_ndcg_at_3_max": -0.101433, + "nauc_ndcg_at_3_std": -0.159956, + "nauc_ndcg_at_3_diff1": 0.045357, + "nauc_ndcg_at_5_max": -0.077888, + "nauc_ndcg_at_5_std": -0.143172, + "nauc_ndcg_at_5_diff1": 0.058039, + "nauc_ndcg_at_10_max": -0.068837, + "nauc_ndcg_at_10_std": -0.146382, + "nauc_ndcg_at_10_diff1": 0.058981, + "nauc_ndcg_at_20_max": -0.064883, + "nauc_ndcg_at_20_std": -0.150601, + "nauc_ndcg_at_20_diff1": 0.070764, + "nauc_ndcg_at_100_max": -0.076064, + "nauc_ndcg_at_100_std": -0.146032, + "nauc_ndcg_at_100_diff1": 0.068368, + "nauc_ndcg_at_1000_max": -0.079675, + "nauc_ndcg_at_1000_std": -0.147678, + "nauc_ndcg_at_1000_diff1": 0.066924, + "nauc_map_at_1_max": -0.089609, + "nauc_map_at_1_std": -0.154713, + "nauc_map_at_1_diff1": 0.105282, + "nauc_map_at_3_max": -0.099739, + "nauc_map_at_3_std": -0.1586, + "nauc_map_at_3_diff1": 0.05803, + "nauc_map_at_5_max": -0.087295, + "nauc_map_at_5_std": -0.149587, + "nauc_map_at_5_diff1": 0.065526, + "nauc_map_at_10_max": -0.085304, + "nauc_map_at_10_std": -0.151347, + "nauc_map_at_10_diff1": 0.0657, + "nauc_map_at_20_max": -0.084568, + "nauc_map_at_20_std": -0.152259, + "nauc_map_at_20_diff1": 0.068523, + "nauc_map_at_100_max": -0.085852, + "nauc_map_at_100_std": -0.151479, + "nauc_map_at_100_diff1": 0.068393, + "nauc_map_at_1000_max": -0.086002, + "nauc_map_at_1000_std": -0.151546, + "nauc_map_at_1000_diff1": 0.068336, + "nauc_recall_at_1_max": -0.089609, + "nauc_recall_at_1_std": -0.154713, + "nauc_recall_at_1_diff1": 0.105282, + "nauc_recall_at_3_max": -0.106027, + "nauc_recall_at_3_std": -0.164081, + "nauc_recall_at_3_diff1": 0.007963, + "nauc_recall_at_5_max": -0.038487, + "nauc_recall_at_5_std": -0.11686, + "nauc_recall_at_5_diff1": 0.033976, + "nauc_recall_at_10_max": 0.065771, + "nauc_recall_at_10_std": -0.108536, + "nauc_recall_at_10_diff1": 0.023857, + "nauc_recall_at_20_max": 0.318662, + "nauc_recall_at_20_std": -0.136525, + "nauc_recall_at_20_diff1": 0.183976, + "nauc_recall_at_100_max": 0.502233, + "nauc_recall_at_100_std": 0.328504, + "nauc_recall_at_100_diff1": 0.290713, + "nauc_recall_at_1000_max": 0.03312, + "nauc_recall_at_1000_std": 0.580858, + "nauc_recall_at_1000_diff1": 0.072317, + "nauc_precision_at_1_max": -0.089609, + "nauc_precision_at_1_std": -0.154713, + "nauc_precision_at_1_diff1": 0.105282, + "nauc_precision_at_3_max": -0.106027, + "nauc_precision_at_3_std": -0.164081, + "nauc_precision_at_3_diff1": 0.007963, + "nauc_precision_at_5_max": -0.038487, + "nauc_precision_at_5_std": -0.11686, + "nauc_precision_at_5_diff1": 0.033976, + "nauc_precision_at_10_max": 0.065771, + "nauc_precision_at_10_std": -0.108536, + "nauc_precision_at_10_diff1": 0.023857, + "nauc_precision_at_20_max": 0.318662, + "nauc_precision_at_20_std": -0.136525, + "nauc_precision_at_20_diff1": 0.183976, + "nauc_precision_at_100_max": 0.502233, + "nauc_precision_at_100_std": 0.328504, + "nauc_precision_at_100_diff1": 0.290713, + "nauc_precision_at_1000_max": 0.03312, + "nauc_precision_at_1000_std": 0.580858, + "nauc_precision_at_1000_diff1": 0.072317, + "nauc_mrr_at_1_max": -0.093616, + "nauc_mrr_at_1_std": -0.152803, + "nauc_mrr_at_1_diff1": 0.096532, + "nauc_mrr_at_3_max": -0.107114, + "nauc_mrr_at_3_std": -0.158022, + "nauc_mrr_at_3_diff1": 0.044973, + "nauc_mrr_at_5_max": -0.094152, + "nauc_mrr_at_5_std": -0.149176, + "nauc_mrr_at_5_diff1": 0.054589, + "nauc_mrr_at_10_max": -0.092945, + "nauc_mrr_at_10_std": -0.150808, + "nauc_mrr_at_10_diff1": 0.053944, + "nauc_mrr_at_20_max": -0.091773, + "nauc_mrr_at_20_std": -0.151243, + "nauc_mrr_at_20_diff1": 0.056982, + "nauc_mrr_at_100_max": -0.093101, + "nauc_mrr_at_100_std": -0.150463, + "nauc_mrr_at_100_diff1": 0.056787, + "nauc_mrr_at_1000_max": -0.093251, + "nauc_mrr_at_1000_std": -0.15053, + "nauc_mrr_at_1000_diff1": 0.056728, + "main_score": 0.5616, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 43.68112850189209, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/BrightLongRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/BrightLongRetrieval.json new file mode 100644 index 0000000000..b77f0bbc11 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/BrightLongRetrieval.json @@ -0,0 +1,1187 @@ +{ + "dataset_revision": "a75a0eb", + "task_name": "BrightLongRetrieval", + "mteb_version": "1.38.9", + "scores": { + "long": [ + { + "ndcg_at_1": 0.3301, + "ndcg_at_3": 0.40047, + "ndcg_at_5": 0.44123, + "ndcg_at_10": 0.50473, + "ndcg_at_20": 0.53539, + "ndcg_at_100": 0.56041, + "ndcg_at_1000": 0.56388, + "map_at_1": 0.25324, + "map_at_3": 0.36516, + "map_at_5": 0.3904, + "map_at_10": 0.41768, + "map_at_20": 0.42729, + "map_at_100": 0.43169, + "map_at_1000": 0.43187, + "recall_at_1": 0.25324, + "recall_at_3": 0.46117, + "recall_at_5": 0.56068, + "recall_at_10": 0.74757, + "recall_at_20": 0.85761, + "recall_at_100": 0.97573, + "recall_at_1000": 1.0, + "precision_at_1": 0.3301, + "precision_at_3": 0.19741, + "precision_at_5": 0.14175, + "precision_at_10": 0.09417, + "precision_at_20": 0.05534, + "precision_at_100": 0.01272, + "precision_at_1000": 0.0013, + "mrr_at_1": 0.320388, + "mrr_at_3": 0.398058, + "mrr_at_5": 0.417961, + "mrr_at_10": 0.444822, + "mrr_at_20": 0.451709, + "mrr_at_100": 0.455158, + "mrr_at_1000": 0.455317, + "nauc_ndcg_at_1_max": 0.065331, + "nauc_ndcg_at_1_std": 0.086307, + "nauc_ndcg_at_1_diff1": 0.228822, + "nauc_ndcg_at_3_max": 0.100874, + "nauc_ndcg_at_3_std": 0.066849, + "nauc_ndcg_at_3_diff1": 0.144636, + "nauc_ndcg_at_5_max": 0.075564, + "nauc_ndcg_at_5_std": 0.05031, + "nauc_ndcg_at_5_diff1": 0.146239, + "nauc_ndcg_at_10_max": 0.09758, + "nauc_ndcg_at_10_std": 0.036719, + "nauc_ndcg_at_10_diff1": 0.165361, + "nauc_ndcg_at_20_max": 0.10076, + "nauc_ndcg_at_20_std": 0.033783, + "nauc_ndcg_at_20_diff1": 0.178693, + "nauc_ndcg_at_100_max": 0.107392, + "nauc_ndcg_at_100_std": 0.054927, + "nauc_ndcg_at_100_diff1": 0.180499, + "nauc_ndcg_at_1000_max": 0.097829, + "nauc_ndcg_at_1000_std": 0.056799, + "nauc_ndcg_at_1000_diff1": 0.175673, + "nauc_map_at_1_max": 0.107989, + "nauc_map_at_1_std": 0.050028, + "nauc_map_at_1_diff1": 0.208658, + "nauc_map_at_3_max": 0.123682, + "nauc_map_at_3_std": 0.061872, + "nauc_map_at_3_diff1": 0.146655, + "nauc_map_at_5_max": 0.108063, + "nauc_map_at_5_std": 0.052941, + "nauc_map_at_5_diff1": 0.147284, + "nauc_map_at_10_max": 0.11472, + "nauc_map_at_10_std": 0.046917, + "nauc_map_at_10_diff1": 0.150679, + "nauc_map_at_20_max": 0.114435, + "nauc_map_at_20_std": 0.047013, + "nauc_map_at_20_diff1": 0.155496, + "nauc_map_at_100_max": 0.115742, + "nauc_map_at_100_std": 0.050867, + "nauc_map_at_100_diff1": 0.15649, + "nauc_map_at_1000_max": 0.115337, + "nauc_map_at_1000_std": 0.051025, + "nauc_map_at_1000_diff1": 0.156309, + "nauc_recall_at_1_max": 0.107989, + "nauc_recall_at_1_std": 0.050028, + "nauc_recall_at_1_diff1": 0.208658, + "nauc_recall_at_3_max": 0.090662, + "nauc_recall_at_3_std": 0.06949, + "nauc_recall_at_3_diff1": 0.106298, + "nauc_recall_at_5_max": 0.02363, + "nauc_recall_at_5_std": 0.024262, + "nauc_recall_at_5_diff1": 0.095998, + "nauc_recall_at_10_max": 0.096113, + "nauc_recall_at_10_std": -0.056915, + "nauc_recall_at_10_diff1": 0.145172, + "nauc_recall_at_20_max": 0.146805, + "nauc_recall_at_20_std": -0.151944, + "nauc_recall_at_20_diff1": 0.235454, + "nauc_recall_at_100_max": 0.799496, + "nauc_recall_at_100_std": -0.061767, + "nauc_recall_at_100_diff1": 0.537002, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.065331, + "nauc_precision_at_1_std": 0.086307, + "nauc_precision_at_1_diff1": 0.228822, + "nauc_precision_at_3_max": 0.038988, + "nauc_precision_at_3_std": 0.111755, + "nauc_precision_at_3_diff1": 0.100735, + "nauc_precision_at_5_max": -0.016285, + "nauc_precision_at_5_std": 0.085733, + "nauc_precision_at_5_diff1": 0.086761, + "nauc_precision_at_10_max": -0.010593, + "nauc_precision_at_10_std": 0.068866, + "nauc_precision_at_10_diff1": 0.143465, + "nauc_precision_at_20_max": -0.059938, + "nauc_precision_at_20_std": 0.060032, + "nauc_precision_at_20_diff1": 0.108977, + "nauc_precision_at_100_max": -0.092666, + "nauc_precision_at_100_std": 0.15376, + "nauc_precision_at_100_diff1": 0.114464, + "nauc_precision_at_1000_max": -0.147405, + "nauc_precision_at_1000_std": 0.152398, + "nauc_precision_at_1000_diff1": 0.076319, + "nauc_mrr_at_1_max": 0.080997, + "nauc_mrr_at_1_std": 0.102075, + "nauc_mrr_at_1_diff1": 0.257657, + "nauc_mrr_at_3_max": 0.07106, + "nauc_mrr_at_3_std": 0.07392, + "nauc_mrr_at_3_diff1": 0.206392, + "nauc_mrr_at_5_max": 0.059495, + "nauc_mrr_at_5_std": 0.059199, + "nauc_mrr_at_5_diff1": 0.206724, + "nauc_mrr_at_10_max": 0.070789, + "nauc_mrr_at_10_std": 0.060214, + "nauc_mrr_at_10_diff1": 0.218905, + "nauc_mrr_at_20_max": 0.073333, + "nauc_mrr_at_20_std": 0.059676, + "nauc_mrr_at_20_diff1": 0.222528, + "nauc_mrr_at_100_max": 0.073178, + "nauc_mrr_at_100_std": 0.061524, + "nauc_mrr_at_100_diff1": 0.220546, + "nauc_mrr_at_1000_max": 0.072793, + "nauc_mrr_at_1000_std": 0.061739, + "nauc_mrr_at_1000_diff1": 0.22046, + "main_score": 0.50473, + "hf_subset": "biology", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.43103, + "ndcg_at_3": 0.47372, + "ndcg_at_5": 0.5163, + "ndcg_at_10": 0.54874, + "ndcg_at_20": 0.57754, + "ndcg_at_100": 0.60986, + "ndcg_at_1000": 0.62046, + "map_at_1": 0.33477, + "map_at_3": 0.42325, + "map_at_5": 0.45256, + "map_at_10": 0.46887, + "map_at_20": 0.47997, + "map_at_100": 0.48686, + "map_at_1000": 0.4873, + "recall_at_1": 0.33477, + "recall_at_3": 0.51365, + "recall_at_5": 0.60129, + "recall_at_10": 0.69181, + "recall_at_20": 0.7852, + "recall_at_100": 0.92385, + "recall_at_1000": 1.0, + "precision_at_1": 0.43103, + "precision_at_3": 0.24138, + "precision_at_5": 0.17759, + "precision_at_10": 0.10345, + "precision_at_20": 0.06078, + "precision_at_100": 0.01483, + "precision_at_1000": 0.00161, + "mrr_at_1": 0.431034, + "mrr_at_3": 0.53592, + "mrr_at_5": 0.550144, + "mrr_at_10": 0.559062, + "mrr_at_20": 0.563695, + "mrr_at_100": 0.566634, + "mrr_at_1000": 0.566747, + "nauc_ndcg_at_1_max": 0.3866, + "nauc_ndcg_at_1_std": -0.020471, + "nauc_ndcg_at_1_diff1": 0.375249, + "nauc_ndcg_at_3_max": 0.332504, + "nauc_ndcg_at_3_std": -0.08356, + "nauc_ndcg_at_3_diff1": 0.290001, + "nauc_ndcg_at_5_max": 0.369899, + "nauc_ndcg_at_5_std": -0.075146, + "nauc_ndcg_at_5_diff1": 0.287876, + "nauc_ndcg_at_10_max": 0.358992, + "nauc_ndcg_at_10_std": -0.083561, + "nauc_ndcg_at_10_diff1": 0.278927, + "nauc_ndcg_at_20_max": 0.34978, + "nauc_ndcg_at_20_std": -0.058896, + "nauc_ndcg_at_20_diff1": 0.282844, + "nauc_ndcg_at_100_max": 0.355595, + "nauc_ndcg_at_100_std": -0.076049, + "nauc_ndcg_at_100_diff1": 0.281355, + "nauc_ndcg_at_1000_max": 0.354733, + "nauc_ndcg_at_1000_std": -0.062431, + "nauc_ndcg_at_1000_diff1": 0.282458, + "nauc_map_at_1_max": 0.422006, + "nauc_map_at_1_std": -0.004998, + "nauc_map_at_1_diff1": 0.370071, + "nauc_map_at_3_max": 0.351769, + "nauc_map_at_3_std": -0.072655, + "nauc_map_at_3_diff1": 0.309562, + "nauc_map_at_5_max": 0.369436, + "nauc_map_at_5_std": -0.066362, + "nauc_map_at_5_diff1": 0.301173, + "nauc_map_at_10_max": 0.365599, + "nauc_map_at_10_std": -0.072142, + "nauc_map_at_10_diff1": 0.293179, + "nauc_map_at_20_max": 0.365079, + "nauc_map_at_20_std": -0.062466, + "nauc_map_at_20_diff1": 0.295866, + "nauc_map_at_100_max": 0.365254, + "nauc_map_at_100_std": -0.063718, + "nauc_map_at_100_diff1": 0.295036, + "nauc_map_at_1000_max": 0.365189, + "nauc_map_at_1000_std": -0.063063, + "nauc_map_at_1000_diff1": 0.29491, + "nauc_recall_at_1_max": 0.422006, + "nauc_recall_at_1_std": -0.004998, + "nauc_recall_at_1_diff1": 0.370071, + "nauc_recall_at_3_max": 0.295004, + "nauc_recall_at_3_std": -0.113734, + "nauc_recall_at_3_diff1": 0.233132, + "nauc_recall_at_5_max": 0.37522, + "nauc_recall_at_5_std": -0.101024, + "nauc_recall_at_5_diff1": 0.235101, + "nauc_recall_at_10_max": 0.318668, + "nauc_recall_at_10_std": -0.132815, + "nauc_recall_at_10_diff1": 0.203731, + "nauc_recall_at_20_max": 0.249728, + "nauc_recall_at_20_std": -0.065819, + "nauc_recall_at_20_diff1": 0.188749, + "nauc_recall_at_100_max": 0.330229, + "nauc_recall_at_100_std": -0.394329, + "nauc_recall_at_100_diff1": 0.15505, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.3866, + "nauc_precision_at_1_std": -0.020471, + "nauc_precision_at_1_diff1": 0.375249, + "nauc_precision_at_3_max": 0.095679, + "nauc_precision_at_3_std": -0.171969, + "nauc_precision_at_3_diff1": 0.132132, + "nauc_precision_at_5_max": 0.11205, + "nauc_precision_at_5_std": -0.073121, + "nauc_precision_at_5_diff1": 0.050428, + "nauc_precision_at_10_max": 0.021731, + "nauc_precision_at_10_std": -0.07499, + "nauc_precision_at_10_diff1": -0.063555, + "nauc_precision_at_20_max": -0.065105, + "nauc_precision_at_20_std": 0.054679, + "nauc_precision_at_20_diff1": -0.104713, + "nauc_precision_at_100_max": -0.190915, + "nauc_precision_at_100_std": 0.044996, + "nauc_precision_at_100_diff1": -0.19259, + "nauc_precision_at_1000_max": -0.253656, + "nauc_precision_at_1000_std": 0.095361, + "nauc_precision_at_1000_diff1": -0.258662, + "nauc_mrr_at_1_max": 0.3866, + "nauc_mrr_at_1_std": -0.020471, + "nauc_mrr_at_1_diff1": 0.375249, + "nauc_mrr_at_3_max": 0.32426, + "nauc_mrr_at_3_std": -0.064008, + "nauc_mrr_at_3_diff1": 0.288287, + "nauc_mrr_at_5_max": 0.35316, + "nauc_mrr_at_5_std": -0.063014, + "nauc_mrr_at_5_diff1": 0.307253, + "nauc_mrr_at_10_max": 0.353807, + "nauc_mrr_at_10_std": -0.063057, + "nauc_mrr_at_10_diff1": 0.312469, + "nauc_mrr_at_20_max": 0.347555, + "nauc_mrr_at_20_std": -0.058803, + "nauc_mrr_at_20_diff1": 0.31199, + "nauc_mrr_at_100_max": 0.350357, + "nauc_mrr_at_100_std": -0.061337, + "nauc_mrr_at_100_diff1": 0.310826, + "nauc_mrr_at_1000_max": 0.350266, + "nauc_mrr_at_1000_std": -0.061106, + "nauc_mrr_at_1000_diff1": 0.310865, + "main_score": 0.54874, + "hf_subset": "earth_science", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.14563, + "ndcg_at_3": 0.2504, + "ndcg_at_5": 0.27652, + "ndcg_at_10": 0.31009, + "ndcg_at_20": 0.32009, + "ndcg_at_100": 0.35829, + "ndcg_at_1000": 0.39197, + "map_at_1": 0.14078, + "map_at_3": 0.22006, + "map_at_5": 0.23414, + "map_at_10": 0.24758, + "map_at_20": 0.25041, + "map_at_100": 0.25569, + "map_at_1000": 0.25702, + "recall_at_1": 0.14078, + "recall_at_3": 0.33495, + "recall_at_5": 0.39806, + "recall_at_10": 0.50162, + "recall_at_20": 0.54045, + "recall_at_100": 0.74434, + "recall_at_1000": 1.0, + "precision_at_1": 0.14563, + "precision_at_3": 0.11327, + "precision_at_5": 0.08155, + "precision_at_10": 0.05243, + "precision_at_20": 0.02816, + "precision_at_100": 0.00777, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.135922, + "mrr_at_3": 0.220065, + "mrr_at_5": 0.235113, + "mrr_at_10": 0.248756, + "mrr_at_20": 0.251586, + "mrr_at_100": 0.25682, + "mrr_at_1000": 0.257997, + "nauc_ndcg_at_1_max": 0.231722, + "nauc_ndcg_at_1_std": 0.073452, + "nauc_ndcg_at_1_diff1": 0.268649, + "nauc_ndcg_at_3_max": 0.164807, + "nauc_ndcg_at_3_std": 0.043317, + "nauc_ndcg_at_3_diff1": 0.002255, + "nauc_ndcg_at_5_max": 0.20078, + "nauc_ndcg_at_5_std": 0.075952, + "nauc_ndcg_at_5_diff1": 0.020894, + "nauc_ndcg_at_10_max": 0.255913, + "nauc_ndcg_at_10_std": 0.110182, + "nauc_ndcg_at_10_diff1": 0.0017, + "nauc_ndcg_at_20_max": 0.261847, + "nauc_ndcg_at_20_std": 0.092472, + "nauc_ndcg_at_20_diff1": 0.003764, + "nauc_ndcg_at_100_max": 0.252079, + "nauc_ndcg_at_100_std": 0.087433, + "nauc_ndcg_at_100_diff1": 0.020439, + "nauc_ndcg_at_1000_max": 0.239751, + "nauc_ndcg_at_1000_std": 0.081421, + "nauc_ndcg_at_1000_diff1": 0.039889, + "nauc_map_at_1_max": 0.256789, + "nauc_map_at_1_std": 0.062805, + "nauc_map_at_1_diff1": 0.236796, + "nauc_map_at_3_max": 0.190119, + "nauc_map_at_3_std": 0.041925, + "nauc_map_at_3_diff1": 0.04881, + "nauc_map_at_5_max": 0.211668, + "nauc_map_at_5_std": 0.059272, + "nauc_map_at_5_diff1": 0.057561, + "nauc_map_at_10_max": 0.234389, + "nauc_map_at_10_std": 0.074197, + "nauc_map_at_10_diff1": 0.048388, + "nauc_map_at_20_max": 0.236399, + "nauc_map_at_20_std": 0.068889, + "nauc_map_at_20_diff1": 0.049911, + "nauc_map_at_100_max": 0.236434, + "nauc_map_at_100_std": 0.066298, + "nauc_map_at_100_diff1": 0.05329, + "nauc_map_at_1000_max": 0.235588, + "nauc_map_at_1000_std": 0.066235, + "nauc_map_at_1000_diff1": 0.054416, + "nauc_recall_at_1_max": 0.256789, + "nauc_recall_at_1_std": 0.062805, + "nauc_recall_at_1_diff1": 0.236796, + "nauc_recall_at_3_max": 0.116937, + "nauc_recall_at_3_std": 0.040844, + "nauc_recall_at_3_diff1": -0.126372, + "nauc_recall_at_5_max": 0.195169, + "nauc_recall_at_5_std": 0.103698, + "nauc_recall_at_5_diff1": -0.08218, + "nauc_recall_at_10_max": 0.335678, + "nauc_recall_at_10_std": 0.185384, + "nauc_recall_at_10_diff1": -0.13634, + "nauc_recall_at_20_max": 0.35639, + "nauc_recall_at_20_std": 0.127639, + "nauc_recall_at_20_diff1": -0.139074, + "nauc_recall_at_100_max": 0.314872, + "nauc_recall_at_100_std": 0.137092, + "nauc_recall_at_100_diff1": -0.119507, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.231722, + "nauc_precision_at_1_std": 0.073452, + "nauc_precision_at_1_diff1": 0.268649, + "nauc_precision_at_3_max": 0.104716, + "nauc_precision_at_3_std": 0.048515, + "nauc_precision_at_3_diff1": -0.102058, + "nauc_precision_at_5_max": 0.169996, + "nauc_precision_at_5_std": 0.134611, + "nauc_precision_at_5_diff1": -0.051132, + "nauc_precision_at_10_max": 0.339317, + "nauc_precision_at_10_std": 0.250563, + "nauc_precision_at_10_diff1": -0.080141, + "nauc_precision_at_20_max": 0.360181, + "nauc_precision_at_20_std": 0.197879, + "nauc_precision_at_20_diff1": -0.080834, + "nauc_precision_at_100_max": 0.270789, + "nauc_precision_at_100_std": 0.258384, + "nauc_precision_at_100_diff1": -0.012362, + "nauc_precision_at_1000_max": -0.138536, + "nauc_precision_at_1000_std": 0.331646, + "nauc_precision_at_1000_diff1": 0.329622, + "nauc_mrr_at_1_max": 0.28906, + "nauc_mrr_at_1_std": 0.025448, + "nauc_mrr_at_1_diff1": 0.3279, + "nauc_mrr_at_3_max": 0.192408, + "nauc_mrr_at_3_std": 0.031788, + "nauc_mrr_at_3_diff1": 0.097099, + "nauc_mrr_at_5_max": 0.210542, + "nauc_mrr_at_5_std": 0.055334, + "nauc_mrr_at_5_diff1": 0.10749, + "nauc_mrr_at_10_max": 0.234135, + "nauc_mrr_at_10_std": 0.071092, + "nauc_mrr_at_10_diff1": 0.099119, + "nauc_mrr_at_20_max": 0.236145, + "nauc_mrr_at_20_std": 0.06577, + "nauc_mrr_at_20_diff1": 0.10083, + "nauc_mrr_at_100_max": 0.236391, + "nauc_mrr_at_100_std": 0.062326, + "nauc_mrr_at_100_diff1": 0.104617, + "nauc_mrr_at_1000_max": 0.2357, + "nauc_mrr_at_1000_std": 0.062069, + "nauc_mrr_at_1000_diff1": 0.105512, + "main_score": 0.31009, + "hf_subset": "economics", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.17822, + "ndcg_at_3": 0.24212, + "ndcg_at_5": 0.25702, + "ndcg_at_10": 0.26466, + "ndcg_at_20": 0.28254, + "ndcg_at_100": 0.30058, + "ndcg_at_1000": 0.36492, + "map_at_1": 0.15545, + "map_at_3": 0.2198, + "map_at_5": 0.22871, + "map_at_10": 0.23131, + "map_at_20": 0.23642, + "map_at_100": 0.2385, + "map_at_1000": 0.24109, + "recall_at_1": 0.15545, + "recall_at_3": 0.28416, + "recall_at_5": 0.32376, + "recall_at_10": 0.34851, + "recall_at_20": 0.41617, + "recall_at_100": 0.51848, + "recall_at_1000": 1.0, + "precision_at_1": 0.17822, + "precision_at_3": 0.10891, + "precision_at_5": 0.07327, + "precision_at_10": 0.0396, + "precision_at_20": 0.02376, + "precision_at_100": 0.00584, + "precision_at_1000": 0.00115, + "mrr_at_1": 0.178218, + "mrr_at_3": 0.239274, + "mrr_at_5": 0.248185, + "mrr_at_10": 0.251403, + "mrr_at_20": 0.256403, + "mrr_at_100": 0.258369, + "mrr_at_1000": 0.260653, + "nauc_ndcg_at_1_max": 0.190136, + "nauc_ndcg_at_1_std": -0.113914, + "nauc_ndcg_at_1_diff1": 0.205968, + "nauc_ndcg_at_3_max": 0.260839, + "nauc_ndcg_at_3_std": -0.063579, + "nauc_ndcg_at_3_diff1": 0.162854, + "nauc_ndcg_at_5_max": 0.237087, + "nauc_ndcg_at_5_std": -0.056412, + "nauc_ndcg_at_5_diff1": 0.145037, + "nauc_ndcg_at_10_max": 0.242776, + "nauc_ndcg_at_10_std": -0.055714, + "nauc_ndcg_at_10_diff1": 0.122801, + "nauc_ndcg_at_20_max": 0.23679, + "nauc_ndcg_at_20_std": -0.050554, + "nauc_ndcg_at_20_diff1": 0.104725, + "nauc_ndcg_at_100_max": 0.228908, + "nauc_ndcg_at_100_std": -0.047978, + "nauc_ndcg_at_100_diff1": 0.087112, + "nauc_ndcg_at_1000_max": 0.230415, + "nauc_ndcg_at_1000_std": -0.06155, + "nauc_ndcg_at_1000_diff1": 0.116431, + "nauc_map_at_1_max": 0.173285, + "nauc_map_at_1_std": -0.066944, + "nauc_map_at_1_diff1": 0.16296, + "nauc_map_at_3_max": 0.244694, + "nauc_map_at_3_std": -0.06403, + "nauc_map_at_3_diff1": 0.154419, + "nauc_map_at_5_max": 0.232328, + "nauc_map_at_5_std": -0.061012, + "nauc_map_at_5_diff1": 0.142985, + "nauc_map_at_10_max": 0.234447, + "nauc_map_at_10_std": -0.060695, + "nauc_map_at_10_diff1": 0.134968, + "nauc_map_at_20_max": 0.233146, + "nauc_map_at_20_std": -0.059511, + "nauc_map_at_20_diff1": 0.130078, + "nauc_map_at_100_max": 0.231482, + "nauc_map_at_100_std": -0.0588, + "nauc_map_at_100_diff1": 0.127786, + "nauc_map_at_1000_max": 0.231552, + "nauc_map_at_1000_std": -0.060226, + "nauc_map_at_1000_diff1": 0.128821, + "nauc_recall_at_1_max": 0.173285, + "nauc_recall_at_1_std": -0.066944, + "nauc_recall_at_1_diff1": 0.16296, + "nauc_recall_at_3_max": 0.285795, + "nauc_recall_at_3_std": -0.037738, + "nauc_recall_at_3_diff1": 0.147694, + "nauc_recall_at_5_max": 0.233798, + "nauc_recall_at_5_std": -0.030688, + "nauc_recall_at_5_diff1": 0.108293, + "nauc_recall_at_10_max": 0.251635, + "nauc_recall_at_10_std": -0.027241, + "nauc_recall_at_10_diff1": 0.046525, + "nauc_recall_at_20_max": 0.233833, + "nauc_recall_at_20_std": -0.011289, + "nauc_recall_at_20_diff1": -0.013891, + "nauc_recall_at_100_max": 0.202897, + "nauc_recall_at_100_std": -0.001712, + "nauc_recall_at_100_diff1": -0.10683, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.190136, + "nauc_precision_at_1_std": -0.113914, + "nauc_precision_at_1_diff1": 0.205968, + "nauc_precision_at_3_max": 0.314588, + "nauc_precision_at_3_std": -0.060631, + "nauc_precision_at_3_diff1": 0.172817, + "nauc_precision_at_5_max": 0.271607, + "nauc_precision_at_5_std": -0.054459, + "nauc_precision_at_5_diff1": 0.139146, + "nauc_precision_at_10_max": 0.285059, + "nauc_precision_at_10_std": -0.055535, + "nauc_precision_at_10_diff1": 0.071991, + "nauc_precision_at_20_max": 0.269907, + "nauc_precision_at_20_std": -0.032259, + "nauc_precision_at_20_diff1": 0.036093, + "nauc_precision_at_100_max": 0.222542, + "nauc_precision_at_100_std": -0.014739, + "nauc_precision_at_100_diff1": -0.043906, + "nauc_precision_at_1000_max": 0.053364, + "nauc_precision_at_1000_std": -0.061751, + "nauc_precision_at_1000_diff1": 0.014164, + "nauc_mrr_at_1_max": 0.190136, + "nauc_mrr_at_1_std": -0.113914, + "nauc_mrr_at_1_diff1": 0.205968, + "nauc_mrr_at_3_max": 0.245907, + "nauc_mrr_at_3_std": -0.086106, + "nauc_mrr_at_3_diff1": 0.197069, + "nauc_mrr_at_5_max": 0.234295, + "nauc_mrr_at_5_std": -0.083165, + "nauc_mrr_at_5_diff1": 0.186128, + "nauc_mrr_at_10_max": 0.235943, + "nauc_mrr_at_10_std": -0.083535, + "nauc_mrr_at_10_diff1": 0.176726, + "nauc_mrr_at_20_max": 0.231103, + "nauc_mrr_at_20_std": -0.08159, + "nauc_mrr_at_20_diff1": 0.169295, + "nauc_mrr_at_100_max": 0.229848, + "nauc_mrr_at_100_std": -0.081155, + "nauc_mrr_at_100_diff1": 0.167229, + "nauc_mrr_at_1000_max": 0.229947, + "nauc_mrr_at_1000_std": -0.082398, + "nauc_mrr_at_1000_diff1": 0.168483, + "main_score": 0.26466, + "hf_subset": "psychology", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.12871, + "ndcg_at_3": 0.18043, + "ndcg_at_5": 0.20366, + "ndcg_at_10": 0.23457, + "ndcg_at_20": 0.26066, + "ndcg_at_100": 0.28868, + "ndcg_at_1000": 0.33649, + "map_at_1": 0.12871, + "map_at_3": 0.16832, + "map_at_5": 0.18119, + "map_at_10": 0.19376, + "map_at_20": 0.20172, + "map_at_100": 0.20504, + "map_at_1000": 0.20666, + "recall_at_1": 0.12871, + "recall_at_3": 0.21287, + "recall_at_5": 0.26733, + "recall_at_10": 0.36139, + "recall_at_20": 0.4604, + "recall_at_100": 0.61881, + "recall_at_1000": 1.0, + "precision_at_1": 0.12871, + "precision_at_3": 0.07261, + "precision_at_5": 0.05545, + "precision_at_10": 0.03861, + "precision_at_20": 0.02475, + "precision_at_100": 0.00663, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.128713, + "mrr_at_3": 0.169967, + "mrr_at_5": 0.183828, + "mrr_at_10": 0.196503, + "mrr_at_20": 0.203057, + "mrr_at_100": 0.206163, + "mrr_at_1000": 0.207721, + "nauc_ndcg_at_1_max": 0.276821, + "nauc_ndcg_at_1_std": 0.038722, + "nauc_ndcg_at_1_diff1": 0.3916, + "nauc_ndcg_at_3_max": 0.318485, + "nauc_ndcg_at_3_std": 0.060991, + "nauc_ndcg_at_3_diff1": 0.395941, + "nauc_ndcg_at_5_max": 0.332518, + "nauc_ndcg_at_5_std": 0.047183, + "nauc_ndcg_at_5_diff1": 0.373797, + "nauc_ndcg_at_10_max": 0.307342, + "nauc_ndcg_at_10_std": 0.074871, + "nauc_ndcg_at_10_diff1": 0.294468, + "nauc_ndcg_at_20_max": 0.319405, + "nauc_ndcg_at_20_std": 0.100628, + "nauc_ndcg_at_20_diff1": 0.298064, + "nauc_ndcg_at_100_max": 0.295231, + "nauc_ndcg_at_100_std": 0.094696, + "nauc_ndcg_at_100_diff1": 0.287229, + "nauc_ndcg_at_1000_max": 0.305167, + "nauc_ndcg_at_1000_std": 0.085766, + "nauc_ndcg_at_1000_diff1": 0.316669, + "nauc_map_at_1_max": 0.276821, + "nauc_map_at_1_std": 0.038722, + "nauc_map_at_1_diff1": 0.3916, + "nauc_map_at_3_max": 0.306611, + "nauc_map_at_3_std": 0.053867, + "nauc_map_at_3_diff1": 0.386797, + "nauc_map_at_5_max": 0.312468, + "nauc_map_at_5_std": 0.045003, + "nauc_map_at_5_diff1": 0.375399, + "nauc_map_at_10_max": 0.302842, + "nauc_map_at_10_std": 0.053885, + "nauc_map_at_10_diff1": 0.340286, + "nauc_map_at_20_max": 0.306738, + "nauc_map_at_20_std": 0.064015, + "nauc_map_at_20_diff1": 0.340191, + "nauc_map_at_100_max": 0.304426, + "nauc_map_at_100_std": 0.065188, + "nauc_map_at_100_diff1": 0.341367, + "nauc_map_at_1000_max": 0.304086, + "nauc_map_at_1000_std": 0.064603, + "nauc_map_at_1000_diff1": 0.341979, + "nauc_recall_at_1_max": 0.276821, + "nauc_recall_at_1_std": 0.038722, + "nauc_recall_at_1_diff1": 0.3916, + "nauc_recall_at_3_max": 0.336235, + "nauc_recall_at_3_std": 0.065689, + "nauc_recall_at_3_diff1": 0.409028, + "nauc_recall_at_5_max": 0.362656, + "nauc_recall_at_5_std": 0.042263, + "nauc_recall_at_5_diff1": 0.365213, + "nauc_recall_at_10_max": 0.290016, + "nauc_recall_at_10_std": 0.105876, + "nauc_recall_at_10_diff1": 0.168777, + "nauc_recall_at_20_max": 0.327901, + "nauc_recall_at_20_std": 0.183403, + "nauc_recall_at_20_diff1": 0.182411, + "nauc_recall_at_100_max": 0.202377, + "nauc_recall_at_100_std": 0.145595, + "nauc_recall_at_100_diff1": 0.077969, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.276821, + "nauc_precision_at_1_std": 0.038722, + "nauc_precision_at_1_diff1": 0.3916, + "nauc_precision_at_3_max": 0.362066, + "nauc_precision_at_3_std": 0.094048, + "nauc_precision_at_3_diff1": 0.43418, + "nauc_precision_at_5_max": 0.408667, + "nauc_precision_at_5_std": 0.062249, + "nauc_precision_at_5_diff1": 0.370274, + "nauc_precision_at_10_max": 0.361598, + "nauc_precision_at_10_std": 0.160164, + "nauc_precision_at_10_diff1": 0.138637, + "nauc_precision_at_20_max": 0.383112, + "nauc_precision_at_20_std": 0.258567, + "nauc_precision_at_20_diff1": 0.122885, + "nauc_precision_at_100_max": 0.283664, + "nauc_precision_at_100_std": 0.238831, + "nauc_precision_at_100_diff1": 0.042859, + "nauc_precision_at_1000_max": 0.452424, + "nauc_precision_at_1000_std": 0.459731, + "nauc_precision_at_1000_diff1": -0.110319, + "nauc_mrr_at_1_max": 0.276821, + "nauc_mrr_at_1_std": 0.038722, + "nauc_mrr_at_1_diff1": 0.3916, + "nauc_mrr_at_3_max": 0.31696, + "nauc_mrr_at_3_std": 0.065151, + "nauc_mrr_at_3_diff1": 0.396848, + "nauc_mrr_at_5_max": 0.328492, + "nauc_mrr_at_5_std": 0.054508, + "nauc_mrr_at_5_diff1": 0.380569, + "nauc_mrr_at_10_max": 0.3145, + "nauc_mrr_at_10_std": 0.072815, + "nauc_mrr_at_10_diff1": 0.347457, + "nauc_mrr_at_20_max": 0.315572, + "nauc_mrr_at_20_std": 0.073598, + "nauc_mrr_at_20_diff1": 0.351026, + "nauc_mrr_at_100_max": 0.311476, + "nauc_mrr_at_100_std": 0.072809, + "nauc_mrr_at_100_diff1": 0.350554, + "nauc_mrr_at_1000_max": 0.311323, + "nauc_mrr_at_1000_std": 0.072432, + "nauc_mrr_at_1000_diff1": 0.351244, + "main_score": 0.23457, + "hf_subset": "robotics", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.1453, + "ndcg_at_3": 0.26307, + "ndcg_at_5": 0.30004, + "ndcg_at_10": 0.31835, + "ndcg_at_20": 0.34239, + "ndcg_at_100": 0.3892, + "ndcg_at_1000": 0.40876, + "map_at_1": 0.12821, + "map_at_3": 0.22578, + "map_at_5": 0.24672, + "map_at_10": 0.25458, + "map_at_20": 0.2612, + "map_at_100": 0.26813, + "map_at_1000": 0.26891, + "recall_at_1": 0.12821, + "recall_at_3": 0.35897, + "recall_at_5": 0.44872, + "recall_at_10": 0.50427, + "recall_at_20": 0.59829, + "recall_at_100": 0.84188, + "recall_at_1000": 0.99145, + "precision_at_1": 0.1453, + "precision_at_3": 0.13105, + "precision_at_5": 0.09744, + "precision_at_10": 0.05556, + "precision_at_20": 0.03291, + "precision_at_100": 0.00932, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.145299, + "mrr_at_3": 0.239316, + "mrr_at_5": 0.258547, + "mrr_at_10": 0.266595, + "mrr_at_20": 0.271532, + "mrr_at_100": 0.27774, + "mrr_at_1000": 0.278467, + "nauc_ndcg_at_1_max": 0.082325, + "nauc_ndcg_at_1_std": -0.016051, + "nauc_ndcg_at_1_diff1": 0.197354, + "nauc_ndcg_at_3_max": 0.258164, + "nauc_ndcg_at_3_std": -0.102391, + "nauc_ndcg_at_3_diff1": 0.235508, + "nauc_ndcg_at_5_max": 0.254636, + "nauc_ndcg_at_5_std": -0.050628, + "nauc_ndcg_at_5_diff1": 0.228768, + "nauc_ndcg_at_10_max": 0.265652, + "nauc_ndcg_at_10_std": -0.06902, + "nauc_ndcg_at_10_diff1": 0.244086, + "nauc_ndcg_at_20_max": 0.245425, + "nauc_ndcg_at_20_std": -0.079782, + "nauc_ndcg_at_20_diff1": 0.237588, + "nauc_ndcg_at_100_max": 0.242244, + "nauc_ndcg_at_100_std": -0.062004, + "nauc_ndcg_at_100_diff1": 0.239948, + "nauc_ndcg_at_1000_max": 0.238879, + "nauc_ndcg_at_1000_std": -0.060783, + "nauc_ndcg_at_1000_diff1": 0.236792, + "nauc_map_at_1_max": 0.082706, + "nauc_map_at_1_std": 0.018652, + "nauc_map_at_1_diff1": 0.238179, + "nauc_map_at_3_max": 0.224804, + "nauc_map_at_3_std": -0.074846, + "nauc_map_at_3_diff1": 0.238865, + "nauc_map_at_5_max": 0.225664, + "nauc_map_at_5_std": -0.041949, + "nauc_map_at_5_diff1": 0.234702, + "nauc_map_at_10_max": 0.231606, + "nauc_map_at_10_std": -0.050259, + "nauc_map_at_10_diff1": 0.241459, + "nauc_map_at_20_max": 0.225811, + "nauc_map_at_20_std": -0.053835, + "nauc_map_at_20_diff1": 0.239356, + "nauc_map_at_100_max": 0.223307, + "nauc_map_at_100_std": -0.051598, + "nauc_map_at_100_diff1": 0.238408, + "nauc_map_at_1000_max": 0.223208, + "nauc_map_at_1000_std": -0.051651, + "nauc_map_at_1000_diff1": 0.238275, + "nauc_recall_at_1_max": 0.082706, + "nauc_recall_at_1_std": 0.018652, + "nauc_recall_at_1_diff1": 0.238179, + "nauc_recall_at_3_max": 0.334014, + "nauc_recall_at_3_std": -0.15334, + "nauc_recall_at_3_diff1": 0.229517, + "nauc_recall_at_5_max": 0.31729, + "nauc_recall_at_5_std": -0.051954, + "nauc_recall_at_5_diff1": 0.213344, + "nauc_recall_at_10_max": 0.341692, + "nauc_recall_at_10_std": -0.09934, + "nauc_recall_at_10_diff1": 0.253667, + "nauc_recall_at_20_max": 0.275592, + "nauc_recall_at_20_std": -0.135768, + "nauc_recall_at_20_diff1": 0.23369, + "nauc_recall_at_100_max": 0.335664, + "nauc_recall_at_100_std": -0.044595, + "nauc_recall_at_100_diff1": 0.264757, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.722475, + "nauc_recall_at_1000_diff1": -0.168193, + "nauc_precision_at_1_max": 0.082325, + "nauc_precision_at_1_std": -0.016051, + "nauc_precision_at_1_diff1": 0.197354, + "nauc_precision_at_3_max": 0.360656, + "nauc_precision_at_3_std": -0.176835, + "nauc_precision_at_3_diff1": 0.224321, + "nauc_precision_at_5_max": 0.342302, + "nauc_precision_at_5_std": -0.075662, + "nauc_precision_at_5_diff1": 0.202748, + "nauc_precision_at_10_max": 0.352647, + "nauc_precision_at_10_std": -0.12267, + "nauc_precision_at_10_diff1": 0.221652, + "nauc_precision_at_20_max": 0.278205, + "nauc_precision_at_20_std": -0.151984, + "nauc_precision_at_20_diff1": 0.177445, + "nauc_precision_at_100_max": 0.269626, + "nauc_precision_at_100_std": -0.109743, + "nauc_precision_at_100_diff1": 0.199506, + "nauc_precision_at_1000_max": 0.19956, + "nauc_precision_at_1000_std": -0.104768, + "nauc_precision_at_1000_diff1": 0.105692, + "nauc_mrr_at_1_max": 0.082325, + "nauc_mrr_at_1_std": -0.016051, + "nauc_mrr_at_1_diff1": 0.197354, + "nauc_mrr_at_3_max": 0.209064, + "nauc_mrr_at_3_std": -0.087118, + "nauc_mrr_at_3_diff1": 0.220959, + "nauc_mrr_at_5_max": 0.2068, + "nauc_mrr_at_5_std": -0.05932, + "nauc_mrr_at_5_diff1": 0.21805, + "nauc_mrr_at_10_max": 0.210294, + "nauc_mrr_at_10_std": -0.067978, + "nauc_mrr_at_10_diff1": 0.221859, + "nauc_mrr_at_20_max": 0.208293, + "nauc_mrr_at_20_std": -0.069865, + "nauc_mrr_at_20_diff1": 0.223652, + "nauc_mrr_at_100_max": 0.205821, + "nauc_mrr_at_100_std": -0.06578, + "nauc_mrr_at_100_diff1": 0.223276, + "nauc_mrr_at_1000_max": 0.205664, + "nauc_mrr_at_1000_std": -0.065934, + "nauc_mrr_at_1000_diff1": 0.223057, + "main_score": 0.31835, + "hf_subset": "stackoverflow", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.22222, + "ndcg_at_3": 0.30382, + "ndcg_at_5": 0.31566, + "ndcg_at_10": 0.3465, + "ndcg_at_20": 0.36394, + "ndcg_at_100": 0.39497, + "ndcg_at_1000": 0.43025, + "map_at_1": 0.18472, + "map_at_3": 0.26713, + "map_at_5": 0.27833, + "map_at_10": 0.29093, + "map_at_20": 0.29653, + "map_at_100": 0.30011, + "map_at_1000": 0.30156, + "recall_at_1": 0.18472, + "recall_at_3": 0.36435, + "recall_at_5": 0.40093, + "recall_at_10": 0.49815, + "recall_at_20": 0.56019, + "recall_at_100": 0.73611, + "recall_at_1000": 1.0, + "precision_at_1": 0.22222, + "precision_at_3": 0.14506, + "precision_at_5": 0.09815, + "precision_at_10": 0.06019, + "precision_at_20": 0.03426, + "precision_at_100": 0.0088, + "precision_at_1000": 0.00119, + "mrr_at_1": 0.212963, + "mrr_at_3": 0.29321, + "mrr_at_5": 0.299228, + "mrr_at_10": 0.310016, + "mrr_at_20": 0.315125, + "mrr_at_100": 0.318404, + "mrr_at_1000": 0.319624, + "nauc_ndcg_at_1_max": 0.100414, + "nauc_ndcg_at_1_std": 0.031003, + "nauc_ndcg_at_1_diff1": 0.08493, + "nauc_ndcg_at_3_max": 0.26512, + "nauc_ndcg_at_3_std": 0.055293, + "nauc_ndcg_at_3_diff1": 0.127225, + "nauc_ndcg_at_5_max": 0.238013, + "nauc_ndcg_at_5_std": 0.073628, + "nauc_ndcg_at_5_diff1": 0.141331, + "nauc_ndcg_at_10_max": 0.226393, + "nauc_ndcg_at_10_std": 0.073486, + "nauc_ndcg_at_10_diff1": 0.107856, + "nauc_ndcg_at_20_max": 0.213678, + "nauc_ndcg_at_20_std": 0.091854, + "nauc_ndcg_at_20_diff1": 0.097815, + "nauc_ndcg_at_100_max": 0.220165, + "nauc_ndcg_at_100_std": 0.085938, + "nauc_ndcg_at_100_diff1": 0.10495, + "nauc_ndcg_at_1000_max": 0.21599, + "nauc_ndcg_at_1000_std": 0.073836, + "nauc_ndcg_at_1000_diff1": 0.116228, + "nauc_map_at_1_max": 0.082346, + "nauc_map_at_1_std": 0.039623, + "nauc_map_at_1_diff1": 0.129312, + "nauc_map_at_3_max": 0.232097, + "nauc_map_at_3_std": 0.049835, + "nauc_map_at_3_diff1": 0.141937, + "nauc_map_at_5_max": 0.206171, + "nauc_map_at_5_std": 0.051119, + "nauc_map_at_5_diff1": 0.138579, + "nauc_map_at_10_max": 0.204401, + "nauc_map_at_10_std": 0.055549, + "nauc_map_at_10_diff1": 0.122224, + "nauc_map_at_20_max": 0.199327, + "nauc_map_at_20_std": 0.060724, + "nauc_map_at_20_diff1": 0.116886, + "nauc_map_at_100_max": 0.201089, + "nauc_map_at_100_std": 0.059401, + "nauc_map_at_100_diff1": 0.118961, + "nauc_map_at_1000_max": 0.200733, + "nauc_map_at_1000_std": 0.059008, + "nauc_map_at_1000_diff1": 0.119907, + "nauc_recall_at_1_max": 0.082346, + "nauc_recall_at_1_std": 0.039623, + "nauc_recall_at_1_diff1": 0.129312, + "nauc_recall_at_3_max": 0.361776, + "nauc_recall_at_3_std": 0.095704, + "nauc_recall_at_3_diff1": 0.145277, + "nauc_recall_at_5_max": 0.287594, + "nauc_recall_at_5_std": 0.127438, + "nauc_recall_at_5_diff1": 0.159299, + "nauc_recall_at_10_max": 0.252408, + "nauc_recall_at_10_std": 0.118936, + "nauc_recall_at_10_diff1": 0.069771, + "nauc_recall_at_20_max": 0.219232, + "nauc_recall_at_20_std": 0.196052, + "nauc_recall_at_20_diff1": 0.028538, + "nauc_recall_at_100_max": 0.248024, + "nauc_recall_at_100_std": 0.199658, + "nauc_recall_at_100_diff1": 0.031801, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.100414, + "nauc_precision_at_1_std": 0.031003, + "nauc_precision_at_1_diff1": 0.08493, + "nauc_precision_at_3_max": 0.289249, + "nauc_precision_at_3_std": 0.040234, + "nauc_precision_at_3_diff1": 0.045991, + "nauc_precision_at_5_max": 0.1927, + "nauc_precision_at_5_std": 0.038528, + "nauc_precision_at_5_diff1": 0.036631, + "nauc_precision_at_10_max": 0.154607, + "nauc_precision_at_10_std": 0.034182, + "nauc_precision_at_10_diff1": -0.054093, + "nauc_precision_at_20_max": 0.099294, + "nauc_precision_at_20_std": 0.072315, + "nauc_precision_at_20_diff1": -0.068347, + "nauc_precision_at_100_max": 0.047985, + "nauc_precision_at_100_std": 0.041126, + "nauc_precision_at_100_diff1": -0.093313, + "nauc_precision_at_1000_max": -0.114451, + "nauc_precision_at_1000_std": -0.059769, + "nauc_precision_at_1000_diff1": -0.137232, + "nauc_mrr_at_1_max": 0.129221, + "nauc_mrr_at_1_std": -0.004116, + "nauc_mrr_at_1_diff1": 0.117537, + "nauc_mrr_at_3_max": 0.259753, + "nauc_mrr_at_3_std": 0.034685, + "nauc_mrr_at_3_diff1": 0.133329, + "nauc_mrr_at_5_max": 0.244477, + "nauc_mrr_at_5_std": 0.046252, + "nauc_mrr_at_5_diff1": 0.136954, + "nauc_mrr_at_10_max": 0.236644, + "nauc_mrr_at_10_std": 0.042961, + "nauc_mrr_at_10_diff1": 0.126931, + "nauc_mrr_at_20_max": 0.232911, + "nauc_mrr_at_20_std": 0.046065, + "nauc_mrr_at_20_diff1": 0.127314, + "nauc_mrr_at_100_max": 0.235459, + "nauc_mrr_at_100_std": 0.045182, + "nauc_mrr_at_100_diff1": 0.129508, + "nauc_mrr_at_1000_max": 0.23518, + "nauc_mrr_at_1000_std": 0.04464, + "nauc_mrr_at_1000_diff1": 0.130237, + "main_score": 0.3465, + "hf_subset": "sustainable_living", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.16964, + "ndcg_at_3": 0.1649, + "ndcg_at_5": 0.15525, + "ndcg_at_10": 0.16153, + "ndcg_at_20": 0.20787, + "ndcg_at_100": 0.36391, + "ndcg_at_1000": 0.4373, + "map_at_1": 0.02838, + "map_at_3": 0.05538, + "map_at_5": 0.06681, + "map_at_10": 0.07889, + "map_at_20": 0.09241, + "map_at_100": 0.12883, + "map_at_1000": 0.13958, + "recall_at_1": 0.02838, + "recall_at_3": 0.08213, + "recall_at_5": 0.11885, + "recall_at_10": 0.16721, + "recall_at_20": 0.26603, + "recall_at_100": 0.71895, + "recall_at_1000": 1.0, + "precision_at_1": 0.16964, + "precision_at_3": 0.16369, + "precision_at_5": 0.14286, + "precision_at_10": 0.10089, + "precision_at_20": 0.08393, + "precision_at_100": 0.04795, + "precision_at_1000": 0.00687, + "mrr_at_1": 0.169643, + "mrr_at_3": 0.278274, + "mrr_at_5": 0.312649, + "mrr_at_10": 0.33728, + "mrr_at_20": 0.348132, + "mrr_at_100": 0.352394, + "mrr_at_1000": 0.352394, + "nauc_ndcg_at_1_max": 8.9e-05, + "nauc_ndcg_at_1_std": -0.195872, + "nauc_ndcg_at_1_diff1": -0.011135, + "nauc_ndcg_at_3_max": 0.026087, + "nauc_ndcg_at_3_std": -0.116735, + "nauc_ndcg_at_3_diff1": -0.065445, + "nauc_ndcg_at_5_max": 0.000548, + "nauc_ndcg_at_5_std": -0.10284, + "nauc_ndcg_at_5_diff1": -0.069478, + "nauc_ndcg_at_10_max": 0.044334, + "nauc_ndcg_at_10_std": -0.062284, + "nauc_ndcg_at_10_diff1": -0.079428, + "nauc_ndcg_at_20_max": 0.026015, + "nauc_ndcg_at_20_std": -0.05774, + "nauc_ndcg_at_20_diff1": -0.108315, + "nauc_ndcg_at_100_max": 0.005714, + "nauc_ndcg_at_100_std": -0.057065, + "nauc_ndcg_at_100_diff1": -0.080429, + "nauc_ndcg_at_1000_max": 0.01343, + "nauc_ndcg_at_1000_std": -0.069972, + "nauc_ndcg_at_1000_diff1": -0.042088, + "nauc_map_at_1_max": 0.082466, + "nauc_map_at_1_std": -0.160361, + "nauc_map_at_1_diff1": 0.020186, + "nauc_map_at_3_max": 0.028642, + "nauc_map_at_3_std": -0.114556, + "nauc_map_at_3_diff1": -0.055732, + "nauc_map_at_5_max": 0.022781, + "nauc_map_at_5_std": -0.114897, + "nauc_map_at_5_diff1": -0.074818, + "nauc_map_at_10_max": 0.034915, + "nauc_map_at_10_std": -0.102854, + "nauc_map_at_10_diff1": -0.068443, + "nauc_map_at_20_max": 0.026091, + "nauc_map_at_20_std": -0.091011, + "nauc_map_at_20_diff1": -0.089899, + "nauc_map_at_100_max": 0.026511, + "nauc_map_at_100_std": -0.08049, + "nauc_map_at_100_diff1": -0.071731, + "nauc_map_at_1000_max": 0.026629, + "nauc_map_at_1000_std": -0.084461, + "nauc_map_at_1000_diff1": -0.066887, + "nauc_recall_at_1_max": 0.082466, + "nauc_recall_at_1_std": -0.160361, + "nauc_recall_at_1_diff1": 0.020186, + "nauc_recall_at_3_max": 0.024915, + "nauc_recall_at_3_std": -0.108825, + "nauc_recall_at_3_diff1": -0.073521, + "nauc_recall_at_5_max": 0.012005, + "nauc_recall_at_5_std": -0.074721, + "nauc_recall_at_5_diff1": -0.101153, + "nauc_recall_at_10_max": 0.044796, + "nauc_recall_at_10_std": -0.003202, + "nauc_recall_at_10_diff1": -0.103567, + "nauc_recall_at_20_max": 0.008848, + "nauc_recall_at_20_std": 0.007287, + "nauc_recall_at_20_diff1": -0.156356, + "nauc_recall_at_100_max": -0.022175, + "nauc_recall_at_100_std": 0.000851, + "nauc_recall_at_100_diff1": -0.119717, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 8.9e-05, + "nauc_precision_at_1_std": -0.195872, + "nauc_precision_at_1_diff1": -0.011135, + "nauc_precision_at_3_max": 0.031067, + "nauc_precision_at_3_std": -0.084825, + "nauc_precision_at_3_diff1": -0.079346, + "nauc_precision_at_5_max": -0.018154, + "nauc_precision_at_5_std": -0.058127, + "nauc_precision_at_5_diff1": -0.094506, + "nauc_precision_at_10_max": 0.017615, + "nauc_precision_at_10_std": 0.030866, + "nauc_precision_at_10_diff1": -0.086879, + "nauc_precision_at_20_max": -0.016426, + "nauc_precision_at_20_std": 0.044528, + "nauc_precision_at_20_diff1": -0.115044, + "nauc_precision_at_100_max": -0.126185, + "nauc_precision_at_100_std": 0.085115, + "nauc_precision_at_100_diff1": 0.04863, + "nauc_precision_at_1000_max": -0.075692, + "nauc_precision_at_1000_std": 0.050746, + "nauc_precision_at_1000_diff1": 0.159941, + "nauc_mrr_at_1_max": 8.9e-05, + "nauc_mrr_at_1_std": -0.195872, + "nauc_mrr_at_1_diff1": -0.011135, + "nauc_mrr_at_3_max": 0.051087, + "nauc_mrr_at_3_std": -0.156749, + "nauc_mrr_at_3_diff1": -0.041657, + "nauc_mrr_at_5_max": 0.03024, + "nauc_mrr_at_5_std": -0.126123, + "nauc_mrr_at_5_diff1": -0.04244, + "nauc_mrr_at_10_max": 0.046461, + "nauc_mrr_at_10_std": -0.096869, + "nauc_mrr_at_10_diff1": -0.039753, + "nauc_mrr_at_20_max": 0.036504, + "nauc_mrr_at_20_std": -0.108246, + "nauc_mrr_at_20_diff1": -0.039919, + "nauc_mrr_at_100_max": 0.037745, + "nauc_mrr_at_100_std": -0.11473, + "nauc_mrr_at_100_diff1": -0.039777, + "nauc_mrr_at_1000_max": 0.037745, + "nauc_mrr_at_1000_std": -0.11473, + "nauc_mrr_at_1000_diff1": -0.039777, + "main_score": 0.16153, + "hf_subset": "pony", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 496.9581935405731, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/BrightRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/BrightRetrieval.json new file mode 100644 index 0000000000..d47d517b52 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/BrightRetrieval.json @@ -0,0 +1,1775 @@ +{ + "dataset_revision": "a75a0eb", + "task_name": "BrightRetrieval", + "mteb_version": "1.38.9", + "scores": { + "standard": [ + { + "ndcg_at_1": 0.2233, + "ndcg_at_3": 0.1929, + "ndcg_at_5": 0.19352, + "ndcg_at_10": 0.21564, + "ndcg_at_20": 0.24474, + "ndcg_at_100": 0.29516, + "ndcg_at_1000": 0.34979, + "map_at_1": 0.06825, + "map_at_3": 0.11759, + "map_at_5": 0.13408, + "map_at_10": 0.15287, + "map_at_20": 0.1653, + "map_at_100": 0.17604, + "map_at_1000": 0.17887, + "recall_at_1": 0.06825, + "recall_at_3": 0.13631, + "recall_at_5": 0.17612, + "recall_at_10": 0.24948, + "recall_at_20": 0.32702, + "recall_at_100": 0.51462, + "recall_at_1000": 0.82611, + "precision_at_1": 0.2233, + "precision_at_3": 0.16505, + "precision_at_5": 0.12816, + "precision_at_10": 0.09223, + "precision_at_20": 0.06165, + "precision_at_100": 0.01961, + "precision_at_1000": 0.00299, + "mrr_at_1": 0.23301, + "mrr_at_3": 0.268608, + "mrr_at_5": 0.280744, + "mrr_at_10": 0.291432, + "mrr_at_20": 0.29924, + "mrr_at_100": 0.302907, + "mrr_at_1000": 0.304191, + "nauc_ndcg_at_1_max": 0.128452, + "nauc_ndcg_at_1_std": -0.069596, + "nauc_ndcg_at_1_diff1": 0.421115, + "nauc_ndcg_at_3_max": 0.103735, + "nauc_ndcg_at_3_std": -0.123265, + "nauc_ndcg_at_3_diff1": 0.398346, + "nauc_ndcg_at_5_max": 0.126257, + "nauc_ndcg_at_5_std": -0.146942, + "nauc_ndcg_at_5_diff1": 0.386112, + "nauc_ndcg_at_10_max": 0.112555, + "nauc_ndcg_at_10_std": -0.184416, + "nauc_ndcg_at_10_diff1": 0.376829, + "nauc_ndcg_at_20_max": 0.113159, + "nauc_ndcg_at_20_std": -0.155407, + "nauc_ndcg_at_20_diff1": 0.352739, + "nauc_ndcg_at_100_max": 0.167199, + "nauc_ndcg_at_100_std": -0.137182, + "nauc_ndcg_at_100_diff1": 0.374871, + "nauc_ndcg_at_1000_max": 0.159366, + "nauc_ndcg_at_1000_std": -0.105222, + "nauc_ndcg_at_1000_diff1": 0.345117, + "nauc_map_at_1_max": 0.171655, + "nauc_map_at_1_std": -0.116992, + "nauc_map_at_1_diff1": 0.45876, + "nauc_map_at_3_max": 0.147329, + "nauc_map_at_3_std": -0.143083, + "nauc_map_at_3_diff1": 0.452243, + "nauc_map_at_5_max": 0.153771, + "nauc_map_at_5_std": -0.146232, + "nauc_map_at_5_diff1": 0.446557, + "nauc_map_at_10_max": 0.131648, + "nauc_map_at_10_std": -0.177326, + "nauc_map_at_10_diff1": 0.43028, + "nauc_map_at_20_max": 0.123828, + "nauc_map_at_20_std": -0.164549, + "nauc_map_at_20_diff1": 0.414513, + "nauc_map_at_100_max": 0.133561, + "nauc_map_at_100_std": -0.159936, + "nauc_map_at_100_diff1": 0.413015, + "nauc_map_at_1000_max": 0.1334, + "nauc_map_at_1000_std": -0.157937, + "nauc_map_at_1000_diff1": 0.410754, + "nauc_recall_at_1_max": 0.171655, + "nauc_recall_at_1_std": -0.116992, + "nauc_recall_at_1_diff1": 0.45876, + "nauc_recall_at_3_max": 0.134853, + "nauc_recall_at_3_std": -0.180651, + "nauc_recall_at_3_diff1": 0.392429, + "nauc_recall_at_5_max": 0.135465, + "nauc_recall_at_5_std": -0.148063, + "nauc_recall_at_5_diff1": 0.345878, + "nauc_recall_at_10_max": 0.074735, + "nauc_recall_at_10_std": -0.235183, + "nauc_recall_at_10_diff1": 0.294451, + "nauc_recall_at_20_max": 0.070618, + "nauc_recall_at_20_std": -0.149143, + "nauc_recall_at_20_diff1": 0.255067, + "nauc_recall_at_100_max": 0.245921, + "nauc_recall_at_100_std": -0.107333, + "nauc_recall_at_100_diff1": 0.338209, + "nauc_recall_at_1000_max": 0.256287, + "nauc_recall_at_1000_std": 0.212247, + "nauc_recall_at_1000_diff1": 0.090522, + "nauc_precision_at_1_max": 0.128452, + "nauc_precision_at_1_std": -0.069596, + "nauc_precision_at_1_diff1": 0.421115, + "nauc_precision_at_3_max": 0.085946, + "nauc_precision_at_3_std": -0.126001, + "nauc_precision_at_3_diff1": 0.367748, + "nauc_precision_at_5_max": 0.095476, + "nauc_precision_at_5_std": -0.147227, + "nauc_precision_at_5_diff1": 0.343352, + "nauc_precision_at_10_max": 0.023414, + "nauc_precision_at_10_std": -0.222023, + "nauc_precision_at_10_diff1": 0.288267, + "nauc_precision_at_20_max": 0.020659, + "nauc_precision_at_20_std": -0.157844, + "nauc_precision_at_20_diff1": 0.21998, + "nauc_precision_at_100_max": 0.114262, + "nauc_precision_at_100_std": -0.073248, + "nauc_precision_at_100_diff1": 0.210845, + "nauc_precision_at_1000_max": 0.01338, + "nauc_precision_at_1000_std": 0.016443, + "nauc_precision_at_1000_diff1": -0.011478, + "nauc_mrr_at_1_max": 0.162152, + "nauc_mrr_at_1_std": -0.085789, + "nauc_mrr_at_1_diff1": 0.382122, + "nauc_mrr_at_3_max": 0.152391, + "nauc_mrr_at_3_std": -0.106758, + "nauc_mrr_at_3_diff1": 0.338114, + "nauc_mrr_at_5_max": 0.146909, + "nauc_mrr_at_5_std": -0.105713, + "nauc_mrr_at_5_diff1": 0.327306, + "nauc_mrr_at_10_max": 0.14833, + "nauc_mrr_at_10_std": -0.116376, + "nauc_mrr_at_10_diff1": 0.322122, + "nauc_mrr_at_20_max": 0.151877, + "nauc_mrr_at_20_std": -0.110396, + "nauc_mrr_at_20_diff1": 0.318564, + "nauc_mrr_at_100_max": 0.156213, + "nauc_mrr_at_100_std": -0.111878, + "nauc_mrr_at_100_diff1": 0.324658, + "nauc_mrr_at_1000_max": 0.155416, + "nauc_mrr_at_1000_std": -0.111298, + "nauc_mrr_at_1000_diff1": 0.324031, + "main_score": 0.21564, + "hf_subset": "biology", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.37069, + "ndcg_at_3": 0.34143, + "ndcg_at_5": 0.34083, + "ndcg_at_10": 0.34201, + "ndcg_at_20": 0.36587, + "ndcg_at_100": 0.41121, + "ndcg_at_1000": 0.45752, + "map_at_1": 0.16787, + "map_at_3": 0.23163, + "map_at_5": 0.25901, + "map_at_10": 0.26941, + "map_at_20": 0.28215, + "map_at_100": 0.2919, + "map_at_1000": 0.29514, + "recall_at_1": 0.16787, + "recall_at_3": 0.25698, + "recall_at_5": 0.31711, + "recall_at_10": 0.35108, + "recall_at_20": 0.41875, + "recall_at_100": 0.57077, + "recall_at_1000": 0.78865, + "precision_at_1": 0.37069, + "precision_at_3": 0.24425, + "precision_at_5": 0.19483, + "precision_at_10": 0.11638, + "precision_at_20": 0.07457, + "precision_at_100": 0.02345, + "precision_at_1000": 0.00375, + "mrr_at_1": 0.37069, + "mrr_at_3": 0.439655, + "mrr_at_5": 0.457759, + "mrr_at_10": 0.466147, + "mrr_at_20": 0.470562, + "mrr_at_100": 0.473909, + "mrr_at_1000": 0.474409, + "nauc_ndcg_at_1_max": 0.38099, + "nauc_ndcg_at_1_std": 0.053966, + "nauc_ndcg_at_1_diff1": 0.226086, + "nauc_ndcg_at_3_max": 0.352651, + "nauc_ndcg_at_3_std": -0.037554, + "nauc_ndcg_at_3_diff1": 0.251456, + "nauc_ndcg_at_5_max": 0.369133, + "nauc_ndcg_at_5_std": -0.008258, + "nauc_ndcg_at_5_diff1": 0.287706, + "nauc_ndcg_at_10_max": 0.375664, + "nauc_ndcg_at_10_std": 0.014086, + "nauc_ndcg_at_10_diff1": 0.279302, + "nauc_ndcg_at_20_max": 0.371454, + "nauc_ndcg_at_20_std": -0.010534, + "nauc_ndcg_at_20_diff1": 0.296649, + "nauc_ndcg_at_100_max": 0.360326, + "nauc_ndcg_at_100_std": 0.010624, + "nauc_ndcg_at_100_diff1": 0.265559, + "nauc_ndcg_at_1000_max": 0.342362, + "nauc_ndcg_at_1000_std": 0.018153, + "nauc_ndcg_at_1000_diff1": 0.256771, + "nauc_map_at_1_max": 0.386164, + "nauc_map_at_1_std": 0.21079, + "nauc_map_at_1_diff1": 0.406186, + "nauc_map_at_3_max": 0.396796, + "nauc_map_at_3_std": 0.068508, + "nauc_map_at_3_diff1": 0.318863, + "nauc_map_at_5_max": 0.390137, + "nauc_map_at_5_std": 0.034858, + "nauc_map_at_5_diff1": 0.318299, + "nauc_map_at_10_max": 0.398999, + "nauc_map_at_10_std": 0.035698, + "nauc_map_at_10_diff1": 0.318934, + "nauc_map_at_20_max": 0.398965, + "nauc_map_at_20_std": 0.020606, + "nauc_map_at_20_diff1": 0.327566, + "nauc_map_at_100_max": 0.396975, + "nauc_map_at_100_std": 0.025648, + "nauc_map_at_100_diff1": 0.320615, + "nauc_map_at_1000_max": 0.395238, + "nauc_map_at_1000_std": 0.026307, + "nauc_map_at_1000_diff1": 0.319468, + "nauc_recall_at_1_max": 0.386164, + "nauc_recall_at_1_std": 0.21079, + "nauc_recall_at_1_diff1": 0.406186, + "nauc_recall_at_3_max": 0.373982, + "nauc_recall_at_3_std": 0.039409, + "nauc_recall_at_3_diff1": 0.285154, + "nauc_recall_at_5_max": 0.351648, + "nauc_recall_at_5_std": 0.008532, + "nauc_recall_at_5_diff1": 0.277748, + "nauc_recall_at_10_max": 0.349755, + "nauc_recall_at_10_std": 0.032991, + "nauc_recall_at_10_diff1": 0.261815, + "nauc_recall_at_20_max": 0.336261, + "nauc_recall_at_20_std": -0.046129, + "nauc_recall_at_20_diff1": 0.306678, + "nauc_recall_at_100_max": 0.28531, + "nauc_recall_at_100_std": 0.012312, + "nauc_recall_at_100_diff1": 0.176895, + "nauc_recall_at_1000_max": 0.143505, + "nauc_recall_at_1000_std": 0.060119, + "nauc_recall_at_1000_diff1": 0.047898, + "nauc_precision_at_1_max": 0.38099, + "nauc_precision_at_1_std": 0.053966, + "nauc_precision_at_1_diff1": 0.226086, + "nauc_precision_at_3_max": 0.239294, + "nauc_precision_at_3_std": -0.229234, + "nauc_precision_at_3_diff1": 0.070457, + "nauc_precision_at_5_max": 0.188063, + "nauc_precision_at_5_std": -0.242443, + "nauc_precision_at_5_diff1": 0.091378, + "nauc_precision_at_10_max": 0.179915, + "nauc_precision_at_10_std": -0.186613, + "nauc_precision_at_10_diff1": 0.070983, + "nauc_precision_at_20_max": 0.125562, + "nauc_precision_at_20_std": -0.231114, + "nauc_precision_at_20_diff1": 0.065738, + "nauc_precision_at_100_max": -0.004856, + "nauc_precision_at_100_std": -0.10906, + "nauc_precision_at_100_diff1": -0.063501, + "nauc_precision_at_1000_max": -0.140658, + "nauc_precision_at_1000_std": -0.055425, + "nauc_precision_at_1000_diff1": -0.095017, + "nauc_mrr_at_1_max": 0.38099, + "nauc_mrr_at_1_std": 0.053966, + "nauc_mrr_at_1_diff1": 0.226086, + "nauc_mrr_at_3_max": 0.340299, + "nauc_mrr_at_3_std": -0.015158, + "nauc_mrr_at_3_diff1": 0.203222, + "nauc_mrr_at_5_max": 0.342415, + "nauc_mrr_at_5_std": 0.000185, + "nauc_mrr_at_5_diff1": 0.210855, + "nauc_mrr_at_10_max": 0.335775, + "nauc_mrr_at_10_std": 0.001094, + "nauc_mrr_at_10_diff1": 0.204906, + "nauc_mrr_at_20_max": 0.335188, + "nauc_mrr_at_20_std": -0.005775, + "nauc_mrr_at_20_diff1": 0.204772, + "nauc_mrr_at_100_max": 0.335825, + "nauc_mrr_at_100_std": -0.003505, + "nauc_mrr_at_100_diff1": 0.203303, + "nauc_mrr_at_1000_max": 0.336223, + "nauc_mrr_at_1000_std": -0.003091, + "nauc_mrr_at_1000_diff1": 0.203886, + "main_score": 0.34201, + "hf_subset": "earth_science", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.17476, + "ndcg_at_3": 0.17503, + "ndcg_at_5": 0.18078, + "ndcg_at_10": 0.19042, + "ndcg_at_20": 0.20341, + "ndcg_at_100": 0.25692, + "ndcg_at_1000": 0.31799, + "map_at_1": 0.05046, + "map_at_3": 0.07849, + "map_at_5": 0.08916, + "map_at_10": 0.10567, + "map_at_20": 0.11682, + "map_at_100": 0.13572, + "map_at_1000": 0.14205, + "recall_at_1": 0.05046, + "recall_at_3": 0.10854, + "recall_at_5": 0.14066, + "recall_at_10": 0.1999, + "recall_at_20": 0.27249, + "recall_at_100": 0.49751, + "recall_at_1000": 0.78209, + "precision_at_1": 0.17476, + "precision_at_3": 0.13916, + "precision_at_5": 0.12621, + "precision_at_10": 0.1, + "precision_at_20": 0.07379, + "precision_at_100": 0.03243, + "precision_at_1000": 0.00588, + "mrr_at_1": 0.174757, + "mrr_at_3": 0.224919, + "mrr_at_5": 0.24288, + "mrr_at_10": 0.251241, + "mrr_at_20": 0.255053, + "mrr_at_100": 0.260317, + "mrr_at_1000": 0.26134, + "nauc_ndcg_at_1_max": -0.107385, + "nauc_ndcg_at_1_std": -0.118901, + "nauc_ndcg_at_1_diff1": 0.053929, + "nauc_ndcg_at_3_max": 0.03973, + "nauc_ndcg_at_3_std": -0.053892, + "nauc_ndcg_at_3_diff1": 0.072843, + "nauc_ndcg_at_5_max": 0.078942, + "nauc_ndcg_at_5_std": -0.064283, + "nauc_ndcg_at_5_diff1": 0.104218, + "nauc_ndcg_at_10_max": 0.105666, + "nauc_ndcg_at_10_std": -0.023441, + "nauc_ndcg_at_10_diff1": 0.06724, + "nauc_ndcg_at_20_max": 0.090768, + "nauc_ndcg_at_20_std": -0.016375, + "nauc_ndcg_at_20_diff1": 0.046855, + "nauc_ndcg_at_100_max": 0.047942, + "nauc_ndcg_at_100_std": -0.022249, + "nauc_ndcg_at_100_diff1": 0.054866, + "nauc_ndcg_at_1000_max": 0.05464, + "nauc_ndcg_at_1000_std": -0.03046, + "nauc_ndcg_at_1000_diff1": 0.024457, + "nauc_map_at_1_max": -0.200947, + "nauc_map_at_1_std": -0.116397, + "nauc_map_at_1_diff1": 0.134034, + "nauc_map_at_3_max": -0.067718, + "nauc_map_at_3_std": -0.090889, + "nauc_map_at_3_diff1": 0.08902, + "nauc_map_at_5_max": -0.046809, + "nauc_map_at_5_std": -0.098456, + "nauc_map_at_5_diff1": 0.124237, + "nauc_map_at_10_max": -0.00145, + "nauc_map_at_10_std": -0.05515, + "nauc_map_at_10_diff1": 0.091185, + "nauc_map_at_20_max": 0.012723, + "nauc_map_at_20_std": -0.047467, + "nauc_map_at_20_diff1": 0.077031, + "nauc_map_at_100_max": 0.047366, + "nauc_map_at_100_std": -0.014366, + "nauc_map_at_100_diff1": 0.075553, + "nauc_map_at_1000_max": 0.05816, + "nauc_map_at_1000_std": -0.009433, + "nauc_map_at_1000_diff1": 0.074306, + "nauc_recall_at_1_max": -0.200947, + "nauc_recall_at_1_std": -0.116397, + "nauc_recall_at_1_diff1": 0.134034, + "nauc_recall_at_3_max": 0.039618, + "nauc_recall_at_3_std": -0.052719, + "nauc_recall_at_3_diff1": 0.045384, + "nauc_recall_at_5_max": 0.100265, + "nauc_recall_at_5_std": -0.070773, + "nauc_recall_at_5_diff1": 0.157996, + "nauc_recall_at_10_max": 0.11126, + "nauc_recall_at_10_std": 0.008061, + "nauc_recall_at_10_diff1": 0.047163, + "nauc_recall_at_20_max": 0.081852, + "nauc_recall_at_20_std": -0.002547, + "nauc_recall_at_20_diff1": -0.00742, + "nauc_recall_at_100_max": 0.071189, + "nauc_recall_at_100_std": 0.099118, + "nauc_recall_at_100_diff1": 0.076616, + "nauc_recall_at_1000_max": 0.020125, + "nauc_recall_at_1000_std": 0.204792, + "nauc_recall_at_1000_diff1": -0.122288, + "nauc_precision_at_1_max": -0.107385, + "nauc_precision_at_1_std": -0.118901, + "nauc_precision_at_1_diff1": 0.053929, + "nauc_precision_at_3_max": 0.109984, + "nauc_precision_at_3_std": -0.022683, + "nauc_precision_at_3_diff1": 0.050083, + "nauc_precision_at_5_max": 0.163606, + "nauc_precision_at_5_std": -0.031541, + "nauc_precision_at_5_diff1": 0.068037, + "nauc_precision_at_10_max": 0.231529, + "nauc_precision_at_10_std": 0.035769, + "nauc_precision_at_10_diff1": 0.010011, + "nauc_precision_at_20_max": 0.230563, + "nauc_precision_at_20_std": 0.07867, + "nauc_precision_at_20_diff1": -0.026009, + "nauc_precision_at_100_max": 0.271723, + "nauc_precision_at_100_std": 0.133534, + "nauc_precision_at_100_diff1": 0.046661, + "nauc_precision_at_1000_max": 0.25527, + "nauc_precision_at_1000_std": 0.060344, + "nauc_precision_at_1000_diff1": -0.012496, + "nauc_mrr_at_1_max": -0.107385, + "nauc_mrr_at_1_std": -0.118901, + "nauc_mrr_at_1_diff1": 0.053929, + "nauc_mrr_at_3_max": -0.020399, + "nauc_mrr_at_3_std": -0.102907, + "nauc_mrr_at_3_diff1": 0.040185, + "nauc_mrr_at_5_max": 0.01088, + "nauc_mrr_at_5_std": -0.109205, + "nauc_mrr_at_5_diff1": 0.055886, + "nauc_mrr_at_10_max": -0.004035, + "nauc_mrr_at_10_std": -0.118188, + "nauc_mrr_at_10_diff1": 0.040161, + "nauc_mrr_at_20_max": -0.009438, + "nauc_mrr_at_20_std": -0.114331, + "nauc_mrr_at_20_diff1": 0.037312, + "nauc_mrr_at_100_max": -0.008771, + "nauc_mrr_at_100_std": -0.112589, + "nauc_mrr_at_100_diff1": 0.038964, + "nauc_mrr_at_1000_max": -0.008735, + "nauc_mrr_at_1000_std": -0.112608, + "nauc_mrr_at_1000_diff1": 0.039308, + "main_score": 0.19042, + "hf_subset": "economics", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.18812, + "ndcg_at_3": 0.2008, + "ndcg_at_5": 0.20501, + "ndcg_at_10": 0.21194, + "ndcg_at_20": 0.22698, + "ndcg_at_100": 0.26487, + "ndcg_at_1000": 0.32258, + "map_at_1": 0.07677, + "map_at_3": 0.1195, + "map_at_5": 0.13201, + "map_at_10": 0.1484, + "map_at_20": 0.16416, + "map_at_100": 0.17293, + "map_at_1000": 0.17694, + "recall_at_1": 0.07677, + "recall_at_3": 0.15057, + "recall_at_5": 0.18165, + "recall_at_10": 0.22971, + "recall_at_20": 0.29433, + "recall_at_100": 0.45319, + "recall_at_1000": 0.74386, + "precision_at_1": 0.18812, + "precision_at_3": 0.14521, + "precision_at_5": 0.12079, + "precision_at_10": 0.08911, + "precision_at_20": 0.06485, + "precision_at_100": 0.02168, + "precision_at_1000": 0.00433, + "mrr_at_1": 0.188119, + "mrr_at_3": 0.235974, + "mrr_at_5": 0.251815, + "mrr_at_10": 0.260832, + "mrr_at_20": 0.264522, + "mrr_at_100": 0.268451, + "mrr_at_1000": 0.269733, + "nauc_ndcg_at_1_max": -0.053609, + "nauc_ndcg_at_1_std": -0.039127, + "nauc_ndcg_at_1_diff1": 0.195798, + "nauc_ndcg_at_3_max": 0.003866, + "nauc_ndcg_at_3_std": 0.00604, + "nauc_ndcg_at_3_diff1": 0.203061, + "nauc_ndcg_at_5_max": 0.017137, + "nauc_ndcg_at_5_std": 0.023284, + "nauc_ndcg_at_5_diff1": 0.211169, + "nauc_ndcg_at_10_max": 0.010622, + "nauc_ndcg_at_10_std": 0.033833, + "nauc_ndcg_at_10_diff1": 0.205534, + "nauc_ndcg_at_20_max": 0.018419, + "nauc_ndcg_at_20_std": 0.046653, + "nauc_ndcg_at_20_diff1": 0.212058, + "nauc_ndcg_at_100_max": 0.062169, + "nauc_ndcg_at_100_std": 0.098522, + "nauc_ndcg_at_100_diff1": 0.218055, + "nauc_ndcg_at_1000_max": 0.070772, + "nauc_ndcg_at_1000_std": 0.127874, + "nauc_ndcg_at_1000_diff1": 0.225176, + "nauc_map_at_1_max": -0.331234, + "nauc_map_at_1_std": -0.085413, + "nauc_map_at_1_diff1": 0.098402, + "nauc_map_at_3_max": -0.175013, + "nauc_map_at_3_std": -0.049034, + "nauc_map_at_3_diff1": 0.135368, + "nauc_map_at_5_max": -0.127215, + "nauc_map_at_5_std": -0.029502, + "nauc_map_at_5_diff1": 0.1554, + "nauc_map_at_10_max": -0.085143, + "nauc_map_at_10_std": 0.000255, + "nauc_map_at_10_diff1": 0.173997, + "nauc_map_at_20_max": -0.061834, + "nauc_map_at_20_std": 0.012018, + "nauc_map_at_20_diff1": 0.199215, + "nauc_map_at_100_max": -0.03428, + "nauc_map_at_100_std": 0.032308, + "nauc_map_at_100_diff1": 0.202378, + "nauc_map_at_1000_max": -0.029305, + "nauc_map_at_1000_std": 0.038854, + "nauc_map_at_1000_diff1": 0.202515, + "nauc_recall_at_1_max": -0.331234, + "nauc_recall_at_1_std": -0.085413, + "nauc_recall_at_1_diff1": 0.098402, + "nauc_recall_at_3_max": -0.077673, + "nauc_recall_at_3_std": 0.002062, + "nauc_recall_at_3_diff1": 0.121469, + "nauc_recall_at_5_max": -0.003148, + "nauc_recall_at_5_std": 0.019321, + "nauc_recall_at_5_diff1": 0.173142, + "nauc_recall_at_10_max": 0.024112, + "nauc_recall_at_10_std": 0.065231, + "nauc_recall_at_10_diff1": 0.14925, + "nauc_recall_at_20_max": 0.08931, + "nauc_recall_at_20_std": 0.111016, + "nauc_recall_at_20_diff1": 0.167919, + "nauc_recall_at_100_max": 0.180885, + "nauc_recall_at_100_std": 0.254408, + "nauc_recall_at_100_diff1": 0.148123, + "nauc_recall_at_1000_max": 0.126234, + "nauc_recall_at_1000_std": 0.387289, + "nauc_recall_at_1000_diff1": 0.129289, + "nauc_precision_at_1_max": -0.053609, + "nauc_precision_at_1_std": -0.039127, + "nauc_precision_at_1_diff1": 0.195798, + "nauc_precision_at_3_max": 0.144732, + "nauc_precision_at_3_std": 0.039178, + "nauc_precision_at_3_diff1": 0.261247, + "nauc_precision_at_5_max": 0.214513, + "nauc_precision_at_5_std": 0.083643, + "nauc_precision_at_5_diff1": 0.286252, + "nauc_precision_at_10_max": 0.224837, + "nauc_precision_at_10_std": 0.102534, + "nauc_precision_at_10_diff1": 0.304258, + "nauc_precision_at_20_max": 0.210991, + "nauc_precision_at_20_std": 0.108147, + "nauc_precision_at_20_diff1": 0.316437, + "nauc_precision_at_100_max": 0.336717, + "nauc_precision_at_100_std": 0.23353, + "nauc_precision_at_100_diff1": 0.183905, + "nauc_precision_at_1000_max": 0.295224, + "nauc_precision_at_1000_std": 0.242803, + "nauc_precision_at_1000_diff1": 0.073522, + "nauc_mrr_at_1_max": -0.053609, + "nauc_mrr_at_1_std": -0.039127, + "nauc_mrr_at_1_diff1": 0.195798, + "nauc_mrr_at_3_max": -0.007287, + "nauc_mrr_at_3_std": -0.016342, + "nauc_mrr_at_3_diff1": 0.233729, + "nauc_mrr_at_5_max": 0.000446, + "nauc_mrr_at_5_std": -0.011918, + "nauc_mrr_at_5_diff1": 0.247336, + "nauc_mrr_at_10_max": 0.011617, + "nauc_mrr_at_10_std": -0.003642, + "nauc_mrr_at_10_diff1": 0.235275, + "nauc_mrr_at_20_max": 0.013966, + "nauc_mrr_at_20_std": 0.000262, + "nauc_mrr_at_20_diff1": 0.236615, + "nauc_mrr_at_100_max": 0.01633, + "nauc_mrr_at_100_std": 0.004908, + "nauc_mrr_at_100_diff1": 0.235608, + "nauc_mrr_at_1000_max": 0.015964, + "nauc_mrr_at_1000_std": 0.004559, + "nauc_mrr_at_1000_diff1": 0.236424, + "main_score": 0.21194, + "hf_subset": "psychology", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.13861, + "ndcg_at_3": 0.14334, + "ndcg_at_5": 0.14466, + "ndcg_at_10": 0.15369, + "ndcg_at_20": 0.1604, + "ndcg_at_100": 0.19605, + "ndcg_at_1000": 0.2486, + "map_at_1": 0.04378, + "map_at_3": 0.0773, + "map_at_5": 0.08922, + "map_at_10": 0.10021, + "map_at_20": 0.10463, + "map_at_100": 0.11169, + "map_at_1000": 0.11491, + "recall_at_1": 0.04378, + "recall_at_3": 0.1058, + "recall_at_5": 0.13431, + "recall_at_10": 0.18472, + "recall_at_20": 0.22241, + "recall_at_100": 0.34798, + "recall_at_1000": 0.63096, + "precision_at_1": 0.13861, + "precision_at_3": 0.11221, + "precision_at_5": 0.09109, + "precision_at_10": 0.06436, + "precision_at_20": 0.0401, + "precision_at_100": 0.01564, + "precision_at_1000": 0.00301, + "mrr_at_1": 0.138614, + "mrr_at_3": 0.19802, + "mrr_at_5": 0.205446, + "mrr_at_10": 0.21159, + "mrr_at_20": 0.214847, + "mrr_at_100": 0.218282, + "mrr_at_1000": 0.219425, + "nauc_ndcg_at_1_max": 0.327933, + "nauc_ndcg_at_1_std": -0.129265, + "nauc_ndcg_at_1_diff1": 0.487362, + "nauc_ndcg_at_3_max": 0.2853, + "nauc_ndcg_at_3_std": -0.053133, + "nauc_ndcg_at_3_diff1": 0.421097, + "nauc_ndcg_at_5_max": 0.277803, + "nauc_ndcg_at_5_std": -0.094188, + "nauc_ndcg_at_5_diff1": 0.402449, + "nauc_ndcg_at_10_max": 0.315021, + "nauc_ndcg_at_10_std": -0.011948, + "nauc_ndcg_at_10_diff1": 0.347987, + "nauc_ndcg_at_20_max": 0.306512, + "nauc_ndcg_at_20_std": 0.015651, + "nauc_ndcg_at_20_diff1": 0.337724, + "nauc_ndcg_at_100_max": 0.323324, + "nauc_ndcg_at_100_std": 0.080406, + "nauc_ndcg_at_100_diff1": 0.299508, + "nauc_ndcg_at_1000_max": 0.315728, + "nauc_ndcg_at_1000_std": 0.123251, + "nauc_ndcg_at_1000_diff1": 0.26178, + "nauc_map_at_1_max": 0.267547, + "nauc_map_at_1_std": -0.106449, + "nauc_map_at_1_diff1": 0.42277, + "nauc_map_at_3_max": 0.27592, + "nauc_map_at_3_std": -0.046936, + "nauc_map_at_3_diff1": 0.377737, + "nauc_map_at_5_max": 0.264296, + "nauc_map_at_5_std": -0.099122, + "nauc_map_at_5_diff1": 0.379183, + "nauc_map_at_10_max": 0.297433, + "nauc_map_at_10_std": -0.054716, + "nauc_map_at_10_diff1": 0.361359, + "nauc_map_at_20_max": 0.29123, + "nauc_map_at_20_std": -0.050325, + "nauc_map_at_20_diff1": 0.364162, + "nauc_map_at_100_max": 0.29629, + "nauc_map_at_100_std": -0.029484, + "nauc_map_at_100_diff1": 0.354803, + "nauc_map_at_1000_max": 0.29819, + "nauc_map_at_1000_std": -0.0206, + "nauc_map_at_1000_diff1": 0.349529, + "nauc_recall_at_1_max": 0.267547, + "nauc_recall_at_1_std": -0.106449, + "nauc_recall_at_1_diff1": 0.42277, + "nauc_recall_at_3_max": 0.27616, + "nauc_recall_at_3_std": 0.018688, + "nauc_recall_at_3_diff1": 0.357224, + "nauc_recall_at_5_max": 0.231384, + "nauc_recall_at_5_std": -0.073222, + "nauc_recall_at_5_diff1": 0.318806, + "nauc_recall_at_10_max": 0.301408, + "nauc_recall_at_10_std": 0.098619, + "nauc_recall_at_10_diff1": 0.207819, + "nauc_recall_at_20_max": 0.259465, + "nauc_recall_at_20_std": 0.125366, + "nauc_recall_at_20_diff1": 0.204274, + "nauc_recall_at_100_max": 0.28458, + "nauc_recall_at_100_std": 0.214549, + "nauc_recall_at_100_diff1": 0.141506, + "nauc_recall_at_1000_max": 0.198464, + "nauc_recall_at_1000_std": 0.275868, + "nauc_recall_at_1000_diff1": -0.021661, + "nauc_precision_at_1_max": 0.327933, + "nauc_precision_at_1_std": -0.129265, + "nauc_precision_at_1_diff1": 0.487362, + "nauc_precision_at_3_max": 0.299684, + "nauc_precision_at_3_std": -0.069059, + "nauc_precision_at_3_diff1": 0.42308, + "nauc_precision_at_5_max": 0.314599, + "nauc_precision_at_5_std": -0.127287, + "nauc_precision_at_5_diff1": 0.422567, + "nauc_precision_at_10_max": 0.357075, + "nauc_precision_at_10_std": -0.020584, + "nauc_precision_at_10_diff1": 0.369318, + "nauc_precision_at_20_max": 0.290092, + "nauc_precision_at_20_std": 0.035174, + "nauc_precision_at_20_diff1": 0.318074, + "nauc_precision_at_100_max": 0.293228, + "nauc_precision_at_100_std": 0.254408, + "nauc_precision_at_100_diff1": 0.136784, + "nauc_precision_at_1000_max": 0.230554, + "nauc_precision_at_1000_std": 0.380313, + "nauc_precision_at_1000_diff1": 0.015675, + "nauc_mrr_at_1_max": 0.327933, + "nauc_mrr_at_1_std": -0.129265, + "nauc_mrr_at_1_diff1": 0.487362, + "nauc_mrr_at_3_max": 0.300805, + "nauc_mrr_at_3_std": -0.041982, + "nauc_mrr_at_3_diff1": 0.419294, + "nauc_mrr_at_5_max": 0.305849, + "nauc_mrr_at_5_std": -0.044494, + "nauc_mrr_at_5_diff1": 0.402476, + "nauc_mrr_at_10_max": 0.313027, + "nauc_mrr_at_10_std": -0.015874, + "nauc_mrr_at_10_diff1": 0.385778, + "nauc_mrr_at_20_max": 0.309021, + "nauc_mrr_at_20_std": -0.013615, + "nauc_mrr_at_20_diff1": 0.383482, + "nauc_mrr_at_100_max": 0.309974, + "nauc_mrr_at_100_std": -0.011275, + "nauc_mrr_at_100_diff1": 0.38282, + "nauc_mrr_at_1000_max": 0.309412, + "nauc_mrr_at_1000_std": -0.01199, + "nauc_mrr_at_1000_diff1": 0.382795, + "main_score": 0.15369, + "hf_subset": "robotics", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.11111, + "ndcg_at_3": 0.09315, + "ndcg_at_5": 0.10579, + "ndcg_at_10": 0.12011, + "ndcg_at_20": 0.1485, + "ndcg_at_100": 0.19825, + "ndcg_at_1000": 0.25419, + "map_at_1": 0.03207, + "map_at_3": 0.05076, + "map_at_5": 0.06045, + "map_at_10": 0.07083, + "map_at_20": 0.08052, + "map_at_100": 0.09164, + "map_at_1000": 0.09566, + "recall_at_1": 0.03207, + "recall_at_3": 0.06838, + "recall_at_5": 0.09733, + "recall_at_10": 0.14303, + "recall_at_20": 0.23124, + "recall_at_100": 0.40553, + "recall_at_1000": 0.6691, + "precision_at_1": 0.11111, + "precision_at_3": 0.07692, + "precision_at_5": 0.07863, + "precision_at_10": 0.06325, + "precision_at_20": 0.05, + "precision_at_100": 0.0206, + "precision_at_1000": 0.00397, + "mrr_at_1": 0.111111, + "mrr_at_3": 0.138177, + "mrr_at_5": 0.159117, + "mrr_at_10": 0.169288, + "mrr_at_20": 0.18036, + "mrr_at_100": 0.18543, + "mrr_at_1000": 0.186287, + "nauc_ndcg_at_1_max": 0.13563, + "nauc_ndcg_at_1_std": 0.152476, + "nauc_ndcg_at_1_diff1": 0.213775, + "nauc_ndcg_at_3_max": 0.061271, + "nauc_ndcg_at_3_std": 0.092826, + "nauc_ndcg_at_3_diff1": 0.201039, + "nauc_ndcg_at_5_max": 0.075011, + "nauc_ndcg_at_5_std": 0.076718, + "nauc_ndcg_at_5_diff1": 0.148661, + "nauc_ndcg_at_10_max": 0.117401, + "nauc_ndcg_at_10_std": 0.143345, + "nauc_ndcg_at_10_diff1": 0.090849, + "nauc_ndcg_at_20_max": 0.135849, + "nauc_ndcg_at_20_std": 0.14011, + "nauc_ndcg_at_20_diff1": 0.131288, + "nauc_ndcg_at_100_max": 0.187536, + "nauc_ndcg_at_100_std": 0.223302, + "nauc_ndcg_at_100_diff1": 0.118276, + "nauc_ndcg_at_1000_max": 0.219782, + "nauc_ndcg_at_1000_std": 0.337585, + "nauc_ndcg_at_1000_diff1": 0.105031, + "nauc_map_at_1_max": 0.082438, + "nauc_map_at_1_std": 0.225138, + "nauc_map_at_1_diff1": 0.149471, + "nauc_map_at_3_max": 0.053978, + "nauc_map_at_3_std": 0.115549, + "nauc_map_at_3_diff1": 0.195586, + "nauc_map_at_5_max": 0.079443, + "nauc_map_at_5_std": 0.094802, + "nauc_map_at_5_diff1": 0.165241, + "nauc_map_at_10_max": 0.096883, + "nauc_map_at_10_std": 0.115916, + "nauc_map_at_10_diff1": 0.130413, + "nauc_map_at_20_max": 0.105285, + "nauc_map_at_20_std": 0.122375, + "nauc_map_at_20_diff1": 0.141287, + "nauc_map_at_100_max": 0.138253, + "nauc_map_at_100_std": 0.173193, + "nauc_map_at_100_diff1": 0.134649, + "nauc_map_at_1000_max": 0.141559, + "nauc_map_at_1000_std": 0.192781, + "nauc_map_at_1000_diff1": 0.136102, + "nauc_recall_at_1_max": 0.082438, + "nauc_recall_at_1_std": 0.225138, + "nauc_recall_at_1_diff1": 0.149471, + "nauc_recall_at_3_max": -0.003433, + "nauc_recall_at_3_std": 0.006593, + "nauc_recall_at_3_diff1": 0.169403, + "nauc_recall_at_5_max": 0.052262, + "nauc_recall_at_5_std": -0.012617, + "nauc_recall_at_5_diff1": 0.098296, + "nauc_recall_at_10_max": 0.108175, + "nauc_recall_at_10_std": 0.104404, + "nauc_recall_at_10_diff1": -0.015622, + "nauc_recall_at_20_max": 0.105418, + "nauc_recall_at_20_std": 0.055726, + "nauc_recall_at_20_diff1": 0.112054, + "nauc_recall_at_100_max": 0.197591, + "nauc_recall_at_100_std": 0.179644, + "nauc_recall_at_100_diff1": 0.138848, + "nauc_recall_at_1000_max": 0.333513, + "nauc_recall_at_1000_std": 0.529455, + "nauc_recall_at_1000_diff1": 0.092582, + "nauc_precision_at_1_max": 0.13563, + "nauc_precision_at_1_std": 0.152476, + "nauc_precision_at_1_diff1": 0.213775, + "nauc_precision_at_3_max": 0.090464, + "nauc_precision_at_3_std": 0.082181, + "nauc_precision_at_3_diff1": 0.223271, + "nauc_precision_at_5_max": 0.105228, + "nauc_precision_at_5_std": 0.081457, + "nauc_precision_at_5_diff1": 0.122582, + "nauc_precision_at_10_max": 0.130343, + "nauc_precision_at_10_std": 0.192694, + "nauc_precision_at_10_diff1": 0.012604, + "nauc_precision_at_20_max": 0.196208, + "nauc_precision_at_20_std": 0.273697, + "nauc_precision_at_20_diff1": 0.041604, + "nauc_precision_at_100_max": 0.192041, + "nauc_precision_at_100_std": 0.413154, + "nauc_precision_at_100_diff1": 0.032904, + "nauc_precision_at_1000_max": 0.121832, + "nauc_precision_at_1000_std": 0.436252, + "nauc_precision_at_1000_diff1": 0.072809, + "nauc_mrr_at_1_max": 0.13563, + "nauc_mrr_at_1_std": 0.152476, + "nauc_mrr_at_1_diff1": 0.213775, + "nauc_mrr_at_3_max": 0.128547, + "nauc_mrr_at_3_std": 0.104003, + "nauc_mrr_at_3_diff1": 0.187815, + "nauc_mrr_at_5_max": 0.122615, + "nauc_mrr_at_5_std": 0.113509, + "nauc_mrr_at_5_diff1": 0.169877, + "nauc_mrr_at_10_max": 0.130033, + "nauc_mrr_at_10_std": 0.139243, + "nauc_mrr_at_10_diff1": 0.143595, + "nauc_mrr_at_20_max": 0.122596, + "nauc_mrr_at_20_std": 0.124783, + "nauc_mrr_at_20_diff1": 0.149659, + "nauc_mrr_at_100_max": 0.123354, + "nauc_mrr_at_100_std": 0.126148, + "nauc_mrr_at_100_diff1": 0.155612, + "nauc_mrr_at_1000_max": 0.124415, + "nauc_mrr_at_1000_std": 0.127701, + "nauc_mrr_at_1000_diff1": 0.155256, + "main_score": 0.12011, + "hf_subset": "stackoverflow", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.14815, + "ndcg_at_3": 0.13605, + "ndcg_at_5": 0.13221, + "ndcg_at_10": 0.14953, + "ndcg_at_20": 0.18024, + "ndcg_at_100": 0.23257, + "ndcg_at_1000": 0.28997, + "map_at_1": 0.05444, + "map_at_3": 0.0794, + "map_at_5": 0.08602, + "map_at_10": 0.09855, + "map_at_20": 0.11212, + "map_at_100": 0.12169, + "map_at_1000": 0.12571, + "recall_at_1": 0.05444, + "recall_at_3": 0.10288, + "recall_at_5": 0.12135, + "recall_at_10": 0.17593, + "recall_at_20": 0.2698, + "recall_at_100": 0.46902, + "recall_at_1000": 0.7459, + "precision_at_1": 0.14815, + "precision_at_3": 0.10494, + "precision_at_5": 0.07963, + "precision_at_10": 0.06389, + "precision_at_20": 0.05, + "precision_at_100": 0.01898, + "precision_at_1000": 0.00376, + "mrr_at_1": 0.148148, + "mrr_at_3": 0.199074, + "mrr_at_5": 0.207407, + "mrr_at_10": 0.219878, + "mrr_at_20": 0.225764, + "mrr_at_100": 0.231374, + "mrr_at_1000": 0.23205, + "nauc_ndcg_at_1_max": 0.370313, + "nauc_ndcg_at_1_std": -0.154812, + "nauc_ndcg_at_1_diff1": 0.193062, + "nauc_ndcg_at_3_max": 0.485158, + "nauc_ndcg_at_3_std": -0.089211, + "nauc_ndcg_at_3_diff1": 0.198335, + "nauc_ndcg_at_5_max": 0.543684, + "nauc_ndcg_at_5_std": -0.076719, + "nauc_ndcg_at_5_diff1": 0.206244, + "nauc_ndcg_at_10_max": 0.574134, + "nauc_ndcg_at_10_std": -0.073867, + "nauc_ndcg_at_10_diff1": 0.251493, + "nauc_ndcg_at_20_max": 0.559249, + "nauc_ndcg_at_20_std": -0.105866, + "nauc_ndcg_at_20_diff1": 0.270988, + "nauc_ndcg_at_100_max": 0.539527, + "nauc_ndcg_at_100_std": -0.057697, + "nauc_ndcg_at_100_diff1": 0.198342, + "nauc_ndcg_at_1000_max": 0.498847, + "nauc_ndcg_at_1000_std": -0.055938, + "nauc_ndcg_at_1000_diff1": 0.174115, + "nauc_map_at_1_max": 0.474521, + "nauc_map_at_1_std": -0.16781, + "nauc_map_at_1_diff1": 0.302802, + "nauc_map_at_3_max": 0.571942, + "nauc_map_at_3_std": -0.078889, + "nauc_map_at_3_diff1": 0.208266, + "nauc_map_at_5_max": 0.583962, + "nauc_map_at_5_std": -0.085355, + "nauc_map_at_5_diff1": 0.219941, + "nauc_map_at_10_max": 0.594006, + "nauc_map_at_10_std": -0.086358, + "nauc_map_at_10_diff1": 0.244519, + "nauc_map_at_20_max": 0.584912, + "nauc_map_at_20_std": -0.100909, + "nauc_map_at_20_diff1": 0.248727, + "nauc_map_at_100_max": 0.575216, + "nauc_map_at_100_std": -0.090172, + "nauc_map_at_100_diff1": 0.227707, + "nauc_map_at_1000_max": 0.571608, + "nauc_map_at_1000_std": -0.09011, + "nauc_map_at_1000_diff1": 0.222963, + "nauc_recall_at_1_max": 0.474521, + "nauc_recall_at_1_std": -0.16781, + "nauc_recall_at_1_diff1": 0.302802, + "nauc_recall_at_3_max": 0.600089, + "nauc_recall_at_3_std": 0.011153, + "nauc_recall_at_3_diff1": 0.185281, + "nauc_recall_at_5_max": 0.604405, + "nauc_recall_at_5_std": -0.013399, + "nauc_recall_at_5_diff1": 0.194541, + "nauc_recall_at_10_max": 0.627381, + "nauc_recall_at_10_std": -0.008567, + "nauc_recall_at_10_diff1": 0.289618, + "nauc_recall_at_20_max": 0.556829, + "nauc_recall_at_20_std": -0.099557, + "nauc_recall_at_20_diff1": 0.3332, + "nauc_recall_at_100_max": 0.507868, + "nauc_recall_at_100_std": 0.067122, + "nauc_recall_at_100_diff1": 0.123862, + "nauc_recall_at_1000_max": 0.424232, + "nauc_recall_at_1000_std": 0.179047, + "nauc_recall_at_1000_diff1": 0.069597, + "nauc_precision_at_1_max": 0.370313, + "nauc_precision_at_1_std": -0.154812, + "nauc_precision_at_1_diff1": 0.193062, + "nauc_precision_at_3_max": 0.462882, + "nauc_precision_at_3_std": -0.086701, + "nauc_precision_at_3_diff1": 0.171399, + "nauc_precision_at_5_max": 0.504739, + "nauc_precision_at_5_std": -0.07377, + "nauc_precision_at_5_diff1": 0.153139, + "nauc_precision_at_10_max": 0.466864, + "nauc_precision_at_10_std": -0.090206, + "nauc_precision_at_10_diff1": 0.158738, + "nauc_precision_at_20_max": 0.367437, + "nauc_precision_at_20_std": -0.112265, + "nauc_precision_at_20_diff1": 0.106227, + "nauc_precision_at_100_max": 0.20464, + "nauc_precision_at_100_std": -0.082133, + "nauc_precision_at_100_diff1": -0.067531, + "nauc_precision_at_1000_max": 0.043209, + "nauc_precision_at_1000_std": -0.103242, + "nauc_precision_at_1000_diff1": -0.156789, + "nauc_mrr_at_1_max": 0.370313, + "nauc_mrr_at_1_std": -0.154812, + "nauc_mrr_at_1_diff1": 0.193062, + "nauc_mrr_at_3_max": 0.401405, + "nauc_mrr_at_3_std": -0.110941, + "nauc_mrr_at_3_diff1": 0.222626, + "nauc_mrr_at_5_max": 0.405518, + "nauc_mrr_at_5_std": -0.114316, + "nauc_mrr_at_5_diff1": 0.21267, + "nauc_mrr_at_10_max": 0.404897, + "nauc_mrr_at_10_std": -0.114033, + "nauc_mrr_at_10_diff1": 0.208458, + "nauc_mrr_at_20_max": 0.404131, + "nauc_mrr_at_20_std": -0.118175, + "nauc_mrr_at_20_diff1": 0.214253, + "nauc_mrr_at_100_max": 0.403839, + "nauc_mrr_at_100_std": -0.112254, + "nauc_mrr_at_100_diff1": 0.20734, + "nauc_mrr_at_1000_max": 0.403362, + "nauc_mrr_at_1000_std": -0.112328, + "nauc_mrr_at_1000_diff1": 0.20751, + "main_score": 0.14953, + "hf_subset": "sustainable_living", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.10714, + "ndcg_at_3": 0.07662, + "ndcg_at_5": 0.07986, + "ndcg_at_10": 0.07379, + "ndcg_at_20": 0.06896, + "ndcg_at_100": 0.11848, + "ndcg_at_1000": 0.25111, + "map_at_1": 0.0059, + "map_at_3": 0.00831, + "map_at_5": 0.01053, + "map_at_10": 0.01321, + "map_at_20": 0.01592, + "map_at_100": 0.02305, + "map_at_1000": 0.03296, + "recall_at_1": 0.0059, + "recall_at_3": 0.01117, + "recall_at_5": 0.02002, + "recall_at_10": 0.03436, + "recall_at_20": 0.05519, + "recall_at_100": 0.17267, + "recall_at_1000": 0.55191, + "precision_at_1": 0.10714, + "precision_at_3": 0.06845, + "precision_at_5": 0.07679, + "precision_at_10": 0.06875, + "precision_at_20": 0.05759, + "precision_at_100": 0.03482, + "precision_at_1000": 0.01188, + "mrr_at_1": 0.107143, + "mrr_at_3": 0.145833, + "mrr_at_5": 0.179315, + "mrr_at_10": 0.207795, + "mrr_at_20": 0.218185, + "mrr_at_100": 0.22597, + "mrr_at_1000": 0.226204, + "nauc_ndcg_at_1_max": 0.051728, + "nauc_ndcg_at_1_std": -0.281421, + "nauc_ndcg_at_1_diff1": 0.073832, + "nauc_ndcg_at_3_max": 0.110946, + "nauc_ndcg_at_3_std": -0.239255, + "nauc_ndcg_at_3_diff1": 0.036084, + "nauc_ndcg_at_5_max": 0.069486, + "nauc_ndcg_at_5_std": -0.128282, + "nauc_ndcg_at_5_diff1": 0.073763, + "nauc_ndcg_at_10_max": 0.015418, + "nauc_ndcg_at_10_std": -0.186379, + "nauc_ndcg_at_10_diff1": 0.092514, + "nauc_ndcg_at_20_max": -0.020054, + "nauc_ndcg_at_20_std": -0.140539, + "nauc_ndcg_at_20_diff1": 0.055219, + "nauc_ndcg_at_100_max": 0.101718, + "nauc_ndcg_at_100_std": -0.006577, + "nauc_ndcg_at_100_diff1": 0.106751, + "nauc_ndcg_at_1000_max": 0.13456, + "nauc_ndcg_at_1000_std": 0.359076, + "nauc_ndcg_at_1000_diff1": 0.137669, + "nauc_map_at_1_max": 0.095774, + "nauc_map_at_1_std": -0.214183, + "nauc_map_at_1_diff1": -0.034126, + "nauc_map_at_3_max": 0.079939, + "nauc_map_at_3_std": -0.177032, + "nauc_map_at_3_diff1": -0.063959, + "nauc_map_at_5_max": 0.06602, + "nauc_map_at_5_std": -0.113989, + "nauc_map_at_5_diff1": -0.046535, + "nauc_map_at_10_max": 0.031067, + "nauc_map_at_10_std": -0.127521, + "nauc_map_at_10_diff1": -0.040282, + "nauc_map_at_20_max": 0.014905, + "nauc_map_at_20_std": -0.11815, + "nauc_map_at_20_diff1": -0.029806, + "nauc_map_at_100_max": 0.054996, + "nauc_map_at_100_std": -0.065533, + "nauc_map_at_100_diff1": 0.013436, + "nauc_map_at_1000_max": 0.084828, + "nauc_map_at_1000_std": 0.069661, + "nauc_map_at_1000_diff1": 0.034347, + "nauc_recall_at_1_max": 0.095774, + "nauc_recall_at_1_std": -0.214183, + "nauc_recall_at_1_diff1": -0.034126, + "nauc_recall_at_3_max": 0.097508, + "nauc_recall_at_3_std": -0.147474, + "nauc_recall_at_3_diff1": -0.070339, + "nauc_recall_at_5_max": 0.066742, + "nauc_recall_at_5_std": 0.019189, + "nauc_recall_at_5_diff1": -0.025419, + "nauc_recall_at_10_max": 0.006186, + "nauc_recall_at_10_std": -0.03734, + "nauc_recall_at_10_diff1": 0.023822, + "nauc_recall_at_20_max": -0.050771, + "nauc_recall_at_20_std": -0.024268, + "nauc_recall_at_20_diff1": 0.008863, + "nauc_recall_at_100_max": 0.235688, + "nauc_recall_at_100_std": 0.063938, + "nauc_recall_at_100_diff1": 0.178867, + "nauc_recall_at_1000_max": 0.286969, + "nauc_recall_at_1000_std": 0.582346, + "nauc_recall_at_1000_diff1": 0.16537, + "nauc_precision_at_1_max": 0.051728, + "nauc_precision_at_1_std": -0.281421, + "nauc_precision_at_1_diff1": 0.073832, + "nauc_precision_at_3_max": 0.142551, + "nauc_precision_at_3_std": -0.219284, + "nauc_precision_at_3_diff1": 0.017343, + "nauc_precision_at_5_max": 0.054699, + "nauc_precision_at_5_std": -0.035033, + "nauc_precision_at_5_diff1": 0.08168, + "nauc_precision_at_10_max": -0.033761, + "nauc_precision_at_10_std": -0.15616, + "nauc_precision_at_10_diff1": 0.119253, + "nauc_precision_at_20_max": -0.107223, + "nauc_precision_at_20_std": -0.11941, + "nauc_precision_at_20_diff1": 0.103886, + "nauc_precision_at_100_max": -0.015165, + "nauc_precision_at_100_std": 0.104299, + "nauc_precision_at_100_diff1": 0.160493, + "nauc_precision_at_1000_max": -0.036834, + "nauc_precision_at_1000_std": 0.488935, + "nauc_precision_at_1000_diff1": 0.164372, + "nauc_mrr_at_1_max": 0.051728, + "nauc_mrr_at_1_std": -0.281421, + "nauc_mrr_at_1_diff1": 0.073832, + "nauc_mrr_at_3_max": 0.107585, + "nauc_mrr_at_3_std": -0.243911, + "nauc_mrr_at_3_diff1": 0.056337, + "nauc_mrr_at_5_max": 0.092013, + "nauc_mrr_at_5_std": -0.167562, + "nauc_mrr_at_5_diff1": 0.079773, + "nauc_mrr_at_10_max": 0.076828, + "nauc_mrr_at_10_std": -0.195667, + "nauc_mrr_at_10_diff1": 0.091142, + "nauc_mrr_at_20_max": 0.072094, + "nauc_mrr_at_20_std": -0.197519, + "nauc_mrr_at_20_diff1": 0.087538, + "nauc_mrr_at_100_max": 0.074842, + "nauc_mrr_at_100_std": -0.19984, + "nauc_mrr_at_100_diff1": 0.084196, + "nauc_mrr_at_1000_max": 0.07478, + "nauc_mrr_at_1000_std": -0.200241, + "nauc_mrr_at_1000_diff1": 0.084147, + "main_score": 0.07379, + "hf_subset": "pony", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.0, + "ndcg_at_3": 0.10055, + "ndcg_at_5": 0.12656, + "ndcg_at_10": 0.15116, + "ndcg_at_20": 0.17937, + "ndcg_at_100": 0.20879, + "ndcg_at_1000": 0.25481, + "map_at_1": 0.0, + "map_at_3": 0.06442, + "map_at_5": 0.0825, + "map_at_10": 0.09723, + "map_at_20": 0.10566, + "map_at_100": 0.10936, + "map_at_1000": 0.11109, + "recall_at_1": 0.0, + "recall_at_3": 0.13345, + "recall_at_5": 0.19073, + "recall_at_10": 0.24871, + "recall_at_20": 0.34812, + "recall_at_100": 0.49249, + "recall_at_1000": 0.82641, + "precision_at_1": 0.0, + "precision_at_3": 0.0892, + "precision_at_5": 0.08169, + "precision_at_10": 0.05563, + "precision_at_20": 0.03556, + "precision_at_100": 0.00951, + "precision_at_1000": 0.00151, + "mrr_at_1": 0.0, + "mrr_at_3": 0.110329, + "mrr_at_5": 0.131103, + "mrr_at_10": 0.137838, + "mrr_at_20": 0.145427, + "mrr_at_100": 0.149741, + "mrr_at_1000": 0.150719, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_3_max": -0.126361, + "nauc_ndcg_at_3_std": 0.240787, + "nauc_ndcg_at_3_diff1": -0.396904, + "nauc_ndcg_at_5_max": -0.110238, + "nauc_ndcg_at_5_std": 0.252373, + "nauc_ndcg_at_5_diff1": -0.387875, + "nauc_ndcg_at_10_max": -0.160141, + "nauc_ndcg_at_10_std": 0.25108, + "nauc_ndcg_at_10_diff1": -0.43073, + "nauc_ndcg_at_20_max": -0.202709, + "nauc_ndcg_at_20_std": 0.287184, + "nauc_ndcg_at_20_diff1": -0.469119, + "nauc_ndcg_at_100_max": -0.233497, + "nauc_ndcg_at_100_std": 0.298718, + "nauc_ndcg_at_100_diff1": -0.483908, + "nauc_ndcg_at_1000_max": -0.201689, + "nauc_ndcg_at_1000_std": 0.316569, + "nauc_ndcg_at_1000_diff1": -0.474098, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_3_max": -0.124643, + "nauc_map_at_3_std": 0.205612, + "nauc_map_at_3_diff1": -0.380333, + "nauc_map_at_5_max": -0.093543, + "nauc_map_at_5_std": 0.238766, + "nauc_map_at_5_diff1": -0.38482, + "nauc_map_at_10_max": -0.130566, + "nauc_map_at_10_std": 0.265584, + "nauc_map_at_10_diff1": -0.418978, + "nauc_map_at_20_max": -0.150099, + "nauc_map_at_20_std": 0.2851, + "nauc_map_at_20_diff1": -0.434772, + "nauc_map_at_100_max": -0.155504, + "nauc_map_at_100_std": 0.288525, + "nauc_map_at_100_diff1": -0.438957, + "nauc_map_at_1000_max": -0.152631, + "nauc_map_at_1000_std": 0.289079, + "nauc_map_at_1000_diff1": -0.437036, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_3_max": -0.134013, + "nauc_recall_at_3_std": 0.211784, + "nauc_recall_at_3_diff1": -0.381506, + "nauc_recall_at_5_max": -0.106331, + "nauc_recall_at_5_std": 0.235184, + "nauc_recall_at_5_diff1": -0.369744, + "nauc_recall_at_10_max": -0.187032, + "nauc_recall_at_10_std": 0.213573, + "nauc_recall_at_10_diff1": -0.426628, + "nauc_recall_at_20_max": -0.255759, + "nauc_recall_at_20_std": 0.26194, + "nauc_recall_at_20_diff1": -0.470766, + "nauc_recall_at_100_max": -0.330175, + "nauc_recall_at_100_std": 0.271694, + "nauc_recall_at_100_diff1": -0.466688, + "nauc_recall_at_1000_max": -0.184597, + "nauc_recall_at_1000_std": 0.430862, + "nauc_recall_at_1000_diff1": -0.410733, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_3_max": -0.120319, + "nauc_precision_at_3_std": 0.297081, + "nauc_precision_at_3_diff1": -0.4089, + "nauc_precision_at_5_max": -0.085205, + "nauc_precision_at_5_std": 0.356739, + "nauc_precision_at_5_diff1": -0.417972, + "nauc_precision_at_10_max": -0.149626, + "nauc_precision_at_10_std": 0.339146, + "nauc_precision_at_10_diff1": -0.446751, + "nauc_precision_at_20_max": -0.185476, + "nauc_precision_at_20_std": 0.378829, + "nauc_precision_at_20_diff1": -0.465697, + "nauc_precision_at_100_max": -0.232936, + "nauc_precision_at_100_std": 0.37204, + "nauc_precision_at_100_diff1": -0.462557, + "nauc_precision_at_1000_max": -0.143817, + "nauc_precision_at_1000_std": 0.292861, + "nauc_precision_at_1000_diff1": -0.347738, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_3_max": -0.138605, + "nauc_mrr_at_3_std": 0.23854, + "nauc_mrr_at_3_diff1": -0.393365, + "nauc_mrr_at_5_max": -0.147903, + "nauc_mrr_at_5_std": 0.252166, + "nauc_mrr_at_5_diff1": -0.396916, + "nauc_mrr_at_10_max": -0.160634, + "nauc_mrr_at_10_std": 0.25036, + "nauc_mrr_at_10_diff1": -0.416567, + "nauc_mrr_at_20_max": -0.174473, + "nauc_mrr_at_20_std": 0.25344, + "nauc_mrr_at_20_diff1": -0.430687, + "nauc_mrr_at_100_max": -0.177583, + "nauc_mrr_at_100_std": 0.252211, + "nauc_mrr_at_100_diff1": -0.430771, + "nauc_mrr_at_1000_max": -0.176489, + "nauc_mrr_at_1000_std": 0.252676, + "nauc_mrr_at_1000_diff1": -0.430028, + "main_score": 0.15116, + "hf_subset": "leetcode", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.0, + "ndcg_at_3": 0.01012, + "ndcg_at_5": 0.01313, + "ndcg_at_10": 0.025, + "ndcg_at_20": 0.03369, + "ndcg_at_100": 0.05981, + "ndcg_at_1000": 0.11278, + "map_at_1": 0.0, + "map_at_3": 0.00236, + "map_at_5": 0.00419, + "map_at_10": 0.00932, + "map_at_20": 0.01218, + "map_at_100": 0.01716, + "map_at_1000": 0.01936, + "recall_at_1": 0.0, + "recall_at_3": 0.00515, + "recall_at_5": 0.01357, + "recall_at_10": 0.04231, + "recall_at_20": 0.06483, + "recall_at_100": 0.15267, + "recall_at_1000": 0.42273, + "precision_at_1": 0.0, + "precision_at_3": 0.01201, + "precision_at_5": 0.01441, + "precision_at_10": 0.01892, + "precision_at_20": 0.01441, + "precision_at_100": 0.00694, + "precision_at_1000": 0.00201, + "mrr_at_1": 0.0, + "mrr_at_3": 0.016517, + "mrr_at_5": 0.024174, + "mrr_at_10": 0.030291, + "mrr_at_20": 0.033006, + "mrr_at_100": 0.037012, + "mrr_at_1000": 0.038328, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_3_max": 0.287859, + "nauc_ndcg_at_3_std": 0.303441, + "nauc_ndcg_at_3_diff1": -0.43809, + "nauc_ndcg_at_5_max": 0.326074, + "nauc_ndcg_at_5_std": 0.245855, + "nauc_ndcg_at_5_diff1": -0.176033, + "nauc_ndcg_at_10_max": 0.257219, + "nauc_ndcg_at_10_std": 0.077875, + "nauc_ndcg_at_10_diff1": -0.140352, + "nauc_ndcg_at_20_max": 0.238458, + "nauc_ndcg_at_20_std": 0.131361, + "nauc_ndcg_at_20_diff1": -0.170118, + "nauc_ndcg_at_100_max": 0.16863, + "nauc_ndcg_at_100_std": 0.235245, + "nauc_ndcg_at_100_diff1": -0.192661, + "nauc_ndcg_at_1000_max": 0.176034, + "nauc_ndcg_at_1000_std": 0.361166, + "nauc_ndcg_at_1000_diff1": -0.199991, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_3_max": 0.304775, + "nauc_map_at_3_std": 0.302727, + "nauc_map_at_3_diff1": -0.43614, + "nauc_map_at_5_max": 0.335095, + "nauc_map_at_5_std": 0.244583, + "nauc_map_at_5_diff1": -0.139374, + "nauc_map_at_10_max": 0.287398, + "nauc_map_at_10_std": 0.102646, + "nauc_map_at_10_diff1": -0.166122, + "nauc_map_at_20_max": 0.287989, + "nauc_map_at_20_std": 0.148119, + "nauc_map_at_20_diff1": -0.204183, + "nauc_map_at_100_max": 0.226676, + "nauc_map_at_100_std": 0.204874, + "nauc_map_at_100_diff1": -0.223451, + "nauc_map_at_1000_max": 0.227219, + "nauc_map_at_1000_std": 0.227054, + "nauc_map_at_1000_diff1": -0.225885, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_3_max": 0.262283, + "nauc_recall_at_3_std": 0.304519, + "nauc_recall_at_3_diff1": -0.441037, + "nauc_recall_at_5_max": 0.297039, + "nauc_recall_at_5_std": 0.24165, + "nauc_recall_at_5_diff1": -0.058829, + "nauc_recall_at_10_max": 0.213823, + "nauc_recall_at_10_std": -0.007454, + "nauc_recall_at_10_diff1": -0.061598, + "nauc_recall_at_20_max": 0.174771, + "nauc_recall_at_20_std": 0.082445, + "nauc_recall_at_20_diff1": -0.115091, + "nauc_recall_at_100_max": 0.09769, + "nauc_recall_at_100_std": 0.219084, + "nauc_recall_at_100_diff1": -0.146661, + "nauc_recall_at_1000_max": 0.118635, + "nauc_recall_at_1000_std": 0.377193, + "nauc_recall_at_1000_diff1": -0.135004, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_3_max": 0.262283, + "nauc_precision_at_3_std": 0.304519, + "nauc_precision_at_3_diff1": -0.441037, + "nauc_precision_at_5_max": 0.317573, + "nauc_precision_at_5_std": 0.241165, + "nauc_precision_at_5_diff1": -0.174078, + "nauc_precision_at_10_max": 0.274198, + "nauc_precision_at_10_std": 0.096239, + "nauc_precision_at_10_diff1": -0.19269, + "nauc_precision_at_20_max": 0.291599, + "nauc_precision_at_20_std": 0.16592, + "nauc_precision_at_20_diff1": -0.220605, + "nauc_precision_at_100_max": 0.187538, + "nauc_precision_at_100_std": 0.318009, + "nauc_precision_at_100_diff1": -0.253499, + "nauc_precision_at_1000_max": 0.157559, + "nauc_precision_at_1000_std": 0.48666, + "nauc_precision_at_1000_diff1": -0.242148, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_3_max": 0.304775, + "nauc_mrr_at_3_std": 0.302727, + "nauc_mrr_at_3_diff1": -0.43614, + "nauc_mrr_at_5_max": 0.338029, + "nauc_mrr_at_5_std": 0.256022, + "nauc_mrr_at_5_diff1": -0.251659, + "nauc_mrr_at_10_max": 0.29067, + "nauc_mrr_at_10_std": 0.172804, + "nauc_mrr_at_10_diff1": -0.20742, + "nauc_mrr_at_20_max": 0.278014, + "nauc_mrr_at_20_std": 0.182843, + "nauc_mrr_at_20_diff1": -0.204951, + "nauc_mrr_at_100_max": 0.267854, + "nauc_mrr_at_100_std": 0.231964, + "nauc_mrr_at_100_diff1": -0.213262, + "nauc_mrr_at_1000_max": 0.266738, + "nauc_mrr_at_1000_std": 0.230681, + "nauc_mrr_at_1000_diff1": -0.210791, + "main_score": 0.025, + "hf_subset": "aops", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.06154, + "ndcg_at_3": 0.06468, + "ndcg_at_5": 0.06468, + "ndcg_at_10": 0.0872, + "ndcg_at_20": 0.09349, + "ndcg_at_100": 0.11537, + "ndcg_at_1000": 0.15778, + "map_at_1": 0.0359, + "map_at_3": 0.05598, + "map_at_5": 0.05598, + "map_at_10": 0.06534, + "map_at_20": 0.06695, + "map_at_100": 0.0698, + "map_at_1000": 0.07152, + "recall_at_1": 0.0359, + "recall_at_3": 0.07179, + "recall_at_5": 0.07179, + "recall_at_10": 0.13846, + "recall_at_20": 0.16154, + "recall_at_100": 0.27051, + "recall_at_1000": 0.55051, + "precision_at_1": 0.06154, + "precision_at_3": 0.04103, + "precision_at_5": 0.02462, + "precision_at_10": 0.02154, + "precision_at_20": 0.01231, + "precision_at_100": 0.00415, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.061538, + "mrr_at_3": 0.074359, + "mrr_at_5": 0.074359, + "mrr_at_10": 0.083632, + "mrr_at_20": 0.085757, + "mrr_at_100": 0.088807, + "mrr_at_1000": 0.090279, + "nauc_ndcg_at_1_max": 0.403761, + "nauc_ndcg_at_1_std": 0.211221, + "nauc_ndcg_at_1_diff1": -0.189764, + "nauc_ndcg_at_3_max": 0.359507, + "nauc_ndcg_at_3_std": 0.37894, + "nauc_ndcg_at_3_diff1": -0.164181, + "nauc_ndcg_at_5_max": 0.359507, + "nauc_ndcg_at_5_std": 0.37894, + "nauc_ndcg_at_5_diff1": -0.164181, + "nauc_ndcg_at_10_max": 0.305541, + "nauc_ndcg_at_10_std": 0.277967, + "nauc_ndcg_at_10_diff1": -0.11051, + "nauc_ndcg_at_20_max": 0.289251, + "nauc_ndcg_at_20_std": 0.295932, + "nauc_ndcg_at_20_diff1": -0.130235, + "nauc_ndcg_at_100_max": 0.324455, + "nauc_ndcg_at_100_std": 0.289166, + "nauc_ndcg_at_100_diff1": -0.121617, + "nauc_ndcg_at_1000_max": 0.33065, + "nauc_ndcg_at_1000_std": 0.228312, + "nauc_ndcg_at_1000_diff1": -0.099178, + "nauc_map_at_1_max": 0.443185, + "nauc_map_at_1_std": 0.363156, + "nauc_map_at_1_diff1": -0.246419, + "nauc_map_at_3_max": 0.406683, + "nauc_map_at_3_std": 0.425645, + "nauc_map_at_3_diff1": -0.115888, + "nauc_map_at_5_max": 0.406683, + "nauc_map_at_5_std": 0.425645, + "nauc_map_at_5_diff1": -0.115888, + "nauc_map_at_10_max": 0.372412, + "nauc_map_at_10_std": 0.368135, + "nauc_map_at_10_diff1": -0.107726, + "nauc_map_at_20_max": 0.366134, + "nauc_map_at_20_std": 0.371501, + "nauc_map_at_20_diff1": -0.115141, + "nauc_map_at_100_max": 0.370381, + "nauc_map_at_100_std": 0.370741, + "nauc_map_at_100_diff1": -0.115787, + "nauc_map_at_1000_max": 0.369244, + "nauc_map_at_1000_std": 0.365218, + "nauc_map_at_1000_diff1": -0.115412, + "nauc_recall_at_1_max": 0.443185, + "nauc_recall_at_1_std": 0.363156, + "nauc_recall_at_1_diff1": -0.246419, + "nauc_recall_at_3_max": 0.308663, + "nauc_recall_at_3_std": 0.458663, + "nauc_recall_at_3_diff1": -0.165668, + "nauc_recall_at_5_max": 0.308663, + "nauc_recall_at_5_std": 0.458663, + "nauc_recall_at_5_diff1": -0.165668, + "nauc_recall_at_10_max": 0.212373, + "nauc_recall_at_10_std": 0.210794, + "nauc_recall_at_10_diff1": -0.034305, + "nauc_recall_at_20_max": 0.188653, + "nauc_recall_at_20_std": 0.251725, + "nauc_recall_at_20_diff1": -0.088916, + "nauc_recall_at_100_max": 0.322904, + "nauc_recall_at_100_std": 0.238281, + "nauc_recall_at_100_diff1": -0.05835, + "nauc_recall_at_1000_max": 0.392387, + "nauc_recall_at_1000_std": 0.073819, + "nauc_recall_at_1000_diff1": 0.076277, + "nauc_precision_at_1_max": 0.403761, + "nauc_precision_at_1_std": 0.211221, + "nauc_precision_at_1_diff1": -0.189764, + "nauc_precision_at_3_max": 0.357975, + "nauc_precision_at_3_std": 0.282612, + "nauc_precision_at_3_diff1": -0.142659, + "nauc_precision_at_5_max": 0.357975, + "nauc_precision_at_5_std": 0.282612, + "nauc_precision_at_5_diff1": -0.142659, + "nauc_precision_at_10_max": 0.313443, + "nauc_precision_at_10_std": 0.092965, + "nauc_precision_at_10_diff1": -0.068261, + "nauc_precision_at_20_max": 0.275313, + "nauc_precision_at_20_std": 0.162039, + "nauc_precision_at_20_diff1": -0.103151, + "nauc_precision_at_100_max": 0.282006, + "nauc_precision_at_100_std": 0.147122, + "nauc_precision_at_100_diff1": -0.086548, + "nauc_precision_at_1000_max": 0.198796, + "nauc_precision_at_1000_std": -0.075704, + "nauc_precision_at_1000_diff1": -0.090334, + "nauc_mrr_at_1_max": 0.403761, + "nauc_mrr_at_1_std": 0.211221, + "nauc_mrr_at_1_diff1": -0.189764, + "nauc_mrr_at_3_max": 0.329921, + "nauc_mrr_at_3_std": 0.236152, + "nauc_mrr_at_3_diff1": -0.232152, + "nauc_mrr_at_5_max": 0.329921, + "nauc_mrr_at_5_std": 0.236152, + "nauc_mrr_at_5_diff1": -0.232152, + "nauc_mrr_at_10_max": 0.30527, + "nauc_mrr_at_10_std": 0.214291, + "nauc_mrr_at_10_diff1": -0.201152, + "nauc_mrr_at_20_max": 0.298673, + "nauc_mrr_at_20_std": 0.224237, + "nauc_mrr_at_20_diff1": -0.205322, + "nauc_mrr_at_100_max": 0.299084, + "nauc_mrr_at_100_std": 0.219472, + "nauc_mrr_at_100_diff1": -0.205794, + "nauc_mrr_at_1000_max": 0.299913, + "nauc_mrr_at_1000_std": 0.217932, + "nauc_mrr_at_1000_diff1": -0.205389, + "main_score": 0.0872, + "hf_subset": "theoremqa_theorems", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.01951, + "ndcg_at_3": 0.07226, + "ndcg_at_5": 0.08041, + "ndcg_at_10": 0.09175, + "ndcg_at_20": 0.10598, + "ndcg_at_100": 0.12363, + "ndcg_at_1000": 0.15128, + "map_at_1": 0.0126, + "map_at_3": 0.05271, + "map_at_5": 0.05873, + "map_at_10": 0.06418, + "map_at_20": 0.06875, + "map_at_100": 0.07115, + "map_at_1000": 0.07214, + "recall_at_1": 0.0126, + "recall_at_3": 0.09593, + "recall_at_5": 0.11341, + "recall_at_10": 0.14297, + "recall_at_20": 0.19175, + "recall_at_100": 0.27184, + "recall_at_1000": 0.45301, + "precision_at_1": 0.01951, + "precision_at_3": 0.05691, + "precision_at_5": 0.04195, + "precision_at_10": 0.02683, + "precision_at_20": 0.01756, + "precision_at_100": 0.00522, + "precision_at_1000": 0.00095, + "mrr_at_1": 0.019512, + "mrr_at_3": 0.079675, + "mrr_at_5": 0.084553, + "mrr_at_10": 0.088312, + "mrr_at_20": 0.092047, + "mrr_at_100": 0.094801, + "mrr_at_1000": 0.095618, + "nauc_ndcg_at_1_max": -0.392834, + "nauc_ndcg_at_1_std": -0.500769, + "nauc_ndcg_at_1_diff1": -0.21176, + "nauc_ndcg_at_3_max": 0.373664, + "nauc_ndcg_at_3_std": -0.114528, + "nauc_ndcg_at_3_diff1": 0.30494, + "nauc_ndcg_at_5_max": 0.365522, + "nauc_ndcg_at_5_std": -0.136375, + "nauc_ndcg_at_5_diff1": 0.3172, + "nauc_ndcg_at_10_max": 0.363593, + "nauc_ndcg_at_10_std": -0.115614, + "nauc_ndcg_at_10_diff1": 0.334598, + "nauc_ndcg_at_20_max": 0.373963, + "nauc_ndcg_at_20_std": -0.107489, + "nauc_ndcg_at_20_diff1": 0.351488, + "nauc_ndcg_at_100_max": 0.358772, + "nauc_ndcg_at_100_std": -0.098087, + "nauc_ndcg_at_100_diff1": 0.343341, + "nauc_ndcg_at_1000_max": 0.361812, + "nauc_ndcg_at_1000_std": -0.066995, + "nauc_ndcg_at_1000_diff1": 0.356188, + "nauc_map_at_1_max": -0.290954, + "nauc_map_at_1_std": -0.499504, + "nauc_map_at_1_diff1": -0.273805, + "nauc_map_at_3_max": 0.368085, + "nauc_map_at_3_std": -0.128215, + "nauc_map_at_3_diff1": 0.263788, + "nauc_map_at_5_max": 0.361491, + "nauc_map_at_5_std": -0.153431, + "nauc_map_at_5_diff1": 0.282505, + "nauc_map_at_10_max": 0.367818, + "nauc_map_at_10_std": -0.133426, + "nauc_map_at_10_diff1": 0.300714, + "nauc_map_at_20_max": 0.366242, + "nauc_map_at_20_std": -0.128358, + "nauc_map_at_20_diff1": 0.308595, + "nauc_map_at_100_max": 0.361117, + "nauc_map_at_100_std": -0.126406, + "nauc_map_at_100_diff1": 0.307479, + "nauc_map_at_1000_max": 0.361143, + "nauc_map_at_1000_std": -0.123485, + "nauc_map_at_1000_diff1": 0.308124, + "nauc_recall_at_1_max": -0.290954, + "nauc_recall_at_1_std": -0.499504, + "nauc_recall_at_1_diff1": -0.273805, + "nauc_recall_at_3_max": 0.456666, + "nauc_recall_at_3_std": -0.053947, + "nauc_recall_at_3_diff1": 0.334522, + "nauc_recall_at_5_max": 0.41718, + "nauc_recall_at_5_std": -0.099462, + "nauc_recall_at_5_diff1": 0.341886, + "nauc_recall_at_10_max": 0.40213, + "nauc_recall_at_10_std": -0.077443, + "nauc_recall_at_10_diff1": 0.37145, + "nauc_recall_at_20_max": 0.43077, + "nauc_recall_at_20_std": -0.058393, + "nauc_recall_at_20_diff1": 0.400958, + "nauc_recall_at_100_max": 0.387579, + "nauc_recall_at_100_std": -0.044771, + "nauc_recall_at_100_diff1": 0.369707, + "nauc_recall_at_1000_max": 0.390145, + "nauc_recall_at_1000_std": 0.021442, + "nauc_recall_at_1000_diff1": 0.43029, + "nauc_precision_at_1_max": -0.392834, + "nauc_precision_at_1_std": -0.500769, + "nauc_precision_at_1_diff1": -0.21176, + "nauc_precision_at_3_max": 0.426939, + "nauc_precision_at_3_std": -0.104537, + "nauc_precision_at_3_diff1": 0.414818, + "nauc_precision_at_5_max": 0.396149, + "nauc_precision_at_5_std": -0.146573, + "nauc_precision_at_5_diff1": 0.420166, + "nauc_precision_at_10_max": 0.365491, + "nauc_precision_at_10_std": -0.086386, + "nauc_precision_at_10_diff1": 0.399438, + "nauc_precision_at_20_max": 0.337908, + "nauc_precision_at_20_std": -0.083651, + "nauc_precision_at_20_diff1": 0.395035, + "nauc_precision_at_100_max": 0.24784, + "nauc_precision_at_100_std": -0.071839, + "nauc_precision_at_100_diff1": 0.330561, + "nauc_precision_at_1000_max": 0.208798, + "nauc_precision_at_1000_std": 0.063491, + "nauc_precision_at_1000_diff1": 0.245653, + "nauc_mrr_at_1_max": -0.392834, + "nauc_mrr_at_1_std": -0.500769, + "nauc_mrr_at_1_diff1": -0.21176, + "nauc_mrr_at_3_max": 0.271773, + "nauc_mrr_at_3_std": -0.158767, + "nauc_mrr_at_3_diff1": 0.29411, + "nauc_mrr_at_5_max": 0.262576, + "nauc_mrr_at_5_std": -0.162703, + "nauc_mrr_at_5_diff1": 0.298802, + "nauc_mrr_at_10_max": 0.256405, + "nauc_mrr_at_10_std": -0.156699, + "nauc_mrr_at_10_diff1": 0.297518, + "nauc_mrr_at_20_max": 0.263556, + "nauc_mrr_at_20_std": -0.156673, + "nauc_mrr_at_20_diff1": 0.299734, + "nauc_mrr_at_100_max": 0.261807, + "nauc_mrr_at_100_std": -0.15149, + "nauc_mrr_at_100_diff1": 0.297561, + "nauc_mrr_at_1000_max": 0.262326, + "nauc_mrr_at_1000_std": -0.149341, + "nauc_mrr_at_1000_diff1": 0.298127, + "main_score": 0.09175, + "hf_subset": "theoremqa_questions", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5544.63121509552, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/BuiltBenchRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/BuiltBenchRetrieval.json new file mode 100644 index 0000000000..69c42ac98c --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/BuiltBenchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae611238a58dae85f3130563fe9f9e995444a8d6", + "task_name": "BuiltBenchRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.61677, + "ndcg_at_3": 0.6175, + "ndcg_at_5": 0.63516, + "ndcg_at_10": 0.65303, + "ndcg_at_20": 0.67711, + "ndcg_at_100": 0.72685, + "ndcg_at_1000": 0.74786, + "map_at_1": 0.30378, + "map_at_3": 0.42349, + "map_at_5": 0.47596, + "map_at_10": 0.52631, + "map_at_20": 0.56285, + "map_at_100": 0.60001, + "map_at_1000": 0.60571, + "recall_at_1": 0.30378, + "recall_at_3": 0.46576, + "recall_at_5": 0.55933, + "recall_at_10": 0.65728, + "recall_at_20": 0.75742, + "recall_at_100": 0.9211, + "recall_at_1000": 0.99624, + "precision_at_1": 0.61677, + "precision_at_3": 0.43613, + "precision_at_5": 0.36766, + "precision_at_10": 0.26916, + "precision_at_20": 0.18847, + "precision_at_100": 0.06368, + "precision_at_1000": 0.00817, + "mrr_at_1": 0.616766, + "mrr_at_3": 0.689122, + "mrr_at_5": 0.701547, + "mrr_at_10": 0.707474, + "mrr_at_20": 0.710176, + "mrr_at_100": 0.711842, + "mrr_at_1000": 0.711999, + "nauc_ndcg_at_1_max": 0.418036, + "nauc_ndcg_at_1_std": -0.061098, + "nauc_ndcg_at_1_diff1": 0.498647, + "nauc_ndcg_at_3_max": 0.39041, + "nauc_ndcg_at_3_std": -0.098457, + "nauc_ndcg_at_3_diff1": 0.404455, + "nauc_ndcg_at_5_max": 0.409219, + "nauc_ndcg_at_5_std": -0.10969, + "nauc_ndcg_at_5_diff1": 0.414251, + "nauc_ndcg_at_10_max": 0.410797, + "nauc_ndcg_at_10_std": -0.094426, + "nauc_ndcg_at_10_diff1": 0.417761, + "nauc_ndcg_at_20_max": 0.429667, + "nauc_ndcg_at_20_std": -0.06326, + "nauc_ndcg_at_20_diff1": 0.430708, + "nauc_ndcg_at_100_max": 0.419131, + "nauc_ndcg_at_100_std": -0.006453, + "nauc_ndcg_at_100_diff1": 0.430054, + "nauc_ndcg_at_1000_max": 0.426857, + "nauc_ndcg_at_1000_std": -0.036203, + "nauc_ndcg_at_1000_diff1": 0.423331, + "nauc_map_at_1_max": 0.349919, + "nauc_map_at_1_std": -0.202123, + "nauc_map_at_1_diff1": 0.502346, + "nauc_map_at_3_max": 0.367065, + "nauc_map_at_3_std": -0.199259, + "nauc_map_at_3_diff1": 0.465468, + "nauc_map_at_5_max": 0.384527, + "nauc_map_at_5_std": -0.18851, + "nauc_map_at_5_diff1": 0.455027, + "nauc_map_at_10_max": 0.389791, + "nauc_map_at_10_std": -0.161264, + "nauc_map_at_10_diff1": 0.442775, + "nauc_map_at_20_max": 0.408581, + "nauc_map_at_20_std": -0.125615, + "nauc_map_at_20_diff1": 0.438605, + "nauc_map_at_100_max": 0.419483, + "nauc_map_at_100_std": -0.088689, + "nauc_map_at_100_diff1": 0.424937, + "nauc_map_at_1000_max": 0.420802, + "nauc_map_at_1000_std": -0.090081, + "nauc_map_at_1000_diff1": 0.423714, + "nauc_recall_at_1_max": 0.349919, + "nauc_recall_at_1_std": -0.202123, + "nauc_recall_at_1_diff1": 0.502346, + "nauc_recall_at_3_max": 0.334123, + "nauc_recall_at_3_std": -0.200488, + "nauc_recall_at_3_diff1": 0.404286, + "nauc_recall_at_5_max": 0.348323, + "nauc_recall_at_5_std": -0.173537, + "nauc_recall_at_5_diff1": 0.392651, + "nauc_recall_at_10_max": 0.332391, + "nauc_recall_at_10_std": -0.122822, + "nauc_recall_at_10_diff1": 0.369219, + "nauc_recall_at_20_max": 0.376717, + "nauc_recall_at_20_std": -0.052848, + "nauc_recall_at_20_diff1": 0.362602, + "nauc_recall_at_100_max": 0.335235, + "nauc_recall_at_100_std": 0.321527, + "nauc_recall_at_100_diff1": 0.395096, + "nauc_recall_at_1000_max": 0.506204, + "nauc_recall_at_1000_std": 0.534665, + "nauc_recall_at_1000_diff1": 0.011705, + "nauc_precision_at_1_max": 0.418036, + "nauc_precision_at_1_std": -0.061098, + "nauc_precision_at_1_diff1": 0.498647, + "nauc_precision_at_3_max": 0.151685, + "nauc_precision_at_3_std": 0.081369, + "nauc_precision_at_3_diff1": 0.053111, + "nauc_precision_at_5_max": 0.08723, + "nauc_precision_at_5_std": 0.113875, + "nauc_precision_at_5_diff1": -0.059468, + "nauc_precision_at_10_max": 0.025804, + "nauc_precision_at_10_std": 0.160971, + "nauc_precision_at_10_diff1": -0.124948, + "nauc_precision_at_20_max": 6.4e-05, + "nauc_precision_at_20_std": 0.209796, + "nauc_precision_at_20_diff1": -0.163452, + "nauc_precision_at_100_max": -0.05396, + "nauc_precision_at_100_std": 0.213256, + "nauc_precision_at_100_diff1": -0.198083, + "nauc_precision_at_1000_max": -0.071461, + "nauc_precision_at_1000_std": 0.113277, + "nauc_precision_at_1000_diff1": -0.200799, + "nauc_mrr_at_1_max": 0.418036, + "nauc_mrr_at_1_std": -0.061098, + "nauc_mrr_at_1_diff1": 0.498647, + "nauc_mrr_at_3_max": 0.398304, + "nauc_mrr_at_3_std": -0.034158, + "nauc_mrr_at_3_diff1": 0.46048, + "nauc_mrr_at_5_max": 0.407195, + "nauc_mrr_at_5_std": -0.037918, + "nauc_mrr_at_5_diff1": 0.458954, + "nauc_mrr_at_10_max": 0.410145, + "nauc_mrr_at_10_std": -0.035382, + "nauc_mrr_at_10_diff1": 0.462887, + "nauc_mrr_at_20_max": 0.410036, + "nauc_mrr_at_20_std": -0.036436, + "nauc_mrr_at_20_diff1": 0.463163, + "nauc_mrr_at_100_max": 0.41019, + "nauc_mrr_at_100_std": -0.035831, + "nauc_mrr_at_100_diff1": 0.464356, + "nauc_mrr_at_1000_max": 0.409905, + "nauc_mrr_at_1000_std": -0.036177, + "nauc_mrr_at_1000_diff1": 0.464386, + "main_score": 0.65303, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.8277056217193604, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackAndroidRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackAndroidRetrieval.json new file mode 100644 index 0000000000..7ce115afc9 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackAndroidRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f46a197baaae43b4f621051089b82a364682dfeb", + "task_name": "CQADupstackAndroidRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.37768, + "ndcg_at_3": 0.4252, + "ndcg_at_5": 0.44905, + "ndcg_at_10": 0.47872, + "ndcg_at_20": 0.49561, + "ndcg_at_100": 0.53216, + "ndcg_at_1000": 0.55049, + "map_at_1": 0.30931, + "map_at_3": 0.38062, + "map_at_5": 0.3994, + "map_at_10": 0.41596, + "map_at_20": 0.42255, + "map_at_100": 0.42944, + "map_at_1000": 0.43059, + "recall_at_1": 0.30931, + "recall_at_3": 0.45391, + "recall_at_5": 0.51831, + "recall_at_10": 0.60575, + "recall_at_20": 0.6645, + "recall_at_100": 0.8352, + "recall_at_1000": 0.94945, + "precision_at_1": 0.37768, + "precision_at_3": 0.19886, + "precision_at_5": 0.14363, + "precision_at_10": 0.08941, + "precision_at_20": 0.05179, + "precision_at_100": 0.01456, + "precision_at_1000": 0.00192, + "mrr_at_1": 0.377682, + "mrr_at_3": 0.443491, + "mrr_at_5": 0.459514, + "mrr_at_10": 0.472552, + "mrr_at_20": 0.476172, + "mrr_at_100": 0.479416, + "mrr_at_1000": 0.479763, + "nauc_ndcg_at_1_max": 0.342789, + "nauc_ndcg_at_1_std": -0.059432, + "nauc_ndcg_at_1_diff1": 0.539974, + "nauc_ndcg_at_3_max": 0.333505, + "nauc_ndcg_at_3_std": -0.047635, + "nauc_ndcg_at_3_diff1": 0.496371, + "nauc_ndcg_at_5_max": 0.334385, + "nauc_ndcg_at_5_std": -0.045757, + "nauc_ndcg_at_5_diff1": 0.498078, + "nauc_ndcg_at_10_max": 0.335462, + "nauc_ndcg_at_10_std": -0.040342, + "nauc_ndcg_at_10_diff1": 0.487398, + "nauc_ndcg_at_20_max": 0.332992, + "nauc_ndcg_at_20_std": -0.038472, + "nauc_ndcg_at_20_diff1": 0.48491, + "nauc_ndcg_at_100_max": 0.325716, + "nauc_ndcg_at_100_std": -0.028396, + "nauc_ndcg_at_100_diff1": 0.492784, + "nauc_ndcg_at_1000_max": 0.332633, + "nauc_ndcg_at_1000_std": -0.02972, + "nauc_ndcg_at_1000_diff1": 0.500068, + "nauc_map_at_1_max": 0.342015, + "nauc_map_at_1_std": -0.087672, + "nauc_map_at_1_diff1": 0.568109, + "nauc_map_at_3_max": 0.337145, + "nauc_map_at_3_std": -0.066113, + "nauc_map_at_3_diff1": 0.517567, + "nauc_map_at_5_max": 0.339986, + "nauc_map_at_5_std": -0.063487, + "nauc_map_at_5_diff1": 0.516601, + "nauc_map_at_10_max": 0.341658, + "nauc_map_at_10_std": -0.059412, + "nauc_map_at_10_diff1": 0.509079, + "nauc_map_at_20_max": 0.341745, + "nauc_map_at_20_std": -0.057938, + "nauc_map_at_20_diff1": 0.506794, + "nauc_map_at_100_max": 0.339671, + "nauc_map_at_100_std": -0.055239, + "nauc_map_at_100_diff1": 0.507325, + "nauc_map_at_1000_max": 0.340381, + "nauc_map_at_1000_std": -0.055441, + "nauc_map_at_1000_diff1": 0.507471, + "nauc_recall_at_1_max": 0.342015, + "nauc_recall_at_1_std": -0.087672, + "nauc_recall_at_1_diff1": 0.568109, + "nauc_recall_at_3_max": 0.316674, + "nauc_recall_at_3_std": -0.041836, + "nauc_recall_at_3_diff1": 0.464175, + "nauc_recall_at_5_max": 0.305387, + "nauc_recall_at_5_std": -0.026632, + "nauc_recall_at_5_diff1": 0.453663, + "nauc_recall_at_10_max": 0.296349, + "nauc_recall_at_10_std": 0.006325, + "nauc_recall_at_10_diff1": 0.404242, + "nauc_recall_at_20_max": 0.284642, + "nauc_recall_at_20_std": 0.02312, + "nauc_recall_at_20_diff1": 0.378435, + "nauc_recall_at_100_max": 0.205406, + "nauc_recall_at_100_std": 0.158683, + "nauc_recall_at_100_diff1": 0.384593, + "nauc_recall_at_1000_max": 0.185047, + "nauc_recall_at_1000_std": 0.546474, + "nauc_recall_at_1000_diff1": 0.608359, + "nauc_precision_at_1_max": 0.342789, + "nauc_precision_at_1_std": -0.059432, + "nauc_precision_at_1_diff1": 0.539974, + "nauc_precision_at_3_max": 0.277514, + "nauc_precision_at_3_std": 0.00752, + "nauc_precision_at_3_diff1": 0.322611, + "nauc_precision_at_5_max": 0.237087, + "nauc_precision_at_5_std": 0.034244, + "nauc_precision_at_5_diff1": 0.242024, + "nauc_precision_at_10_max": 0.188448, + "nauc_precision_at_10_std": 0.057007, + "nauc_precision_at_10_diff1": 0.117164, + "nauc_precision_at_20_max": 0.126984, + "nauc_precision_at_20_std": 0.067671, + "nauc_precision_at_20_diff1": 0.029238, + "nauc_precision_at_100_max": 0.019055, + "nauc_precision_at_100_std": 0.057108, + "nauc_precision_at_100_diff1": -0.104515, + "nauc_precision_at_1000_max": 0.02412, + "nauc_precision_at_1000_std": -0.051649, + "nauc_precision_at_1000_diff1": -0.173237, + "nauc_mrr_at_1_max": 0.342789, + "nauc_mrr_at_1_std": -0.059432, + "nauc_mrr_at_1_diff1": 0.539974, + "nauc_mrr_at_3_max": 0.333857, + "nauc_mrr_at_3_std": -0.053091, + "nauc_mrr_at_3_diff1": 0.503088, + "nauc_mrr_at_5_max": 0.330761, + "nauc_mrr_at_5_std": -0.048409, + "nauc_mrr_at_5_diff1": 0.503001, + "nauc_mrr_at_10_max": 0.330136, + "nauc_mrr_at_10_std": -0.045228, + "nauc_mrr_at_10_diff1": 0.499507, + "nauc_mrr_at_20_max": 0.329303, + "nauc_mrr_at_20_std": -0.045443, + "nauc_mrr_at_20_diff1": 0.499932, + "nauc_mrr_at_100_max": 0.328786, + "nauc_mrr_at_100_std": -0.044676, + "nauc_mrr_at_100_diff1": 0.500627, + "nauc_mrr_at_1000_max": 0.328991, + "nauc_mrr_at_1000_std": -0.044693, + "nauc_mrr_at_1000_diff1": 0.500794, + "main_score": 0.47872, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 56.30307221412659, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackEnglishRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackEnglishRetrieval.json new file mode 100644 index 0000000000..22cf36e785 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackEnglishRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0", + "task_name": "CQADupstackEnglishRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.40764, + "ndcg_at_3": 0.45196, + "ndcg_at_5": 0.46765, + "ndcg_at_10": 0.48848, + "ndcg_at_20": 0.50394, + "ndcg_at_100": 0.53066, + "ndcg_at_1000": 0.55085, + "map_at_1": 0.32526, + "map_at_3": 0.40484, + "map_at_5": 0.41971, + "map_at_10": 0.43198, + "map_at_20": 0.43832, + "map_at_100": 0.44415, + "map_at_1000": 0.44545, + "recall_at_1": 0.32526, + "recall_at_3": 0.47046, + "recall_at_5": 0.51796, + "recall_at_10": 0.5842, + "recall_at_20": 0.64084, + "recall_at_100": 0.76217, + "recall_at_1000": 0.88894, + "precision_at_1": 0.40764, + "precision_at_3": 0.2172, + "precision_at_5": 0.15019, + "precision_at_10": 0.08975, + "precision_at_20": 0.05226, + "precision_at_100": 0.01427, + "precision_at_1000": 0.0019, + "mrr_at_1": 0.407643, + "mrr_at_3": 0.474522, + "mrr_at_5": 0.485255, + "mrr_at_10": 0.494383, + "mrr_at_20": 0.497976, + "mrr_at_100": 0.500469, + "mrr_at_1000": 0.500875, + "nauc_ndcg_at_1_max": 0.408, + "nauc_ndcg_at_1_std": 0.006449, + "nauc_ndcg_at_1_diff1": 0.57585, + "nauc_ndcg_at_3_max": 0.396424, + "nauc_ndcg_at_3_std": 0.001846, + "nauc_ndcg_at_3_diff1": 0.514193, + "nauc_ndcg_at_5_max": 0.389673, + "nauc_ndcg_at_5_std": 0.009426, + "nauc_ndcg_at_5_diff1": 0.512184, + "nauc_ndcg_at_10_max": 0.382618, + "nauc_ndcg_at_10_std": 0.007178, + "nauc_ndcg_at_10_diff1": 0.509565, + "nauc_ndcg_at_20_max": 0.385589, + "nauc_ndcg_at_20_std": 0.014133, + "nauc_ndcg_at_20_diff1": 0.510704, + "nauc_ndcg_at_100_max": 0.386944, + "nauc_ndcg_at_100_std": 0.033571, + "nauc_ndcg_at_100_diff1": 0.507052, + "nauc_ndcg_at_1000_max": 0.392421, + "nauc_ndcg_at_1000_std": 0.042794, + "nauc_ndcg_at_1000_diff1": 0.510311, + "nauc_map_at_1_max": 0.357553, + "nauc_map_at_1_std": -0.066539, + "nauc_map_at_1_diff1": 0.604239, + "nauc_map_at_3_max": 0.378253, + "nauc_map_at_3_std": -0.042734, + "nauc_map_at_3_diff1": 0.539398, + "nauc_map_at_5_max": 0.380078, + "nauc_map_at_5_std": -0.028481, + "nauc_map_at_5_diff1": 0.533191, + "nauc_map_at_10_max": 0.380563, + "nauc_map_at_10_std": -0.024182, + "nauc_map_at_10_diff1": 0.530853, + "nauc_map_at_20_max": 0.383387, + "nauc_map_at_20_std": -0.018254, + "nauc_map_at_20_diff1": 0.530103, + "nauc_map_at_100_max": 0.385022, + "nauc_map_at_100_std": -0.010615, + "nauc_map_at_100_diff1": 0.528387, + "nauc_map_at_1000_max": 0.385334, + "nauc_map_at_1000_std": -0.009011, + "nauc_map_at_1000_diff1": 0.528174, + "nauc_recall_at_1_max": 0.357553, + "nauc_recall_at_1_std": -0.066539, + "nauc_recall_at_1_diff1": 0.604239, + "nauc_recall_at_3_max": 0.355805, + "nauc_recall_at_3_std": -0.03524, + "nauc_recall_at_3_diff1": 0.477162, + "nauc_recall_at_5_max": 0.343039, + "nauc_recall_at_5_std": -0.00275, + "nauc_recall_at_5_diff1": 0.450687, + "nauc_recall_at_10_max": 0.322724, + "nauc_recall_at_10_std": 0.00441, + "nauc_recall_at_10_diff1": 0.425535, + "nauc_recall_at_20_max": 0.328061, + "nauc_recall_at_20_std": 0.033713, + "nauc_recall_at_20_diff1": 0.420272, + "nauc_recall_at_100_max": 0.310323, + "nauc_recall_at_100_std": 0.150391, + "nauc_recall_at_100_diff1": 0.379562, + "nauc_recall_at_1000_max": 0.366648, + "nauc_recall_at_1000_std": 0.357989, + "nauc_recall_at_1000_diff1": 0.38564, + "nauc_precision_at_1_max": 0.408, + "nauc_precision_at_1_std": 0.006449, + "nauc_precision_at_1_diff1": 0.57585, + "nauc_precision_at_3_max": 0.356351, + "nauc_precision_at_3_std": 0.109521, + "nauc_precision_at_3_diff1": 0.267299, + "nauc_precision_at_5_max": 0.316579, + "nauc_precision_at_5_std": 0.167785, + "nauc_precision_at_5_diff1": 0.191239, + "nauc_precision_at_10_max": 0.251605, + "nauc_precision_at_10_std": 0.199984, + "nauc_precision_at_10_diff1": 0.094527, + "nauc_precision_at_20_max": 0.211791, + "nauc_precision_at_20_std": 0.250609, + "nauc_precision_at_20_diff1": 0.022854, + "nauc_precision_at_100_max": 0.106886, + "nauc_precision_at_100_std": 0.344739, + "nauc_precision_at_100_diff1": -0.113277, + "nauc_precision_at_1000_max": 0.035866, + "nauc_precision_at_1000_std": 0.325535, + "nauc_precision_at_1000_diff1": -0.17608, + "nauc_mrr_at_1_max": 0.408, + "nauc_mrr_at_1_std": 0.006449, + "nauc_mrr_at_1_diff1": 0.57585, + "nauc_mrr_at_3_max": 0.411824, + "nauc_mrr_at_3_std": 0.033196, + "nauc_mrr_at_3_diff1": 0.536979, + "nauc_mrr_at_5_max": 0.408126, + "nauc_mrr_at_5_std": 0.03647, + "nauc_mrr_at_5_diff1": 0.533083, + "nauc_mrr_at_10_max": 0.406554, + "nauc_mrr_at_10_std": 0.036812, + "nauc_mrr_at_10_diff1": 0.531365, + "nauc_mrr_at_20_max": 0.40645, + "nauc_mrr_at_20_std": 0.037902, + "nauc_mrr_at_20_diff1": 0.532122, + "nauc_mrr_at_100_max": 0.406336, + "nauc_mrr_at_100_std": 0.038874, + "nauc_mrr_at_100_diff1": 0.531868, + "nauc_mrr_at_1000_max": 0.406433, + "nauc_mrr_at_1000_std": 0.038992, + "nauc_mrr_at_1000_diff1": 0.531984, + "main_score": 0.48848, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 83.83057737350464, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackGamingRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackGamingRetrieval.json new file mode 100644 index 0000000000..d41a020f7d --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackGamingRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4885aa143210c98657558c04aaf3dc47cfb54340", + "task_name": "CQADupstackGamingRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.4721, + "ndcg_at_3": 0.53338, + "ndcg_at_5": 0.55621, + "ndcg_at_10": 0.58282, + "ndcg_at_20": 0.60042, + "ndcg_at_100": 0.62396, + "ndcg_at_1000": 0.63572, + "map_at_1": 0.418, + "map_at_3": 0.49996, + "map_at_5": 0.51612, + "map_at_10": 0.5296, + "map_at_20": 0.53542, + "map_at_100": 0.53958, + "map_at_1000": 0.54016, + "recall_at_1": 0.418, + "recall_at_3": 0.57752, + "recall_at_5": 0.63343, + "recall_at_10": 0.70935, + "recall_at_20": 0.77367, + "recall_at_100": 0.8855, + "recall_at_1000": 0.9683, + "precision_at_1": 0.4721, + "precision_at_3": 0.23197, + "precision_at_5": 0.15636, + "precision_at_10": 0.0906, + "precision_at_20": 0.0505, + "precision_at_100": 0.01203, + "precision_at_1000": 0.00136, + "mrr_at_1": 0.4721, + "mrr_at_3": 0.541588, + "mrr_at_5": 0.552121, + "mrr_at_10": 0.561451, + "mrr_at_20": 0.565938, + "mrr_at_100": 0.568726, + "mrr_at_1000": 0.569006, + "nauc_ndcg_at_1_max": 0.390033, + "nauc_ndcg_at_1_std": -0.112575, + "nauc_ndcg_at_1_diff1": 0.582904, + "nauc_ndcg_at_3_max": 0.422759, + "nauc_ndcg_at_3_std": -0.093403, + "nauc_ndcg_at_3_diff1": 0.53911, + "nauc_ndcg_at_5_max": 0.429388, + "nauc_ndcg_at_5_std": -0.087495, + "nauc_ndcg_at_5_diff1": 0.533479, + "nauc_ndcg_at_10_max": 0.438479, + "nauc_ndcg_at_10_std": -0.081529, + "nauc_ndcg_at_10_diff1": 0.534135, + "nauc_ndcg_at_20_max": 0.441926, + "nauc_ndcg_at_20_std": -0.071694, + "nauc_ndcg_at_20_diff1": 0.528945, + "nauc_ndcg_at_100_max": 0.443869, + "nauc_ndcg_at_100_std": -0.058143, + "nauc_ndcg_at_100_diff1": 0.533564, + "nauc_ndcg_at_1000_max": 0.439392, + "nauc_ndcg_at_1000_std": -0.063848, + "nauc_ndcg_at_1000_diff1": 0.537249, + "nauc_map_at_1_max": 0.357559, + "nauc_map_at_1_std": -0.132017, + "nauc_map_at_1_diff1": 0.588034, + "nauc_map_at_3_max": 0.410014, + "nauc_map_at_3_std": -0.108526, + "nauc_map_at_3_diff1": 0.55268, + "nauc_map_at_5_max": 0.415859, + "nauc_map_at_5_std": -0.102244, + "nauc_map_at_5_diff1": 0.548448, + "nauc_map_at_10_max": 0.421925, + "nauc_map_at_10_std": -0.096439, + "nauc_map_at_10_diff1": 0.548606, + "nauc_map_at_20_max": 0.422972, + "nauc_map_at_20_std": -0.092791, + "nauc_map_at_20_diff1": 0.546864, + "nauc_map_at_100_max": 0.423764, + "nauc_map_at_100_std": -0.089694, + "nauc_map_at_100_diff1": 0.547448, + "nauc_map_at_1000_max": 0.42371, + "nauc_map_at_1000_std": -0.08964, + "nauc_map_at_1000_diff1": 0.547615, + "nauc_recall_at_1_max": 0.357559, + "nauc_recall_at_1_std": -0.132017, + "nauc_recall_at_1_diff1": 0.588034, + "nauc_recall_at_3_max": 0.429245, + "nauc_recall_at_3_std": -0.086936, + "nauc_recall_at_3_diff1": 0.499, + "nauc_recall_at_5_max": 0.443605, + "nauc_recall_at_5_std": -0.068581, + "nauc_recall_at_5_diff1": 0.472862, + "nauc_recall_at_10_max": 0.472211, + "nauc_recall_at_10_std": -0.048289, + "nauc_recall_at_10_diff1": 0.46205, + "nauc_recall_at_20_max": 0.499479, + "nauc_recall_at_20_std": 0.006965, + "nauc_recall_at_20_diff1": 0.423555, + "nauc_recall_at_100_max": 0.562536, + "nauc_recall_at_100_std": 0.195881, + "nauc_recall_at_100_diff1": 0.402937, + "nauc_recall_at_1000_max": 0.600498, + "nauc_recall_at_1000_std": 0.379476, + "nauc_recall_at_1000_diff1": 0.350184, + "nauc_precision_at_1_max": 0.390033, + "nauc_precision_at_1_std": -0.112575, + "nauc_precision_at_1_diff1": 0.582904, + "nauc_precision_at_3_max": 0.401525, + "nauc_precision_at_3_std": 0.006755, + "nauc_precision_at_3_diff1": 0.342164, + "nauc_precision_at_5_max": 0.350082, + "nauc_precision_at_5_std": 0.044527, + "nauc_precision_at_5_diff1": 0.250629, + "nauc_precision_at_10_max": 0.301364, + "nauc_precision_at_10_std": 0.089652, + "nauc_precision_at_10_diff1": 0.154893, + "nauc_precision_at_20_max": 0.253639, + "nauc_precision_at_20_std": 0.154804, + "nauc_precision_at_20_diff1": 0.054882, + "nauc_precision_at_100_max": 0.134474, + "nauc_precision_at_100_std": 0.249341, + "nauc_precision_at_100_diff1": -0.082627, + "nauc_precision_at_1000_max": 0.033214, + "nauc_precision_at_1000_std": 0.21072, + "nauc_precision_at_1000_diff1": -0.144931, + "nauc_mrr_at_1_max": 0.390033, + "nauc_mrr_at_1_std": -0.112575, + "nauc_mrr_at_1_diff1": 0.582904, + "nauc_mrr_at_3_max": 0.421046, + "nauc_mrr_at_3_std": -0.087979, + "nauc_mrr_at_3_diff1": 0.550943, + "nauc_mrr_at_5_max": 0.422449, + "nauc_mrr_at_5_std": -0.086078, + "nauc_mrr_at_5_diff1": 0.547322, + "nauc_mrr_at_10_max": 0.42416, + "nauc_mrr_at_10_std": -0.086859, + "nauc_mrr_at_10_diff1": 0.548299, + "nauc_mrr_at_20_max": 0.424804, + "nauc_mrr_at_20_std": -0.084908, + "nauc_mrr_at_20_diff1": 0.547874, + "nauc_mrr_at_100_max": 0.424773, + "nauc_mrr_at_100_std": -0.083866, + "nauc_mrr_at_100_diff1": 0.548797, + "nauc_mrr_at_1000_max": 0.424606, + "nauc_mrr_at_1000_std": -0.083981, + "nauc_mrr_at_1000_diff1": 0.548868, + "main_score": 0.58282, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 92.46583867073059, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackGisRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackGisRetrieval.json new file mode 100644 index 0000000000..380439c019 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackGisRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "5003b3064772da1887988e05400cf3806fe491f2", + "task_name": "CQADupstackGisRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.29266, + "ndcg_at_3": 0.34031, + "ndcg_at_5": 0.36494, + "ndcg_at_10": 0.3927, + "ndcg_at_20": 0.4137, + "ndcg_at_100": 0.44474, + "ndcg_at_1000": 0.46637, + "map_at_1": 0.27712, + "map_at_3": 0.32186, + "map_at_5": 0.33609, + "map_at_10": 0.34814, + "map_at_20": 0.35422, + "map_at_100": 0.35865, + "map_at_1000": 0.35954, + "recall_at_1": 0.27712, + "recall_at_3": 0.37521, + "recall_at_5": 0.43259, + "recall_at_10": 0.51434, + "recall_at_20": 0.59401, + "recall_at_100": 0.75502, + "recall_at_1000": 0.9135, + "precision_at_1": 0.29266, + "precision_at_3": 0.13748, + "precision_at_5": 0.09695, + "precision_at_10": 0.05864, + "precision_at_20": 0.03401, + "precision_at_100": 0.00886, + "precision_at_1000": 0.00112, + "mrr_at_1": 0.292655, + "mrr_at_3": 0.338795, + "mrr_at_5": 0.353202, + "mrr_at_10": 0.364772, + "mrr_at_20": 0.370454, + "mrr_at_100": 0.374292, + "mrr_at_1000": 0.374954, + "nauc_ndcg_at_1_max": 0.312512, + "nauc_ndcg_at_1_std": -0.026883, + "nauc_ndcg_at_1_diff1": 0.554573, + "nauc_ndcg_at_3_max": 0.311028, + "nauc_ndcg_at_3_std": -0.019553, + "nauc_ndcg_at_3_diff1": 0.494557, + "nauc_ndcg_at_5_max": 0.321054, + "nauc_ndcg_at_5_std": -0.012784, + "nauc_ndcg_at_5_diff1": 0.489058, + "nauc_ndcg_at_10_max": 0.317861, + "nauc_ndcg_at_10_std": -0.001691, + "nauc_ndcg_at_10_diff1": 0.467432, + "nauc_ndcg_at_20_max": 0.315255, + "nauc_ndcg_at_20_std": -0.008162, + "nauc_ndcg_at_20_diff1": 0.459777, + "nauc_ndcg_at_100_max": 0.315045, + "nauc_ndcg_at_100_std": 0.008253, + "nauc_ndcg_at_100_diff1": 0.459098, + "nauc_ndcg_at_1000_max": 0.323508, + "nauc_ndcg_at_1000_std": 0.000429, + "nauc_ndcg_at_1000_diff1": 0.466055, + "nauc_map_at_1_max": 0.305298, + "nauc_map_at_1_std": -0.03382, + "nauc_map_at_1_diff1": 0.55147, + "nauc_map_at_3_max": 0.308507, + "nauc_map_at_3_std": -0.025472, + "nauc_map_at_3_diff1": 0.507527, + "nauc_map_at_5_max": 0.316734, + "nauc_map_at_5_std": -0.021516, + "nauc_map_at_5_diff1": 0.504548, + "nauc_map_at_10_max": 0.315056, + "nauc_map_at_10_std": -0.017805, + "nauc_map_at_10_diff1": 0.495868, + "nauc_map_at_20_max": 0.314259, + "nauc_map_at_20_std": -0.019392, + "nauc_map_at_20_diff1": 0.493899, + "nauc_map_at_100_max": 0.314311, + "nauc_map_at_100_std": -0.016796, + "nauc_map_at_100_diff1": 0.493559, + "nauc_map_at_1000_max": 0.314853, + "nauc_map_at_1000_std": -0.016974, + "nauc_map_at_1000_diff1": 0.493835, + "nauc_recall_at_1_max": 0.305298, + "nauc_recall_at_1_std": -0.03382, + "nauc_recall_at_1_diff1": 0.55147, + "nauc_recall_at_3_max": 0.302554, + "nauc_recall_at_3_std": -0.011277, + "nauc_recall_at_3_diff1": 0.454112, + "nauc_recall_at_5_max": 0.321503, + "nauc_recall_at_5_std": 0.002893, + "nauc_recall_at_5_diff1": 0.432353, + "nauc_recall_at_10_max": 0.307602, + "nauc_recall_at_10_std": 0.039465, + "nauc_recall_at_10_diff1": 0.360273, + "nauc_recall_at_20_max": 0.296026, + "nauc_recall_at_20_std": 0.017559, + "nauc_recall_at_20_diff1": 0.318543, + "nauc_recall_at_100_max": 0.285846, + "nauc_recall_at_100_std": 0.154327, + "nauc_recall_at_100_diff1": 0.266128, + "nauc_recall_at_1000_max": 0.428316, + "nauc_recall_at_1000_std": 0.163663, + "nauc_recall_at_1000_diff1": 0.195264, + "nauc_precision_at_1_max": 0.312512, + "nauc_precision_at_1_std": -0.026883, + "nauc_precision_at_1_diff1": 0.554573, + "nauc_precision_at_3_max": 0.323174, + "nauc_precision_at_3_std": 0.006647, + "nauc_precision_at_3_diff1": 0.427816, + "nauc_precision_at_5_max": 0.34691, + "nauc_precision_at_5_std": 0.023792, + "nauc_precision_at_5_diff1": 0.402873, + "nauc_precision_at_10_max": 0.339908, + "nauc_precision_at_10_std": 0.048736, + "nauc_precision_at_10_diff1": 0.314921, + "nauc_precision_at_20_max": 0.313374, + "nauc_precision_at_20_std": 0.026373, + "nauc_precision_at_20_diff1": 0.249161, + "nauc_precision_at_100_max": 0.259025, + "nauc_precision_at_100_std": 0.094817, + "nauc_precision_at_100_diff1": 0.122399, + "nauc_precision_at_1000_max": 0.176928, + "nauc_precision_at_1000_std": 0.024135, + "nauc_precision_at_1000_diff1": -0.052259, + "nauc_mrr_at_1_max": 0.312512, + "nauc_mrr_at_1_std": -0.026883, + "nauc_mrr_at_1_diff1": 0.554573, + "nauc_mrr_at_3_max": 0.314974, + "nauc_mrr_at_3_std": -0.015, + "nauc_mrr_at_3_diff1": 0.511864, + "nauc_mrr_at_5_max": 0.319076, + "nauc_mrr_at_5_std": -0.011602, + "nauc_mrr_at_5_diff1": 0.508872, + "nauc_mrr_at_10_max": 0.318665, + "nauc_mrr_at_10_std": -0.006893, + "nauc_mrr_at_10_diff1": 0.499764, + "nauc_mrr_at_20_max": 0.317942, + "nauc_mrr_at_20_std": -0.009487, + "nauc_mrr_at_20_diff1": 0.497806, + "nauc_mrr_at_100_max": 0.317456, + "nauc_mrr_at_100_std": -0.007712, + "nauc_mrr_at_100_diff1": 0.497997, + "nauc_mrr_at_1000_max": 0.317787, + "nauc_mrr_at_1000_std": -0.007946, + "nauc_mrr_at_1000_diff1": 0.498306, + "main_score": 0.3927, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 141.64193749427795, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackMathematicaRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackMathematicaRetrieval.json new file mode 100644 index 0000000000..f7efbd2af2 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackMathematicaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "90fceea13679c63fe563ded68f3b6f06e50061de", + "task_name": "CQADupstackMathematicaRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.22886, + "ndcg_at_3": 0.25991, + "ndcg_at_5": 0.28362, + "ndcg_at_10": 0.30424, + "ndcg_at_20": 0.32648, + "ndcg_at_100": 0.36564, + "ndcg_at_1000": 0.39322, + "map_at_1": 0.18683, + "map_at_3": 0.23252, + "map_at_5": 0.24696, + "map_at_10": 0.25671, + "map_at_20": 0.26292, + "map_at_100": 0.26894, + "map_at_1000": 0.27009, + "recall_at_1": 0.18683, + "recall_at_3": 0.28622, + "recall_at_5": 0.3442, + "recall_at_10": 0.40515, + "recall_at_20": 0.48692, + "recall_at_100": 0.67665, + "recall_at_1000": 0.87339, + "precision_at_1": 0.22886, + "precision_at_3": 0.12189, + "precision_at_5": 0.09005, + "precision_at_10": 0.05435, + "precision_at_20": 0.03308, + "precision_at_100": 0.00975, + "precision_at_1000": 0.00135, + "mrr_at_1": 0.228856, + "mrr_at_3": 0.279022, + "mrr_at_5": 0.295066, + "mrr_at_10": 0.30261, + "mrr_at_20": 0.308744, + "mrr_at_100": 0.313421, + "mrr_at_1000": 0.31407, + "nauc_ndcg_at_1_max": 0.277969, + "nauc_ndcg_at_1_std": -0.00326, + "nauc_ndcg_at_1_diff1": 0.431221, + "nauc_ndcg_at_3_max": 0.291361, + "nauc_ndcg_at_3_std": -0.005452, + "nauc_ndcg_at_3_diff1": 0.406662, + "nauc_ndcg_at_5_max": 0.287393, + "nauc_ndcg_at_5_std": -0.002825, + "nauc_ndcg_at_5_diff1": 0.372894, + "nauc_ndcg_at_10_max": 0.273332, + "nauc_ndcg_at_10_std": 0.003558, + "nauc_ndcg_at_10_diff1": 0.369033, + "nauc_ndcg_at_20_max": 0.280095, + "nauc_ndcg_at_20_std": 0.013206, + "nauc_ndcg_at_20_diff1": 0.356104, + "nauc_ndcg_at_100_max": 0.282981, + "nauc_ndcg_at_100_std": 0.029876, + "nauc_ndcg_at_100_diff1": 0.350591, + "nauc_ndcg_at_1000_max": 0.28597, + "nauc_ndcg_at_1000_std": 0.03057, + "nauc_ndcg_at_1000_diff1": 0.3559, + "nauc_map_at_1_max": 0.256004, + "nauc_map_at_1_std": -0.010372, + "nauc_map_at_1_diff1": 0.411673, + "nauc_map_at_3_max": 0.279235, + "nauc_map_at_3_std": -0.006875, + "nauc_map_at_3_diff1": 0.402158, + "nauc_map_at_5_max": 0.279046, + "nauc_map_at_5_std": -0.005692, + "nauc_map_at_5_diff1": 0.385115, + "nauc_map_at_10_max": 0.274092, + "nauc_map_at_10_std": -0.003812, + "nauc_map_at_10_diff1": 0.383885, + "nauc_map_at_20_max": 0.276257, + "nauc_map_at_20_std": -0.000865, + "nauc_map_at_20_diff1": 0.380236, + "nauc_map_at_100_max": 0.277032, + "nauc_map_at_100_std": 0.001965, + "nauc_map_at_100_diff1": 0.379329, + "nauc_map_at_1000_max": 0.277234, + "nauc_map_at_1000_std": 0.002192, + "nauc_map_at_1000_diff1": 0.379607, + "nauc_recall_at_1_max": 0.256004, + "nauc_recall_at_1_std": -0.010372, + "nauc_recall_at_1_diff1": 0.411673, + "nauc_recall_at_3_max": 0.285515, + "nauc_recall_at_3_std": -0.003295, + "nauc_recall_at_3_diff1": 0.374387, + "nauc_recall_at_5_max": 0.277299, + "nauc_recall_at_5_std": 0.000649, + "nauc_recall_at_5_diff1": 0.306693, + "nauc_recall_at_10_max": 0.245389, + "nauc_recall_at_10_std": 0.020807, + "nauc_recall_at_10_diff1": 0.303071, + "nauc_recall_at_20_max": 0.259837, + "nauc_recall_at_20_std": 0.051809, + "nauc_recall_at_20_diff1": 0.249626, + "nauc_recall_at_100_max": 0.260053, + "nauc_recall_at_100_std": 0.148634, + "nauc_recall_at_100_diff1": 0.186284, + "nauc_recall_at_1000_max": 0.305821, + "nauc_recall_at_1000_std": 0.284245, + "nauc_recall_at_1000_diff1": 0.114811, + "nauc_precision_at_1_max": 0.277969, + "nauc_precision_at_1_std": -0.00326, + "nauc_precision_at_1_diff1": 0.431221, + "nauc_precision_at_3_max": 0.312755, + "nauc_precision_at_3_std": -0.000939, + "nauc_precision_at_3_diff1": 0.391257, + "nauc_precision_at_5_max": 0.293898, + "nauc_precision_at_5_std": 0.003153, + "nauc_precision_at_5_diff1": 0.309658, + "nauc_precision_at_10_max": 0.237997, + "nauc_precision_at_10_std": 0.016961, + "nauc_precision_at_10_diff1": 0.280832, + "nauc_precision_at_20_max": 0.239196, + "nauc_precision_at_20_std": 0.049968, + "nauc_precision_at_20_diff1": 0.217248, + "nauc_precision_at_100_max": 0.166034, + "nauc_precision_at_100_std": 0.104676, + "nauc_precision_at_100_diff1": 0.107927, + "nauc_precision_at_1000_max": 0.073917, + "nauc_precision_at_1000_std": 0.055549, + "nauc_precision_at_1000_diff1": 0.016797, + "nauc_mrr_at_1_max": 0.277969, + "nauc_mrr_at_1_std": -0.00326, + "nauc_mrr_at_1_diff1": 0.431221, + "nauc_mrr_at_3_max": 0.297352, + "nauc_mrr_at_3_std": -0.006013, + "nauc_mrr_at_3_diff1": 0.417607, + "nauc_mrr_at_5_max": 0.294551, + "nauc_mrr_at_5_std": -0.002511, + "nauc_mrr_at_5_diff1": 0.401746, + "nauc_mrr_at_10_max": 0.289027, + "nauc_mrr_at_10_std": 0.00155, + "nauc_mrr_at_10_diff1": 0.400569, + "nauc_mrr_at_20_max": 0.2914, + "nauc_mrr_at_20_std": 0.004025, + "nauc_mrr_at_20_diff1": 0.397537, + "nauc_mrr_at_100_max": 0.29175, + "nauc_mrr_at_100_std": 0.005198, + "nauc_mrr_at_100_diff1": 0.39718, + "nauc_mrr_at_1000_max": 0.291842, + "nauc_mrr_at_1000_std": 0.005167, + "nauc_mrr_at_1000_diff1": 0.397372, + "main_score": 0.30424, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 85.23565435409546, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackPhysicsRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackPhysicsRetrieval.json new file mode 100644 index 0000000000..b557fe6485 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackPhysicsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4", + "task_name": "CQADupstackPhysicsRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.33975, + "ndcg_at_3": 0.39173, + "ndcg_at_5": 0.41405, + "ndcg_at_10": 0.44292, + "ndcg_at_20": 0.46348, + "ndcg_at_100": 0.49829, + "ndcg_at_1000": 0.5194, + "map_at_1": 0.2831, + "map_at_3": 0.35225, + "map_at_5": 0.36779, + "map_at_10": 0.38301, + "map_at_20": 0.38968, + "map_at_100": 0.39594, + "map_at_1000": 0.39707, + "recall_at_1": 0.2831, + "recall_at_3": 0.42771, + "recall_at_5": 0.48412, + "recall_at_10": 0.56788, + "recall_at_20": 0.64019, + "recall_at_100": 0.80094, + "recall_at_1000": 0.93769, + "precision_at_1": 0.33975, + "precision_at_3": 0.18287, + "precision_at_5": 0.12974, + "precision_at_10": 0.07998, + "precision_at_20": 0.04658, + "precision_at_100": 0.01266, + "precision_at_1000": 0.00163, + "mrr_at_1": 0.33975, + "mrr_at_3": 0.409047, + "mrr_at_5": 0.424013, + "mrr_at_10": 0.435363, + "mrr_at_20": 0.440627, + "mrr_at_100": 0.443945, + "mrr_at_1000": 0.444449, + "nauc_ndcg_at_1_max": 0.331142, + "nauc_ndcg_at_1_std": -0.080807, + "nauc_ndcg_at_1_diff1": 0.553758, + "nauc_ndcg_at_3_max": 0.306721, + "nauc_ndcg_at_3_std": -0.083797, + "nauc_ndcg_at_3_diff1": 0.480193, + "nauc_ndcg_at_5_max": 0.314762, + "nauc_ndcg_at_5_std": -0.073833, + "nauc_ndcg_at_5_diff1": 0.478911, + "nauc_ndcg_at_10_max": 0.310324, + "nauc_ndcg_at_10_std": -0.06653, + "nauc_ndcg_at_10_diff1": 0.469834, + "nauc_ndcg_at_20_max": 0.309393, + "nauc_ndcg_at_20_std": -0.065317, + "nauc_ndcg_at_20_diff1": 0.465958, + "nauc_ndcg_at_100_max": 0.304722, + "nauc_ndcg_at_100_std": -0.043258, + "nauc_ndcg_at_100_diff1": 0.471477, + "nauc_ndcg_at_1000_max": 0.311529, + "nauc_ndcg_at_1000_std": -0.04307, + "nauc_ndcg_at_1000_diff1": 0.476994, + "nauc_map_at_1_max": 0.327756, + "nauc_map_at_1_std": -0.117348, + "nauc_map_at_1_diff1": 0.564072, + "nauc_map_at_3_max": 0.314959, + "nauc_map_at_3_std": -0.097715, + "nauc_map_at_3_diff1": 0.504639, + "nauc_map_at_5_max": 0.317909, + "nauc_map_at_5_std": -0.090075, + "nauc_map_at_5_diff1": 0.502958, + "nauc_map_at_10_max": 0.317889, + "nauc_map_at_10_std": -0.08187, + "nauc_map_at_10_diff1": 0.499163, + "nauc_map_at_20_max": 0.31791, + "nauc_map_at_20_std": -0.081152, + "nauc_map_at_20_diff1": 0.498298, + "nauc_map_at_100_max": 0.317205, + "nauc_map_at_100_std": -0.076906, + "nauc_map_at_100_diff1": 0.498688, + "nauc_map_at_1000_max": 0.317514, + "nauc_map_at_1000_std": -0.076502, + "nauc_map_at_1000_diff1": 0.498856, + "nauc_recall_at_1_max": 0.327756, + "nauc_recall_at_1_std": -0.117348, + "nauc_recall_at_1_diff1": 0.564072, + "nauc_recall_at_3_max": 0.28331, + "nauc_recall_at_3_std": -0.096029, + "nauc_recall_at_3_diff1": 0.415687, + "nauc_recall_at_5_max": 0.290821, + "nauc_recall_at_5_std": -0.068997, + "nauc_recall_at_5_diff1": 0.39768, + "nauc_recall_at_10_max": 0.269513, + "nauc_recall_at_10_std": -0.042168, + "nauc_recall_at_10_diff1": 0.355428, + "nauc_recall_at_20_max": 0.257217, + "nauc_recall_at_20_std": -0.033926, + "nauc_recall_at_20_diff1": 0.323401, + "nauc_recall_at_100_max": 0.199616, + "nauc_recall_at_100_std": 0.128556, + "nauc_recall_at_100_diff1": 0.319044, + "nauc_recall_at_1000_max": 0.283365, + "nauc_recall_at_1000_std": 0.33909, + "nauc_recall_at_1000_diff1": 0.347902, + "nauc_precision_at_1_max": 0.331142, + "nauc_precision_at_1_std": -0.080807, + "nauc_precision_at_1_diff1": 0.553758, + "nauc_precision_at_3_max": 0.25314, + "nauc_precision_at_3_std": -0.010952, + "nauc_precision_at_3_diff1": 0.345346, + "nauc_precision_at_5_max": 0.249503, + "nauc_precision_at_5_std": 0.045989, + "nauc_precision_at_5_diff1": 0.295965, + "nauc_precision_at_10_max": 0.181666, + "nauc_precision_at_10_std": 0.095356, + "nauc_precision_at_10_diff1": 0.199285, + "nauc_precision_at_20_max": 0.134246, + "nauc_precision_at_20_std": 0.113215, + "nauc_precision_at_20_diff1": 0.12992, + "nauc_precision_at_100_max": 0.00704, + "nauc_precision_at_100_std": 0.213215, + "nauc_precision_at_100_diff1": -0.014112, + "nauc_precision_at_1000_max": -0.068387, + "nauc_precision_at_1000_std": 0.199341, + "nauc_precision_at_1000_diff1": -0.125446, + "nauc_mrr_at_1_max": 0.331142, + "nauc_mrr_at_1_std": -0.080807, + "nauc_mrr_at_1_diff1": 0.553758, + "nauc_mrr_at_3_max": 0.312364, + "nauc_mrr_at_3_std": -0.074282, + "nauc_mrr_at_3_diff1": 0.495839, + "nauc_mrr_at_5_max": 0.316963, + "nauc_mrr_at_5_std": -0.064709, + "nauc_mrr_at_5_diff1": 0.492711, + "nauc_mrr_at_10_max": 0.313466, + "nauc_mrr_at_10_std": -0.064022, + "nauc_mrr_at_10_diff1": 0.49098, + "nauc_mrr_at_20_max": 0.312921, + "nauc_mrr_at_20_std": -0.063912, + "nauc_mrr_at_20_diff1": 0.489671, + "nauc_mrr_at_100_max": 0.312461, + "nauc_mrr_at_100_std": -0.063107, + "nauc_mrr_at_100_diff1": 0.490566, + "nauc_mrr_at_1000_max": 0.312828, + "nauc_mrr_at_1000_std": -0.063173, + "nauc_mrr_at_1000_diff1": 0.490825, + "main_score": 0.44292, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 158.5953254699707, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackProgrammersRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackProgrammersRetrieval.json new file mode 100644 index 0000000000..088fee0300 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackProgrammersRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6184bc1440d2dbc7612be22b50686b8826d22b32", + "task_name": "CQADupstackProgrammersRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.32534, + "ndcg_at_3": 0.37032, + "ndcg_at_5": 0.39015, + "ndcg_at_10": 0.41335, + "ndcg_at_20": 0.43604, + "ndcg_at_100": 0.46965, + "ndcg_at_1000": 0.49621, + "map_at_1": 0.26897, + "map_at_3": 0.33521, + "map_at_5": 0.35016, + "map_at_10": 0.36127, + "map_at_20": 0.36856, + "map_at_100": 0.37437, + "map_at_1000": 0.37563, + "recall_at_1": 0.26897, + "recall_at_3": 0.39869, + "recall_at_5": 0.45032, + "recall_at_10": 0.51901, + "recall_at_20": 0.60064, + "recall_at_100": 0.7566, + "recall_at_1000": 0.93858, + "precision_at_1": 0.32534, + "precision_at_3": 0.17466, + "precision_at_5": 0.12237, + "precision_at_10": 0.0726, + "precision_at_20": 0.04315, + "precision_at_100": 0.01177, + "precision_at_1000": 0.00158, + "mrr_at_1": 0.325342, + "mrr_at_3": 0.387367, + "mrr_at_5": 0.401123, + "mrr_at_10": 0.411209, + "mrr_at_20": 0.416528, + "mrr_at_100": 0.420292, + "mrr_at_1000": 0.420962, + "nauc_ndcg_at_1_max": 0.474918, + "nauc_ndcg_at_1_std": 0.048555, + "nauc_ndcg_at_1_diff1": 0.578801, + "nauc_ndcg_at_3_max": 0.453774, + "nauc_ndcg_at_3_std": 0.051628, + "nauc_ndcg_at_3_diff1": 0.502378, + "nauc_ndcg_at_5_max": 0.449869, + "nauc_ndcg_at_5_std": 0.051872, + "nauc_ndcg_at_5_diff1": 0.487856, + "nauc_ndcg_at_10_max": 0.45243, + "nauc_ndcg_at_10_std": 0.06247, + "nauc_ndcg_at_10_diff1": 0.486751, + "nauc_ndcg_at_20_max": 0.449079, + "nauc_ndcg_at_20_std": 0.076376, + "nauc_ndcg_at_20_diff1": 0.476092, + "nauc_ndcg_at_100_max": 0.447228, + "nauc_ndcg_at_100_std": 0.089776, + "nauc_ndcg_at_100_diff1": 0.480459, + "nauc_ndcg_at_1000_max": 0.451551, + "nauc_ndcg_at_1000_std": 0.085726, + "nauc_ndcg_at_1000_diff1": 0.491432, + "nauc_map_at_1_max": 0.437548, + "nauc_map_at_1_std": -0.022578, + "nauc_map_at_1_diff1": 0.581227, + "nauc_map_at_3_max": 0.449425, + "nauc_map_at_3_std": 0.01831, + "nauc_map_at_3_diff1": 0.516768, + "nauc_map_at_5_max": 0.44989, + "nauc_map_at_5_std": 0.027319, + "nauc_map_at_5_diff1": 0.506676, + "nauc_map_at_10_max": 0.451374, + "nauc_map_at_10_std": 0.034127, + "nauc_map_at_10_diff1": 0.506204, + "nauc_map_at_20_max": 0.450725, + "nauc_map_at_20_std": 0.039688, + "nauc_map_at_20_diff1": 0.503215, + "nauc_map_at_100_max": 0.450511, + "nauc_map_at_100_std": 0.042743, + "nauc_map_at_100_diff1": 0.503441, + "nauc_map_at_1000_max": 0.450669, + "nauc_map_at_1000_std": 0.042986, + "nauc_map_at_1000_diff1": 0.50387, + "nauc_recall_at_1_max": 0.437548, + "nauc_recall_at_1_std": -0.022578, + "nauc_recall_at_1_diff1": 0.581227, + "nauc_recall_at_3_max": 0.433888, + "nauc_recall_at_3_std": 0.049107, + "nauc_recall_at_3_diff1": 0.450646, + "nauc_recall_at_5_max": 0.423337, + "nauc_recall_at_5_std": 0.066073, + "nauc_recall_at_5_diff1": 0.410401, + "nauc_recall_at_10_max": 0.421491, + "nauc_recall_at_10_std": 0.097561, + "nauc_recall_at_10_diff1": 0.401762, + "nauc_recall_at_20_max": 0.399367, + "nauc_recall_at_20_std": 0.14464, + "nauc_recall_at_20_diff1": 0.343487, + "nauc_recall_at_100_max": 0.375094, + "nauc_recall_at_100_std": 0.248722, + "nauc_recall_at_100_diff1": 0.334172, + "nauc_recall_at_1000_max": 0.413103, + "nauc_recall_at_1000_std": 0.446415, + "nauc_recall_at_1000_diff1": 0.408224, + "nauc_precision_at_1_max": 0.474918, + "nauc_precision_at_1_std": 0.048555, + "nauc_precision_at_1_diff1": 0.578801, + "nauc_precision_at_3_max": 0.415684, + "nauc_precision_at_3_std": 0.154714, + "nauc_precision_at_3_diff1": 0.347858, + "nauc_precision_at_5_max": 0.367281, + "nauc_precision_at_5_std": 0.166875, + "nauc_precision_at_5_diff1": 0.277005, + "nauc_precision_at_10_max": 0.314083, + "nauc_precision_at_10_std": 0.190256, + "nauc_precision_at_10_diff1": 0.220903, + "nauc_precision_at_20_max": 0.245206, + "nauc_precision_at_20_std": 0.22068, + "nauc_precision_at_20_diff1": 0.13592, + "nauc_precision_at_100_max": 0.08187, + "nauc_precision_at_100_std": 0.208819, + "nauc_precision_at_100_diff1": 0.011791, + "nauc_precision_at_1000_max": -0.055806, + "nauc_precision_at_1000_std": 0.11137, + "nauc_precision_at_1000_diff1": -0.102341, + "nauc_mrr_at_1_max": 0.474918, + "nauc_mrr_at_1_std": 0.048555, + "nauc_mrr_at_1_diff1": 0.578801, + "nauc_mrr_at_3_max": 0.470039, + "nauc_mrr_at_3_std": 0.079959, + "nauc_mrr_at_3_diff1": 0.526831, + "nauc_mrr_at_5_max": 0.463345, + "nauc_mrr_at_5_std": 0.078565, + "nauc_mrr_at_5_diff1": 0.516074, + "nauc_mrr_at_10_max": 0.463865, + "nauc_mrr_at_10_std": 0.081217, + "nauc_mrr_at_10_diff1": 0.516489, + "nauc_mrr_at_20_max": 0.463694, + "nauc_mrr_at_20_std": 0.083624, + "nauc_mrr_at_20_diff1": 0.514805, + "nauc_mrr_at_100_max": 0.463421, + "nauc_mrr_at_100_std": 0.084532, + "nauc_mrr_at_100_diff1": 0.515214, + "nauc_mrr_at_1000_max": 0.463573, + "nauc_mrr_at_1000_std": 0.084386, + "nauc_mrr_at_1000_diff1": 0.515545, + "main_score": 0.41335, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 121.19785666465759, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackRetrieval.json new file mode 100644 index 0000000000..a91857f9ea --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackRetrieval.json @@ -0,0 +1,19 @@ +{ + "dataset_revision": "1", + "task_name": "CQADupstackRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_10": 0.40803, + "main_score": 0.40803, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1702.6613976955414, + "kg_co2_emissions": NaN +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackStatsRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackStatsRetrieval.json new file mode 100644 index 0000000000..6bec7377c3 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackStatsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a", + "task_name": "CQADupstackStatsRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.27607, + "ndcg_at_3": 0.30652, + "ndcg_at_5": 0.33001, + "ndcg_at_10": 0.35054, + "ndcg_at_20": 0.36711, + "ndcg_at_100": 0.39524, + "ndcg_at_1000": 0.42123, + "map_at_1": 0.24441, + "map_at_3": 0.28619, + "map_at_5": 0.30056, + "map_at_10": 0.30991, + "map_at_20": 0.31469, + "map_at_100": 0.31888, + "map_at_1000": 0.3198, + "recall_at_1": 0.24441, + "recall_at_3": 0.32872, + "recall_at_5": 0.38587, + "recall_at_10": 0.44732, + "recall_at_20": 0.51111, + "recall_at_100": 0.65257, + "recall_at_1000": 0.84847, + "precision_at_1": 0.27607, + "precision_at_3": 0.12883, + "precision_at_5": 0.09202, + "precision_at_10": 0.05445, + "precision_at_20": 0.03129, + "precision_at_100": 0.00827, + "precision_at_1000": 0.00114, + "mrr_at_1": 0.276074, + "mrr_at_3": 0.316462, + "mrr_at_5": 0.330726, + "mrr_at_10": 0.339148, + "mrr_at_20": 0.343375, + "mrr_at_100": 0.346872, + "mrr_at_1000": 0.347525, + "nauc_ndcg_at_1_max": 0.338775, + "nauc_ndcg_at_1_std": -0.036769, + "nauc_ndcg_at_1_diff1": 0.642336, + "nauc_ndcg_at_3_max": 0.345861, + "nauc_ndcg_at_3_std": -0.05356, + "nauc_ndcg_at_3_diff1": 0.572209, + "nauc_ndcg_at_5_max": 0.339004, + "nauc_ndcg_at_5_std": -0.035209, + "nauc_ndcg_at_5_diff1": 0.558772, + "nauc_ndcg_at_10_max": 0.331175, + "nauc_ndcg_at_10_std": -0.016564, + "nauc_ndcg_at_10_diff1": 0.552891, + "nauc_ndcg_at_20_max": 0.32427, + "nauc_ndcg_at_20_std": -0.008526, + "nauc_ndcg_at_20_diff1": 0.543298, + "nauc_ndcg_at_100_max": 0.333168, + "nauc_ndcg_at_100_std": 0.012896, + "nauc_ndcg_at_100_diff1": 0.534917, + "nauc_ndcg_at_1000_max": 0.339641, + "nauc_ndcg_at_1000_std": 0.006799, + "nauc_ndcg_at_1000_diff1": 0.545685, + "nauc_map_at_1_max": 0.337098, + "nauc_map_at_1_std": -0.08545, + "nauc_map_at_1_diff1": 0.666459, + "nauc_map_at_3_max": 0.345108, + "nauc_map_at_3_std": -0.069352, + "nauc_map_at_3_diff1": 0.599003, + "nauc_map_at_5_max": 0.34099, + "nauc_map_at_5_std": -0.053521, + "nauc_map_at_5_diff1": 0.589303, + "nauc_map_at_10_max": 0.337112, + "nauc_map_at_10_std": -0.044627, + "nauc_map_at_10_diff1": 0.586287, + "nauc_map_at_20_max": 0.335215, + "nauc_map_at_20_std": -0.041609, + "nauc_map_at_20_diff1": 0.583179, + "nauc_map_at_100_max": 0.336337, + "nauc_map_at_100_std": -0.039012, + "nauc_map_at_100_diff1": 0.581842, + "nauc_map_at_1000_max": 0.336811, + "nauc_map_at_1000_std": -0.039046, + "nauc_map_at_1000_diff1": 0.58223, + "nauc_recall_at_1_max": 0.337098, + "nauc_recall_at_1_std": -0.08545, + "nauc_recall_at_1_diff1": 0.666459, + "nauc_recall_at_3_max": 0.339811, + "nauc_recall_at_3_std": -0.05768, + "nauc_recall_at_3_diff1": 0.518429, + "nauc_recall_at_5_max": 0.320984, + "nauc_recall_at_5_std": -0.004947, + "nauc_recall_at_5_diff1": 0.479044, + "nauc_recall_at_10_max": 0.299735, + "nauc_recall_at_10_std": 0.048666, + "nauc_recall_at_10_diff1": 0.453897, + "nauc_recall_at_20_max": 0.266521, + "nauc_recall_at_20_std": 0.075352, + "nauc_recall_at_20_diff1": 0.412761, + "nauc_recall_at_100_max": 0.30243, + "nauc_recall_at_100_std": 0.222066, + "nauc_recall_at_100_diff1": 0.332989, + "nauc_recall_at_1000_max": 0.346031, + "nauc_recall_at_1000_std": 0.295269, + "nauc_recall_at_1000_diff1": 0.346466, + "nauc_precision_at_1_max": 0.338775, + "nauc_precision_at_1_std": -0.036769, + "nauc_precision_at_1_diff1": 0.642336, + "nauc_precision_at_3_max": 0.334472, + "nauc_precision_at_3_std": 0.013746, + "nauc_precision_at_3_diff1": 0.465166, + "nauc_precision_at_5_max": 0.305643, + "nauc_precision_at_5_std": 0.067722, + "nauc_precision_at_5_diff1": 0.411441, + "nauc_precision_at_10_max": 0.259676, + "nauc_precision_at_10_std": 0.131044, + "nauc_precision_at_10_diff1": 0.352996, + "nauc_precision_at_20_max": 0.224582, + "nauc_precision_at_20_std": 0.168333, + "nauc_precision_at_20_diff1": 0.279953, + "nauc_precision_at_100_max": 0.201957, + "nauc_precision_at_100_std": 0.247235, + "nauc_precision_at_100_diff1": 0.14107, + "nauc_precision_at_1000_max": 0.123347, + "nauc_precision_at_1000_std": 0.173695, + "nauc_precision_at_1000_diff1": -0.017463, + "nauc_mrr_at_1_max": 0.338775, + "nauc_mrr_at_1_std": -0.036769, + "nauc_mrr_at_1_diff1": 0.642336, + "nauc_mrr_at_3_max": 0.343301, + "nauc_mrr_at_3_std": -0.038952, + "nauc_mrr_at_3_diff1": 0.583457, + "nauc_mrr_at_5_max": 0.339933, + "nauc_mrr_at_5_std": -0.031229, + "nauc_mrr_at_5_diff1": 0.574636, + "nauc_mrr_at_10_max": 0.338652, + "nauc_mrr_at_10_std": -0.022878, + "nauc_mrr_at_10_diff1": 0.570654, + "nauc_mrr_at_20_max": 0.337355, + "nauc_mrr_at_20_std": -0.021958, + "nauc_mrr_at_20_diff1": 0.568957, + "nauc_mrr_at_100_max": 0.337944, + "nauc_mrr_at_100_std": -0.019679, + "nauc_mrr_at_100_diff1": 0.568233, + "nauc_mrr_at_1000_max": 0.338042, + "nauc_mrr_at_1000_std": -0.019907, + "nauc_mrr_at_1000_diff1": 0.568521, + "main_score": 0.35054, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 199.97041988372803, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackTexRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackTexRetrieval.json new file mode 100644 index 0000000000..014de569d7 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackTexRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "46989137a86843e03a6195de44b09deda022eec7", + "task_name": "CQADupstackTexRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.23641, + "ndcg_at_3": 0.2614, + "ndcg_at_5": 0.27753, + "ndcg_at_10": 0.29653, + "ndcg_at_20": 0.31595, + "ndcg_at_100": 0.34916, + "ndcg_at_1000": 0.38129, + "map_at_1": 0.19815, + "map_at_3": 0.23771, + "map_at_5": 0.24859, + "map_at_10": 0.25726, + "map_at_20": 0.26305, + "map_at_100": 0.26803, + "map_at_1000": 0.26936, + "recall_at_1": 0.19815, + "recall_at_3": 0.27952, + "recall_at_5": 0.32178, + "recall_at_10": 0.37879, + "recall_at_20": 0.4512, + "recall_at_100": 0.61685, + "recall_at_1000": 0.84672, + "precision_at_1": 0.23641, + "precision_at_3": 0.11711, + "precision_at_5": 0.08293, + "precision_at_10": 0.05021, + "precision_at_20": 0.03059, + "precision_at_100": 0.00888, + "precision_at_1000": 0.00134, + "mrr_at_1": 0.236407, + "mrr_at_3": 0.274776, + "mrr_at_5": 0.285977, + "mrr_at_10": 0.294523, + "mrr_at_20": 0.29971, + "mrr_at_100": 0.303851, + "mrr_at_1000": 0.304705, + "nauc_ndcg_at_1_max": 0.307972, + "nauc_ndcg_at_1_std": -0.026604, + "nauc_ndcg_at_1_diff1": 0.492426, + "nauc_ndcg_at_3_max": 0.296419, + "nauc_ndcg_at_3_std": -0.024017, + "nauc_ndcg_at_3_diff1": 0.439942, + "nauc_ndcg_at_5_max": 0.299317, + "nauc_ndcg_at_5_std": -0.019744, + "nauc_ndcg_at_5_diff1": 0.427792, + "nauc_ndcg_at_10_max": 0.301372, + "nauc_ndcg_at_10_std": -0.011362, + "nauc_ndcg_at_10_diff1": 0.41592, + "nauc_ndcg_at_20_max": 0.298093, + "nauc_ndcg_at_20_std": 0.003292, + "nauc_ndcg_at_20_diff1": 0.409807, + "nauc_ndcg_at_100_max": 0.297988, + "nauc_ndcg_at_100_std": 0.012282, + "nauc_ndcg_at_100_diff1": 0.403419, + "nauc_ndcg_at_1000_max": 0.296764, + "nauc_ndcg_at_1000_std": 0.017746, + "nauc_ndcg_at_1000_diff1": 0.405913, + "nauc_map_at_1_max": 0.299839, + "nauc_map_at_1_std": -0.027457, + "nauc_map_at_1_diff1": 0.502769, + "nauc_map_at_3_max": 0.298524, + "nauc_map_at_3_std": -0.025849, + "nauc_map_at_3_diff1": 0.457963, + "nauc_map_at_5_max": 0.29977, + "nauc_map_at_5_std": -0.023727, + "nauc_map_at_5_diff1": 0.450004, + "nauc_map_at_10_max": 0.300534, + "nauc_map_at_10_std": -0.020008, + "nauc_map_at_10_diff1": 0.443343, + "nauc_map_at_20_max": 0.299799, + "nauc_map_at_20_std": -0.015272, + "nauc_map_at_20_diff1": 0.44092, + "nauc_map_at_100_max": 0.299668, + "nauc_map_at_100_std": -0.013422, + "nauc_map_at_100_diff1": 0.439408, + "nauc_map_at_1000_max": 0.299464, + "nauc_map_at_1000_std": -0.013136, + "nauc_map_at_1000_diff1": 0.439285, + "nauc_recall_at_1_max": 0.299839, + "nauc_recall_at_1_std": -0.027457, + "nauc_recall_at_1_diff1": 0.502769, + "nauc_recall_at_3_max": 0.290545, + "nauc_recall_at_3_std": -0.024237, + "nauc_recall_at_3_diff1": 0.408404, + "nauc_recall_at_5_max": 0.288785, + "nauc_recall_at_5_std": -0.016828, + "nauc_recall_at_5_diff1": 0.374742, + "nauc_recall_at_10_max": 0.289604, + "nauc_recall_at_10_std": 0.00662, + "nauc_recall_at_10_diff1": 0.340228, + "nauc_recall_at_20_max": 0.27579, + "nauc_recall_at_20_std": 0.056587, + "nauc_recall_at_20_diff1": 0.312381, + "nauc_recall_at_100_max": 0.273196, + "nauc_recall_at_100_std": 0.10675, + "nauc_recall_at_100_diff1": 0.257046, + "nauc_recall_at_1000_max": 0.270902, + "nauc_recall_at_1000_std": 0.281627, + "nauc_recall_at_1000_diff1": 0.183408, + "nauc_precision_at_1_max": 0.307972, + "nauc_precision_at_1_std": -0.026604, + "nauc_precision_at_1_diff1": 0.492426, + "nauc_precision_at_3_max": 0.280422, + "nauc_precision_at_3_std": -0.015333, + "nauc_precision_at_3_diff1": 0.362876, + "nauc_precision_at_5_max": 0.270757, + "nauc_precision_at_5_std": 0.00028, + "nauc_precision_at_5_diff1": 0.309518, + "nauc_precision_at_10_max": 0.255144, + "nauc_precision_at_10_std": 0.025734, + "nauc_precision_at_10_diff1": 0.249012, + "nauc_precision_at_20_max": 0.210762, + "nauc_precision_at_20_std": 0.072581, + "nauc_precision_at_20_diff1": 0.188277, + "nauc_precision_at_100_max": 0.117009, + "nauc_precision_at_100_std": 0.100648, + "nauc_precision_at_100_diff1": 0.050442, + "nauc_precision_at_1000_max": -0.01783, + "nauc_precision_at_1000_std": 0.089427, + "nauc_precision_at_1000_diff1": -0.095525, + "nauc_mrr_at_1_max": 0.307972, + "nauc_mrr_at_1_std": -0.026604, + "nauc_mrr_at_1_diff1": 0.492426, + "nauc_mrr_at_3_max": 0.302208, + "nauc_mrr_at_3_std": -0.022296, + "nauc_mrr_at_3_diff1": 0.451836, + "nauc_mrr_at_5_max": 0.300033, + "nauc_mrr_at_5_std": -0.019871, + "nauc_mrr_at_5_diff1": 0.442973, + "nauc_mrr_at_10_max": 0.300292, + "nauc_mrr_at_10_std": -0.016789, + "nauc_mrr_at_10_diff1": 0.43782, + "nauc_mrr_at_20_max": 0.299177, + "nauc_mrr_at_20_std": -0.013587, + "nauc_mrr_at_20_diff1": 0.436512, + "nauc_mrr_at_100_max": 0.299354, + "nauc_mrr_at_100_std": -0.01269, + "nauc_mrr_at_100_diff1": 0.435884, + "nauc_mrr_at_1000_max": 0.299475, + "nauc_mrr_at_1000_std": -0.012666, + "nauc_mrr_at_1000_diff1": 0.436075, + "main_score": 0.29653, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 332.89063692092896, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackUnixRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackUnixRetrieval.json new file mode 100644 index 0000000000..7951d4d9dd --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackUnixRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53", + "task_name": "CQADupstackUnixRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.32369, + "ndcg_at_3": 0.3599, + "ndcg_at_5": 0.37785, + "ndcg_at_10": 0.40374, + "ndcg_at_20": 0.42659, + "ndcg_at_100": 0.45777, + "ndcg_at_1000": 0.4822, + "map_at_1": 0.27948, + "map_at_3": 0.3331, + "map_at_5": 0.34502, + "map_at_10": 0.35699, + "map_at_20": 0.36379, + "map_at_100": 0.36843, + "map_at_1000": 0.36953, + "recall_at_1": 0.27948, + "recall_at_3": 0.38674, + "recall_at_5": 0.43291, + "recall_at_10": 0.50748, + "recall_at_20": 0.59179, + "recall_at_100": 0.74244, + "recall_at_1000": 0.9127, + "precision_at_1": 0.32369, + "precision_at_3": 0.15858, + "precision_at_5": 0.10821, + "precision_at_10": 0.06483, + "precision_at_20": 0.03853, + "precision_at_100": 0.0104, + "precision_at_1000": 0.00137, + "mrr_at_1": 0.323694, + "mrr_at_3": 0.374845, + "mrr_at_5": 0.386039, + "mrr_at_10": 0.396357, + "mrr_at_20": 0.402119, + "mrr_at_100": 0.40569, + "mrr_at_1000": 0.406312, + "nauc_ndcg_at_1_max": 0.452386, + "nauc_ndcg_at_1_std": -0.094075, + "nauc_ndcg_at_1_diff1": 0.615308, + "nauc_ndcg_at_3_max": 0.427688, + "nauc_ndcg_at_3_std": -0.077476, + "nauc_ndcg_at_3_diff1": 0.533216, + "nauc_ndcg_at_5_max": 0.427704, + "nauc_ndcg_at_5_std": -0.073002, + "nauc_ndcg_at_5_diff1": 0.520616, + "nauc_ndcg_at_10_max": 0.414983, + "nauc_ndcg_at_10_std": -0.050663, + "nauc_ndcg_at_10_diff1": 0.51239, + "nauc_ndcg_at_20_max": 0.408576, + "nauc_ndcg_at_20_std": -0.058279, + "nauc_ndcg_at_20_diff1": 0.505527, + "nauc_ndcg_at_100_max": 0.411232, + "nauc_ndcg_at_100_std": -0.043243, + "nauc_ndcg_at_100_diff1": 0.504414, + "nauc_ndcg_at_1000_max": 0.410828, + "nauc_ndcg_at_1000_std": -0.039348, + "nauc_ndcg_at_1000_diff1": 0.510911, + "nauc_map_at_1_max": 0.465717, + "nauc_map_at_1_std": -0.094659, + "nauc_map_at_1_diff1": 0.610186, + "nauc_map_at_3_max": 0.440922, + "nauc_map_at_3_std": -0.080803, + "nauc_map_at_3_diff1": 0.551331, + "nauc_map_at_5_max": 0.439955, + "nauc_map_at_5_std": -0.077917, + "nauc_map_at_5_diff1": 0.542101, + "nauc_map_at_10_max": 0.433238, + "nauc_map_at_10_std": -0.067326, + "nauc_map_at_10_diff1": 0.538422, + "nauc_map_at_20_max": 0.43119, + "nauc_map_at_20_std": -0.06999, + "nauc_map_at_20_diff1": 0.536515, + "nauc_map_at_100_max": 0.431428, + "nauc_map_at_100_std": -0.068122, + "nauc_map_at_100_diff1": 0.536648, + "nauc_map_at_1000_max": 0.43128, + "nauc_map_at_1000_std": -0.067835, + "nauc_map_at_1000_diff1": 0.536795, + "nauc_recall_at_1_max": 0.465717, + "nauc_recall_at_1_std": -0.094659, + "nauc_recall_at_1_diff1": 0.610186, + "nauc_recall_at_3_max": 0.405679, + "nauc_recall_at_3_std": -0.064726, + "nauc_recall_at_3_diff1": 0.482535, + "nauc_recall_at_5_max": 0.393861, + "nauc_recall_at_5_std": -0.057738, + "nauc_recall_at_5_diff1": 0.44157, + "nauc_recall_at_10_max": 0.355322, + "nauc_recall_at_10_std": 0.006462, + "nauc_recall_at_10_diff1": 0.412931, + "nauc_recall_at_20_max": 0.324222, + "nauc_recall_at_20_std": -0.016997, + "nauc_recall_at_20_diff1": 0.374537, + "nauc_recall_at_100_max": 0.332658, + "nauc_recall_at_100_std": 0.096163, + "nauc_recall_at_100_diff1": 0.326489, + "nauc_recall_at_1000_max": 0.298528, + "nauc_recall_at_1000_std": 0.385683, + "nauc_recall_at_1000_diff1": 0.288626, + "nauc_precision_at_1_max": 0.452386, + "nauc_precision_at_1_std": -0.094075, + "nauc_precision_at_1_diff1": 0.615308, + "nauc_precision_at_3_max": 0.340893, + "nauc_precision_at_3_std": -0.059523, + "nauc_precision_at_3_diff1": 0.413357, + "nauc_precision_at_5_max": 0.313439, + "nauc_precision_at_5_std": -0.03702, + "nauc_precision_at_5_diff1": 0.356439, + "nauc_precision_at_10_max": 0.234953, + "nauc_precision_at_10_std": 0.029586, + "nauc_precision_at_10_diff1": 0.287662, + "nauc_precision_at_20_max": 0.158857, + "nauc_precision_at_20_std": 0.003019, + "nauc_precision_at_20_diff1": 0.200656, + "nauc_precision_at_100_max": 0.015624, + "nauc_precision_at_100_std": 0.078774, + "nauc_precision_at_100_diff1": 0.032752, + "nauc_precision_at_1000_max": -0.157692, + "nauc_precision_at_1000_std": 0.078741, + "nauc_precision_at_1000_diff1": -0.126513, + "nauc_mrr_at_1_max": 0.452386, + "nauc_mrr_at_1_std": -0.094075, + "nauc_mrr_at_1_diff1": 0.615308, + "nauc_mrr_at_3_max": 0.428805, + "nauc_mrr_at_3_std": -0.078473, + "nauc_mrr_at_3_diff1": 0.554433, + "nauc_mrr_at_5_max": 0.425593, + "nauc_mrr_at_5_std": -0.077061, + "nauc_mrr_at_5_diff1": 0.545829, + "nauc_mrr_at_10_max": 0.420418, + "nauc_mrr_at_10_std": -0.070097, + "nauc_mrr_at_10_diff1": 0.542038, + "nauc_mrr_at_20_max": 0.41935, + "nauc_mrr_at_20_std": -0.071884, + "nauc_mrr_at_20_diff1": 0.540918, + "nauc_mrr_at_100_max": 0.41979, + "nauc_mrr_at_100_std": -0.070271, + "nauc_mrr_at_100_diff1": 0.541354, + "nauc_mrr_at_1000_max": 0.419887, + "nauc_mrr_at_1000_std": -0.070184, + "nauc_mrr_at_1000_diff1": 0.541752, + "main_score": 0.40374, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 178.872700214386, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackWebmastersRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackWebmastersRetrieval.json new file mode 100644 index 0000000000..7b715a2859 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackWebmastersRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "160c094312a0e1facb97e55eeddb698c0abe3571", + "task_name": "CQADupstackWebmastersRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.3083, + "ndcg_at_3": 0.36442, + "ndcg_at_5": 0.37449, + "ndcg_at_10": 0.40011, + "ndcg_at_20": 0.41629, + "ndcg_at_100": 0.45629, + "ndcg_at_1000": 0.48408, + "map_at_1": 0.26966, + "map_at_3": 0.3296, + "map_at_5": 0.33821, + "map_at_10": 0.35117, + "map_at_20": 0.35772, + "map_at_100": 0.36567, + "map_at_1000": 0.36794, + "recall_at_1": 0.26966, + "recall_at_3": 0.38879, + "recall_at_5": 0.41751, + "recall_at_10": 0.49325, + "recall_at_20": 0.55524, + "recall_at_100": 0.75303, + "recall_at_1000": 0.93161, + "precision_at_1": 0.3083, + "precision_at_3": 0.16733, + "precision_at_5": 0.11423, + "precision_at_10": 0.07372, + "precision_at_20": 0.04536, + "precision_at_100": 0.01453, + "precision_at_1000": 0.00237, + "mrr_at_1": 0.3083, + "mrr_at_3": 0.365613, + "mrr_at_5": 0.373419, + "mrr_at_10": 0.384815, + "mrr_at_20": 0.389321, + "mrr_at_100": 0.394084, + "mrr_at_1000": 0.394634, + "nauc_ndcg_at_1_max": 0.337816, + "nauc_ndcg_at_1_std": -0.029596, + "nauc_ndcg_at_1_diff1": 0.553884, + "nauc_ndcg_at_3_max": 0.327833, + "nauc_ndcg_at_3_std": -0.015028, + "nauc_ndcg_at_3_diff1": 0.509244, + "nauc_ndcg_at_5_max": 0.321099, + "nauc_ndcg_at_5_std": -0.003244, + "nauc_ndcg_at_5_diff1": 0.511891, + "nauc_ndcg_at_10_max": 0.331826, + "nauc_ndcg_at_10_std": 0.013976, + "nauc_ndcg_at_10_diff1": 0.501263, + "nauc_ndcg_at_20_max": 0.333105, + "nauc_ndcg_at_20_std": 0.023115, + "nauc_ndcg_at_20_diff1": 0.496236, + "nauc_ndcg_at_100_max": 0.335661, + "nauc_ndcg_at_100_std": 0.025046, + "nauc_ndcg_at_100_diff1": 0.495239, + "nauc_ndcg_at_1000_max": 0.329531, + "nauc_ndcg_at_1000_std": 0.028787, + "nauc_ndcg_at_1000_diff1": 0.499497, + "nauc_map_at_1_max": 0.32466, + "nauc_map_at_1_std": -0.070202, + "nauc_map_at_1_diff1": 0.555772, + "nauc_map_at_3_max": 0.325576, + "nauc_map_at_3_std": -0.04551, + "nauc_map_at_3_diff1": 0.520068, + "nauc_map_at_5_max": 0.326033, + "nauc_map_at_5_std": -0.040775, + "nauc_map_at_5_diff1": 0.518245, + "nauc_map_at_10_max": 0.331407, + "nauc_map_at_10_std": -0.028968, + "nauc_map_at_10_diff1": 0.513029, + "nauc_map_at_20_max": 0.332845, + "nauc_map_at_20_std": -0.020849, + "nauc_map_at_20_diff1": 0.514389, + "nauc_map_at_100_max": 0.331951, + "nauc_map_at_100_std": -0.015583, + "nauc_map_at_100_diff1": 0.516019, + "nauc_map_at_1000_max": 0.329366, + "nauc_map_at_1000_std": -0.012949, + "nauc_map_at_1000_diff1": 0.516013, + "nauc_recall_at_1_max": 0.32466, + "nauc_recall_at_1_std": -0.070202, + "nauc_recall_at_1_diff1": 0.555772, + "nauc_recall_at_3_max": 0.320896, + "nauc_recall_at_3_std": -0.00912, + "nauc_recall_at_3_diff1": 0.486184, + "nauc_recall_at_5_max": 0.318009, + "nauc_recall_at_5_std": 0.013743, + "nauc_recall_at_5_diff1": 0.480217, + "nauc_recall_at_10_max": 0.342852, + "nauc_recall_at_10_std": 0.06458, + "nauc_recall_at_10_diff1": 0.4508, + "nauc_recall_at_20_max": 0.345603, + "nauc_recall_at_20_std": 0.112777, + "nauc_recall_at_20_diff1": 0.432033, + "nauc_recall_at_100_max": 0.373103, + "nauc_recall_at_100_std": 0.179284, + "nauc_recall_at_100_diff1": 0.399859, + "nauc_recall_at_1000_max": 0.423774, + "nauc_recall_at_1000_std": 0.450838, + "nauc_recall_at_1000_diff1": 0.499563, + "nauc_precision_at_1_max": 0.337816, + "nauc_precision_at_1_std": -0.029596, + "nauc_precision_at_1_diff1": 0.553884, + "nauc_precision_at_3_max": 0.277277, + "nauc_precision_at_3_std": 0.040643, + "nauc_precision_at_3_diff1": 0.368707, + "nauc_precision_at_5_max": 0.230599, + "nauc_precision_at_5_std": 0.089954, + "nauc_precision_at_5_diff1": 0.320642, + "nauc_precision_at_10_max": 0.203399, + "nauc_precision_at_10_std": 0.182112, + "nauc_precision_at_10_diff1": 0.235923, + "nauc_precision_at_20_max": 0.137141, + "nauc_precision_at_20_std": 0.251389, + "nauc_precision_at_20_diff1": 0.19099, + "nauc_precision_at_100_max": -0.037854, + "nauc_precision_at_100_std": 0.308719, + "nauc_precision_at_100_diff1": 0.080478, + "nauc_precision_at_1000_max": -0.235722, + "nauc_precision_at_1000_std": 0.27079, + "nauc_precision_at_1000_diff1": -0.104494, + "nauc_mrr_at_1_max": 0.337816, + "nauc_mrr_at_1_std": -0.029596, + "nauc_mrr_at_1_diff1": 0.553884, + "nauc_mrr_at_3_max": 0.32513, + "nauc_mrr_at_3_std": -0.010602, + "nauc_mrr_at_3_diff1": 0.515801, + "nauc_mrr_at_5_max": 0.32094, + "nauc_mrr_at_5_std": -0.002177, + "nauc_mrr_at_5_diff1": 0.512058, + "nauc_mrr_at_10_max": 0.324732, + "nauc_mrr_at_10_std": 0.004853, + "nauc_mrr_at_10_diff1": 0.508308, + "nauc_mrr_at_20_max": 0.325037, + "nauc_mrr_at_20_std": 0.006826, + "nauc_mrr_at_20_diff1": 0.506942, + "nauc_mrr_at_100_max": 0.324959, + "nauc_mrr_at_100_std": 0.005608, + "nauc_mrr_at_100_diff1": 0.507022, + "nauc_mrr_at_1000_max": 0.325026, + "nauc_mrr_at_1000_std": 0.005607, + "nauc_mrr_at_1000_diff1": 0.507273, + "main_score": 0.40011, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 56.67860698699951, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackWordpressRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackWordpressRetrieval.json new file mode 100644 index 0000000000..299cc75d20 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CQADupstackWordpressRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4", + "task_name": "CQADupstackWordpressRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.26617, + "ndcg_at_3": 0.30321, + "ndcg_at_5": 0.32324, + "ndcg_at_10": 0.34221, + "ndcg_at_20": 0.36062, + "ndcg_at_100": 0.38947, + "ndcg_at_1000": 0.4183, + "map_at_1": 0.24273, + "map_at_3": 0.28371, + "map_at_5": 0.29615, + "map_at_10": 0.30448, + "map_at_20": 0.30987, + "map_at_100": 0.31382, + "map_at_1000": 0.31495, + "recall_at_1": 0.24273, + "recall_at_3": 0.33521, + "recall_at_5": 0.38401, + "recall_at_10": 0.4402, + "recall_at_20": 0.5089, + "recall_at_100": 0.65798, + "recall_at_1000": 0.87305, + "precision_at_1": 0.26617, + "precision_at_3": 0.12384, + "precision_at_5": 0.08651, + "precision_at_10": 0.05028, + "precision_at_20": 0.02948, + "precision_at_100": 0.00789, + "precision_at_1000": 0.00115, + "mrr_at_1": 0.266174, + "mrr_at_3": 0.307147, + "mrr_at_5": 0.317961, + "mrr_at_10": 0.325062, + "mrr_at_20": 0.330052, + "mrr_at_100": 0.334003, + "mrr_at_1000": 0.334879, + "nauc_ndcg_at_1_max": 0.384991, + "nauc_ndcg_at_1_std": -0.017377, + "nauc_ndcg_at_1_diff1": 0.573581, + "nauc_ndcg_at_3_max": 0.36327, + "nauc_ndcg_at_3_std": -0.023931, + "nauc_ndcg_at_3_diff1": 0.495941, + "nauc_ndcg_at_5_max": 0.354395, + "nauc_ndcg_at_5_std": -0.021974, + "nauc_ndcg_at_5_diff1": 0.492125, + "nauc_ndcg_at_10_max": 0.360311, + "nauc_ndcg_at_10_std": -0.019504, + "nauc_ndcg_at_10_diff1": 0.479662, + "nauc_ndcg_at_20_max": 0.350737, + "nauc_ndcg_at_20_std": -0.010893, + "nauc_ndcg_at_20_diff1": 0.470549, + "nauc_ndcg_at_100_max": 0.348146, + "nauc_ndcg_at_100_std": 0.02221, + "nauc_ndcg_at_100_diff1": 0.468957, + "nauc_ndcg_at_1000_max": 0.357534, + "nauc_ndcg_at_1000_std": 0.022061, + "nauc_ndcg_at_1000_diff1": 0.483723, + "nauc_map_at_1_max": 0.357102, + "nauc_map_at_1_std": -0.026086, + "nauc_map_at_1_diff1": 0.571856, + "nauc_map_at_3_max": 0.356136, + "nauc_map_at_3_std": -0.028323, + "nauc_map_at_3_diff1": 0.513441, + "nauc_map_at_5_max": 0.354242, + "nauc_map_at_5_std": -0.027293, + "nauc_map_at_5_diff1": 0.511341, + "nauc_map_at_10_max": 0.358003, + "nauc_map_at_10_std": -0.024831, + "nauc_map_at_10_diff1": 0.505743, + "nauc_map_at_20_max": 0.356061, + "nauc_map_at_20_std": -0.022425, + "nauc_map_at_20_diff1": 0.503695, + "nauc_map_at_100_max": 0.355448, + "nauc_map_at_100_std": -0.017666, + "nauc_map_at_100_diff1": 0.50353, + "nauc_map_at_1000_max": 0.355749, + "nauc_map_at_1000_std": -0.017435, + "nauc_map_at_1000_diff1": 0.504116, + "nauc_recall_at_1_max": 0.357102, + "nauc_recall_at_1_std": -0.026086, + "nauc_recall_at_1_diff1": 0.571856, + "nauc_recall_at_3_max": 0.347683, + "nauc_recall_at_3_std": -0.029912, + "nauc_recall_at_3_diff1": 0.441813, + "nauc_recall_at_5_max": 0.331102, + "nauc_recall_at_5_std": -0.020331, + "nauc_recall_at_5_diff1": 0.432979, + "nauc_recall_at_10_max": 0.341928, + "nauc_recall_at_10_std": -0.016704, + "nauc_recall_at_10_diff1": 0.392869, + "nauc_recall_at_20_max": 0.302508, + "nauc_recall_at_20_std": 0.009732, + "nauc_recall_at_20_diff1": 0.352735, + "nauc_recall_at_100_max": 0.275942, + "nauc_recall_at_100_std": 0.207057, + "nauc_recall_at_100_diff1": 0.314389, + "nauc_recall_at_1000_max": 0.39465, + "nauc_recall_at_1000_std": 0.408161, + "nauc_recall_at_1000_diff1": 0.406975, + "nauc_precision_at_1_max": 0.384991, + "nauc_precision_at_1_std": -0.017377, + "nauc_precision_at_1_diff1": 0.573581, + "nauc_precision_at_3_max": 0.384542, + "nauc_precision_at_3_std": -0.018112, + "nauc_precision_at_3_diff1": 0.429887, + "nauc_precision_at_5_max": 0.370013, + "nauc_precision_at_5_std": -0.014068, + "nauc_precision_at_5_diff1": 0.405565, + "nauc_precision_at_10_max": 0.374307, + "nauc_precision_at_10_std": 0.009582, + "nauc_precision_at_10_diff1": 0.352247, + "nauc_precision_at_20_max": 0.333559, + "nauc_precision_at_20_std": 0.05535, + "nauc_precision_at_20_diff1": 0.295795, + "nauc_precision_at_100_max": 0.272301, + "nauc_precision_at_100_std": 0.238588, + "nauc_precision_at_100_diff1": 0.207829, + "nauc_precision_at_1000_max": 0.071985, + "nauc_precision_at_1000_std": 0.166709, + "nauc_precision_at_1000_diff1": 0.087586, + "nauc_mrr_at_1_max": 0.384991, + "nauc_mrr_at_1_std": -0.017377, + "nauc_mrr_at_1_diff1": 0.573581, + "nauc_mrr_at_3_max": 0.379925, + "nauc_mrr_at_3_std": -0.012315, + "nauc_mrr_at_3_diff1": 0.515462, + "nauc_mrr_at_5_max": 0.37299, + "nauc_mrr_at_5_std": -0.009821, + "nauc_mrr_at_5_diff1": 0.511664, + "nauc_mrr_at_10_max": 0.374474, + "nauc_mrr_at_10_std": -0.011517, + "nauc_mrr_at_10_diff1": 0.506565, + "nauc_mrr_at_20_max": 0.37051, + "nauc_mrr_at_20_std": -0.008199, + "nauc_mrr_at_20_diff1": 0.503976, + "nauc_mrr_at_100_max": 0.369838, + "nauc_mrr_at_100_std": -0.004851, + "nauc_mrr_at_100_diff1": 0.503924, + "nauc_mrr_at_1000_max": 0.370053, + "nauc_mrr_at_1000_std": -0.004989, + "nauc_mrr_at_1000_diff1": 0.504633, + "main_score": 0.34221, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 194.97877144813538, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ChemHotpotQARetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ChemHotpotQARetrieval.json new file mode 100644 index 0000000000..f5e8faee34 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ChemHotpotQARetrieval.json @@ -0,0 +1,456 @@ +{ + "dataset_revision": "1840e8a5ac6ec752bbdd97d543ead0189bc7c25b", + "task_name": "ChemHotpotQARetrieval", + "mteb_version": "1.38.9", + "scores": { + "train": [ + { + "ndcg_at_1": 0.82778, + "ndcg_at_3": 0.85858, + "ndcg_at_5": 0.87436, + "ndcg_at_10": 0.87939, + "ndcg_at_20": 0.88094, + "ndcg_at_100": 0.88586, + "ndcg_at_1000": 0.88942, + "map_at_1": 0.80833, + "map_at_3": 0.8463, + "map_at_5": 0.85491, + "map_at_10": 0.85745, + "map_at_20": 0.85796, + "map_at_100": 0.85853, + "map_at_1000": 0.85867, + "recall_at_1": 0.80833, + "recall_at_3": 0.88611, + "recall_at_5": 0.925, + "recall_at_10": 0.93889, + "recall_at_20": 0.94444, + "recall_at_100": 0.97222, + "recall_at_1000": 1.0, + "precision_at_1": 0.82778, + "precision_at_3": 0.30556, + "precision_at_5": 0.19111, + "precision_at_10": 0.09722, + "precision_at_20": 0.04889, + "precision_at_100": 0.01006, + "precision_at_1000": 0.00104, + "mrr_at_1": 0.827778, + "mrr_at_3": 0.855556, + "mrr_at_5": 0.864167, + "mrr_at_10": 0.866019, + "mrr_at_20": 0.866524, + "mrr_at_100": 0.867095, + "mrr_at_1000": 0.867173, + "nauc_ndcg_at_1_max": 0.80818, + "nauc_ndcg_at_1_std": -0.538893, + "nauc_ndcg_at_1_diff1": 0.931806, + "nauc_ndcg_at_3_max": 0.711677, + "nauc_ndcg_at_3_std": -0.451123, + "nauc_ndcg_at_3_diff1": 0.826861, + "nauc_ndcg_at_5_max": 0.710484, + "nauc_ndcg_at_5_std": -0.499966, + "nauc_ndcg_at_5_diff1": 0.812497, + "nauc_ndcg_at_10_max": 0.71238, + "nauc_ndcg_at_10_std": -0.527488, + "nauc_ndcg_at_10_diff1": 0.818988, + "nauc_ndcg_at_20_max": 0.717192, + "nauc_ndcg_at_20_std": -0.544462, + "nauc_ndcg_at_20_diff1": 0.816529, + "nauc_ndcg_at_100_max": 0.709318, + "nauc_ndcg_at_100_std": -0.539411, + "nauc_ndcg_at_100_diff1": 0.811909, + "nauc_ndcg_at_1000_max": 0.72581, + "nauc_ndcg_at_1000_std": -0.527082, + "nauc_ndcg_at_1000_diff1": 0.831452, + "nauc_map_at_1_max": 0.647011, + "nauc_map_at_1_std": -0.526108, + "nauc_map_at_1_diff1": 0.822908, + "nauc_map_at_3_max": 0.696041, + "nauc_map_at_3_std": -0.483353, + "nauc_map_at_3_diff1": 0.809564, + "nauc_map_at_5_max": 0.694908, + "nauc_map_at_5_std": -0.507847, + "nauc_map_at_5_diff1": 0.801794, + "nauc_map_at_10_max": 0.696495, + "nauc_map_at_10_std": -0.519743, + "nauc_map_at_10_diff1": 0.805426, + "nauc_map_at_20_max": 0.697759, + "nauc_map_at_20_std": -0.524379, + "nauc_map_at_20_diff1": 0.804699, + "nauc_map_at_100_max": 0.696897, + "nauc_map_at_100_std": -0.523976, + "nauc_map_at_100_diff1": 0.804203, + "nauc_map_at_1000_max": 0.697954, + "nauc_map_at_1000_std": -0.5234, + "nauc_map_at_1000_diff1": 0.805353, + "nauc_recall_at_1_max": 0.647011, + "nauc_recall_at_1_std": -0.526108, + "nauc_recall_at_1_diff1": 0.822908, + "nauc_recall_at_3_max": 0.650934, + "nauc_recall_at_3_std": -0.342887, + "nauc_recall_at_3_diff1": 0.763868, + "nauc_recall_at_5_max": 0.602881, + "nauc_recall_at_5_std": -0.481845, + "nauc_recall_at_5_diff1": 0.66288, + "nauc_recall_at_10_max": 0.589636, + "nauc_recall_at_10_std": -0.624735, + "nauc_recall_at_10_diff1": 0.664757, + "nauc_recall_at_20_max": 0.612792, + "nauc_recall_at_20_std": -0.75943, + "nauc_recall_at_20_diff1": 0.631232, + "nauc_recall_at_100_max": 0.336695, + "nauc_recall_at_100_std": -0.849767, + "nauc_recall_at_100_diff1": 0.344164, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.80818, + "nauc_precision_at_1_std": -0.538893, + "nauc_precision_at_1_diff1": 0.931806, + "nauc_precision_at_3_max": 0.658316, + "nauc_precision_at_3_std": -0.221071, + "nauc_precision_at_3_diff1": 0.588494, + "nauc_precision_at_5_max": 0.633507, + "nauc_precision_at_5_std": -0.279425, + "nauc_precision_at_5_diff1": 0.460948, + "nauc_precision_at_10_max": 0.558507, + "nauc_precision_at_10_std": -0.306229, + "nauc_precision_at_10_diff1": 0.384421, + "nauc_precision_at_20_max": 0.569831, + "nauc_precision_at_20_std": -0.364334, + "nauc_precision_at_20_diff1": 0.349643, + "nauc_precision_at_100_max": 0.444216, + "nauc_precision_at_100_std": -0.244348, + "nauc_precision_at_100_diff1": 0.125758, + "nauc_precision_at_1000_max": 0.467822, + "nauc_precision_at_1000_std": 0.188624, + "nauc_precision_at_1000_diff1": 0.1327, + "nauc_mrr_at_1_max": 0.80818, + "nauc_mrr_at_1_std": -0.538893, + "nauc_mrr_at_1_diff1": 0.931806, + "nauc_mrr_at_3_max": 0.804787, + "nauc_mrr_at_3_std": -0.470113, + "nauc_mrr_at_3_diff1": 0.926024, + "nauc_mrr_at_5_max": 0.810625, + "nauc_mrr_at_5_std": -0.495265, + "nauc_mrr_at_5_diff1": 0.925313, + "nauc_mrr_at_10_max": 0.809778, + "nauc_mrr_at_10_std": -0.504617, + "nauc_mrr_at_10_diff1": 0.92624, + "nauc_mrr_at_20_max": 0.811563, + "nauc_mrr_at_20_std": -0.509463, + "nauc_mrr_at_20_diff1": 0.925948, + "nauc_mrr_at_100_max": 0.811162, + "nauc_mrr_at_100_std": -0.50897, + "nauc_mrr_at_100_diff1": 0.925968, + "nauc_mrr_at_1000_max": 0.811172, + "nauc_mrr_at_1000_std": -0.508701, + "nauc_mrr_at_1000_diff1": 0.926034, + "main_score": 0.87939, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "dev": [ + { + "ndcg_at_1": 0.875, + "ndcg_at_3": 0.875, + "ndcg_at_5": 0.875, + "ndcg_at_10": 0.875, + "ndcg_at_20": 0.90625, + "ndcg_at_100": 0.90625, + "ndcg_at_1000": 0.90625, + "map_at_1": 0.875, + "map_at_3": 0.875, + "map_at_5": 0.875, + "map_at_10": 0.875, + "map_at_20": 0.88333, + "map_at_100": 0.88333, + "map_at_1000": 0.88333, + "recall_at_1": 0.875, + "recall_at_3": 0.875, + "recall_at_5": 0.875, + "recall_at_10": 0.875, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.875, + "precision_at_3": 0.29167, + "precision_at_5": 0.175, + "precision_at_10": 0.0875, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.875, + "mrr_at_3": 0.875, + "mrr_at_5": 0.875, + "mrr_at_10": 0.875, + "mrr_at_20": 0.883333, + "mrr_at_100": 0.883333, + "mrr_at_1000": 0.883333, + "nauc_ndcg_at_1_max": 1.0, + "nauc_ndcg_at_1_std": -0.185434, + "nauc_ndcg_at_1_diff1": 0.865546, + "nauc_ndcg_at_3_max": 1.0, + "nauc_ndcg_at_3_std": -0.185434, + "nauc_ndcg_at_3_diff1": 0.865546, + "nauc_ndcg_at_5_max": 1.0, + "nauc_ndcg_at_5_std": -0.185434, + "nauc_ndcg_at_5_diff1": 0.865546, + "nauc_ndcg_at_10_max": 1.0, + "nauc_ndcg_at_10_std": -0.185434, + "nauc_ndcg_at_10_diff1": 0.865546, + "nauc_ndcg_at_20_max": 1.0, + "nauc_ndcg_at_20_std": -0.185434, + "nauc_ndcg_at_20_diff1": 0.865546, + "nauc_ndcg_at_100_max": 1.0, + "nauc_ndcg_at_100_std": -0.185434, + "nauc_ndcg_at_100_diff1": 0.865546, + "nauc_ndcg_at_1000_max": 1.0, + "nauc_ndcg_at_1000_std": -0.185434, + "nauc_ndcg_at_1000_diff1": 0.865546, + "nauc_map_at_1_max": 1.0, + "nauc_map_at_1_std": -0.185434, + "nauc_map_at_1_diff1": 0.865546, + "nauc_map_at_3_max": 1.0, + "nauc_map_at_3_std": -0.185434, + "nauc_map_at_3_diff1": 0.865546, + "nauc_map_at_5_max": 1.0, + "nauc_map_at_5_std": -0.185434, + "nauc_map_at_5_diff1": 0.865546, + "nauc_map_at_10_max": 1.0, + "nauc_map_at_10_std": -0.185434, + "nauc_map_at_10_diff1": 0.865546, + "nauc_map_at_20_max": 1.0, + "nauc_map_at_20_std": -0.185434, + "nauc_map_at_20_diff1": 0.865546, + "nauc_map_at_100_max": 1.0, + "nauc_map_at_100_std": -0.185434, + "nauc_map_at_100_diff1": 0.865546, + "nauc_map_at_1000_max": 1.0, + "nauc_map_at_1000_std": -0.185434, + "nauc_map_at_1000_diff1": 0.865546, + "nauc_recall_at_1_max": 1.0, + "nauc_recall_at_1_std": -0.185434, + "nauc_recall_at_1_diff1": 0.865546, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": -0.185434, + "nauc_recall_at_3_diff1": 0.865546, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": -0.185434, + "nauc_recall_at_5_diff1": 0.865546, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": -0.185434, + "nauc_recall_at_10_diff1": 0.865546, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 1.0, + "nauc_precision_at_1_std": -0.185434, + "nauc_precision_at_1_diff1": 0.865546, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": -0.185434, + "nauc_precision_at_3_diff1": 0.865546, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": -0.185434, + "nauc_precision_at_5_diff1": 0.865546, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": -0.185434, + "nauc_precision_at_10_diff1": 0.865546, + "nauc_precision_at_20_max": NaN, + "nauc_precision_at_20_std": NaN, + "nauc_precision_at_20_diff1": NaN, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 1.0, + "nauc_mrr_at_1_std": -0.185434, + "nauc_mrr_at_1_diff1": 0.865546, + "nauc_mrr_at_3_max": 1.0, + "nauc_mrr_at_3_std": -0.185434, + "nauc_mrr_at_3_diff1": 0.865546, + "nauc_mrr_at_5_max": 1.0, + "nauc_mrr_at_5_std": -0.185434, + "nauc_mrr_at_5_diff1": 0.865546, + "nauc_mrr_at_10_max": 1.0, + "nauc_mrr_at_10_std": -0.185434, + "nauc_mrr_at_10_diff1": 0.865546, + "nauc_mrr_at_20_max": 1.0, + "nauc_mrr_at_20_std": -0.185434, + "nauc_mrr_at_20_diff1": 0.865546, + "nauc_mrr_at_100_max": 1.0, + "nauc_mrr_at_100_std": -0.185434, + "nauc_mrr_at_100_diff1": 0.865546, + "nauc_mrr_at_1000_max": 1.0, + "nauc_mrr_at_1000_std": -0.185434, + "nauc_mrr_at_1000_diff1": 0.865546, + "main_score": 0.875, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test": [ + { + "ndcg_at_1": 0.77778, + "ndcg_at_3": 0.81283, + "ndcg_at_5": 0.86068, + "ndcg_at_10": 0.86068, + "ndcg_at_20": 0.8749, + "ndcg_at_100": 0.8749, + "ndcg_at_1000": 0.8749, + "map_at_1": 0.77778, + "map_at_3": 0.80556, + "map_at_5": 0.83333, + "map_at_10": 0.83333, + "map_at_20": 0.8373, + "map_at_100": 0.8373, + "map_at_1000": 0.8373, + "recall_at_1": 0.77778, + "recall_at_3": 0.83333, + "recall_at_5": 0.94444, + "recall_at_10": 0.94444, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.77778, + "precision_at_3": 0.27778, + "precision_at_5": 0.18889, + "precision_at_10": 0.09444, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.777778, + "mrr_at_3": 0.805556, + "mrr_at_5": 0.833333, + "mrr_at_10": 0.833333, + "mrr_at_20": 0.837302, + "mrr_at_100": 0.837302, + "mrr_at_1000": 0.837302, + "nauc_ndcg_at_1_max": 0.839967, + "nauc_ndcg_at_1_std": -0.145416, + "nauc_ndcg_at_1_diff1": 0.959505, + "nauc_ndcg_at_3_max": 0.786291, + "nauc_ndcg_at_3_std": -0.051565, + "nauc_ndcg_at_3_diff1": 0.926356, + "nauc_ndcg_at_5_max": 0.765418, + "nauc_ndcg_at_5_std": -0.379099, + "nauc_ndcg_at_5_diff1": 0.95275, + "nauc_ndcg_at_10_max": 0.765418, + "nauc_ndcg_at_10_std": -0.379099, + "nauc_ndcg_at_10_diff1": 0.95275, + "nauc_ndcg_at_20_max": 0.790914, + "nauc_ndcg_at_20_std": -0.225096, + "nauc_ndcg_at_20_diff1": 0.947066, + "nauc_ndcg_at_100_max": 0.790914, + "nauc_ndcg_at_100_std": -0.225096, + "nauc_ndcg_at_100_diff1": 0.947066, + "nauc_ndcg_at_1000_max": 0.790914, + "nauc_ndcg_at_1000_std": -0.225096, + "nauc_ndcg_at_1000_diff1": 0.947066, + "nauc_map_at_1_max": 0.839967, + "nauc_map_at_1_std": -0.145416, + "nauc_map_at_1_diff1": 0.959505, + "nauc_map_at_3_max": 0.798894, + "nauc_map_at_3_std": -0.0736, + "nauc_map_at_3_diff1": 0.934139, + "nauc_map_at_5_max": 0.790781, + "nauc_map_at_5_std": -0.237104, + "nauc_map_at_5_diff1": 0.948305, + "nauc_map_at_10_max": 0.790781, + "nauc_map_at_10_std": -0.237104, + "nauc_map_at_10_diff1": 0.948305, + "nauc_map_at_20_max": 0.796915, + "nauc_map_at_20_std": -0.200342, + "nauc_map_at_20_diff1": 0.946968, + "nauc_map_at_100_max": 0.796915, + "nauc_map_at_100_std": -0.200342, + "nauc_map_at_100_diff1": 0.946968, + "nauc_map_at_1000_max": 0.796915, + "nauc_map_at_1000_std": -0.200342, + "nauc_map_at_1000_diff1": 0.946968, + "nauc_recall_at_1_max": 0.839967, + "nauc_recall_at_1_std": -0.145416, + "nauc_recall_at_1_diff1": 0.959505, + "nauc_recall_at_3_max": 0.745393, + "nauc_recall_at_3_std": 0.019945, + "nauc_recall_at_3_diff1": 0.901099, + "nauc_recall_at_5_max": 0.553491, + "nauc_recall_at_5_std": -1.659191, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 0.553491, + "nauc_recall_at_10_std": -1.659191, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.839967, + "nauc_precision_at_1_std": -0.145416, + "nauc_precision_at_1_diff1": 0.959505, + "nauc_precision_at_3_max": 0.745393, + "nauc_precision_at_3_std": 0.019945, + "nauc_precision_at_3_diff1": 0.901099, + "nauc_precision_at_5_max": 0.553491, + "nauc_precision_at_5_std": -1.659191, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 0.553491, + "nauc_precision_at_10_std": -1.659191, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.839967, + "nauc_mrr_at_1_std": -0.145416, + "nauc_mrr_at_1_diff1": 0.959505, + "nauc_mrr_at_3_max": 0.798894, + "nauc_mrr_at_3_std": -0.0736, + "nauc_mrr_at_3_diff1": 0.934139, + "nauc_mrr_at_5_max": 0.790781, + "nauc_mrr_at_5_std": -0.237104, + "nauc_mrr_at_5_diff1": 0.948305, + "nauc_mrr_at_10_max": 0.790781, + "nauc_mrr_at_10_std": -0.237104, + "nauc_mrr_at_10_diff1": 0.948305, + "nauc_mrr_at_20_max": 0.796915, + "nauc_mrr_at_20_std": -0.200342, + "nauc_mrr_at_20_diff1": 0.946968, + "nauc_mrr_at_100_max": 0.796915, + "nauc_mrr_at_100_std": -0.200342, + "nauc_mrr_at_100_diff1": 0.946968, + "nauc_mrr_at_1000_max": 0.796915, + "nauc_mrr_at_1000_std": -0.200342, + "nauc_mrr_at_1000_diff1": 0.946968, + "main_score": 0.86068, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 52.666677951812744, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ChemNQRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ChemNQRetrieval.json new file mode 100644 index 0000000000..fc86c63263 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ChemNQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "5d958fb6b31055495347724d46431ba41309b03a", + "task_name": "ChemNQRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.51852, + "ndcg_at_3": 0.56311, + "ndcg_at_5": 0.653, + "ndcg_at_10": 0.67377, + "ndcg_at_20": 0.68467, + "ndcg_at_100": 0.69869, + "ndcg_at_1000": 0.70354, + "map_at_1": 0.46296, + "map_at_3": 0.52469, + "map_at_5": 0.58488, + "map_at_10": 0.59634, + "map_at_20": 0.59992, + "map_at_100": 0.60263, + "map_at_1000": 0.60281, + "recall_at_1": 0.46296, + "recall_at_3": 0.60494, + "recall_at_5": 0.80864, + "recall_at_10": 0.8642, + "recall_at_20": 0.89506, + "recall_at_100": 0.96296, + "recall_at_1000": 1.0, + "precision_at_1": 0.51852, + "precision_at_3": 0.23457, + "precision_at_5": 0.2, + "precision_at_10": 0.10741, + "precision_at_20": 0.05741, + "precision_at_100": 0.01259, + "precision_at_1000": 0.0013, + "mrr_at_1": 0.518519, + "mrr_at_3": 0.598765, + "mrr_at_5": 0.630247, + "mrr_at_10": 0.63642, + "mrr_at_20": 0.639787, + "mrr_at_100": 0.640473, + "mrr_at_1000": 0.640661, + "nauc_ndcg_at_1_max": 0.481952, + "nauc_ndcg_at_1_std": -0.401135, + "nauc_ndcg_at_1_diff1": 0.106015, + "nauc_ndcg_at_3_max": 0.163875, + "nauc_ndcg_at_3_std": -0.448411, + "nauc_ndcg_at_3_diff1": 0.035633, + "nauc_ndcg_at_5_max": 0.233419, + "nauc_ndcg_at_5_std": -0.458909, + "nauc_ndcg_at_5_diff1": 0.057573, + "nauc_ndcg_at_10_max": 0.264786, + "nauc_ndcg_at_10_std": -0.470619, + "nauc_ndcg_at_10_diff1": 0.075524, + "nauc_ndcg_at_20_max": 0.289188, + "nauc_ndcg_at_20_std": -0.451114, + "nauc_ndcg_at_20_diff1": 0.04476, + "nauc_ndcg_at_100_max": 0.30052, + "nauc_ndcg_at_100_std": -0.446928, + "nauc_ndcg_at_100_diff1": 0.005543, + "nauc_ndcg_at_1000_max": 0.287901, + "nauc_ndcg_at_1000_std": -0.444131, + "nauc_ndcg_at_1000_diff1": 0.018407, + "nauc_map_at_1_max": 0.304389, + "nauc_map_at_1_std": -0.44226, + "nauc_map_at_1_diff1": -0.04502, + "nauc_map_at_3_max": 0.17871, + "nauc_map_at_3_std": -0.462719, + "nauc_map_at_3_diff1": -0.021404, + "nauc_map_at_5_max": 0.215222, + "nauc_map_at_5_std": -0.466841, + "nauc_map_at_5_diff1": -0.012655, + "nauc_map_at_10_max": 0.235105, + "nauc_map_at_10_std": -0.462188, + "nauc_map_at_10_diff1": 0.003392, + "nauc_map_at_20_max": 0.238238, + "nauc_map_at_20_std": -0.459561, + "nauc_map_at_20_diff1": -0.00538, + "nauc_map_at_100_max": 0.2441, + "nauc_map_at_100_std": -0.454672, + "nauc_map_at_100_diff1": -0.012513, + "nauc_map_at_1000_max": 0.243685, + "nauc_map_at_1000_std": -0.454594, + "nauc_map_at_1000_diff1": -0.01214, + "nauc_recall_at_1_max": 0.304389, + "nauc_recall_at_1_std": -0.44226, + "nauc_recall_at_1_diff1": -0.04502, + "nauc_recall_at_3_max": -0.007099, + "nauc_recall_at_3_std": -0.440446, + "nauc_recall_at_3_diff1": 0.080683, + "nauc_recall_at_5_max": 0.167075, + "nauc_recall_at_5_std": -0.437585, + "nauc_recall_at_5_diff1": 0.223895, + "nauc_recall_at_10_max": 0.293329, + "nauc_recall_at_10_std": -0.547273, + "nauc_recall_at_10_diff1": 0.341441, + "nauc_recall_at_20_max": 0.473353, + "nauc_recall_at_20_std": -0.433419, + "nauc_recall_at_20_diff1": 0.216061, + "nauc_recall_at_100_max": 0.867647, + "nauc_recall_at_100_std": -0.572623, + "nauc_recall_at_100_diff1": -0.572623, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.481952, + "nauc_precision_at_1_std": -0.401135, + "nauc_precision_at_1_diff1": 0.106015, + "nauc_precision_at_3_max": 0.068641, + "nauc_precision_at_3_std": -0.357069, + "nauc_precision_at_3_diff1": 0.259784, + "nauc_precision_at_5_max": 0.090417, + "nauc_precision_at_5_std": -0.128204, + "nauc_precision_at_5_diff1": 0.179308, + "nauc_precision_at_10_max": 0.167536, + "nauc_precision_at_10_std": -0.028683, + "nauc_precision_at_10_diff1": 0.268983, + "nauc_precision_at_20_max": 0.194938, + "nauc_precision_at_20_std": 0.105382, + "nauc_precision_at_20_diff1": 0.140678, + "nauc_precision_at_100_max": 0.127981, + "nauc_precision_at_100_std": 0.203426, + "nauc_precision_at_100_diff1": -0.047333, + "nauc_precision_at_1000_max": 0.057606, + "nauc_precision_at_1000_std": 0.277263, + "nauc_precision_at_1000_diff1": 0.002645, + "nauc_mrr_at_1_max": 0.481952, + "nauc_mrr_at_1_std": -0.401135, + "nauc_mrr_at_1_diff1": 0.106015, + "nauc_mrr_at_3_max": 0.324615, + "nauc_mrr_at_3_std": -0.412282, + "nauc_mrr_at_3_diff1": 0.129249, + "nauc_mrr_at_5_max": 0.352402, + "nauc_mrr_at_5_std": -0.426726, + "nauc_mrr_at_5_diff1": 0.127848, + "nauc_mrr_at_10_max": 0.35125, + "nauc_mrr_at_10_std": -0.451046, + "nauc_mrr_at_10_diff1": 0.114229, + "nauc_mrr_at_20_max": 0.370397, + "nauc_mrr_at_20_std": -0.43525, + "nauc_mrr_at_20_diff1": 0.104977, + "nauc_mrr_at_100_max": 0.369512, + "nauc_mrr_at_100_std": -0.436625, + "nauc_mrr_at_100_diff1": 0.103846, + "nauc_mrr_at_1000_max": 0.36917, + "nauc_mrr_at_1000_std": -0.436532, + "nauc_mrr_at_1000_diff1": 0.104311, + "main_score": 0.67377, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 48.363860845565796, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ClimateFEVER.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ClimateFEVER.json new file mode 100644 index 0000000000..b6c4b522f9 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ClimateFEVER.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380", + "task_name": "ClimateFEVER", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.36091, + "ndcg_at_3": 0.30577, + "ndcg_at_5": 0.32816, + "ndcg_at_10": 0.36686, + "ndcg_at_20": 0.39529, + "ndcg_at_100": 0.43826, + "ndcg_at_1000": 0.46786, + "map_at_1": 0.16074, + "map_at_3": 0.22525, + "map_at_5": 0.24932, + "map_at_10": 0.27011, + "map_at_20": 0.28055, + "map_at_100": 0.28968, + "map_at_1000": 0.29146, + "recall_at_1": 0.16074, + "recall_at_3": 0.27438, + "recall_at_5": 0.34077, + "recall_at_10": 0.42554, + "recall_at_20": 0.50521, + "recall_at_100": 0.66809, + "recall_at_1000": 0.83191, + "precision_at_1": 0.36091, + "precision_at_3": 0.22562, + "precision_at_5": 0.17407, + "precision_at_10": 0.11336, + "precision_at_20": 0.06902, + "precision_at_100": 0.01898, + "precision_at_1000": 0.00246, + "mrr_at_1": 0.360912, + "mrr_at_3": 0.45418, + "mrr_at_5": 0.474636, + "mrr_at_10": 0.486086, + "mrr_at_20": 0.491687, + "mrr_at_100": 0.494119, + "mrr_at_1000": 0.494381, + "nauc_ndcg_at_1_max": 0.276809, + "nauc_ndcg_at_1_std": 0.111023, + "nauc_ndcg_at_1_diff1": 0.316089, + "nauc_ndcg_at_3_max": 0.291749, + "nauc_ndcg_at_3_std": 0.113829, + "nauc_ndcg_at_3_diff1": 0.250161, + "nauc_ndcg_at_5_max": 0.307711, + "nauc_ndcg_at_5_std": 0.13385, + "nauc_ndcg_at_5_diff1": 0.240501, + "nauc_ndcg_at_10_max": 0.32489, + "nauc_ndcg_at_10_std": 0.165653, + "nauc_ndcg_at_10_diff1": 0.232538, + "nauc_ndcg_at_20_max": 0.340406, + "nauc_ndcg_at_20_std": 0.178235, + "nauc_ndcg_at_20_diff1": 0.231049, + "nauc_ndcg_at_100_max": 0.350076, + "nauc_ndcg_at_100_std": 0.197088, + "nauc_ndcg_at_100_diff1": 0.228276, + "nauc_ndcg_at_1000_max": 0.351, + "nauc_ndcg_at_1000_std": 0.205021, + "nauc_ndcg_at_1000_diff1": 0.233946, + "nauc_map_at_1_max": 0.257122, + "nauc_map_at_1_std": 0.069352, + "nauc_map_at_1_diff1": 0.34045, + "nauc_map_at_3_max": 0.27762, + "nauc_map_at_3_std": 0.094869, + "nauc_map_at_3_diff1": 0.264645, + "nauc_map_at_5_max": 0.29108, + "nauc_map_at_5_std": 0.10896, + "nauc_map_at_5_diff1": 0.254589, + "nauc_map_at_10_max": 0.304299, + "nauc_map_at_10_std": 0.131392, + "nauc_map_at_10_diff1": 0.249507, + "nauc_map_at_20_max": 0.310278, + "nauc_map_at_20_std": 0.138245, + "nauc_map_at_20_diff1": 0.247975, + "nauc_map_at_100_max": 0.313934, + "nauc_map_at_100_std": 0.143518, + "nauc_map_at_100_diff1": 0.246731, + "nauc_map_at_1000_max": 0.313885, + "nauc_map_at_1000_std": 0.144275, + "nauc_map_at_1000_diff1": 0.247029, + "nauc_recall_at_1_max": 0.257122, + "nauc_recall_at_1_std": 0.069352, + "nauc_recall_at_1_diff1": 0.34045, + "nauc_recall_at_3_max": 0.273664, + "nauc_recall_at_3_std": 0.109263, + "nauc_recall_at_3_diff1": 0.21103, + "nauc_recall_at_5_max": 0.292761, + "nauc_recall_at_5_std": 0.139695, + "nauc_recall_at_5_diff1": 0.179123, + "nauc_recall_at_10_max": 0.308627, + "nauc_recall_at_10_std": 0.193623, + "nauc_recall_at_10_diff1": 0.15201, + "nauc_recall_at_20_max": 0.335705, + "nauc_recall_at_20_std": 0.216131, + "nauc_recall_at_20_diff1": 0.136887, + "nauc_recall_at_100_max": 0.352399, + "nauc_recall_at_100_std": 0.280907, + "nauc_recall_at_100_diff1": 0.10247, + "nauc_recall_at_1000_max": 0.397733, + "nauc_recall_at_1000_std": 0.409314, + "nauc_recall_at_1000_diff1": 0.091762, + "nauc_precision_at_1_max": 0.276809, + "nauc_precision_at_1_std": 0.111023, + "nauc_precision_at_1_diff1": 0.316089, + "nauc_precision_at_3_max": 0.292795, + "nauc_precision_at_3_std": 0.132268, + "nauc_precision_at_3_diff1": 0.150712, + "nauc_precision_at_5_max": 0.28322, + "nauc_precision_at_5_std": 0.172502, + "nauc_precision_at_5_diff1": 0.106479, + "nauc_precision_at_10_max": 0.279777, + "nauc_precision_at_10_std": 0.227233, + "nauc_precision_at_10_diff1": 0.069732, + "nauc_precision_at_20_max": 0.286808, + "nauc_precision_at_20_std": 0.24015, + "nauc_precision_at_20_diff1": 0.044752, + "nauc_precision_at_100_max": 0.229487, + "nauc_precision_at_100_std": 0.25256, + "nauc_precision_at_100_diff1": -0.010854, + "nauc_precision_at_1000_max": 0.123567, + "nauc_precision_at_1000_std": 0.221236, + "nauc_precision_at_1000_diff1": -0.04507, + "nauc_mrr_at_1_max": 0.276809, + "nauc_mrr_at_1_std": 0.111023, + "nauc_mrr_at_1_diff1": 0.316089, + "nauc_mrr_at_3_max": 0.309122, + "nauc_mrr_at_3_std": 0.129752, + "nauc_mrr_at_3_diff1": 0.271488, + "nauc_mrr_at_5_max": 0.314892, + "nauc_mrr_at_5_std": 0.144662, + "nauc_mrr_at_5_diff1": 0.267872, + "nauc_mrr_at_10_max": 0.316254, + "nauc_mrr_at_10_std": 0.146701, + "nauc_mrr_at_10_diff1": 0.267538, + "nauc_mrr_at_20_max": 0.3184, + "nauc_mrr_at_20_std": 0.147629, + "nauc_mrr_at_20_diff1": 0.269677, + "nauc_mrr_at_100_max": 0.317339, + "nauc_mrr_at_100_std": 0.147052, + "nauc_mrr_at_100_diff1": 0.270086, + "nauc_mrr_at_1000_max": 0.317282, + "nauc_mrr_at_1000_std": 0.146892, + "nauc_mrr_at_1000_diff1": 0.270179, + "main_score": 0.36686, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11025.669432640076, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ClimateFEVER.v2.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ClimateFEVER.v2.json new file mode 100644 index 0000000000..03ef4b23e0 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ClimateFEVER.v2.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e438c9586767800aeb10dbe8a245c41dbea4e5f4", + "task_name": "ClimateFEVER.v2", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.24365, + "ndcg_at_3": 0.23588, + "ndcg_at_5": 0.24679, + "ndcg_at_10": 0.30326, + "ndcg_at_20": 0.34697, + "ndcg_at_100": 0.41124, + "ndcg_at_1000": 0.44618, + "map_at_1": 0.06315, + "map_at_3": 0.13219, + "map_at_5": 0.17405, + "map_at_10": 0.20938, + "map_at_20": 0.22832, + "map_at_100": 0.24382, + "map_at_1000": 0.24655, + "recall_at_1": 0.06315, + "recall_at_3": 0.17229, + "recall_at_5": 0.26321, + "recall_at_10": 0.38248, + "recall_at_20": 0.49709, + "recall_at_100": 0.73176, + "recall_at_1000": 0.91834, + "precision_at_1": 0.26189, + "precision_at_3": 0.23062, + "precision_at_5": 0.20625, + "precision_at_10": 0.14795, + "precision_at_20": 0.09606, + "precision_at_100": 0.02831, + "precision_at_1000": 0.00358, + "mrr_at_1": 0.261889, + "mrr_at_3": 0.350706, + "mrr_at_5": 0.373963, + "mrr_at_10": 0.390156, + "mrr_at_20": 0.396949, + "mrr_at_100": 0.400165, + "mrr_at_1000": 0.40043, + "nauc_ndcg_at_1_max": -0.006573, + "nauc_ndcg_at_1_std": -0.028154, + "nauc_ndcg_at_1_diff1": 0.086385, + "nauc_ndcg_at_3_max": 0.051897, + "nauc_ndcg_at_3_std": -0.030997, + "nauc_ndcg_at_3_diff1": 0.068453, + "nauc_ndcg_at_5_max": 0.105715, + "nauc_ndcg_at_5_std": -0.010425, + "nauc_ndcg_at_5_diff1": 0.079438, + "nauc_ndcg_at_10_max": 0.129158, + "nauc_ndcg_at_10_std": 0.026665, + "nauc_ndcg_at_10_diff1": 0.076708, + "nauc_ndcg_at_20_max": 0.120305, + "nauc_ndcg_at_20_std": 0.040116, + "nauc_ndcg_at_20_diff1": 0.075165, + "nauc_ndcg_at_100_max": 0.097379, + "nauc_ndcg_at_100_std": 0.050916, + "nauc_ndcg_at_100_diff1": 0.081322, + "nauc_ndcg_at_1000_max": 0.060598, + "nauc_ndcg_at_1000_std": 0.00657, + "nauc_ndcg_at_1000_diff1": 0.066459, + "nauc_map_at_1_max": 0.013879, + "nauc_map_at_1_std": -0.01146, + "nauc_map_at_1_diff1": 0.077695, + "nauc_map_at_3_max": 0.078428, + "nauc_map_at_3_std": -0.021581, + "nauc_map_at_3_diff1": 0.071568, + "nauc_map_at_5_max": 0.11697, + "nauc_map_at_5_std": -0.017713, + "nauc_map_at_5_diff1": 0.074437, + "nauc_map_at_10_max": 0.130493, + "nauc_map_at_10_std": 0.01525, + "nauc_map_at_10_diff1": 0.068553, + "nauc_map_at_20_max": 0.127371, + "nauc_map_at_20_std": 0.02337, + "nauc_map_at_20_diff1": 0.068194, + "nauc_map_at_100_max": 0.122307, + "nauc_map_at_100_std": 0.030288, + "nauc_map_at_100_diff1": 0.067887, + "nauc_map_at_1000_max": 0.119607, + "nauc_map_at_1000_std": 0.027676, + "nauc_map_at_1000_diff1": 0.066841, + "nauc_recall_at_1_max": 0.013879, + "nauc_recall_at_1_std": -0.01146, + "nauc_recall_at_1_diff1": 0.077695, + "nauc_recall_at_3_max": 0.109989, + "nauc_recall_at_3_std": -0.011085, + "nauc_recall_at_3_diff1": 0.072808, + "nauc_recall_at_5_max": 0.175281, + "nauc_recall_at_5_std": 0.004609, + "nauc_recall_at_5_diff1": 0.074578, + "nauc_recall_at_10_max": 0.224303, + "nauc_recall_at_10_std": 0.074155, + "nauc_recall_at_10_diff1": 0.078551, + "nauc_recall_at_20_max": 0.214265, + "nauc_recall_at_20_std": 0.110742, + "nauc_recall_at_20_diff1": 0.076609, + "nauc_recall_at_100_max": 0.188513, + "nauc_recall_at_100_std": 0.204908, + "nauc_recall_at_100_diff1": 0.11299, + "nauc_recall_at_1000_max": -0.087155, + "nauc_recall_at_1000_std": -0.097422, + "nauc_recall_at_1000_diff1": 0.002294, + "nauc_precision_at_1_max": -0.001009, + "nauc_precision_at_1_std": -0.019457, + "nauc_precision_at_1_diff1": 0.080708, + "nauc_precision_at_3_max": 0.064702, + "nauc_precision_at_3_std": -0.025379, + "nauc_precision_at_3_diff1": 0.060787, + "nauc_precision_at_5_max": 0.096689, + "nauc_precision_at_5_std": -0.018844, + "nauc_precision_at_5_diff1": 0.06069, + "nauc_precision_at_10_max": 0.095108, + "nauc_precision_at_10_std": 0.033198, + "nauc_precision_at_10_diff1": 0.040404, + "nauc_precision_at_20_max": 0.054646, + "nauc_precision_at_20_std": 0.051775, + "nauc_precision_at_20_diff1": 0.031057, + "nauc_precision_at_100_max": -0.07523, + "nauc_precision_at_100_std": 0.054361, + "nauc_precision_at_100_diff1": 0.00073, + "nauc_precision_at_1000_max": -0.313871, + "nauc_precision_at_1000_std": -0.149848, + "nauc_precision_at_1000_diff1": -0.089961, + "nauc_mrr_at_1_max": -0.001009, + "nauc_mrr_at_1_std": -0.019457, + "nauc_mrr_at_1_diff1": 0.080708, + "nauc_mrr_at_3_max": 0.01947, + "nauc_mrr_at_3_std": -0.016531, + "nauc_mrr_at_3_diff1": 0.072242, + "nauc_mrr_at_5_max": 0.026771, + "nauc_mrr_at_5_std": -0.017083, + "nauc_mrr_at_5_diff1": 0.072656, + "nauc_mrr_at_10_max": 0.027832, + "nauc_mrr_at_10_std": -0.014495, + "nauc_mrr_at_10_diff1": 0.072353, + "nauc_mrr_at_20_max": 0.024043, + "nauc_mrr_at_20_std": -0.014154, + "nauc_mrr_at_20_diff1": 0.071879, + "nauc_mrr_at_100_max": 0.023241, + "nauc_mrr_at_100_std": -0.015464, + "nauc_mrr_at_100_diff1": 0.072916, + "nauc_mrr_at_1000_max": 0.023116, + "nauc_mrr_at_1000_std": -0.015724, + "nauc_mrr_at_1000_diff1": 0.072742, + "main_score": 0.30326, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 14.437301874160767, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ClimateFEVERHardNegatives.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ClimateFEVERHardNegatives.json new file mode 100644 index 0000000000..4d5a846db6 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/ClimateFEVERHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "3a309e201f3c2c4b13bd4a367a8f37eee2ec1d21", + "task_name": "ClimateFEVERHardNegatives", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.36, + "ndcg_at_3": 0.30785, + "ndcg_at_5": 0.33473, + "ndcg_at_10": 0.3737, + "ndcg_at_20": 0.40447, + "ndcg_at_100": 0.44904, + "ndcg_at_1000": 0.48213, + "map_at_1": 0.1616, + "map_at_3": 0.22852, + "map_at_5": 0.2546, + "map_at_10": 0.27573, + "map_at_20": 0.28703, + "map_at_100": 0.29651, + "map_at_1000": 0.29857, + "recall_at_1": 0.1616, + "recall_at_3": 0.27938, + "recall_at_5": 0.35243, + "recall_at_10": 0.43797, + "recall_at_20": 0.523, + "recall_at_100": 0.69253, + "recall_at_1000": 0.87335, + "precision_at_1": 0.36, + "precision_at_3": 0.22767, + "precision_at_5": 0.1784, + "precision_at_10": 0.1154, + "precision_at_20": 0.07125, + "precision_at_100": 0.01968, + "precision_at_1000": 0.0026, + "mrr_at_1": 0.36, + "mrr_at_3": 0.453167, + "mrr_at_5": 0.475817, + "mrr_at_10": 0.487405, + "mrr_at_20": 0.493806, + "mrr_at_100": 0.495962, + "mrr_at_1000": 0.496192, + "nauc_ndcg_at_1_max": 0.259564, + "nauc_ndcg_at_1_std": 0.112411, + "nauc_ndcg_at_1_diff1": 0.31228, + "nauc_ndcg_at_3_max": 0.272764, + "nauc_ndcg_at_3_std": 0.069803, + "nauc_ndcg_at_3_diff1": 0.259546, + "nauc_ndcg_at_5_max": 0.300109, + "nauc_ndcg_at_5_std": 0.085904, + "nauc_ndcg_at_5_diff1": 0.252694, + "nauc_ndcg_at_10_max": 0.322046, + "nauc_ndcg_at_10_std": 0.103687, + "nauc_ndcg_at_10_diff1": 0.251996, + "nauc_ndcg_at_20_max": 0.339148, + "nauc_ndcg_at_20_std": 0.108731, + "nauc_ndcg_at_20_diff1": 0.25073, + "nauc_ndcg_at_100_max": 0.347444, + "nauc_ndcg_at_100_std": 0.124968, + "nauc_ndcg_at_100_diff1": 0.245029, + "nauc_ndcg_at_1000_max": 0.345002, + "nauc_ndcg_at_1000_std": 0.132848, + "nauc_ndcg_at_1000_diff1": 0.252435, + "nauc_map_at_1_max": 0.237638, + "nauc_map_at_1_std": 0.063763, + "nauc_map_at_1_diff1": 0.347234, + "nauc_map_at_3_max": 0.255822, + "nauc_map_at_3_std": 0.062674, + "nauc_map_at_3_diff1": 0.276893, + "nauc_map_at_5_max": 0.275136, + "nauc_map_at_5_std": 0.071052, + "nauc_map_at_5_diff1": 0.268363, + "nauc_map_at_10_max": 0.29307, + "nauc_map_at_10_std": 0.084516, + "nauc_map_at_10_diff1": 0.266501, + "nauc_map_at_20_max": 0.300909, + "nauc_map_at_20_std": 0.087244, + "nauc_map_at_20_diff1": 0.26602, + "nauc_map_at_100_max": 0.302994, + "nauc_map_at_100_std": 0.091654, + "nauc_map_at_100_diff1": 0.263573, + "nauc_map_at_1000_max": 0.303151, + "nauc_map_at_1000_std": 0.092417, + "nauc_map_at_1000_diff1": 0.264062, + "nauc_recall_at_1_max": 0.237638, + "nauc_recall_at_1_std": 0.063763, + "nauc_recall_at_1_diff1": 0.347234, + "nauc_recall_at_3_max": 0.252502, + "nauc_recall_at_3_std": 0.054449, + "nauc_recall_at_3_diff1": 0.219657, + "nauc_recall_at_5_max": 0.29236, + "nauc_recall_at_5_std": 0.075789, + "nauc_recall_at_5_diff1": 0.191897, + "nauc_recall_at_10_max": 0.315348, + "nauc_recall_at_10_std": 0.103759, + "nauc_recall_at_10_diff1": 0.177745, + "nauc_recall_at_20_max": 0.347339, + "nauc_recall_at_20_std": 0.113761, + "nauc_recall_at_20_diff1": 0.161609, + "nauc_recall_at_100_max": 0.367437, + "nauc_recall_at_100_std": 0.163443, + "nauc_recall_at_100_diff1": 0.115596, + "nauc_recall_at_1000_max": 0.427279, + "nauc_recall_at_1000_std": 0.299683, + "nauc_recall_at_1000_diff1": 0.112631, + "nauc_precision_at_1_max": 0.259564, + "nauc_precision_at_1_std": 0.112411, + "nauc_precision_at_1_diff1": 0.31228, + "nauc_precision_at_3_max": 0.278665, + "nauc_precision_at_3_std": 0.063765, + "nauc_precision_at_3_diff1": 0.160248, + "nauc_precision_at_5_max": 0.287975, + "nauc_precision_at_5_std": 0.101594, + "nauc_precision_at_5_diff1": 0.11003, + "nauc_precision_at_10_max": 0.303923, + "nauc_precision_at_10_std": 0.137843, + "nauc_precision_at_10_diff1": 0.094002, + "nauc_precision_at_20_max": 0.3103, + "nauc_precision_at_20_std": 0.135955, + "nauc_precision_at_20_diff1": 0.062053, + "nauc_precision_at_100_max": 0.249427, + "nauc_precision_at_100_std": 0.164988, + "nauc_precision_at_100_diff1": -0.003368, + "nauc_precision_at_1000_max": 0.098279, + "nauc_precision_at_1000_std": 0.145868, + "nauc_precision_at_1000_diff1": -0.052384, + "nauc_mrr_at_1_max": 0.259564, + "nauc_mrr_at_1_std": 0.112411, + "nauc_mrr_at_1_diff1": 0.31228, + "nauc_mrr_at_3_max": 0.29878, + "nauc_mrr_at_3_std": 0.095753, + "nauc_mrr_at_3_diff1": 0.278224, + "nauc_mrr_at_5_max": 0.311039, + "nauc_mrr_at_5_std": 0.108955, + "nauc_mrr_at_5_diff1": 0.276275, + "nauc_mrr_at_10_max": 0.310794, + "nauc_mrr_at_10_std": 0.109621, + "nauc_mrr_at_10_diff1": 0.277495, + "nauc_mrr_at_20_max": 0.312531, + "nauc_mrr_at_20_std": 0.110202, + "nauc_mrr_at_20_diff1": 0.279264, + "nauc_mrr_at_100_max": 0.311376, + "nauc_mrr_at_100_std": 0.110376, + "nauc_mrr_at_100_diff1": 0.278871, + "nauc_mrr_at_1000_max": 0.311198, + "nauc_mrr_at_1000_std": 0.110341, + "nauc_mrr_at_1000_diff1": 0.278935, + "main_score": 0.3737, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 180.33540749549866, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CodeFeedbackMT.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CodeFeedbackMT.json new file mode 100644 index 0000000000..38303a50ce --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CodeFeedbackMT.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b0f12fa0c0dd67f59c95a5c33d02aeeb4c398c5f", + "task_name": "CodeFeedbackMT", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.34458, + "ndcg_at_3": 0.41414, + "ndcg_at_5": 0.43234, + "ndcg_at_10": 0.45041, + "ndcg_at_20": 0.46289, + "ndcg_at_100": 0.4826, + "ndcg_at_1000": 0.49878, + "map_at_1": 0.34458, + "map_at_3": 0.39712, + "map_at_5": 0.40725, + "map_at_10": 0.41472, + "map_at_20": 0.41816, + "map_at_100": 0.42082, + "map_at_1000": 0.42136, + "recall_at_1": 0.34458, + "recall_at_3": 0.46336, + "recall_at_5": 0.50742, + "recall_at_10": 0.56323, + "recall_at_20": 0.61249, + "recall_at_100": 0.71967, + "recall_at_1000": 0.85095, + "precision_at_1": 0.34458, + "precision_at_3": 0.15445, + "precision_at_5": 0.10148, + "precision_at_10": 0.05632, + "precision_at_20": 0.03062, + "precision_at_100": 0.0072, + "precision_at_1000": 0.00085, + "mrr_at_1": 0.344581, + "mrr_at_3": 0.397115, + "mrr_at_5": 0.407253, + "mrr_at_10": 0.414721, + "mrr_at_20": 0.418163, + "mrr_at_100": 0.420821, + "mrr_at_1000": 0.421362, + "nauc_ndcg_at_1_max": 0.280552, + "nauc_ndcg_at_1_std": -0.187731, + "nauc_ndcg_at_1_diff1": 0.602493, + "nauc_ndcg_at_3_max": 0.292248, + "nauc_ndcg_at_3_std": -0.192545, + "nauc_ndcg_at_3_diff1": 0.556765, + "nauc_ndcg_at_5_max": 0.293398, + "nauc_ndcg_at_5_std": -0.188761, + "nauc_ndcg_at_5_diff1": 0.546072, + "nauc_ndcg_at_10_max": 0.296851, + "nauc_ndcg_at_10_std": -0.182631, + "nauc_ndcg_at_10_diff1": 0.540277, + "nauc_ndcg_at_20_max": 0.29735, + "nauc_ndcg_at_20_std": -0.175578, + "nauc_ndcg_at_20_diff1": 0.535887, + "nauc_ndcg_at_100_max": 0.30202, + "nauc_ndcg_at_100_std": -0.160654, + "nauc_ndcg_at_100_diff1": 0.533785, + "nauc_ndcg_at_1000_max": 0.303773, + "nauc_ndcg_at_1000_std": -0.152492, + "nauc_ndcg_at_1000_diff1": 0.536367, + "nauc_map_at_1_max": 0.280552, + "nauc_map_at_1_std": -0.187731, + "nauc_map_at_1_diff1": 0.602493, + "nauc_map_at_3_max": 0.289796, + "nauc_map_at_3_std": -0.191634, + "nauc_map_at_3_diff1": 0.567746, + "nauc_map_at_5_max": 0.290425, + "nauc_map_at_5_std": -0.189655, + "nauc_map_at_5_diff1": 0.561804, + "nauc_map_at_10_max": 0.291868, + "nauc_map_at_10_std": -0.187212, + "nauc_map_at_10_diff1": 0.55955, + "nauc_map_at_20_max": 0.291979, + "nauc_map_at_20_std": -0.185355, + "nauc_map_at_20_diff1": 0.558423, + "nauc_map_at_100_max": 0.29262, + "nauc_map_at_100_std": -0.183477, + "nauc_map_at_100_diff1": 0.558251, + "nauc_map_at_1000_max": 0.29267, + "nauc_map_at_1000_std": -0.183171, + "nauc_map_at_1000_diff1": 0.55833, + "nauc_recall_at_1_max": 0.280552, + "nauc_recall_at_1_std": -0.187731, + "nauc_recall_at_1_diff1": 0.602493, + "nauc_recall_at_3_max": 0.299178, + "nauc_recall_at_3_std": -0.195071, + "nauc_recall_at_3_diff1": 0.525074, + "nauc_recall_at_5_max": 0.302127, + "nauc_recall_at_5_std": -0.18533, + "nauc_recall_at_5_diff1": 0.498475, + "nauc_recall_at_10_max": 0.313576, + "nauc_recall_at_10_std": -0.164892, + "nauc_recall_at_10_diff1": 0.476763, + "nauc_recall_at_20_max": 0.316889, + "nauc_recall_at_20_std": -0.132757, + "nauc_recall_at_20_diff1": 0.453653, + "nauc_recall_at_100_max": 0.354469, + "nauc_recall_at_100_std": -0.013381, + "nauc_recall_at_100_diff1": 0.417284, + "nauc_recall_at_1000_max": 0.423065, + "nauc_recall_at_1000_std": 0.227612, + "nauc_recall_at_1000_diff1": 0.384414, + "nauc_precision_at_1_max": 0.280552, + "nauc_precision_at_1_std": -0.187731, + "nauc_precision_at_1_diff1": 0.602493, + "nauc_precision_at_3_max": 0.299178, + "nauc_precision_at_3_std": -0.195071, + "nauc_precision_at_3_diff1": 0.525074, + "nauc_precision_at_5_max": 0.302127, + "nauc_precision_at_5_std": -0.18533, + "nauc_precision_at_5_diff1": 0.498475, + "nauc_precision_at_10_max": 0.313576, + "nauc_precision_at_10_std": -0.164892, + "nauc_precision_at_10_diff1": 0.476763, + "nauc_precision_at_20_max": 0.316889, + "nauc_precision_at_20_std": -0.132757, + "nauc_precision_at_20_diff1": 0.453653, + "nauc_precision_at_100_max": 0.354469, + "nauc_precision_at_100_std": -0.013381, + "nauc_precision_at_100_diff1": 0.417284, + "nauc_precision_at_1000_max": 0.423065, + "nauc_precision_at_1000_std": 0.227612, + "nauc_precision_at_1000_diff1": 0.384414, + "nauc_mrr_at_1_max": 0.280552, + "nauc_mrr_at_1_std": -0.187731, + "nauc_mrr_at_1_diff1": 0.602493, + "nauc_mrr_at_3_max": 0.289796, + "nauc_mrr_at_3_std": -0.191634, + "nauc_mrr_at_3_diff1": 0.567746, + "nauc_mrr_at_5_max": 0.290425, + "nauc_mrr_at_5_std": -0.189655, + "nauc_mrr_at_5_diff1": 0.561804, + "nauc_mrr_at_10_max": 0.291868, + "nauc_mrr_at_10_std": -0.187212, + "nauc_mrr_at_10_diff1": 0.55955, + "nauc_mrr_at_20_max": 0.291979, + "nauc_mrr_at_20_std": -0.185355, + "nauc_mrr_at_20_diff1": 0.558423, + "nauc_mrr_at_100_max": 0.29262, + "nauc_mrr_at_100_std": -0.183477, + "nauc_mrr_at_100_diff1": 0.558251, + "nauc_mrr_at_1000_max": 0.29267, + "nauc_mrr_at_1000_std": -0.183171, + "nauc_mrr_at_1000_diff1": 0.55833, + "main_score": 0.45041, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 503.46369528770447, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CodeFeedbackST.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CodeFeedbackST.json new file mode 100644 index 0000000000..bb5081843c --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CodeFeedbackST.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "d213819e87aab9010628da8b73ab4eb337c89340", + "task_name": "CodeFeedbackST", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.60004, + "ndcg_at_3": 0.71217, + "ndcg_at_5": 0.73422, + "ndcg_at_10": 0.74999, + "ndcg_at_20": 0.75801, + "ndcg_at_100": 0.76661, + "ndcg_at_1000": 0.76977, + "map_at_1": 0.60004, + "map_at_3": 0.68527, + "map_at_5": 0.6976, + "map_at_10": 0.70424, + "map_at_20": 0.70647, + "map_at_100": 0.70768, + "map_at_1000": 0.7078, + "recall_at_1": 0.60004, + "recall_at_3": 0.78975, + "recall_at_5": 0.84287, + "recall_at_10": 0.89095, + "recall_at_20": 0.92247, + "recall_at_100": 0.96857, + "recall_at_1000": 0.9932, + "precision_at_1": 0.60004, + "precision_at_3": 0.26325, + "precision_at_5": 0.16857, + "precision_at_10": 0.08909, + "precision_at_20": 0.04612, + "precision_at_100": 0.00969, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.599853, + "mrr_at_3": 0.68522, + "mrr_at_5": 0.69755, + "mrr_at_10": 0.704202, + "mrr_at_20": 0.706421, + "mrr_at_100": 0.70763, + "mrr_at_1000": 0.707753, + "nauc_ndcg_at_1_max": 0.271414, + "nauc_ndcg_at_1_std": -0.281886, + "nauc_ndcg_at_1_diff1": 0.740834, + "nauc_ndcg_at_3_max": 0.309297, + "nauc_ndcg_at_3_std": -0.316779, + "nauc_ndcg_at_3_diff1": 0.689362, + "nauc_ndcg_at_5_max": 0.315985, + "nauc_ndcg_at_5_std": -0.319771, + "nauc_ndcg_at_5_diff1": 0.685186, + "nauc_ndcg_at_10_max": 0.314121, + "nauc_ndcg_at_10_std": -0.316267, + "nauc_ndcg_at_10_diff1": 0.686832, + "nauc_ndcg_at_20_max": 0.312129, + "nauc_ndcg_at_20_std": -0.31286, + "nauc_ndcg_at_20_diff1": 0.691106, + "nauc_ndcg_at_100_max": 0.310853, + "nauc_ndcg_at_100_std": -0.30471, + "nauc_ndcg_at_100_diff1": 0.695454, + "nauc_ndcg_at_1000_max": 0.308455, + "nauc_ndcg_at_1000_std": -0.305219, + "nauc_ndcg_at_1000_diff1": 0.697403, + "nauc_map_at_1_max": 0.271414, + "nauc_map_at_1_std": -0.281886, + "nauc_map_at_1_diff1": 0.740834, + "nauc_map_at_3_max": 0.298699, + "nauc_map_at_3_std": -0.306592, + "nauc_map_at_3_diff1": 0.703203, + "nauc_map_at_5_max": 0.301651, + "nauc_map_at_5_std": -0.307728, + "nauc_map_at_5_diff1": 0.701641, + "nauc_map_at_10_max": 0.300677, + "nauc_map_at_10_std": -0.306237, + "nauc_map_at_10_diff1": 0.702582, + "nauc_map_at_20_max": 0.300107, + "nauc_map_at_20_std": -0.305318, + "nauc_map_at_20_diff1": 0.703681, + "nauc_map_at_100_max": 0.29989, + "nauc_map_at_100_std": -0.304372, + "nauc_map_at_100_diff1": 0.704252, + "nauc_map_at_1000_max": 0.299828, + "nauc_map_at_1000_std": -0.304362, + "nauc_map_at_1000_diff1": 0.704316, + "nauc_recall_at_1_max": 0.271414, + "nauc_recall_at_1_std": -0.281886, + "nauc_recall_at_1_diff1": 0.740834, + "nauc_recall_at_3_max": 0.350216, + "nauc_recall_at_3_std": -0.356355, + "nauc_recall_at_3_diff1": 0.636245, + "nauc_recall_at_5_max": 0.38654, + "nauc_recall_at_5_std": -0.378673, + "nauc_recall_at_5_diff1": 0.605452, + "nauc_recall_at_10_max": 0.402219, + "nauc_recall_at_10_std": -0.379758, + "nauc_recall_at_10_diff1": 0.584972, + "nauc_recall_at_20_max": 0.41577, + "nauc_recall_at_20_std": -0.370171, + "nauc_recall_at_20_diff1": 0.59018, + "nauc_recall_at_100_max": 0.525795, + "nauc_recall_at_100_std": -0.181844, + "nauc_recall_at_100_diff1": 0.585437, + "nauc_recall_at_1000_max": 0.743247, + "nauc_recall_at_1000_std": 0.101653, + "nauc_recall_at_1000_diff1": 0.619393, + "nauc_precision_at_1_max": 0.271414, + "nauc_precision_at_1_std": -0.281886, + "nauc_precision_at_1_diff1": 0.740834, + "nauc_precision_at_3_max": 0.350216, + "nauc_precision_at_3_std": -0.356355, + "nauc_precision_at_3_diff1": 0.636245, + "nauc_precision_at_5_max": 0.38654, + "nauc_precision_at_5_std": -0.378673, + "nauc_precision_at_5_diff1": 0.605452, + "nauc_precision_at_10_max": 0.402219, + "nauc_precision_at_10_std": -0.379758, + "nauc_precision_at_10_diff1": 0.584972, + "nauc_precision_at_20_max": 0.41577, + "nauc_precision_at_20_std": -0.370171, + "nauc_precision_at_20_diff1": 0.59018, + "nauc_precision_at_100_max": 0.525795, + "nauc_precision_at_100_std": -0.181844, + "nauc_precision_at_100_diff1": 0.585437, + "nauc_precision_at_1000_max": 0.743247, + "nauc_precision_at_1000_std": 0.101653, + "nauc_precision_at_1000_diff1": 0.619393, + "nauc_mrr_at_1_max": 0.267339, + "nauc_mrr_at_1_std": -0.281551, + "nauc_mrr_at_1_diff1": 0.741121, + "nauc_mrr_at_3_max": 0.295888, + "nauc_mrr_at_3_std": -0.30689, + "nauc_mrr_at_3_diff1": 0.703444, + "nauc_mrr_at_5_max": 0.298609, + "nauc_mrr_at_5_std": -0.307795, + "nauc_mrr_at_5_diff1": 0.701732, + "nauc_mrr_at_10_max": 0.297661, + "nauc_mrr_at_10_std": -0.306382, + "nauc_mrr_at_10_diff1": 0.702697, + "nauc_mrr_at_20_max": 0.29708, + "nauc_mrr_at_20_std": -0.305468, + "nauc_mrr_at_20_diff1": 0.7038, + "nauc_mrr_at_100_max": 0.296844, + "nauc_mrr_at_100_std": -0.304538, + "nauc_mrr_at_100_diff1": 0.704372, + "nauc_mrr_at_1000_max": 0.29678, + "nauc_mrr_at_1000_std": -0.304528, + "nauc_mrr_at_1000_diff1": 0.704437, + "main_score": 0.74999, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1398.7163407802582, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CosQA.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CosQA.json new file mode 100644 index 0000000000..8d12fd10a9 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/CosQA.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "bc5efb7e9d437246ce393ed19d772e08e4a79535", + "task_name": "CosQA", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.136, + "ndcg_at_3": 0.2169, + "ndcg_at_5": 0.27232, + "ndcg_at_10": 0.32149, + "ndcg_at_20": 0.35561, + "ndcg_at_100": 0.39716, + "ndcg_at_1000": 0.40709, + "map_at_1": 0.136, + "map_at_3": 0.19533, + "map_at_5": 0.22573, + "map_at_10": 0.24642, + "map_at_20": 0.2559, + "map_at_100": 0.26207, + "map_at_1000": 0.26246, + "recall_at_1": 0.136, + "recall_at_3": 0.28, + "recall_at_5": 0.416, + "recall_at_10": 0.566, + "recall_at_20": 0.7, + "recall_at_100": 0.918, + "recall_at_1000": 0.996, + "precision_at_1": 0.136, + "precision_at_3": 0.09333, + "precision_at_5": 0.0832, + "precision_at_10": 0.0566, + "precision_at_20": 0.035, + "precision_at_100": 0.00918, + "precision_at_1000": 0.001, + "mrr_at_1": 0.13, + "mrr_at_3": 0.189667, + "mrr_at_5": 0.213067, + "mrr_at_10": 0.235431, + "mrr_at_20": 0.246691, + "mrr_at_100": 0.252817, + "mrr_at_1000": 0.253222, + "nauc_ndcg_at_1_max": 0.253783, + "nauc_ndcg_at_1_std": -0.15662, + "nauc_ndcg_at_1_diff1": 0.387986, + "nauc_ndcg_at_3_max": 0.248445, + "nauc_ndcg_at_3_std": -0.115243, + "nauc_ndcg_at_3_diff1": 0.220809, + "nauc_ndcg_at_5_max": 0.284733, + "nauc_ndcg_at_5_std": -0.099549, + "nauc_ndcg_at_5_diff1": 0.175058, + "nauc_ndcg_at_10_max": 0.285286, + "nauc_ndcg_at_10_std": -0.102808, + "nauc_ndcg_at_10_diff1": 0.176662, + "nauc_ndcg_at_20_max": 0.28245, + "nauc_ndcg_at_20_std": -0.083337, + "nauc_ndcg_at_20_diff1": 0.167137, + "nauc_ndcg_at_100_max": 0.274579, + "nauc_ndcg_at_100_std": -0.081616, + "nauc_ndcg_at_100_diff1": 0.176841, + "nauc_ndcg_at_1000_max": 0.273697, + "nauc_ndcg_at_1000_std": -0.094675, + "nauc_ndcg_at_1000_diff1": 0.19524, + "nauc_map_at_1_max": 0.253783, + "nauc_map_at_1_std": -0.15662, + "nauc_map_at_1_diff1": 0.387986, + "nauc_map_at_3_max": 0.247992, + "nauc_map_at_3_std": -0.125783, + "nauc_map_at_3_diff1": 0.25606, + "nauc_map_at_5_max": 0.269273, + "nauc_map_at_5_std": -0.11727, + "nauc_map_at_5_diff1": 0.22788, + "nauc_map_at_10_max": 0.267147, + "nauc_map_at_10_std": -0.118589, + "nauc_map_at_10_diff1": 0.22711, + "nauc_map_at_20_max": 0.265608, + "nauc_map_at_20_std": -0.113559, + "nauc_map_at_20_diff1": 0.224749, + "nauc_map_at_100_max": 0.264855, + "nauc_map_at_100_std": -0.113098, + "nauc_map_at_100_diff1": 0.226535, + "nauc_map_at_1000_max": 0.264898, + "nauc_map_at_1000_std": -0.113376, + "nauc_map_at_1000_diff1": 0.227234, + "nauc_recall_at_1_max": 0.253783, + "nauc_recall_at_1_std": -0.15662, + "nauc_recall_at_1_diff1": 0.387986, + "nauc_recall_at_3_max": 0.249986, + "nauc_recall_at_3_std": -0.09005, + "nauc_recall_at_3_diff1": 0.138404, + "nauc_recall_at_5_max": 0.323635, + "nauc_recall_at_5_std": -0.057083, + "nauc_recall_at_5_diff1": 0.053208, + "nauc_recall_at_10_max": 0.338382, + "nauc_recall_at_10_std": -0.063134, + "nauc_recall_at_10_diff1": 0.054787, + "nauc_recall_at_20_max": 0.343942, + "nauc_recall_at_20_std": 0.027667, + "nauc_recall_at_20_diff1": -0.01122, + "nauc_recall_at_100_max": 0.334677, + "nauc_recall_at_100_std": 0.264843, + "nauc_recall_at_100_diff1": -0.227494, + "nauc_recall_at_1000_max": 0.861111, + "nauc_recall_at_1000_std": 0.561391, + "nauc_recall_at_1000_diff1": 0.122782, + "nauc_precision_at_1_max": 0.253783, + "nauc_precision_at_1_std": -0.15662, + "nauc_precision_at_1_diff1": 0.387986, + "nauc_precision_at_3_max": 0.249986, + "nauc_precision_at_3_std": -0.09005, + "nauc_precision_at_3_diff1": 0.138404, + "nauc_precision_at_5_max": 0.323635, + "nauc_precision_at_5_std": -0.057083, + "nauc_precision_at_5_diff1": 0.053208, + "nauc_precision_at_10_max": 0.338382, + "nauc_precision_at_10_std": -0.063134, + "nauc_precision_at_10_diff1": 0.054787, + "nauc_precision_at_20_max": 0.343942, + "nauc_precision_at_20_std": 0.027667, + "nauc_precision_at_20_diff1": -0.01122, + "nauc_precision_at_100_max": 0.334677, + "nauc_precision_at_100_std": 0.264843, + "nauc_precision_at_100_diff1": -0.227494, + "nauc_precision_at_1000_max": 0.861111, + "nauc_precision_at_1000_std": 0.561391, + "nauc_precision_at_1000_diff1": 0.122782, + "nauc_mrr_at_1_max": 0.288003, + "nauc_mrr_at_1_std": -0.169466, + "nauc_mrr_at_1_diff1": 0.444387, + "nauc_mrr_at_3_max": 0.250982, + "nauc_mrr_at_3_std": -0.168699, + "nauc_mrr_at_3_diff1": 0.309156, + "nauc_mrr_at_5_max": 0.254528, + "nauc_mrr_at_5_std": -0.146991, + "nauc_mrr_at_5_diff1": 0.270908, + "nauc_mrr_at_10_max": 0.26496, + "nauc_mrr_at_10_std": -0.148547, + "nauc_mrr_at_10_diff1": 0.266443, + "nauc_mrr_at_20_max": 0.266575, + "nauc_mrr_at_20_std": -0.141486, + "nauc_mrr_at_20_diff1": 0.266906, + "nauc_mrr_at_100_max": 0.266114, + "nauc_mrr_at_100_std": -0.141332, + "nauc_mrr_at_100_diff1": 0.269275, + "nauc_mrr_at_1000_max": 0.266102, + "nauc_mrr_at_1000_std": -0.141685, + "nauc_mrr_at_1000_diff1": 0.270157, + "main_score": 0.32149, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 39.26116156578064, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/DBPedia.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/DBPedia.json new file mode 100644 index 0000000000..9ded50b2a8 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/DBPedia.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659", + "task_name": "DBPedia", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.57, + "ndcg_at_3": 0.4926, + "ndcg_at_5": 0.47051, + "ndcg_at_10": 0.45259, + "ndcg_at_20": 0.4497, + "ndcg_at_100": 0.50589, + "ndcg_at_1000": 0.57634, + "map_at_1": 0.08835, + "map_at_3": 0.1459, + "map_at_5": 0.17425, + "map_at_10": 0.21525, + "map_at_20": 0.25456, + "map_at_100": 0.31773, + "map_at_1000": 0.33669, + "recall_at_1": 0.08835, + "recall_at_3": 0.16014, + "recall_at_5": 0.20395, + "recall_at_10": 0.27513, + "recall_at_20": 0.3586, + "recall_at_100": 0.57937, + "recall_at_1000": 0.80407, + "precision_at_1": 0.6925, + "precision_at_3": 0.53333, + "precision_at_5": 0.4595, + "precision_at_10": 0.36975, + "precision_at_20": 0.28787, + "precision_at_100": 0.12208, + "precision_at_1000": 0.02426, + "mrr_at_1": 0.6925, + "mrr_at_3": 0.76, + "mrr_at_5": 0.769625, + "mrr_at_10": 0.774224, + "mrr_at_20": 0.776169, + "mrr_at_100": 0.776935, + "mrr_at_1000": 0.777009, + "nauc_ndcg_at_1_max": 0.527716, + "nauc_ndcg_at_1_std": 0.233106, + "nauc_ndcg_at_1_diff1": 0.447973, + "nauc_ndcg_at_3_max": 0.45787, + "nauc_ndcg_at_3_std": 0.297221, + "nauc_ndcg_at_3_diff1": 0.285116, + "nauc_ndcg_at_5_max": 0.44429, + "nauc_ndcg_at_5_std": 0.31797, + "nauc_ndcg_at_5_diff1": 0.257371, + "nauc_ndcg_at_10_max": 0.469063, + "nauc_ndcg_at_10_std": 0.297251, + "nauc_ndcg_at_10_diff1": 0.274059, + "nauc_ndcg_at_20_max": 0.487576, + "nauc_ndcg_at_20_std": 0.274663, + "nauc_ndcg_at_20_diff1": 0.289695, + "nauc_ndcg_at_100_max": 0.496038, + "nauc_ndcg_at_100_std": 0.322763, + "nauc_ndcg_at_100_diff1": 0.287895, + "nauc_ndcg_at_1000_max": 0.45804, + "nauc_ndcg_at_1000_std": 0.420038, + "nauc_ndcg_at_1000_diff1": 0.267064, + "nauc_map_at_1_max": 0.402351, + "nauc_map_at_1_std": -0.241569, + "nauc_map_at_1_diff1": 0.412894, + "nauc_map_at_3_max": 0.394449, + "nauc_map_at_3_std": -0.226246, + "nauc_map_at_3_diff1": 0.313813, + "nauc_map_at_5_max": 0.41427, + "nauc_map_at_5_std": -0.175816, + "nauc_map_at_5_diff1": 0.292819, + "nauc_map_at_10_max": 0.436132, + "nauc_map_at_10_std": -0.082328, + "nauc_map_at_10_diff1": 0.273619, + "nauc_map_at_20_max": 0.448856, + "nauc_map_at_20_std": 0.04698, + "nauc_map_at_20_diff1": 0.259262, + "nauc_map_at_100_max": 0.433235, + "nauc_map_at_100_std": 0.224418, + "nauc_map_at_100_diff1": 0.230649, + "nauc_map_at_1000_max": 0.418113, + "nauc_map_at_1000_std": 0.257089, + "nauc_map_at_1000_diff1": 0.218283, + "nauc_recall_at_1_max": 0.402351, + "nauc_recall_at_1_std": -0.241569, + "nauc_recall_at_1_diff1": 0.412894, + "nauc_recall_at_3_max": 0.346341, + "nauc_recall_at_3_std": -0.246908, + "nauc_recall_at_3_diff1": 0.273497, + "nauc_recall_at_5_max": 0.360079, + "nauc_recall_at_5_std": -0.204505, + "nauc_recall_at_5_diff1": 0.248998, + "nauc_recall_at_10_max": 0.379518, + "nauc_recall_at_10_std": -0.133331, + "nauc_recall_at_10_diff1": 0.231558, + "nauc_recall_at_20_max": 0.381022, + "nauc_recall_at_20_std": -0.00463, + "nauc_recall_at_20_diff1": 0.230515, + "nauc_recall_at_100_max": 0.395139, + "nauc_recall_at_100_std": 0.25711, + "nauc_recall_at_100_diff1": 0.192148, + "nauc_recall_at_1000_max": 0.336555, + "nauc_recall_at_1000_std": 0.481404, + "nauc_recall_at_1000_diff1": 0.180956, + "nauc_precision_at_1_max": 0.57291, + "nauc_precision_at_1_std": 0.311541, + "nauc_precision_at_1_diff1": 0.506412, + "nauc_precision_at_3_max": 0.312732, + "nauc_precision_at_3_std": 0.430986, + "nauc_precision_at_3_diff1": 0.109412, + "nauc_precision_at_5_max": 0.226635, + "nauc_precision_at_5_std": 0.491158, + "nauc_precision_at_5_diff1": 0.011747, + "nauc_precision_at_10_max": 0.158519, + "nauc_precision_at_10_std": 0.530989, + "nauc_precision_at_10_diff1": -0.035235, + "nauc_precision_at_20_max": 0.096773, + "nauc_precision_at_20_std": 0.562902, + "nauc_precision_at_20_diff1": -0.04464, + "nauc_precision_at_100_max": -0.039278, + "nauc_precision_at_100_std": 0.501331, + "nauc_precision_at_100_diff1": -0.10605, + "nauc_precision_at_1000_max": -0.237657, + "nauc_precision_at_1000_std": 0.09331, + "nauc_precision_at_1000_diff1": -0.223446, + "nauc_mrr_at_1_max": 0.57291, + "nauc_mrr_at_1_std": 0.311541, + "nauc_mrr_at_1_diff1": 0.506412, + "nauc_mrr_at_3_max": 0.580133, + "nauc_mrr_at_3_std": 0.362823, + "nauc_mrr_at_3_diff1": 0.496387, + "nauc_mrr_at_5_max": 0.577137, + "nauc_mrr_at_5_std": 0.361089, + "nauc_mrr_at_5_diff1": 0.503196, + "nauc_mrr_at_10_max": 0.574837, + "nauc_mrr_at_10_std": 0.35452, + "nauc_mrr_at_10_diff1": 0.499805, + "nauc_mrr_at_20_max": 0.573002, + "nauc_mrr_at_20_std": 0.354453, + "nauc_mrr_at_20_diff1": 0.500155, + "nauc_mrr_at_100_max": 0.573907, + "nauc_mrr_at_100_std": 0.354742, + "nauc_mrr_at_100_diff1": 0.500037, + "nauc_mrr_at_1000_max": 0.573844, + "nauc_mrr_at_1000_std": 0.354582, + "nauc_mrr_at_1000_diff1": 0.500052, + "main_score": 0.45259, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6090.212055683136, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/DBPediaHardNegatives.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/DBPediaHardNegatives.json new file mode 100644 index 0000000000..dd14de925c --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/DBPediaHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "943ec7fdfef3728b2ad1966c5b6479ff9ffd26c9", + "task_name": "DBPediaHardNegatives", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.5875, + "ndcg_at_3": 0.51354, + "ndcg_at_5": 0.49661, + "ndcg_at_10": 0.48481, + "ndcg_at_20": 0.48776, + "ndcg_at_100": 0.57428, + "ndcg_at_1000": 0.65677, + "map_at_1": 0.08869, + "map_at_3": 0.14998, + "map_at_5": 0.18035, + "map_at_10": 0.2257, + "map_at_20": 0.27218, + "map_at_100": 0.36254, + "map_at_1000": 0.40121, + "recall_at_1": 0.08869, + "recall_at_3": 0.16466, + "recall_at_5": 0.2132, + "recall_at_10": 0.29208, + "recall_at_20": 0.39397, + "recall_at_100": 0.69493, + "recall_at_1000": 0.94971, + "precision_at_1": 0.7125, + "precision_at_3": 0.5625, + "precision_at_5": 0.494, + "precision_at_10": 0.41025, + "precision_at_20": 0.32787, + "precision_at_100": 0.1622, + "precision_at_1000": 0.03363, + "mrr_at_1": 0.7125, + "mrr_at_3": 0.782917, + "mrr_at_5": 0.795542, + "mrr_at_10": 0.798211, + "mrr_at_20": 0.79981, + "mrr_at_100": 0.800371, + "mrr_at_1000": 0.800433, + "nauc_ndcg_at_1_max": 0.489025, + "nauc_ndcg_at_1_std": 0.133886, + "nauc_ndcg_at_1_diff1": 0.430865, + "nauc_ndcg_at_3_max": 0.405786, + "nauc_ndcg_at_3_std": 0.169015, + "nauc_ndcg_at_3_diff1": 0.255886, + "nauc_ndcg_at_5_max": 0.384688, + "nauc_ndcg_at_5_std": 0.168497, + "nauc_ndcg_at_5_diff1": 0.225767, + "nauc_ndcg_at_10_max": 0.412936, + "nauc_ndcg_at_10_std": 0.13406, + "nauc_ndcg_at_10_diff1": 0.241786, + "nauc_ndcg_at_20_max": 0.426036, + "nauc_ndcg_at_20_std": 0.096224, + "nauc_ndcg_at_20_diff1": 0.243972, + "nauc_ndcg_at_100_max": 0.452149, + "nauc_ndcg_at_100_std": 0.140635, + "nauc_ndcg_at_100_diff1": 0.266172, + "nauc_ndcg_at_1000_max": 0.374255, + "nauc_ndcg_at_1000_std": 0.275345, + "nauc_ndcg_at_1000_diff1": 0.225421, + "nauc_map_at_1_max": 0.399492, + "nauc_map_at_1_std": -0.295944, + "nauc_map_at_1_diff1": 0.409357, + "nauc_map_at_3_max": 0.391051, + "nauc_map_at_3_std": -0.309037, + "nauc_map_at_3_diff1": 0.308957, + "nauc_map_at_5_max": 0.407181, + "nauc_map_at_5_std": -0.283735, + "nauc_map_at_5_diff1": 0.292437, + "nauc_map_at_10_max": 0.429323, + "nauc_map_at_10_std": -0.215447, + "nauc_map_at_10_diff1": 0.272518, + "nauc_map_at_20_max": 0.434977, + "nauc_map_at_20_std": -0.125908, + "nauc_map_at_20_diff1": 0.249454, + "nauc_map_at_100_max": 0.406007, + "nauc_map_at_100_std": 0.047593, + "nauc_map_at_100_diff1": 0.210275, + "nauc_map_at_1000_max": 0.332647, + "nauc_map_at_1000_std": 0.086878, + "nauc_map_at_1000_diff1": 0.158444, + "nauc_recall_at_1_max": 0.399492, + "nauc_recall_at_1_std": -0.295944, + "nauc_recall_at_1_diff1": 0.409357, + "nauc_recall_at_3_max": 0.34292, + "nauc_recall_at_3_std": -0.319509, + "nauc_recall_at_3_diff1": 0.266828, + "nauc_recall_at_5_max": 0.341726, + "nauc_recall_at_5_std": -0.310754, + "nauc_recall_at_5_diff1": 0.24908, + "nauc_recall_at_10_max": 0.366911, + "nauc_recall_at_10_std": -0.26376, + "nauc_recall_at_10_diff1": 0.236627, + "nauc_recall_at_20_max": 0.333547, + "nauc_recall_at_20_std": -0.188066, + "nauc_recall_at_20_diff1": 0.191553, + "nauc_recall_at_100_max": 0.376296, + "nauc_recall_at_100_std": 0.027657, + "nauc_recall_at_100_diff1": 0.212746, + "nauc_recall_at_1000_max": 0.262021, + "nauc_recall_at_1000_std": 0.486191, + "nauc_recall_at_1000_diff1": 0.230874, + "nauc_precision_at_1_max": 0.535163, + "nauc_precision_at_1_std": 0.173766, + "nauc_precision_at_1_diff1": 0.486746, + "nauc_precision_at_3_max": 0.245872, + "nauc_precision_at_3_std": 0.267655, + "nauc_precision_at_3_diff1": 0.065966, + "nauc_precision_at_5_max": 0.143314, + "nauc_precision_at_5_std": 0.319394, + "nauc_precision_at_5_diff1": -0.042924, + "nauc_precision_at_10_max": 0.067924, + "nauc_precision_at_10_std": 0.365787, + "nauc_precision_at_10_diff1": -0.093592, + "nauc_precision_at_20_max": -0.003877, + "nauc_precision_at_20_std": 0.387201, + "nauc_precision_at_20_diff1": -0.114001, + "nauc_precision_at_100_max": -0.162844, + "nauc_precision_at_100_std": 0.327534, + "nauc_precision_at_100_diff1": -0.173371, + "nauc_precision_at_1000_max": -0.314455, + "nauc_precision_at_1000_std": 0.005252, + "nauc_precision_at_1000_diff1": -0.266878, + "nauc_mrr_at_1_max": 0.535163, + "nauc_mrr_at_1_std": 0.173766, + "nauc_mrr_at_1_diff1": 0.486746, + "nauc_mrr_at_3_max": 0.54338, + "nauc_mrr_at_3_std": 0.216376, + "nauc_mrr_at_3_diff1": 0.481685, + "nauc_mrr_at_5_max": 0.531574, + "nauc_mrr_at_5_std": 0.208594, + "nauc_mrr_at_5_diff1": 0.483472, + "nauc_mrr_at_10_max": 0.530699, + "nauc_mrr_at_10_std": 0.206752, + "nauc_mrr_at_10_diff1": 0.482557, + "nauc_mrr_at_20_max": 0.530037, + "nauc_mrr_at_20_std": 0.207388, + "nauc_mrr_at_20_diff1": 0.483053, + "nauc_mrr_at_100_max": 0.531332, + "nauc_mrr_at_100_std": 0.206669, + "nauc_mrr_at_100_diff1": 0.48298, + "nauc_mrr_at_1000_max": 0.531452, + "nauc_mrr_at_1000_std": 0.206598, + "nauc_mrr_at_1000_diff1": 0.483055, + "main_score": 0.48481, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 125.76529216766357, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FEVER.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FEVER.json new file mode 100644 index 0000000000..ee5a340a16 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FEVER.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12", + "task_name": "FEVER", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.84608, + "ndcg_at_3": 0.87985, + "ndcg_at_5": 0.88773, + "ndcg_at_10": 0.89332, + "ndcg_at_20": 0.89653, + "ndcg_at_100": 0.8999, + "ndcg_at_1000": 0.90164, + "map_at_1": 0.78481, + "map_at_3": 0.8508, + "map_at_5": 0.85689, + "map_at_10": 0.86024, + "map_at_20": 0.8616, + "map_at_100": 0.86235, + "map_at_1000": 0.86245, + "recall_at_1": 0.78481, + "recall_at_3": 0.91226, + "recall_at_5": 0.93314, + "recall_at_10": 0.94918, + "recall_at_20": 0.95948, + "recall_at_100": 0.97378, + "recall_at_1000": 0.98435, + "precision_at_1": 0.84608, + "precision_at_3": 0.33293, + "precision_at_5": 0.20561, + "precision_at_10": 0.1057, + "precision_at_20": 0.05397, + "precision_at_100": 0.01113, + "precision_at_1000": 0.00114, + "mrr_at_1": 0.846085, + "mrr_at_3": 0.90189, + "mrr_at_5": 0.905498, + "mrr_at_10": 0.906645, + "mrr_at_20": 0.906941, + "mrr_at_100": 0.907079, + "mrr_at_1000": 0.907085, + "nauc_ndcg_at_1_max": 0.206529, + "nauc_ndcg_at_1_std": -0.094547, + "nauc_ndcg_at_1_diff1": 0.731577, + "nauc_ndcg_at_3_max": 0.271607, + "nauc_ndcg_at_3_std": -0.050444, + "nauc_ndcg_at_3_diff1": 0.51151, + "nauc_ndcg_at_5_max": 0.273036, + "nauc_ndcg_at_5_std": -0.036115, + "nauc_ndcg_at_5_diff1": 0.500655, + "nauc_ndcg_at_10_max": 0.267222, + "nauc_ndcg_at_10_std": -0.035501, + "nauc_ndcg_at_10_diff1": 0.503948, + "nauc_ndcg_at_20_max": 0.265504, + "nauc_ndcg_at_20_std": -0.031659, + "nauc_ndcg_at_20_diff1": 0.511917, + "nauc_ndcg_at_100_max": 0.262236, + "nauc_ndcg_at_100_std": -0.026755, + "nauc_ndcg_at_100_diff1": 0.51787, + "nauc_ndcg_at_1000_max": 0.261273, + "nauc_ndcg_at_1000_std": -0.0282, + "nauc_ndcg_at_1000_diff1": 0.527598, + "nauc_map_at_1_max": 0.171589, + "nauc_map_at_1_std": -0.067943, + "nauc_map_at_1_diff1": 0.56077, + "nauc_map_at_3_max": 0.245343, + "nauc_map_at_3_std": -0.050118, + "nauc_map_at_3_diff1": 0.496569, + "nauc_map_at_5_max": 0.249254, + "nauc_map_at_5_std": -0.04298, + "nauc_map_at_5_diff1": 0.49574, + "nauc_map_at_10_max": 0.247752, + "nauc_map_at_10_std": -0.042663, + "nauc_map_at_10_diff1": 0.498636, + "nauc_map_at_20_max": 0.247596, + "nauc_map_at_20_std": -0.041339, + "nauc_map_at_20_diff1": 0.50136, + "nauc_map_at_100_max": 0.247397, + "nauc_map_at_100_std": -0.04036, + "nauc_map_at_100_diff1": 0.502584, + "nauc_map_at_1000_max": 0.247353, + "nauc_map_at_1000_std": -0.04032, + "nauc_map_at_1000_diff1": 0.502959, + "nauc_recall_at_1_max": 0.171589, + "nauc_recall_at_1_std": -0.067943, + "nauc_recall_at_1_diff1": 0.56077, + "nauc_recall_at_3_max": 0.331239, + "nauc_recall_at_3_std": -0.008626, + "nauc_recall_at_3_diff1": 0.305276, + "nauc_recall_at_5_max": 0.3495, + "nauc_recall_at_5_std": 0.053085, + "nauc_recall_at_5_diff1": 0.217746, + "nauc_recall_at_10_max": 0.344531, + "nauc_recall_at_10_std": 0.078517, + "nauc_recall_at_10_diff1": 0.161529, + "nauc_recall_at_20_max": 0.34896, + "nauc_recall_at_20_std": 0.133719, + "nauc_recall_at_20_diff1": 0.142544, + "nauc_recall_at_100_max": 0.33819, + "nauc_recall_at_100_std": 0.284263, + "nauc_recall_at_100_diff1": 0.035499, + "nauc_recall_at_1000_max": 0.349836, + "nauc_recall_at_1000_std": 0.416343, + "nauc_recall_at_1000_diff1": 0.068869, + "nauc_precision_at_1_max": 0.206529, + "nauc_precision_at_1_std": -0.094547, + "nauc_precision_at_1_diff1": 0.731577, + "nauc_precision_at_3_max": 0.281684, + "nauc_precision_at_3_std": -0.002028, + "nauc_precision_at_3_diff1": 0.128958, + "nauc_precision_at_5_max": 0.216188, + "nauc_precision_at_5_std": 0.057523, + "nauc_precision_at_5_diff1": -0.014624, + "nauc_precision_at_10_max": 0.108407, + "nauc_precision_at_10_std": 0.057562, + "nauc_precision_at_10_diff1": -0.074705, + "nauc_precision_at_20_max": 0.05897, + "nauc_precision_at_20_std": 0.072212, + "nauc_precision_at_20_diff1": -0.083948, + "nauc_precision_at_100_max": 0.002348, + "nauc_precision_at_100_std": 0.083556, + "nauc_precision_at_100_diff1": -0.098227, + "nauc_precision_at_1000_max": -0.02433, + "nauc_precision_at_1000_std": 0.064566, + "nauc_precision_at_1000_diff1": -0.072511, + "nauc_mrr_at_1_max": 0.206529, + "nauc_mrr_at_1_std": -0.094547, + "nauc_mrr_at_1_diff1": 0.731577, + "nauc_mrr_at_3_max": 0.267375, + "nauc_mrr_at_3_std": -0.090278, + "nauc_mrr_at_3_diff1": 0.719426, + "nauc_mrr_at_5_max": 0.262657, + "nauc_mrr_at_5_std": -0.086021, + "nauc_mrr_at_5_diff1": 0.719943, + "nauc_mrr_at_10_max": 0.258611, + "nauc_mrr_at_10_std": -0.08725, + "nauc_mrr_at_10_diff1": 0.72062, + "nauc_mrr_at_20_max": 0.257198, + "nauc_mrr_at_20_std": -0.087251, + "nauc_mrr_at_20_diff1": 0.721171, + "nauc_mrr_at_100_max": 0.256527, + "nauc_mrr_at_100_std": -0.086793, + "nauc_mrr_at_100_diff1": 0.721007, + "nauc_mrr_at_1000_max": 0.256479, + "nauc_mrr_at_1000_std": -0.086856, + "nauc_mrr_at_1000_diff1": 0.721005, + "main_score": 0.89332, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11562.310239315033, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FEVERHardNegatives.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FEVERHardNegatives.json new file mode 100644 index 0000000000..e59ef0fd8a --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FEVERHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "080c9ed6267b65029207906e815d44a9240bafca", + "task_name": "FEVERHardNegatives", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.836, + "ndcg_at_3": 0.87827, + "ndcg_at_5": 0.88638, + "ndcg_at_10": 0.89469, + "ndcg_at_20": 0.89802, + "ndcg_at_100": 0.90112, + "ndcg_at_1000": 0.90287, + "map_at_1": 0.78707, + "map_at_3": 0.85054, + "map_at_5": 0.85678, + "map_at_10": 0.86128, + "map_at_20": 0.86271, + "map_at_100": 0.86341, + "map_at_1000": 0.86354, + "recall_at_1": 0.78707, + "recall_at_3": 0.9144, + "recall_at_5": 0.93569, + "recall_at_10": 0.95905, + "recall_at_20": 0.96998, + "recall_at_100": 0.98226, + "recall_at_1000": 0.99192, + "precision_at_1": 0.836, + "precision_at_3": 0.32667, + "precision_at_5": 0.2022, + "precision_at_10": 0.105, + "precision_at_20": 0.0537, + "precision_at_100": 0.0111, + "precision_at_1000": 0.00114, + "mrr_at_1": 0.836, + "mrr_at_3": 0.894, + "mrr_at_5": 0.8974, + "mrr_at_10": 0.899508, + "mrr_at_20": 0.899713, + "mrr_at_100": 0.899865, + "mrr_at_1000": 0.899879, + "nauc_ndcg_at_1_max": 0.281062, + "nauc_ndcg_at_1_std": -0.139614, + "nauc_ndcg_at_1_diff1": 0.774051, + "nauc_ndcg_at_3_max": 0.328669, + "nauc_ndcg_at_3_std": -0.046886, + "nauc_ndcg_at_3_diff1": 0.628114, + "nauc_ndcg_at_5_max": 0.32593, + "nauc_ndcg_at_5_std": -0.036409, + "nauc_ndcg_at_5_diff1": 0.618888, + "nauc_ndcg_at_10_max": 0.314383, + "nauc_ndcg_at_10_std": -0.047435, + "nauc_ndcg_at_10_diff1": 0.625127, + "nauc_ndcg_at_20_max": 0.30881, + "nauc_ndcg_at_20_std": -0.05455, + "nauc_ndcg_at_20_diff1": 0.632113, + "nauc_ndcg_at_100_max": 0.305381, + "nauc_ndcg_at_100_std": -0.054738, + "nauc_ndcg_at_100_diff1": 0.635323, + "nauc_ndcg_at_1000_max": 0.310273, + "nauc_ndcg_at_1000_std": -0.054294, + "nauc_ndcg_at_1000_diff1": 0.642552, + "nauc_map_at_1_max": 0.218319, + "nauc_map_at_1_std": -0.093372, + "nauc_map_at_1_diff1": 0.647453, + "nauc_map_at_3_max": 0.294468, + "nauc_map_at_3_std": -0.060691, + "nauc_map_at_3_diff1": 0.613301, + "nauc_map_at_5_max": 0.297106, + "nauc_map_at_5_std": -0.055209, + "nauc_map_at_5_diff1": 0.612021, + "nauc_map_at_10_max": 0.29423, + "nauc_map_at_10_std": -0.060682, + "nauc_map_at_10_diff1": 0.615995, + "nauc_map_at_20_max": 0.292537, + "nauc_map_at_20_std": -0.062774, + "nauc_map_at_20_diff1": 0.618644, + "nauc_map_at_100_max": 0.29185, + "nauc_map_at_100_std": -0.062724, + "nauc_map_at_100_diff1": 0.619041, + "nauc_map_at_1000_max": 0.292161, + "nauc_map_at_1000_std": -0.062493, + "nauc_map_at_1000_diff1": 0.619428, + "nauc_recall_at_1_max": 0.218319, + "nauc_recall_at_1_std": -0.093372, + "nauc_recall_at_1_diff1": 0.647453, + "nauc_recall_at_3_max": 0.370531, + "nauc_recall_at_3_std": 0.048279, + "nauc_recall_at_3_diff1": 0.472493, + "nauc_recall_at_5_max": 0.378549, + "nauc_recall_at_5_std": 0.114279, + "nauc_recall_at_5_diff1": 0.39787, + "nauc_recall_at_10_max": 0.343882, + "nauc_recall_at_10_std": 0.124771, + "nauc_recall_at_10_diff1": 0.338948, + "nauc_recall_at_20_max": 0.29425, + "nauc_recall_at_20_std": 0.109126, + "nauc_recall_at_20_diff1": 0.319171, + "nauc_recall_at_100_max": 0.202108, + "nauc_recall_at_100_std": 0.207642, + "nauc_recall_at_100_diff1": 0.193353, + "nauc_recall_at_1000_max": 0.373881, + "nauc_recall_at_1000_std": 0.488155, + "nauc_recall_at_1000_diff1": 0.153892, + "nauc_precision_at_1_max": 0.281062, + "nauc_precision_at_1_std": -0.139614, + "nauc_precision_at_1_diff1": 0.774051, + "nauc_precision_at_3_max": 0.471503, + "nauc_precision_at_3_std": 0.038717, + "nauc_precision_at_3_diff1": 0.3544, + "nauc_precision_at_5_max": 0.304988, + "nauc_precision_at_5_std": 0.111068, + "nauc_precision_at_5_diff1": 0.095088, + "nauc_precision_at_10_max": 0.128991, + "nauc_precision_at_10_std": 0.068934, + "nauc_precision_at_10_diff1": -0.067889, + "nauc_precision_at_20_max": 0.053371, + "nauc_precision_at_20_std": 0.047513, + "nauc_precision_at_20_diff1": -0.102025, + "nauc_precision_at_100_max": -0.013284, + "nauc_precision_at_100_std": 0.057874, + "nauc_precision_at_100_diff1": -0.163447, + "nauc_precision_at_1000_max": -0.01434, + "nauc_precision_at_1000_std": 0.059349, + "nauc_precision_at_1000_diff1": -0.172829, + "nauc_mrr_at_1_max": 0.281062, + "nauc_mrr_at_1_std": -0.139614, + "nauc_mrr_at_1_diff1": 0.774051, + "nauc_mrr_at_3_max": 0.359008, + "nauc_mrr_at_3_std": -0.094446, + "nauc_mrr_at_3_diff1": 0.772492, + "nauc_mrr_at_5_max": 0.354708, + "nauc_mrr_at_5_std": -0.09612, + "nauc_mrr_at_5_diff1": 0.771536, + "nauc_mrr_at_10_max": 0.344814, + "nauc_mrr_at_10_std": -0.102127, + "nauc_mrr_at_10_diff1": 0.770894, + "nauc_mrr_at_20_max": 0.343418, + "nauc_mrr_at_20_std": -0.102969, + "nauc_mrr_at_20_diff1": 0.770473, + "nauc_mrr_at_100_max": 0.342777, + "nauc_mrr_at_100_std": -0.103282, + "nauc_mrr_at_100_diff1": 0.770459, + "nauc_mrr_at_1000_max": 0.342893, + "nauc_mrr_at_1000_std": -0.10336, + "nauc_mrr_at_1000_diff1": 0.770512, + "main_score": 0.89469, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 393.3530397415161, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FaithDial.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FaithDial.json new file mode 100644 index 0000000000..9dd27ca39e --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FaithDial.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "7a414e80725eac766f2602676dc8b39f80b061e4", + "task_name": "FaithDial", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.05289, + "ndcg_at_3": 0.13231, + "ndcg_at_5": 0.18384, + "ndcg_at_10": 0.23155, + "ndcg_at_20": 0.26727, + "ndcg_at_100": 0.32266, + "ndcg_at_1000": 0.33434, + "map_at_1": 0.05289, + "map_at_3": 0.11108, + "map_at_5": 0.13985, + "map_at_10": 0.15947, + "map_at_20": 0.1693, + "map_at_100": 0.17692, + "map_at_1000": 0.17745, + "recall_at_1": 0.05289, + "recall_at_3": 0.19442, + "recall_at_5": 0.31881, + "recall_at_10": 0.4667, + "recall_at_20": 0.60774, + "recall_at_100": 0.90695, + "recall_at_1000": 0.99412, + "precision_at_1": 0.05289, + "precision_at_3": 0.06481, + "precision_at_5": 0.06376, + "precision_at_10": 0.04667, + "precision_at_20": 0.03039, + "precision_at_100": 0.00907, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.053869, + "mrr_at_3": 0.112227, + "mrr_at_5": 0.140728, + "mrr_at_10": 0.161015, + "mrr_at_20": 0.170778, + "mrr_at_100": 0.178196, + "mrr_at_1000": 0.178724, + "nauc_ndcg_at_1_max": -0.267832, + "nauc_ndcg_at_1_std": -0.282771, + "nauc_ndcg_at_1_diff1": 0.216608, + "nauc_ndcg_at_3_max": -0.166694, + "nauc_ndcg_at_3_std": -0.32252, + "nauc_ndcg_at_3_diff1": 0.320261, + "nauc_ndcg_at_5_max": -0.108309, + "nauc_ndcg_at_5_std": -0.388785, + "nauc_ndcg_at_5_diff1": 0.346691, + "nauc_ndcg_at_10_max": -0.091575, + "nauc_ndcg_at_10_std": -0.441979, + "nauc_ndcg_at_10_diff1": 0.331371, + "nauc_ndcg_at_20_max": -0.090998, + "nauc_ndcg_at_20_std": -0.461185, + "nauc_ndcg_at_20_diff1": 0.301869, + "nauc_ndcg_at_100_max": -0.101284, + "nauc_ndcg_at_100_std": -0.412443, + "nauc_ndcg_at_100_diff1": 0.292422, + "nauc_ndcg_at_1000_max": -0.112577, + "nauc_ndcg_at_1000_std": -0.407534, + "nauc_ndcg_at_1000_diff1": 0.304923, + "nauc_map_at_1_max": -0.267832, + "nauc_map_at_1_std": -0.282771, + "nauc_map_at_1_diff1": 0.216608, + "nauc_map_at_3_max": -0.181099, + "nauc_map_at_3_std": -0.314793, + "nauc_map_at_3_diff1": 0.30578, + "nauc_map_at_5_max": -0.139887, + "nauc_map_at_5_std": -0.359848, + "nauc_map_at_5_diff1": 0.325211, + "nauc_map_at_10_max": -0.131607, + "nauc_map_at_10_std": -0.387083, + "nauc_map_at_10_diff1": 0.318689, + "nauc_map_at_20_max": -0.131982, + "nauc_map_at_20_std": -0.392816, + "nauc_map_at_20_diff1": 0.308791, + "nauc_map_at_100_max": -0.133426, + "nauc_map_at_100_std": -0.386055, + "nauc_map_at_100_diff1": 0.307082, + "nauc_map_at_1000_max": -0.13397, + "nauc_map_at_1000_std": -0.385508, + "nauc_map_at_1000_diff1": 0.307624, + "nauc_recall_at_1_max": -0.267832, + "nauc_recall_at_1_std": -0.282771, + "nauc_recall_at_1_diff1": 0.216608, + "nauc_recall_at_3_max": -0.140911, + "nauc_recall_at_3_std": -0.336822, + "nauc_recall_at_3_diff1": 0.346115, + "nauc_recall_at_5_max": -0.054508, + "nauc_recall_at_5_std": -0.439974, + "nauc_recall_at_5_diff1": 0.382282, + "nauc_recall_at_10_max": -0.020815, + "nauc_recall_at_10_std": -0.545352, + "nauc_recall_at_10_diff1": 0.347921, + "nauc_recall_at_20_max": -0.009375, + "nauc_recall_at_20_std": -0.612227, + "nauc_recall_at_20_diff1": 0.268239, + "nauc_recall_at_100_max": 0.043327, + "nauc_recall_at_100_std": -0.403774, + "nauc_recall_at_100_diff1": 0.106132, + "nauc_recall_at_1000_max": -0.29463, + "nauc_recall_at_1000_std": 0.367823, + "nauc_recall_at_1000_diff1": 0.103883, + "nauc_precision_at_1_max": -0.267832, + "nauc_precision_at_1_std": -0.282771, + "nauc_precision_at_1_diff1": 0.216608, + "nauc_precision_at_3_max": -0.140911, + "nauc_precision_at_3_std": -0.336822, + "nauc_precision_at_3_diff1": 0.346115, + "nauc_precision_at_5_max": -0.054508, + "nauc_precision_at_5_std": -0.439974, + "nauc_precision_at_5_diff1": 0.382282, + "nauc_precision_at_10_max": -0.020815, + "nauc_precision_at_10_std": -0.545352, + "nauc_precision_at_10_diff1": 0.347921, + "nauc_precision_at_20_max": -0.009375, + "nauc_precision_at_20_std": -0.612227, + "nauc_precision_at_20_diff1": 0.268239, + "nauc_precision_at_100_max": 0.043327, + "nauc_precision_at_100_std": -0.403774, + "nauc_precision_at_100_diff1": 0.106132, + "nauc_precision_at_1000_max": -0.29463, + "nauc_precision_at_1000_std": 0.367823, + "nauc_precision_at_1000_diff1": 0.103883, + "nauc_mrr_at_1_max": -0.376519, + "nauc_mrr_at_1_std": -0.238818, + "nauc_mrr_at_1_diff1": 0.1929, + "nauc_mrr_at_3_max": -0.223212, + "nauc_mrr_at_3_std": -0.300541, + "nauc_mrr_at_3_diff1": 0.275508, + "nauc_mrr_at_5_max": -0.179891, + "nauc_mrr_at_5_std": -0.348489, + "nauc_mrr_at_5_diff1": 0.299621, + "nauc_mrr_at_10_max": -0.172379, + "nauc_mrr_at_10_std": -0.373198, + "nauc_mrr_at_10_diff1": 0.294396, + "nauc_mrr_at_20_max": -0.172791, + "nauc_mrr_at_20_std": -0.379014, + "nauc_mrr_at_20_diff1": 0.284643, + "nauc_mrr_at_100_max": -0.17448, + "nauc_mrr_at_100_std": -0.372203, + "nauc_mrr_at_100_diff1": 0.283049, + "nauc_mrr_at_1000_max": -0.17512, + "nauc_mrr_at_1000_std": -0.371664, + "nauc_mrr_at_1000_diff1": 0.283546, + "main_score": 0.23155, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.91192102432251, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FeedbackQARetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FeedbackQARetrieval.json new file mode 100644 index 0000000000..b187025b11 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FeedbackQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "1ee1cd0", + "task_name": "FeedbackQARetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.56024, + "ndcg_at_3": 0.65665, + "ndcg_at_5": 0.68354, + "ndcg_at_10": 0.70628, + "ndcg_at_20": 0.71619, + "ndcg_at_100": 0.72917, + "ndcg_at_1000": 0.73285, + "map_at_1": 0.56024, + "map_at_3": 0.6332, + "map_at_5": 0.64813, + "map_at_10": 0.65762, + "map_at_20": 0.66041, + "map_at_100": 0.66222, + "map_at_1000": 0.66236, + "recall_at_1": 0.56024, + "recall_at_3": 0.7244, + "recall_at_5": 0.78966, + "recall_at_10": 0.85944, + "recall_at_20": 0.89809, + "recall_at_100": 0.96787, + "recall_at_1000": 0.99649, + "precision_at_1": 0.56024, + "precision_at_3": 0.24147, + "precision_at_5": 0.15793, + "precision_at_10": 0.08594, + "precision_at_20": 0.0449, + "precision_at_100": 0.00968, + "precision_at_1000": 0.001, + "mrr_at_1": 0.560241, + "mrr_at_3": 0.633199, + "mrr_at_5": 0.648134, + "mrr_at_10": 0.657621, + "mrr_at_20": 0.660409, + "mrr_at_100": 0.662221, + "mrr_at_1000": 0.662364, + "nauc_ndcg_at_1_max": 0.335234, + "nauc_ndcg_at_1_std": -0.238399, + "nauc_ndcg_at_1_diff1": 0.671201, + "nauc_ndcg_at_3_max": 0.326231, + "nauc_ndcg_at_3_std": -0.245644, + "nauc_ndcg_at_3_diff1": 0.609377, + "nauc_ndcg_at_5_max": 0.314177, + "nauc_ndcg_at_5_std": -0.239025, + "nauc_ndcg_at_5_diff1": 0.597164, + "nauc_ndcg_at_10_max": 0.309055, + "nauc_ndcg_at_10_std": -0.240131, + "nauc_ndcg_at_10_diff1": 0.598418, + "nauc_ndcg_at_20_max": 0.300657, + "nauc_ndcg_at_20_std": -0.237318, + "nauc_ndcg_at_20_diff1": 0.596594, + "nauc_ndcg_at_100_max": 0.311794, + "nauc_ndcg_at_100_std": -0.229829, + "nauc_ndcg_at_100_diff1": 0.606298, + "nauc_ndcg_at_1000_max": 0.314357, + "nauc_ndcg_at_1000_std": -0.235285, + "nauc_ndcg_at_1000_diff1": 0.611095, + "nauc_map_at_1_max": 0.335234, + "nauc_map_at_1_std": -0.238399, + "nauc_map_at_1_diff1": 0.671201, + "nauc_map_at_3_max": 0.328807, + "nauc_map_at_3_std": -0.244543, + "nauc_map_at_3_diff1": 0.625639, + "nauc_map_at_5_max": 0.322736, + "nauc_map_at_5_std": -0.241093, + "nauc_map_at_5_diff1": 0.620043, + "nauc_map_at_10_max": 0.321027, + "nauc_map_at_10_std": -0.241471, + "nauc_map_at_10_diff1": 0.621041, + "nauc_map_at_20_max": 0.319158, + "nauc_map_at_20_std": -0.241055, + "nauc_map_at_20_diff1": 0.620857, + "nauc_map_at_100_max": 0.320224, + "nauc_map_at_100_std": -0.240268, + "nauc_map_at_100_diff1": 0.622176, + "nauc_map_at_1000_max": 0.320315, + "nauc_map_at_1000_std": -0.240368, + "nauc_map_at_1000_diff1": 0.622358, + "nauc_recall_at_1_max": 0.335234, + "nauc_recall_at_1_std": -0.238399, + "nauc_recall_at_1_diff1": 0.671201, + "nauc_recall_at_3_max": 0.317216, + "nauc_recall_at_3_std": -0.249038, + "nauc_recall_at_3_diff1": 0.553141, + "nauc_recall_at_5_max": 0.275998, + "nauc_recall_at_5_std": -0.22854, + "nauc_recall_at_5_diff1": 0.499782, + "nauc_recall_at_10_max": 0.232171, + "nauc_recall_at_10_std": -0.230881, + "nauc_recall_at_10_diff1": 0.465503, + "nauc_recall_at_20_max": 0.131364, + "nauc_recall_at_20_std": -0.197594, + "nauc_recall_at_20_diff1": 0.40438, + "nauc_recall_at_100_max": 0.205812, + "nauc_recall_at_100_std": 0.162593, + "nauc_recall_at_100_diff1": 0.33559, + "nauc_recall_at_1000_max": 0.517622, + "nauc_recall_at_1000_std": 0.477295, + "nauc_recall_at_1000_diff1": 0.314509, + "nauc_precision_at_1_max": 0.335234, + "nauc_precision_at_1_std": -0.238399, + "nauc_precision_at_1_diff1": 0.671201, + "nauc_precision_at_3_max": 0.317216, + "nauc_precision_at_3_std": -0.249038, + "nauc_precision_at_3_diff1": 0.553141, + "nauc_precision_at_5_max": 0.275998, + "nauc_precision_at_5_std": -0.22854, + "nauc_precision_at_5_diff1": 0.499782, + "nauc_precision_at_10_max": 0.232171, + "nauc_precision_at_10_std": -0.230881, + "nauc_precision_at_10_diff1": 0.465503, + "nauc_precision_at_20_max": 0.131364, + "nauc_precision_at_20_std": -0.197594, + "nauc_precision_at_20_diff1": 0.40438, + "nauc_precision_at_100_max": 0.205812, + "nauc_precision_at_100_std": 0.162593, + "nauc_precision_at_100_diff1": 0.33559, + "nauc_precision_at_1000_max": 0.517622, + "nauc_precision_at_1000_std": 0.477295, + "nauc_precision_at_1000_diff1": 0.314509, + "nauc_mrr_at_1_max": 0.335234, + "nauc_mrr_at_1_std": -0.238399, + "nauc_mrr_at_1_diff1": 0.671201, + "nauc_mrr_at_3_max": 0.328807, + "nauc_mrr_at_3_std": -0.244543, + "nauc_mrr_at_3_diff1": 0.625639, + "nauc_mrr_at_5_max": 0.322736, + "nauc_mrr_at_5_std": -0.241093, + "nauc_mrr_at_5_diff1": 0.620043, + "nauc_mrr_at_10_max": 0.321027, + "nauc_mrr_at_10_std": -0.241471, + "nauc_mrr_at_10_diff1": 0.621041, + "nauc_mrr_at_20_max": 0.319158, + "nauc_mrr_at_20_std": -0.241055, + "nauc_mrr_at_20_diff1": 0.620857, + "nauc_mrr_at_100_max": 0.320224, + "nauc_mrr_at_100_std": -0.240268, + "nauc_mrr_at_100_diff1": 0.622176, + "nauc_mrr_at_1000_max": 0.320315, + "nauc_mrr_at_1000_std": -0.240368, + "nauc_mrr_at_1000_diff1": 0.622358, + "main_score": 0.56024, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 13.08293890953064, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FiQA2018.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FiQA2018.json new file mode 100644 index 0000000000..26addd8d0c --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/FiQA2018.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "27a168819829fe9bcd655c2df245fb19452e8e06", + "task_name": "FiQA2018", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.40586, + "ndcg_at_3": 0.37272, + "ndcg_at_5": 0.38346, + "ndcg_at_10": 0.40893, + "ndcg_at_20": 0.43943, + "ndcg_at_100": 0.47961, + "ndcg_at_1000": 0.50973, + "map_at_1": 0.20881, + "map_at_3": 0.29031, + "map_at_5": 0.31333, + "map_at_10": 0.33194, + "map_at_20": 0.34356, + "map_at_100": 0.35173, + "map_at_1000": 0.35353, + "recall_at_1": 0.20881, + "recall_at_3": 0.33708, + "recall_at_5": 0.39466, + "recall_at_10": 0.47476, + "recall_at_20": 0.5676, + "recall_at_100": 0.73398, + "recall_at_1000": 0.91295, + "precision_at_1": 0.40586, + "precision_at_3": 0.24743, + "precision_at_5": 0.18056, + "precision_at_10": 0.11111, + "precision_at_20": 0.06829, + "precision_at_100": 0.0184, + "precision_at_1000": 0.00237, + "mrr_at_1": 0.405864, + "mrr_at_3": 0.468879, + "mrr_at_5": 0.480761, + "mrr_at_10": 0.49144, + "mrr_at_20": 0.497661, + "mrr_at_100": 0.500439, + "mrr_at_1000": 0.500865, + "nauc_ndcg_at_1_max": 0.351357, + "nauc_ndcg_at_1_std": -0.025922, + "nauc_ndcg_at_1_diff1": 0.490916, + "nauc_ndcg_at_3_max": 0.331552, + "nauc_ndcg_at_3_std": -0.017491, + "nauc_ndcg_at_3_diff1": 0.415709, + "nauc_ndcg_at_5_max": 0.326792, + "nauc_ndcg_at_5_std": -0.03447, + "nauc_ndcg_at_5_diff1": 0.399461, + "nauc_ndcg_at_10_max": 0.33736, + "nauc_ndcg_at_10_std": -0.026487, + "nauc_ndcg_at_10_diff1": 0.401288, + "nauc_ndcg_at_20_max": 0.344374, + "nauc_ndcg_at_20_std": -0.008536, + "nauc_ndcg_at_20_diff1": 0.401838, + "nauc_ndcg_at_100_max": 0.364012, + "nauc_ndcg_at_100_std": 0.008054, + "nauc_ndcg_at_100_diff1": 0.40594, + "nauc_ndcg_at_1000_max": 0.355989, + "nauc_ndcg_at_1000_std": 0.014361, + "nauc_ndcg_at_1000_diff1": 0.407187, + "nauc_map_at_1_max": 0.272533, + "nauc_map_at_1_std": -0.058526, + "nauc_map_at_1_diff1": 0.479548, + "nauc_map_at_3_max": 0.301679, + "nauc_map_at_3_std": -0.044016, + "nauc_map_at_3_diff1": 0.433021, + "nauc_map_at_5_max": 0.312336, + "nauc_map_at_5_std": -0.048487, + "nauc_map_at_5_diff1": 0.414908, + "nauc_map_at_10_max": 0.321055, + "nauc_map_at_10_std": -0.037161, + "nauc_map_at_10_diff1": 0.410874, + "nauc_map_at_20_max": 0.324619, + "nauc_map_at_20_std": -0.029951, + "nauc_map_at_20_diff1": 0.410465, + "nauc_map_at_100_max": 0.328655, + "nauc_map_at_100_std": -0.026669, + "nauc_map_at_100_diff1": 0.410358, + "nauc_map_at_1000_max": 0.328472, + "nauc_map_at_1000_std": -0.025898, + "nauc_map_at_1000_diff1": 0.41029, + "nauc_recall_at_1_max": 0.272533, + "nauc_recall_at_1_std": -0.058526, + "nauc_recall_at_1_diff1": 0.479548, + "nauc_recall_at_3_max": 0.276014, + "nauc_recall_at_3_std": -0.032472, + "nauc_recall_at_3_diff1": 0.372384, + "nauc_recall_at_5_max": 0.271399, + "nauc_recall_at_5_std": -0.049944, + "nauc_recall_at_5_diff1": 0.314294, + "nauc_recall_at_10_max": 0.286244, + "nauc_recall_at_10_std": -0.027705, + "nauc_recall_at_10_diff1": 0.291067, + "nauc_recall_at_20_max": 0.296989, + "nauc_recall_at_20_std": 0.027666, + "nauc_recall_at_20_diff1": 0.277234, + "nauc_recall_at_100_max": 0.378216, + "nauc_recall_at_100_std": 0.118303, + "nauc_recall_at_100_diff1": 0.272322, + "nauc_recall_at_1000_max": 0.36182, + "nauc_recall_at_1000_std": 0.346626, + "nauc_recall_at_1000_diff1": 0.226243, + "nauc_precision_at_1_max": 0.351357, + "nauc_precision_at_1_std": -0.025922, + "nauc_precision_at_1_diff1": 0.490916, + "nauc_precision_at_3_max": 0.32306, + "nauc_precision_at_3_std": 0.024608, + "nauc_precision_at_3_diff1": 0.277526, + "nauc_precision_at_5_max": 0.305127, + "nauc_precision_at_5_std": 0.02901, + "nauc_precision_at_5_diff1": 0.195603, + "nauc_precision_at_10_max": 0.285742, + "nauc_precision_at_10_std": 0.080636, + "nauc_precision_at_10_diff1": 0.134956, + "nauc_precision_at_20_max": 0.257816, + "nauc_precision_at_20_std": 0.124348, + "nauc_precision_at_20_diff1": 0.088863, + "nauc_precision_at_100_max": 0.211098, + "nauc_precision_at_100_std": 0.165323, + "nauc_precision_at_100_diff1": 0.008604, + "nauc_precision_at_1000_max": 0.096343, + "nauc_precision_at_1000_std": 0.18427, + "nauc_precision_at_1000_diff1": -0.066453, + "nauc_mrr_at_1_max": 0.351357, + "nauc_mrr_at_1_std": -0.025922, + "nauc_mrr_at_1_diff1": 0.490916, + "nauc_mrr_at_3_max": 0.353951, + "nauc_mrr_at_3_std": -0.018393, + "nauc_mrr_at_3_diff1": 0.459892, + "nauc_mrr_at_5_max": 0.351941, + "nauc_mrr_at_5_std": -0.017627, + "nauc_mrr_at_5_diff1": 0.448641, + "nauc_mrr_at_10_max": 0.35536, + "nauc_mrr_at_10_std": -0.016669, + "nauc_mrr_at_10_diff1": 0.449485, + "nauc_mrr_at_20_max": 0.356931, + "nauc_mrr_at_20_std": -0.012029, + "nauc_mrr_at_20_diff1": 0.449632, + "nauc_mrr_at_100_max": 0.357248, + "nauc_mrr_at_100_std": -0.012184, + "nauc_mrr_at_100_diff1": 0.449835, + "nauc_mrr_at_1000_max": 0.357, + "nauc_mrr_at_1000_std": -0.012445, + "nauc_mrr_at_1000_diff1": 0.449971, + "main_score": 0.40893, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 196.07935404777527, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/HagridRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/HagridRetrieval.json new file mode 100644 index 0000000000..a9ed383fa1 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/HagridRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b2a085913606be3c4f2f1a8bff1810e38bade8fa", + "task_name": "HagridRetrieval", + "mteb_version": "1.38.9", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.98387, + "ndcg_at_3": 0.98896, + "ndcg_at_5": 0.98896, + "ndcg_at_10": 0.98896, + "ndcg_at_20": 0.98896, + "ndcg_at_100": 0.98995, + "ndcg_at_1000": 0.99024, + "map_at_1": 0.98387, + "map_at_3": 0.9879, + "map_at_5": 0.9879, + "map_at_10": 0.9879, + "map_at_20": 0.9879, + "map_at_100": 0.98799, + "map_at_1000": 0.98801, + "recall_at_1": 0.98387, + "recall_at_3": 0.99194, + "recall_at_5": 0.99194, + "recall_at_10": 0.99194, + "recall_at_20": 0.99194, + "recall_at_100": 0.99798, + "recall_at_1000": 1.0, + "precision_at_1": 0.98387, + "precision_at_3": 0.33065, + "precision_at_5": 0.19839, + "precision_at_10": 0.09919, + "precision_at_20": 0.0496, + "precision_at_100": 0.00998, + "precision_at_1000": 0.001, + "mrr_at_1": 0.983871, + "mrr_at_3": 0.987903, + "mrr_at_5": 0.987903, + "mrr_at_10": 0.987903, + "mrr_at_20": 0.987903, + "mrr_at_100": 0.987993, + "mrr_at_1000": 0.988009, + "nauc_ndcg_at_1_max": 0.569494, + "nauc_ndcg_at_1_std": 0.040421, + "nauc_ndcg_at_1_diff1": 0.983645, + "nauc_ndcg_at_3_max": 0.716508, + "nauc_ndcg_at_3_std": 0.170588, + "nauc_ndcg_at_3_diff1": 0.976109, + "nauc_ndcg_at_5_max": 0.716508, + "nauc_ndcg_at_5_std": 0.170588, + "nauc_ndcg_at_5_diff1": 0.976109, + "nauc_ndcg_at_10_max": 0.716508, + "nauc_ndcg_at_10_std": 0.170588, + "nauc_ndcg_at_10_diff1": 0.976109, + "nauc_ndcg_at_20_max": 0.716508, + "nauc_ndcg_at_20_std": 0.170588, + "nauc_ndcg_at_20_diff1": 0.976109, + "nauc_ndcg_at_100_max": 0.702823, + "nauc_ndcg_at_100_std": 0.164887, + "nauc_ndcg_at_100_diff1": 0.977922, + "nauc_ndcg_at_1000_max": 0.694013, + "nauc_ndcg_at_1000_std": 0.148366, + "nauc_ndcg_at_1000_diff1": 0.977268, + "nauc_map_at_1_max": 0.569494, + "nauc_map_at_1_std": 0.040421, + "nauc_map_at_1_diff1": 0.983645, + "nauc_map_at_3_max": 0.675831, + "nauc_map_at_3_std": 0.134572, + "nauc_map_at_3_diff1": 0.978194, + "nauc_map_at_5_max": 0.675831, + "nauc_map_at_5_std": 0.134572, + "nauc_map_at_5_diff1": 0.978194, + "nauc_map_at_10_max": 0.675831, + "nauc_map_at_10_std": 0.134572, + "nauc_map_at_10_diff1": 0.978194, + "nauc_map_at_20_max": 0.675831, + "nauc_map_at_20_std": 0.134572, + "nauc_map_at_20_diff1": 0.978194, + "nauc_map_at_100_max": 0.674349, + "nauc_map_at_100_std": 0.132934, + "nauc_map_at_100_diff1": 0.978311, + "nauc_map_at_1000_max": 0.673908, + "nauc_map_at_1000_std": 0.132135, + "nauc_map_at_1000_diff1": 0.978282, + "nauc_recall_at_1_max": 0.569494, + "nauc_recall_at_1_std": 0.040421, + "nauc_recall_at_1_diff1": 0.983645, + "nauc_recall_at_3_max": 0.888504, + "nauc_recall_at_3_std": 0.322874, + "nauc_recall_at_3_diff1": 0.967291, + "nauc_recall_at_5_max": 0.888504, + "nauc_recall_at_5_std": 0.322874, + "nauc_recall_at_5_diff1": 0.967291, + "nauc_recall_at_10_max": 0.888504, + "nauc_recall_at_10_std": 0.322874, + "nauc_recall_at_10_diff1": 0.967291, + "nauc_recall_at_20_max": 0.888504, + "nauc_recall_at_20_std": 0.322874, + "nauc_recall_at_20_diff1": 0.967291, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.722179, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.569494, + "nauc_precision_at_1_std": 0.040421, + "nauc_precision_at_1_diff1": 0.983645, + "nauc_precision_at_3_max": 0.888504, + "nauc_precision_at_3_std": 0.322874, + "nauc_precision_at_3_diff1": 0.967291, + "nauc_precision_at_5_max": 0.888504, + "nauc_precision_at_5_std": 0.322874, + "nauc_precision_at_5_diff1": 0.967291, + "nauc_precision_at_10_max": 0.888504, + "nauc_precision_at_10_std": 0.322874, + "nauc_precision_at_10_diff1": 0.967291, + "nauc_precision_at_20_max": 0.888504, + "nauc_precision_at_20_std": 0.322874, + "nauc_precision_at_20_diff1": 0.967291, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.722179, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.569494, + "nauc_mrr_at_1_std": 0.040421, + "nauc_mrr_at_1_diff1": 0.983645, + "nauc_mrr_at_3_max": 0.675831, + "nauc_mrr_at_3_std": 0.134572, + "nauc_mrr_at_3_diff1": 0.978194, + "nauc_mrr_at_5_max": 0.675831, + "nauc_mrr_at_5_std": 0.134572, + "nauc_mrr_at_5_diff1": 0.978194, + "nauc_mrr_at_10_max": 0.675831, + "nauc_mrr_at_10_std": 0.134572, + "nauc_mrr_at_10_diff1": 0.978194, + "nauc_mrr_at_20_max": 0.675831, + "nauc_mrr_at_20_std": 0.134572, + "nauc_mrr_at_20_diff1": 0.978194, + "nauc_mrr_at_100_max": 0.674389, + "nauc_mrr_at_100_std": 0.132905, + "nauc_mrr_at_100_diff1": 0.978317, + "nauc_mrr_at_1000_max": 0.673944, + "nauc_mrr_at_1000_std": 0.132099, + "nauc_mrr_at_1000_diff1": 0.978287, + "main_score": 0.98896, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.7093515396118164, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/HellaSwag.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/HellaSwag.json new file mode 100644 index 0000000000..8d40ec2317 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/HellaSwag.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a5c990205e017d10761197ccab3000936689c3ae", + "task_name": "HellaSwag", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.15794, + "ndcg_at_3": 0.24682, + "ndcg_at_5": 0.27275, + "ndcg_at_10": 0.29545, + "ndcg_at_20": 0.31261, + "ndcg_at_100": 0.33991, + "ndcg_at_1000": 0.36287, + "map_at_1": 0.15794, + "map_at_3": 0.22466, + "map_at_5": 0.23907, + "map_at_10": 0.24844, + "map_at_20": 0.25316, + "map_at_100": 0.25684, + "map_at_1000": 0.25761, + "recall_at_1": 0.15794, + "recall_at_3": 0.31109, + "recall_at_5": 0.37393, + "recall_at_10": 0.44413, + "recall_at_20": 0.51185, + "recall_at_100": 0.66053, + "recall_at_1000": 0.84635, + "precision_at_1": 0.15794, + "precision_at_3": 0.1037, + "precision_at_5": 0.07479, + "precision_at_10": 0.04441, + "precision_at_20": 0.02559, + "precision_at_100": 0.00661, + "precision_at_1000": 0.00085, + "mrr_at_1": 0.157937, + "mrr_at_3": 0.224656, + "mrr_at_5": 0.239071, + "mrr_at_10": 0.248439, + "mrr_at_20": 0.253163, + "mrr_at_100": 0.256836, + "mrr_at_1000": 0.257612, + "nauc_ndcg_at_1_max": 0.159827, + "nauc_ndcg_at_1_std": -0.107612, + "nauc_ndcg_at_1_diff1": 0.230598, + "nauc_ndcg_at_3_max": 0.190846, + "nauc_ndcg_at_3_std": -0.108283, + "nauc_ndcg_at_3_diff1": 0.197483, + "nauc_ndcg_at_5_max": 0.197298, + "nauc_ndcg_at_5_std": -0.105942, + "nauc_ndcg_at_5_diff1": 0.193836, + "nauc_ndcg_at_10_max": 0.197445, + "nauc_ndcg_at_10_std": -0.099824, + "nauc_ndcg_at_10_diff1": 0.189419, + "nauc_ndcg_at_20_max": 0.200135, + "nauc_ndcg_at_20_std": -0.098363, + "nauc_ndcg_at_20_diff1": 0.191074, + "nauc_ndcg_at_100_max": 0.199762, + "nauc_ndcg_at_100_std": -0.088619, + "nauc_ndcg_at_100_diff1": 0.189458, + "nauc_ndcg_at_1000_max": 0.202907, + "nauc_ndcg_at_1000_std": -0.087476, + "nauc_ndcg_at_1000_diff1": 0.191537, + "nauc_map_at_1_max": 0.159827, + "nauc_map_at_1_std": -0.107612, + "nauc_map_at_1_diff1": 0.230598, + "nauc_map_at_3_max": 0.183684, + "nauc_map_at_3_std": -0.108319, + "nauc_map_at_3_diff1": 0.203961, + "nauc_map_at_5_max": 0.18774, + "nauc_map_at_5_std": -0.107274, + "nauc_map_at_5_diff1": 0.201836, + "nauc_map_at_10_max": 0.187743, + "nauc_map_at_10_std": -0.104606, + "nauc_map_at_10_diff1": 0.199866, + "nauc_map_at_20_max": 0.188342, + "nauc_map_at_20_std": -0.104265, + "nauc_map_at_20_diff1": 0.200332, + "nauc_map_at_100_max": 0.188281, + "nauc_map_at_100_std": -0.103046, + "nauc_map_at_100_diff1": 0.2002, + "nauc_map_at_1000_max": 0.188365, + "nauc_map_at_1000_std": -0.102954, + "nauc_map_at_1000_diff1": 0.200287, + "nauc_recall_at_1_max": 0.159827, + "nauc_recall_at_1_std": -0.107612, + "nauc_recall_at_1_diff1": 0.230598, + "nauc_recall_at_3_max": 0.208374, + "nauc_recall_at_3_std": -0.108139, + "nauc_recall_at_3_diff1": 0.181979, + "nauc_recall_at_5_max": 0.220652, + "nauc_recall_at_5_std": -0.102128, + "nauc_recall_at_5_diff1": 0.174572, + "nauc_recall_at_10_max": 0.221376, + "nauc_recall_at_10_std": -0.085757, + "nauc_recall_at_10_diff1": 0.162878, + "nauc_recall_at_20_max": 0.232497, + "nauc_recall_at_20_std": -0.079805, + "nauc_recall_at_20_diff1": 0.168047, + "nauc_recall_at_100_max": 0.237069, + "nauc_recall_at_100_std": -0.019111, + "nauc_recall_at_100_diff1": 0.152791, + "nauc_recall_at_1000_max": 0.318723, + "nauc_recall_at_1000_std": 0.051101, + "nauc_recall_at_1000_diff1": 0.15184, + "nauc_precision_at_1_max": 0.159827, + "nauc_precision_at_1_std": -0.107612, + "nauc_precision_at_1_diff1": 0.230598, + "nauc_precision_at_3_max": 0.208374, + "nauc_precision_at_3_std": -0.108139, + "nauc_precision_at_3_diff1": 0.181979, + "nauc_precision_at_5_max": 0.220652, + "nauc_precision_at_5_std": -0.102128, + "nauc_precision_at_5_diff1": 0.174572, + "nauc_precision_at_10_max": 0.221376, + "nauc_precision_at_10_std": -0.085757, + "nauc_precision_at_10_diff1": 0.162878, + "nauc_precision_at_20_max": 0.232497, + "nauc_precision_at_20_std": -0.079805, + "nauc_precision_at_20_diff1": 0.168047, + "nauc_precision_at_100_max": 0.237069, + "nauc_precision_at_100_std": -0.019111, + "nauc_precision_at_100_diff1": 0.152791, + "nauc_precision_at_1000_max": 0.318723, + "nauc_precision_at_1000_std": 0.051101, + "nauc_precision_at_1000_diff1": 0.15184, + "nauc_mrr_at_1_max": 0.159827, + "nauc_mrr_at_1_std": -0.107612, + "nauc_mrr_at_1_diff1": 0.230598, + "nauc_mrr_at_3_max": 0.183684, + "nauc_mrr_at_3_std": -0.108319, + "nauc_mrr_at_3_diff1": 0.203961, + "nauc_mrr_at_5_max": 0.18774, + "nauc_mrr_at_5_std": -0.107274, + "nauc_mrr_at_5_diff1": 0.201836, + "nauc_mrr_at_10_max": 0.187737, + "nauc_mrr_at_10_std": -0.104602, + "nauc_mrr_at_10_diff1": 0.199863, + "nauc_mrr_at_20_max": 0.188336, + "nauc_mrr_at_20_std": -0.104261, + "nauc_mrr_at_20_diff1": 0.200329, + "nauc_mrr_at_100_max": 0.188276, + "nauc_mrr_at_100_std": -0.103042, + "nauc_mrr_at_100_diff1": 0.200197, + "nauc_mrr_at_1000_max": 0.18836, + "nauc_mrr_at_1000_std": -0.102951, + "nauc_mrr_at_1000_diff1": 0.200284, + "main_score": 0.29545, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 187.33644032478333, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/HotpotQA.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/HotpotQA.json new file mode 100644 index 0000000000..a3bd2d64fd --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/HotpotQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ab518f4d6fcca38d87c25209f94beba119d02014", + "task_name": "HotpotQA", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.84564, + "ndcg_at_3": 0.69419, + "ndcg_at_5": 0.71792, + "ndcg_at_10": 0.73764, + "ndcg_at_20": 0.74932, + "ndcg_at_100": 0.76543, + "ndcg_at_1000": 0.776, + "map_at_1": 0.42282, + "map_at_3": 0.62377, + "map_at_5": 0.6433, + "map_at_10": 0.65552, + "map_at_20": 0.66039, + "map_at_100": 0.66377, + "map_at_1000": 0.66433, + "recall_at_1": 0.42282, + "recall_at_3": 0.66158, + "recall_at_5": 0.70837, + "recall_at_10": 0.75773, + "recall_at_20": 0.79534, + "recall_at_100": 0.86644, + "recall_at_1000": 0.93646, + "precision_at_1": 0.84564, + "precision_at_3": 0.44105, + "precision_at_5": 0.28335, + "precision_at_10": 0.15155, + "precision_at_20": 0.07953, + "precision_at_100": 0.01733, + "precision_at_1000": 0.00187, + "mrr_at_1": 0.845645, + "mrr_at_3": 0.883727, + "mrr_at_5": 0.887779, + "mrr_at_10": 0.89016, + "mrr_at_20": 0.890916, + "mrr_at_100": 0.891344, + "mrr_at_1000": 0.89139, + "nauc_ndcg_at_1_max": 0.651961, + "nauc_ndcg_at_1_std": -0.257873, + "nauc_ndcg_at_1_diff1": 0.722424, + "nauc_ndcg_at_3_max": 0.376772, + "nauc_ndcg_at_3_std": -0.147174, + "nauc_ndcg_at_3_diff1": 0.109771, + "nauc_ndcg_at_5_max": 0.370736, + "nauc_ndcg_at_5_std": -0.148664, + "nauc_ndcg_at_5_diff1": 0.086884, + "nauc_ndcg_at_10_max": 0.370373, + "nauc_ndcg_at_10_std": -0.131831, + "nauc_ndcg_at_10_diff1": 0.07671, + "nauc_ndcg_at_20_max": 0.370048, + "nauc_ndcg_at_20_std": -0.120867, + "nauc_ndcg_at_20_diff1": 0.072974, + "nauc_ndcg_at_100_max": 0.376366, + "nauc_ndcg_at_100_std": -0.108568, + "nauc_ndcg_at_100_diff1": 0.084012, + "nauc_ndcg_at_1000_max": 0.38235, + "nauc_ndcg_at_1000_std": -0.113507, + "nauc_ndcg_at_1000_diff1": 0.097318, + "nauc_map_at_1_max": 0.651961, + "nauc_map_at_1_std": -0.257873, + "nauc_map_at_1_diff1": 0.722424, + "nauc_map_at_3_max": 0.331136, + "nauc_map_at_3_std": -0.132656, + "nauc_map_at_3_diff1": 0.048343, + "nauc_map_at_5_max": 0.329289, + "nauc_map_at_5_std": -0.134537, + "nauc_map_at_5_diff1": 0.034249, + "nauc_map_at_10_max": 0.330618, + "nauc_map_at_10_std": -0.126183, + "nauc_map_at_10_diff1": 0.030558, + "nauc_map_at_20_max": 0.330797, + "nauc_map_at_20_std": -0.122579, + "nauc_map_at_20_diff1": 0.029783, + "nauc_map_at_100_max": 0.331772, + "nauc_map_at_100_std": -0.120317, + "nauc_map_at_100_diff1": 0.031525, + "nauc_map_at_1000_max": 0.331998, + "nauc_map_at_1000_std": -0.120431, + "nauc_map_at_1000_diff1": 0.032011, + "nauc_recall_at_1_max": 0.651961, + "nauc_recall_at_1_std": -0.257873, + "nauc_recall_at_1_diff1": 0.722424, + "nauc_recall_at_3_max": 0.317571, + "nauc_recall_at_3_std": -0.118171, + "nauc_recall_at_3_diff1": -0.02209, + "nauc_recall_at_5_max": 0.296117, + "nauc_recall_at_5_std": -0.116282, + "nauc_recall_at_5_diff1": -0.074179, + "nauc_recall_at_10_max": 0.282882, + "nauc_recall_at_10_std": -0.06912, + "nauc_recall_at_10_diff1": -0.117162, + "nauc_recall_at_20_max": 0.265549, + "nauc_recall_at_20_std": -0.022381, + "nauc_recall_at_20_diff1": -0.152925, + "nauc_recall_at_100_max": 0.267541, + "nauc_recall_at_100_std": 0.084329, + "nauc_recall_at_100_diff1": -0.169151, + "nauc_recall_at_1000_max": 0.285308, + "nauc_recall_at_1000_std": 0.162051, + "nauc_recall_at_1000_diff1": -0.17553, + "nauc_precision_at_1_max": 0.651961, + "nauc_precision_at_1_std": -0.257873, + "nauc_precision_at_1_diff1": 0.722424, + "nauc_precision_at_3_max": 0.317571, + "nauc_precision_at_3_std": -0.118171, + "nauc_precision_at_3_diff1": -0.02209, + "nauc_precision_at_5_max": 0.296117, + "nauc_precision_at_5_std": -0.116282, + "nauc_precision_at_5_diff1": -0.074179, + "nauc_precision_at_10_max": 0.282882, + "nauc_precision_at_10_std": -0.06912, + "nauc_precision_at_10_diff1": -0.117162, + "nauc_precision_at_20_max": 0.265549, + "nauc_precision_at_20_std": -0.022381, + "nauc_precision_at_20_diff1": -0.152925, + "nauc_precision_at_100_max": 0.267541, + "nauc_precision_at_100_std": 0.084329, + "nauc_precision_at_100_diff1": -0.169151, + "nauc_precision_at_1000_max": 0.285308, + "nauc_precision_at_1000_std": 0.162051, + "nauc_precision_at_1000_diff1": -0.17553, + "nauc_mrr_at_1_max": 0.651961, + "nauc_mrr_at_1_std": -0.257873, + "nauc_mrr_at_1_diff1": 0.722424, + "nauc_mrr_at_3_max": 0.703487, + "nauc_mrr_at_3_std": -0.27519, + "nauc_mrr_at_3_diff1": 0.711684, + "nauc_mrr_at_5_max": 0.701238, + "nauc_mrr_at_5_std": -0.276464, + "nauc_mrr_at_5_diff1": 0.713869, + "nauc_mrr_at_10_max": 0.699311, + "nauc_mrr_at_10_std": -0.274627, + "nauc_mrr_at_10_diff1": 0.714562, + "nauc_mrr_at_20_max": 0.698434, + "nauc_mrr_at_20_std": -0.27414, + "nauc_mrr_at_20_diff1": 0.714511, + "nauc_mrr_at_100_max": 0.697878, + "nauc_mrr_at_100_std": -0.273761, + "nauc_mrr_at_100_diff1": 0.71472, + "nauc_mrr_at_1000_max": 0.697773, + "nauc_mrr_at_1000_std": -0.273763, + "nauc_mrr_at_1000_diff1": 0.714726, + "main_score": 0.73764, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7286.981647729874, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/HotpotQAHardNegatives.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/HotpotQAHardNegatives.json new file mode 100644 index 0000000000..2b6f6cf41b --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/HotpotQAHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "617612fa63afcb60e3b134bed8b7216a99707c37", + "task_name": "HotpotQAHardNegatives", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.836, + "ndcg_at_3": 0.68832, + "ndcg_at_5": 0.71442, + "ndcg_at_10": 0.73921, + "ndcg_at_20": 0.75564, + "ndcg_at_100": 0.77559, + "ndcg_at_1000": 0.78611, + "map_at_1": 0.418, + "map_at_3": 0.61933, + "map_at_5": 0.64053, + "map_at_10": 0.65523, + "map_at_20": 0.66197, + "map_at_100": 0.66637, + "map_at_1000": 0.667, + "recall_at_1": 0.418, + "recall_at_3": 0.6555, + "recall_at_5": 0.707, + "recall_at_10": 0.7695, + "recall_at_20": 0.8225, + "recall_at_100": 0.91, + "recall_at_1000": 0.978, + "precision_at_1": 0.836, + "precision_at_3": 0.437, + "precision_at_5": 0.2828, + "precision_at_10": 0.1539, + "precision_at_20": 0.08225, + "precision_at_100": 0.0182, + "precision_at_1000": 0.00196, + "mrr_at_1": 0.836, + "mrr_at_3": 0.875, + "mrr_at_5": 0.88, + "mrr_at_10": 0.883995, + "mrr_at_20": 0.885244, + "mrr_at_100": 0.885491, + "mrr_at_1000": 0.885508, + "nauc_ndcg_at_1_max": 0.592063, + "nauc_ndcg_at_1_std": -0.321525, + "nauc_ndcg_at_1_diff1": 0.677171, + "nauc_ndcg_at_3_max": 0.347627, + "nauc_ndcg_at_3_std": -0.229505, + "nauc_ndcg_at_3_diff1": 0.099635, + "nauc_ndcg_at_5_max": 0.339282, + "nauc_ndcg_at_5_std": -0.239388, + "nauc_ndcg_at_5_diff1": 0.06643, + "nauc_ndcg_at_10_max": 0.335821, + "nauc_ndcg_at_10_std": -0.234197, + "nauc_ndcg_at_10_diff1": 0.04847, + "nauc_ndcg_at_20_max": 0.333971, + "nauc_ndcg_at_20_std": -0.214295, + "nauc_ndcg_at_20_diff1": 0.044242, + "nauc_ndcg_at_100_max": 0.344058, + "nauc_ndcg_at_100_std": -0.203032, + "nauc_ndcg_at_100_diff1": 0.066455, + "nauc_ndcg_at_1000_max": 0.352014, + "nauc_ndcg_at_1000_std": -0.207183, + "nauc_ndcg_at_1000_diff1": 0.085864, + "nauc_map_at_1_max": 0.592063, + "nauc_map_at_1_std": -0.321525, + "nauc_map_at_1_diff1": 0.677171, + "nauc_map_at_3_max": 0.306044, + "nauc_map_at_3_std": -0.203659, + "nauc_map_at_3_diff1": 0.038721, + "nauc_map_at_5_max": 0.302458, + "nauc_map_at_5_std": -0.21033, + "nauc_map_at_5_diff1": 0.017315, + "nauc_map_at_10_max": 0.303187, + "nauc_map_at_10_std": -0.210064, + "nauc_map_at_10_diff1": 0.010156, + "nauc_map_at_20_max": 0.30263, + "nauc_map_at_20_std": -0.203612, + "nauc_map_at_20_diff1": 0.008952, + "nauc_map_at_100_max": 0.303629, + "nauc_map_at_100_std": -0.201394, + "nauc_map_at_100_diff1": 0.012111, + "nauc_map_at_1000_max": 0.303892, + "nauc_map_at_1000_std": -0.201359, + "nauc_map_at_1000_diff1": 0.012831, + "nauc_recall_at_1_max": 0.592063, + "nauc_recall_at_1_std": -0.321525, + "nauc_recall_at_1_diff1": 0.677171, + "nauc_recall_at_3_max": 0.289927, + "nauc_recall_at_3_std": -0.211391, + "nauc_recall_at_3_diff1": -0.029943, + "nauc_recall_at_5_max": 0.260631, + "nauc_recall_at_5_std": -0.222117, + "nauc_recall_at_5_diff1": -0.103252, + "nauc_recall_at_10_max": 0.231902, + "nauc_recall_at_10_std": -0.198336, + "nauc_recall_at_10_diff1": -0.175852, + "nauc_recall_at_20_max": 0.197702, + "nauc_recall_at_20_std": -0.109171, + "nauc_recall_at_20_diff1": -0.232579, + "nauc_recall_at_100_max": 0.20309, + "nauc_recall_at_100_std": 0.0392, + "nauc_recall_at_100_diff1": -0.247314, + "nauc_recall_at_1000_max": 0.266043, + "nauc_recall_at_1000_std": 0.415468, + "nauc_recall_at_1000_diff1": -0.273215, + "nauc_precision_at_1_max": 0.592063, + "nauc_precision_at_1_std": -0.321525, + "nauc_precision_at_1_diff1": 0.677171, + "nauc_precision_at_3_max": 0.289927, + "nauc_precision_at_3_std": -0.211391, + "nauc_precision_at_3_diff1": -0.029943, + "nauc_precision_at_5_max": 0.260631, + "nauc_precision_at_5_std": -0.222117, + "nauc_precision_at_5_diff1": -0.103252, + "nauc_precision_at_10_max": 0.231902, + "nauc_precision_at_10_std": -0.198336, + "nauc_precision_at_10_diff1": -0.175852, + "nauc_precision_at_20_max": 0.197702, + "nauc_precision_at_20_std": -0.109171, + "nauc_precision_at_20_diff1": -0.232579, + "nauc_precision_at_100_max": 0.20309, + "nauc_precision_at_100_std": 0.0392, + "nauc_precision_at_100_diff1": -0.247314, + "nauc_precision_at_1000_max": 0.266043, + "nauc_precision_at_1000_std": 0.415468, + "nauc_precision_at_1000_diff1": -0.273215, + "nauc_mrr_at_1_max": 0.592063, + "nauc_mrr_at_1_std": -0.321525, + "nauc_mrr_at_1_diff1": 0.677171, + "nauc_mrr_at_3_max": 0.642173, + "nauc_mrr_at_3_std": -0.368071, + "nauc_mrr_at_3_diff1": 0.664971, + "nauc_mrr_at_5_max": 0.643773, + "nauc_mrr_at_5_std": -0.377834, + "nauc_mrr_at_5_diff1": 0.666957, + "nauc_mrr_at_10_max": 0.640611, + "nauc_mrr_at_10_std": -0.372726, + "nauc_mrr_at_10_diff1": 0.669353, + "nauc_mrr_at_20_max": 0.638681, + "nauc_mrr_at_20_std": -0.371011, + "nauc_mrr_at_20_diff1": 0.668354, + "nauc_mrr_at_100_max": 0.638037, + "nauc_mrr_at_100_std": -0.369809, + "nauc_mrr_at_100_diff1": 0.668512, + "nauc_mrr_at_1000_max": 0.637981, + "nauc_mrr_at_1000_std": -0.369812, + "nauc_mrr_at_1000_diff1": 0.668524, + "main_score": 0.73921, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 334.4158687591553, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBNarrativeQARetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBNarrativeQARetrieval.json new file mode 100644 index 0000000000..d4f8e2243a --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBNarrativeQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBNarrativeQARetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.18586, + "ndcg_at_3": 0.22092, + "ndcg_at_5": 0.23187, + "ndcg_at_10": 0.24575, + "ndcg_at_20": 0.25924, + "ndcg_at_100": 0.30072, + "ndcg_at_1000": 0.35304, + "map_at_1": 0.18586, + "map_at_3": 0.21233, + "map_at_5": 0.21842, + "map_at_10": 0.22413, + "map_at_20": 0.2278, + "map_at_100": 0.23282, + "map_at_1000": 0.23498, + "recall_at_1": 0.18586, + "recall_at_3": 0.24577, + "recall_at_5": 0.27227, + "recall_at_10": 0.31534, + "recall_at_20": 0.36893, + "recall_at_100": 0.60341, + "recall_at_1000": 1.0, + "precision_at_1": 0.18586, + "precision_at_3": 0.08192, + "precision_at_5": 0.05445, + "precision_at_10": 0.03153, + "precision_at_20": 0.01845, + "precision_at_100": 0.00603, + "precision_at_1000": 0.001, + "mrr_at_1": 0.185855, + "mrr_at_3": 0.212333, + "mrr_at_5": 0.218424, + "mrr_at_10": 0.224126, + "mrr_at_20": 0.227796, + "mrr_at_100": 0.232815, + "mrr_at_1000": 0.234977, + "nauc_ndcg_at_1_max": 0.524395, + "nauc_ndcg_at_1_std": 0.069637, + "nauc_ndcg_at_1_diff1": 0.632903, + "nauc_ndcg_at_3_max": 0.493083, + "nauc_ndcg_at_3_std": 0.060799, + "nauc_ndcg_at_3_diff1": 0.556402, + "nauc_ndcg_at_5_max": 0.482402, + "nauc_ndcg_at_5_std": 0.058855, + "nauc_ndcg_at_5_diff1": 0.540528, + "nauc_ndcg_at_10_max": 0.471929, + "nauc_ndcg_at_10_std": 0.058898, + "nauc_ndcg_at_10_diff1": 0.518807, + "nauc_ndcg_at_20_max": 0.465434, + "nauc_ndcg_at_20_std": 0.058145, + "nauc_ndcg_at_20_diff1": 0.507107, + "nauc_ndcg_at_100_max": 0.455605, + "nauc_ndcg_at_100_std": 0.064653, + "nauc_ndcg_at_100_diff1": 0.487858, + "nauc_ndcg_at_1000_max": 0.472792, + "nauc_ndcg_at_1000_std": 0.062177, + "nauc_ndcg_at_1000_diff1": 0.520923, + "nauc_map_at_1_max": 0.524395, + "nauc_map_at_1_std": 0.069637, + "nauc_map_at_1_diff1": 0.632903, + "nauc_map_at_3_max": 0.500238, + "nauc_map_at_3_std": 0.062924, + "nauc_map_at_3_diff1": 0.573428, + "nauc_map_at_5_max": 0.493999, + "nauc_map_at_5_std": 0.061846, + "nauc_map_at_5_diff1": 0.564166, + "nauc_map_at_10_max": 0.489318, + "nauc_map_at_10_std": 0.061855, + "nauc_map_at_10_diff1": 0.554515, + "nauc_map_at_20_max": 0.487488, + "nauc_map_at_20_std": 0.061609, + "nauc_map_at_20_diff1": 0.551062, + "nauc_map_at_100_max": 0.485994, + "nauc_map_at_100_std": 0.062582, + "nauc_map_at_100_diff1": 0.548165, + "nauc_map_at_1000_max": 0.486751, + "nauc_map_at_1000_std": 0.062482, + "nauc_map_at_1000_diff1": 0.549532, + "nauc_recall_at_1_max": 0.524395, + "nauc_recall_at_1_std": 0.069637, + "nauc_recall_at_1_diff1": 0.632903, + "nauc_recall_at_3_max": 0.473907, + "nauc_recall_at_3_std": 0.055066, + "nauc_recall_at_3_diff1": 0.51094, + "nauc_recall_at_5_max": 0.45108, + "nauc_recall_at_5_std": 0.050759, + "nauc_recall_at_5_diff1": 0.477468, + "nauc_recall_at_10_max": 0.42425, + "nauc_recall_at_10_std": 0.051225, + "nauc_recall_at_10_diff1": 0.421513, + "nauc_recall_at_20_max": 0.402543, + "nauc_recall_at_20_std": 0.048979, + "nauc_recall_at_20_diff1": 0.383688, + "nauc_recall_at_100_max": 0.343978, + "nauc_recall_at_100_std": 0.080707, + "nauc_recall_at_100_diff1": 0.268658, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.524395, + "nauc_precision_at_1_std": 0.069637, + "nauc_precision_at_1_diff1": 0.632903, + "nauc_precision_at_3_max": 0.473907, + "nauc_precision_at_3_std": 0.055066, + "nauc_precision_at_3_diff1": 0.51094, + "nauc_precision_at_5_max": 0.45108, + "nauc_precision_at_5_std": 0.050759, + "nauc_precision_at_5_diff1": 0.477468, + "nauc_precision_at_10_max": 0.42425, + "nauc_precision_at_10_std": 0.051225, + "nauc_precision_at_10_diff1": 0.421513, + "nauc_precision_at_20_max": 0.402543, + "nauc_precision_at_20_std": 0.048979, + "nauc_precision_at_20_diff1": 0.383688, + "nauc_precision_at_100_max": 0.343978, + "nauc_precision_at_100_std": 0.080707, + "nauc_precision_at_100_diff1": 0.268658, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.524395, + "nauc_mrr_at_1_std": 0.069637, + "nauc_mrr_at_1_diff1": 0.632903, + "nauc_mrr_at_3_max": 0.500238, + "nauc_mrr_at_3_std": 0.062924, + "nauc_mrr_at_3_diff1": 0.573428, + "nauc_mrr_at_5_max": 0.493999, + "nauc_mrr_at_5_std": 0.061846, + "nauc_mrr_at_5_diff1": 0.564166, + "nauc_mrr_at_10_max": 0.489318, + "nauc_mrr_at_10_std": 0.061855, + "nauc_mrr_at_10_diff1": 0.554515, + "nauc_mrr_at_20_max": 0.487488, + "nauc_mrr_at_20_std": 0.061609, + "nauc_mrr_at_20_diff1": 0.551062, + "nauc_mrr_at_100_max": 0.485994, + "nauc_mrr_at_100_std": 0.062582, + "nauc_mrr_at_100_diff1": 0.548165, + "nauc_mrr_at_1000_max": 0.486752, + "nauc_mrr_at_1000_std": 0.062482, + "nauc_mrr_at_1000_diff1": 0.549532, + "main_score": 0.24575, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 179.81693243980408, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBNeedleRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBNeedleRetrieval.json new file mode 100644 index 0000000000..8f368ab81b --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBNeedleRetrieval.json @@ -0,0 +1,1201 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBNeedleRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test_256": [ + { + "ndcg_at_1": 0.9, + "ndcg_at_3": 0.93262, + "ndcg_at_5": 0.94123, + "ndcg_at_10": 0.9479, + "ndcg_at_20": 0.9479, + "ndcg_at_100": 0.9479, + "ndcg_at_1000": 0.9479, + "map_at_1": 0.9, + "map_at_3": 0.92333, + "map_at_5": 0.92833, + "map_at_10": 0.93119, + "map_at_20": 0.93119, + "map_at_100": 0.93119, + "map_at_1000": 0.93119, + "recall_at_1": 0.9, + "recall_at_3": 0.96, + "recall_at_5": 0.98, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.9, + "precision_at_3": 0.32, + "precision_at_5": 0.196, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.9, + "mrr_at_3": 0.923333, + "mrr_at_5": 0.928333, + "mrr_at_10": 0.93119, + "mrr_at_20": 0.93119, + "mrr_at_100": 0.93119, + "mrr_at_1000": 0.93119, + "nauc_ndcg_at_1_max": 0.486088, + "nauc_ndcg_at_1_std": 0.230906, + "nauc_ndcg_at_1_diff1": 0.862745, + "nauc_ndcg_at_3_max": 0.39828, + "nauc_ndcg_at_3_std": 0.150221, + "nauc_ndcg_at_3_diff1": 0.862006, + "nauc_ndcg_at_5_max": 0.43866, + "nauc_ndcg_at_5_std": 0.197351, + "nauc_ndcg_at_5_diff1": 0.882494, + "nauc_ndcg_at_10_max": 0.448971, + "nauc_ndcg_at_10_std": 0.206893, + "nauc_ndcg_at_10_diff1": 0.867458, + "nauc_ndcg_at_20_max": 0.448971, + "nauc_ndcg_at_20_std": 0.206893, + "nauc_ndcg_at_20_diff1": 0.867458, + "nauc_ndcg_at_100_max": 0.448971, + "nauc_ndcg_at_100_std": 0.206893, + "nauc_ndcg_at_100_diff1": 0.867458, + "nauc_ndcg_at_1000_max": 0.448971, + "nauc_ndcg_at_1000_std": 0.206893, + "nauc_ndcg_at_1000_diff1": 0.867458, + "nauc_map_at_1_max": 0.486088, + "nauc_map_at_1_std": 0.230906, + "nauc_map_at_1_diff1": 0.862745, + "nauc_map_at_3_max": 0.426704, + "nauc_map_at_3_std": 0.180997, + "nauc_map_at_3_diff1": 0.862177, + "nauc_map_at_5_max": 0.447908, + "nauc_map_at_5_std": 0.205578, + "nauc_map_at_5_diff1": 0.871941, + "nauc_map_at_10_max": 0.451638, + "nauc_map_at_10_std": 0.209016, + "nauc_map_at_10_diff1": 0.866624, + "nauc_map_at_20_max": 0.451638, + "nauc_map_at_20_std": 0.209016, + "nauc_map_at_20_diff1": 0.866624, + "nauc_map_at_100_max": 0.451638, + "nauc_map_at_100_std": 0.209016, + "nauc_map_at_100_diff1": 0.866624, + "nauc_map_at_1000_max": 0.451638, + "nauc_map_at_1000_std": 0.209016, + "nauc_map_at_1000_diff1": 0.866624, + "nauc_recall_at_1_max": 0.486088, + "nauc_recall_at_1_std": 0.230906, + "nauc_recall_at_1_diff1": 0.862745, + "nauc_recall_at_3_max": 0.24043, + "nauc_recall_at_3_std": -0.024276, + "nauc_recall_at_3_diff1": 0.861111, + "nauc_recall_at_5_max": 0.358077, + "nauc_recall_at_5_std": 0.122782, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.486088, + "nauc_precision_at_1_std": 0.230906, + "nauc_precision_at_1_diff1": 0.862745, + "nauc_precision_at_3_max": 0.24043, + "nauc_precision_at_3_std": -0.024276, + "nauc_precision_at_3_diff1": 0.861111, + "nauc_precision_at_5_max": 0.358077, + "nauc_precision_at_5_std": 0.122782, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.486088, + "nauc_mrr_at_1_std": 0.230906, + "nauc_mrr_at_1_diff1": 0.862745, + "nauc_mrr_at_3_max": 0.426704, + "nauc_mrr_at_3_std": 0.180997, + "nauc_mrr_at_3_diff1": 0.862177, + "nauc_mrr_at_5_max": 0.447908, + "nauc_mrr_at_5_std": 0.205578, + "nauc_mrr_at_5_diff1": 0.871941, + "nauc_mrr_at_10_max": 0.451638, + "nauc_mrr_at_10_std": 0.209016, + "nauc_mrr_at_10_diff1": 0.866624, + "nauc_mrr_at_20_max": 0.451638, + "nauc_mrr_at_20_std": 0.209016, + "nauc_mrr_at_20_diff1": 0.866624, + "nauc_mrr_at_100_max": 0.451638, + "nauc_mrr_at_100_std": 0.209016, + "nauc_mrr_at_100_diff1": 0.866624, + "nauc_mrr_at_1000_max": 0.451638, + "nauc_mrr_at_1000_std": 0.209016, + "nauc_mrr_at_1000_diff1": 0.866624, + "main_score": 0.9, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_512": [ + { + "ndcg_at_1": 0.66, + "ndcg_at_3": 0.79833, + "ndcg_at_5": 0.82417, + "ndcg_at_10": 0.83842, + "ndcg_at_20": 0.83842, + "ndcg_at_100": 0.83842, + "ndcg_at_1000": 0.83842, + "map_at_1": 0.66, + "map_at_3": 0.76333, + "map_at_5": 0.77833, + "map_at_10": 0.785, + "map_at_20": 0.785, + "map_at_100": 0.785, + "map_at_1000": 0.785, + "recall_at_1": 0.66, + "recall_at_3": 0.9, + "recall_at_5": 0.96, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.66, + "precision_at_3": 0.3, + "precision_at_5": 0.192, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.66, + "mrr_at_3": 0.763333, + "mrr_at_5": 0.778333, + "mrr_at_10": 0.785, + "mrr_at_20": 0.785, + "mrr_at_100": 0.785, + "mrr_at_1000": 0.785, + "nauc_ndcg_at_1_max": 0.020877, + "nauc_ndcg_at_1_std": -0.319028, + "nauc_ndcg_at_1_diff1": 0.808391, + "nauc_ndcg_at_3_max": -0.08688, + "nauc_ndcg_at_3_std": -0.469264, + "nauc_ndcg_at_3_diff1": 0.743395, + "nauc_ndcg_at_5_max": -0.083353, + "nauc_ndcg_at_5_std": -0.416736, + "nauc_ndcg_at_5_diff1": 0.722982, + "nauc_ndcg_at_10_max": -0.033855, + "nauc_ndcg_at_10_std": -0.421556, + "nauc_ndcg_at_10_diff1": 0.7606, + "nauc_ndcg_at_20_max": -0.033855, + "nauc_ndcg_at_20_std": -0.421556, + "nauc_ndcg_at_20_diff1": 0.7606, + "nauc_ndcg_at_100_max": -0.033855, + "nauc_ndcg_at_100_std": -0.421556, + "nauc_ndcg_at_100_diff1": 0.7606, + "nauc_ndcg_at_1000_max": -0.033855, + "nauc_ndcg_at_1000_std": -0.421556, + "nauc_ndcg_at_1000_diff1": 0.7606, + "nauc_map_at_1_max": 0.020877, + "nauc_map_at_1_std": -0.319028, + "nauc_map_at_1_diff1": 0.808391, + "nauc_map_at_3_max": -0.052402, + "nauc_map_at_3_std": -0.441435, + "nauc_map_at_3_diff1": 0.758183, + "nauc_map_at_5_max": -0.048041, + "nauc_map_at_5_std": -0.414791, + "nauc_map_at_5_diff1": 0.749849, + "nauc_map_at_10_max": -0.029312, + "nauc_map_at_10_std": -0.416419, + "nauc_map_at_10_diff1": 0.764083, + "nauc_map_at_20_max": -0.029312, + "nauc_map_at_20_std": -0.416419, + "nauc_map_at_20_diff1": 0.764083, + "nauc_map_at_100_max": -0.029312, + "nauc_map_at_100_std": -0.416419, + "nauc_map_at_100_diff1": 0.764083, + "nauc_map_at_1000_max": -0.029312, + "nauc_map_at_1000_std": -0.416419, + "nauc_map_at_1000_diff1": 0.764083, + "nauc_recall_at_1_max": 0.020877, + "nauc_recall_at_1_std": -0.319028, + "nauc_recall_at_1_diff1": 0.808391, + "nauc_recall_at_3_max": -0.290009, + "nauc_recall_at_3_std": -0.6155, + "nauc_recall_at_3_diff1": 0.661531, + "nauc_recall_at_5_max": -0.563492, + "nauc_recall_at_5_std": -0.369981, + "nauc_recall_at_5_diff1": 0.358077, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.020877, + "nauc_precision_at_1_std": -0.319028, + "nauc_precision_at_1_diff1": 0.808391, + "nauc_precision_at_3_max": -0.290009, + "nauc_precision_at_3_std": -0.6155, + "nauc_precision_at_3_diff1": 0.661531, + "nauc_precision_at_5_max": -0.563492, + "nauc_precision_at_5_std": -0.369981, + "nauc_precision_at_5_diff1": 0.358077, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.020877, + "nauc_mrr_at_1_std": -0.319028, + "nauc_mrr_at_1_diff1": 0.808391, + "nauc_mrr_at_3_max": -0.052402, + "nauc_mrr_at_3_std": -0.441435, + "nauc_mrr_at_3_diff1": 0.758183, + "nauc_mrr_at_5_max": -0.048041, + "nauc_mrr_at_5_std": -0.414791, + "nauc_mrr_at_5_diff1": 0.749849, + "nauc_mrr_at_10_max": -0.029312, + "nauc_mrr_at_10_std": -0.416419, + "nauc_mrr_at_10_diff1": 0.764083, + "nauc_mrr_at_20_max": -0.029312, + "nauc_mrr_at_20_std": -0.416419, + "nauc_mrr_at_20_diff1": 0.764083, + "nauc_mrr_at_100_max": -0.029312, + "nauc_mrr_at_100_std": -0.416419, + "nauc_mrr_at_100_diff1": 0.764083, + "nauc_mrr_at_1000_max": -0.029312, + "nauc_mrr_at_1000_std": -0.416419, + "nauc_mrr_at_1000_diff1": 0.764083, + "main_score": 0.66, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_1024": [ + { + "ndcg_at_1": 0.34, + "ndcg_at_3": 0.38262, + "ndcg_at_5": 0.39123, + "ndcg_at_10": 0.39123, + "ndcg_at_20": 0.39123, + "ndcg_at_100": 0.48533, + "ndcg_at_1000": 0.48533, + "map_at_1": 0.34, + "map_at_3": 0.37, + "map_at_5": 0.375, + "map_at_10": 0.375, + "map_at_20": 0.375, + "map_at_100": 0.38434, + "map_at_1000": 0.38434, + "recall_at_1": 0.34, + "recall_at_3": 0.42, + "recall_at_5": 0.44, + "recall_at_10": 0.44, + "recall_at_20": 0.44, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.34, + "precision_at_3": 0.14, + "precision_at_5": 0.088, + "precision_at_10": 0.044, + "precision_at_20": 0.022, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.34, + "mrr_at_3": 0.37, + "mrr_at_5": 0.375, + "mrr_at_10": 0.375, + "mrr_at_20": 0.375, + "mrr_at_100": 0.384633, + "mrr_at_1000": 0.384633, + "nauc_ndcg_at_1_max": 0.220462, + "nauc_ndcg_at_1_std": 0.336363, + "nauc_ndcg_at_1_diff1": 0.784493, + "nauc_ndcg_at_3_max": 0.307543, + "nauc_ndcg_at_3_std": 0.336503, + "nauc_ndcg_at_3_diff1": 0.755288, + "nauc_ndcg_at_5_max": 0.290967, + "nauc_ndcg_at_5_std": 0.379641, + "nauc_ndcg_at_5_diff1": 0.736482, + "nauc_ndcg_at_10_max": 0.290967, + "nauc_ndcg_at_10_std": 0.379641, + "nauc_ndcg_at_10_diff1": 0.736482, + "nauc_ndcg_at_20_max": 0.290967, + "nauc_ndcg_at_20_std": 0.379641, + "nauc_ndcg_at_20_diff1": 0.736482, + "nauc_ndcg_at_100_max": 0.280767, + "nauc_ndcg_at_100_std": 0.362758, + "nauc_ndcg_at_100_diff1": 0.744135, + "nauc_ndcg_at_1000_max": 0.280767, + "nauc_ndcg_at_1000_std": 0.362758, + "nauc_ndcg_at_1000_diff1": 0.744135, + "nauc_map_at_1_max": 0.220462, + "nauc_map_at_1_std": 0.336363, + "nauc_map_at_1_diff1": 0.784493, + "nauc_map_at_3_max": 0.283683, + "nauc_map_at_3_std": 0.332974, + "nauc_map_at_3_diff1": 0.76381, + "nauc_map_at_5_max": 0.274, + "nauc_map_at_5_std": 0.358061, + "nauc_map_at_5_diff1": 0.752885, + "nauc_map_at_10_max": 0.274, + "nauc_map_at_10_std": 0.358061, + "nauc_map_at_10_diff1": 0.752885, + "nauc_map_at_20_max": 0.274, + "nauc_map_at_20_std": 0.358061, + "nauc_map_at_20_diff1": 0.752885, + "nauc_map_at_100_max": 0.27275, + "nauc_map_at_100_std": 0.355399, + "nauc_map_at_100_diff1": 0.753156, + "nauc_map_at_1000_max": 0.27275, + "nauc_map_at_1000_std": 0.355399, + "nauc_map_at_1000_diff1": 0.753156, + "nauc_recall_at_1_max": 0.220462, + "nauc_recall_at_1_std": 0.336363, + "nauc_recall_at_1_diff1": 0.784493, + "nauc_recall_at_3_max": 0.376753, + "nauc_recall_at_3_std": 0.347979, + "nauc_recall_at_3_diff1": 0.730383, + "nauc_recall_at_5_max": 0.338962, + "nauc_recall_at_5_std": 0.447643, + "nauc_recall_at_5_diff1": 0.686799, + "nauc_recall_at_10_max": 0.338962, + "nauc_recall_at_10_std": 0.447643, + "nauc_recall_at_10_diff1": 0.686799, + "nauc_recall_at_20_max": 0.338962, + "nauc_recall_at_20_std": 0.447643, + "nauc_recall_at_20_diff1": 0.686799, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.220462, + "nauc_precision_at_1_std": 0.336363, + "nauc_precision_at_1_diff1": 0.784493, + "nauc_precision_at_3_max": 0.376753, + "nauc_precision_at_3_std": 0.347979, + "nauc_precision_at_3_diff1": 0.730383, + "nauc_precision_at_5_max": 0.338962, + "nauc_precision_at_5_std": 0.447643, + "nauc_precision_at_5_diff1": 0.686799, + "nauc_precision_at_10_max": 0.338962, + "nauc_precision_at_10_std": 0.447643, + "nauc_precision_at_10_diff1": 0.686799, + "nauc_precision_at_20_max": 0.338962, + "nauc_precision_at_20_std": 0.447643, + "nauc_precision_at_20_diff1": 0.686799, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.220462, + "nauc_mrr_at_1_std": 0.336363, + "nauc_mrr_at_1_diff1": 0.784493, + "nauc_mrr_at_3_max": 0.283683, + "nauc_mrr_at_3_std": 0.332974, + "nauc_mrr_at_3_diff1": 0.76381, + "nauc_mrr_at_5_max": 0.274, + "nauc_mrr_at_5_std": 0.358061, + "nauc_mrr_at_5_diff1": 0.752885, + "nauc_mrr_at_10_max": 0.274, + "nauc_mrr_at_10_std": 0.358061, + "nauc_mrr_at_10_diff1": 0.752885, + "nauc_mrr_at_20_max": 0.274, + "nauc_mrr_at_20_std": 0.358061, + "nauc_mrr_at_20_diff1": 0.752885, + "nauc_mrr_at_100_max": 0.270666, + "nauc_mrr_at_100_std": 0.355055, + "nauc_mrr_at_100_diff1": 0.752093, + "nauc_mrr_at_1000_max": 0.270666, + "nauc_mrr_at_1000_std": 0.355055, + "nauc_mrr_at_1000_diff1": 0.752093, + "main_score": 0.34, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_2048": [ + { + "ndcg_at_1": 0.18, + "ndcg_at_3": 0.19, + "ndcg_at_5": 0.19, + "ndcg_at_10": 0.19, + "ndcg_at_20": 0.20011, + "ndcg_at_100": 0.33121, + "ndcg_at_1000": 0.33121, + "map_at_1": 0.18, + "map_at_3": 0.18667, + "map_at_5": 0.18667, + "map_at_10": 0.18667, + "map_at_20": 0.18944, + "map_at_100": 0.204, + "map_at_1000": 0.204, + "recall_at_1": 0.18, + "recall_at_3": 0.2, + "recall_at_5": 0.2, + "recall_at_10": 0.2, + "recall_at_20": 0.24, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.18, + "precision_at_3": 0.06667, + "precision_at_5": 0.04, + "precision_at_10": 0.02, + "precision_at_20": 0.012, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.18, + "mrr_at_3": 0.186667, + "mrr_at_5": 0.186667, + "mrr_at_10": 0.186667, + "mrr_at_20": 0.186667, + "mrr_at_100": 0.203656, + "mrr_at_1000": 0.203656, + "nauc_ndcg_at_1_max": 0.815809, + "nauc_ndcg_at_1_std": 0.577097, + "nauc_ndcg_at_1_diff1": 0.602042, + "nauc_ndcg_at_3_max": 0.75066, + "nauc_ndcg_at_3_std": 0.548736, + "nauc_ndcg_at_3_diff1": 0.572646, + "nauc_ndcg_at_5_max": 0.75066, + "nauc_ndcg_at_5_std": 0.548736, + "nauc_ndcg_at_5_diff1": 0.572646, + "nauc_ndcg_at_10_max": 0.75066, + "nauc_ndcg_at_10_std": 0.548736, + "nauc_ndcg_at_10_diff1": 0.572646, + "nauc_ndcg_at_20_max": 0.706982, + "nauc_ndcg_at_20_std": 0.504182, + "nauc_ndcg_at_20_diff1": 0.569838, + "nauc_ndcg_at_100_max": 0.735999, + "nauc_ndcg_at_100_std": 0.519161, + "nauc_ndcg_at_100_diff1": 0.56591, + "nauc_ndcg_at_1000_max": 0.735999, + "nauc_ndcg_at_1000_std": 0.519161, + "nauc_ndcg_at_1000_diff1": 0.56591, + "nauc_map_at_1_max": 0.815809, + "nauc_map_at_1_std": 0.577097, + "nauc_map_at_1_diff1": 0.602042, + "nauc_map_at_3_max": 0.771767, + "nauc_map_at_3_std": 0.557925, + "nauc_map_at_3_diff1": 0.58217, + "nauc_map_at_5_max": 0.771767, + "nauc_map_at_5_std": 0.557925, + "nauc_map_at_5_diff1": 0.58217, + "nauc_map_at_10_max": 0.771767, + "nauc_map_at_10_std": 0.557925, + "nauc_map_at_10_diff1": 0.58217, + "nauc_map_at_20_max": 0.759205, + "nauc_map_at_20_std": 0.545194, + "nauc_map_at_20_diff1": 0.581776, + "nauc_map_at_100_max": 0.759574, + "nauc_map_at_100_std": 0.542102, + "nauc_map_at_100_diff1": 0.577741, + "nauc_map_at_1000_max": 0.759574, + "nauc_map_at_1000_std": 0.542102, + "nauc_map_at_1000_diff1": 0.577741, + "nauc_recall_at_1_max": 0.815809, + "nauc_recall_at_1_std": 0.577097, + "nauc_recall_at_1_diff1": 0.602042, + "nauc_recall_at_3_max": 0.690701, + "nauc_recall_at_3_std": 0.522635, + "nauc_recall_at_3_diff1": 0.545593, + "nauc_recall_at_5_max": 0.690701, + "nauc_recall_at_5_std": 0.522635, + "nauc_recall_at_5_diff1": 0.545593, + "nauc_recall_at_10_max": 0.690701, + "nauc_recall_at_10_std": 0.522635, + "nauc_recall_at_10_diff1": 0.545593, + "nauc_recall_at_20_max": 0.538641, + "nauc_recall_at_20_std": 0.364905, + "nauc_recall_at_20_diff1": 0.534418, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.815809, + "nauc_precision_at_1_std": 0.577097, + "nauc_precision_at_1_diff1": 0.602042, + "nauc_precision_at_3_max": 0.690701, + "nauc_precision_at_3_std": 0.522635, + "nauc_precision_at_3_diff1": 0.545593, + "nauc_precision_at_5_max": 0.690701, + "nauc_precision_at_5_std": 0.522635, + "nauc_precision_at_5_diff1": 0.545593, + "nauc_precision_at_10_max": 0.690701, + "nauc_precision_at_10_std": 0.522635, + "nauc_precision_at_10_diff1": 0.545593, + "nauc_precision_at_20_max": 0.538641, + "nauc_precision_at_20_std": 0.364905, + "nauc_precision_at_20_diff1": 0.534418, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.815809, + "nauc_mrr_at_1_std": 0.577097, + "nauc_mrr_at_1_diff1": 0.602042, + "nauc_mrr_at_3_max": 0.771767, + "nauc_mrr_at_3_std": 0.557925, + "nauc_mrr_at_3_diff1": 0.58217, + "nauc_mrr_at_5_max": 0.771767, + "nauc_mrr_at_5_std": 0.557925, + "nauc_mrr_at_5_diff1": 0.58217, + "nauc_mrr_at_10_max": 0.771767, + "nauc_mrr_at_10_std": 0.557925, + "nauc_mrr_at_10_diff1": 0.58217, + "nauc_mrr_at_20_max": 0.771767, + "nauc_mrr_at_20_std": 0.557925, + "nauc_mrr_at_20_diff1": 0.58217, + "nauc_mrr_at_100_max": 0.761322, + "nauc_mrr_at_100_std": 0.553546, + "nauc_mrr_at_100_diff1": 0.573335, + "nauc_mrr_at_1000_max": 0.761322, + "nauc_mrr_at_1000_std": 0.553546, + "nauc_mrr_at_1000_diff1": 0.573335, + "main_score": 0.18, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_4096": [ + { + "ndcg_at_1": 0.1, + "ndcg_at_3": 0.11262, + "ndcg_at_5": 0.11262, + "ndcg_at_10": 0.1184, + "ndcg_at_20": 0.14225, + "ndcg_at_100": 0.2728, + "ndcg_at_1000": 0.2728, + "map_at_1": 0.1, + "map_at_3": 0.11, + "map_at_5": 0.11, + "map_at_10": 0.112, + "map_at_20": 0.11781, + "map_at_100": 0.13216, + "map_at_1000": 0.13216, + "recall_at_1": 0.1, + "recall_at_3": 0.12, + "recall_at_5": 0.12, + "recall_at_10": 0.14, + "recall_at_20": 0.24, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.1, + "precision_at_3": 0.04, + "precision_at_5": 0.024, + "precision_at_10": 0.014, + "precision_at_20": 0.012, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.1, + "mrr_at_3": 0.11, + "mrr_at_5": 0.11, + "mrr_at_10": 0.11, + "mrr_at_20": 0.116686, + "mrr_at_100": 0.134688, + "mrr_at_1000": 0.134688, + "nauc_ndcg_at_1_max": 0.339469, + "nauc_ndcg_at_1_std": 0.594312, + "nauc_ndcg_at_1_diff1": 0.61685, + "nauc_ndcg_at_3_max": 0.289333, + "nauc_ndcg_at_3_std": 0.664957, + "nauc_ndcg_at_3_diff1": 0.607014, + "nauc_ndcg_at_5_max": 0.289333, + "nauc_ndcg_at_5_std": 0.664957, + "nauc_ndcg_at_5_diff1": 0.607014, + "nauc_ndcg_at_10_max": 0.258072, + "nauc_ndcg_at_10_std": 0.638636, + "nauc_ndcg_at_10_diff1": 0.58267, + "nauc_ndcg_at_20_max": 0.284407, + "nauc_ndcg_at_20_std": 0.605826, + "nauc_ndcg_at_20_diff1": 0.536846, + "nauc_ndcg_at_100_max": 0.294741, + "nauc_ndcg_at_100_std": 0.614661, + "nauc_ndcg_at_100_diff1": 0.576002, + "nauc_ndcg_at_1000_max": 0.294741, + "nauc_ndcg_at_1000_std": 0.614661, + "nauc_ndcg_at_1000_diff1": 0.576002, + "nauc_map_at_1_max": 0.339469, + "nauc_map_at_1_std": 0.594312, + "nauc_map_at_1_diff1": 0.61685, + "nauc_map_at_3_max": 0.299092, + "nauc_map_at_3_std": 0.651207, + "nauc_map_at_3_diff1": 0.608929, + "nauc_map_at_5_max": 0.299092, + "nauc_map_at_5_std": 0.651207, + "nauc_map_at_5_diff1": 0.608929, + "nauc_map_at_10_max": 0.287733, + "nauc_map_at_10_std": 0.641913, + "nauc_map_at_10_diff1": 0.600153, + "nauc_map_at_20_max": 0.294031, + "nauc_map_at_20_std": 0.632966, + "nauc_map_at_20_diff1": 0.586685, + "nauc_map_at_100_max": 0.296861, + "nauc_map_at_100_std": 0.628705, + "nauc_map_at_100_diff1": 0.591008, + "nauc_map_at_1000_max": 0.296861, + "nauc_map_at_1000_std": 0.628705, + "nauc_map_at_1000_diff1": 0.591008, + "nauc_recall_at_1_max": 0.339469, + "nauc_recall_at_1_std": 0.594312, + "nauc_recall_at_1_diff1": 0.61685, + "nauc_recall_at_3_max": 0.263435, + "nauc_recall_at_3_std": 0.70145, + "nauc_recall_at_3_diff1": 0.601933, + "nauc_recall_at_5_max": 0.263435, + "nauc_recall_at_5_std": 0.70145, + "nauc_recall_at_5_diff1": 0.601933, + "nauc_recall_at_10_max": 0.170224, + "nauc_recall_at_10_std": 0.61687, + "nauc_recall_at_10_diff1": 0.52777, + "nauc_recall_at_20_max": 0.273275, + "nauc_recall_at_20_std": 0.526779, + "nauc_recall_at_20_diff1": 0.412084, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.339469, + "nauc_precision_at_1_std": 0.594312, + "nauc_precision_at_1_diff1": 0.61685, + "nauc_precision_at_3_max": 0.263435, + "nauc_precision_at_3_std": 0.70145, + "nauc_precision_at_3_diff1": 0.601933, + "nauc_precision_at_5_max": 0.263435, + "nauc_precision_at_5_std": 0.70145, + "nauc_precision_at_5_diff1": 0.601933, + "nauc_precision_at_10_max": 0.170224, + "nauc_precision_at_10_std": 0.61687, + "nauc_precision_at_10_diff1": 0.52777, + "nauc_precision_at_20_max": 0.273275, + "nauc_precision_at_20_std": 0.526779, + "nauc_precision_at_20_diff1": 0.412084, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.339469, + "nauc_mrr_at_1_std": 0.594312, + "nauc_mrr_at_1_diff1": 0.61685, + "nauc_mrr_at_3_max": 0.299092, + "nauc_mrr_at_3_std": 0.651207, + "nauc_mrr_at_3_diff1": 0.608929, + "nauc_mrr_at_5_max": 0.299092, + "nauc_mrr_at_5_std": 0.651207, + "nauc_mrr_at_5_diff1": 0.608929, + "nauc_mrr_at_10_max": 0.299092, + "nauc_mrr_at_10_std": 0.651207, + "nauc_mrr_at_10_diff1": 0.608929, + "nauc_mrr_at_20_max": 0.280394, + "nauc_mrr_at_20_std": 0.608059, + "nauc_mrr_at_20_diff1": 0.572258, + "nauc_mrr_at_100_max": 0.272151, + "nauc_mrr_at_100_std": 0.606855, + "nauc_mrr_at_100_diff1": 0.565064, + "nauc_mrr_at_1000_max": 0.272151, + "nauc_mrr_at_1000_std": 0.606855, + "nauc_mrr_at_1000_diff1": 0.565064, + "main_score": 0.1, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_8192": [ + { + "ndcg_at_1": 0.04, + "ndcg_at_3": 0.04, + "ndcg_at_5": 0.04774, + "ndcg_at_10": 0.05405, + "ndcg_at_20": 0.07298, + "ndcg_at_100": 0.21797, + "ndcg_at_1000": 0.21797, + "map_at_1": 0.04, + "map_at_3": 0.04, + "map_at_5": 0.044, + "map_at_10": 0.0465, + "map_at_20": 0.05102, + "map_at_100": 0.0672, + "map_at_1000": 0.0672, + "recall_at_1": 0.04, + "recall_at_3": 0.04, + "recall_at_5": 0.06, + "recall_at_10": 0.08, + "recall_at_20": 0.16, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.04, + "precision_at_3": 0.01333, + "precision_at_5": 0.012, + "precision_at_10": 0.008, + "precision_at_20": 0.008, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.04, + "mrr_at_3": 0.046667, + "mrr_at_5": 0.046667, + "mrr_at_10": 0.049167, + "mrr_at_20": 0.055729, + "mrr_at_100": 0.072842, + "mrr_at_1000": 0.072842, + "nauc_ndcg_at_1_max": -0.200563, + "nauc_ndcg_at_1_std": 1.0, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": -0.200563, + "nauc_ndcg_at_3_std": 1.0, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": -0.261206, + "nauc_ndcg_at_5_std": 0.744774, + "nauc_ndcg_at_5_diff1": 0.804569, + "nauc_ndcg_at_10_max": -0.297806, + "nauc_ndcg_at_10_std": 0.599508, + "nauc_ndcg_at_10_diff1": 0.662189, + "nauc_ndcg_at_20_max": -0.256107, + "nauc_ndcg_at_20_std": 0.512068, + "nauc_ndcg_at_20_diff1": 0.557905, + "nauc_ndcg_at_100_max": -0.166791, + "nauc_ndcg_at_100_std": 0.586182, + "nauc_ndcg_at_100_diff1": 0.64092, + "nauc_ndcg_at_1000_max": -0.166791, + "nauc_ndcg_at_1000_std": 0.586182, + "nauc_ndcg_at_1000_diff1": 0.64092, + "nauc_map_at_1_max": -0.200563, + "nauc_map_at_1_std": 1.0, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": -0.200563, + "nauc_map_at_3_std": 1.0, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": -0.234578, + "nauc_map_at_5_std": 0.856843, + "nauc_map_at_5_diff1": 0.890382, + "nauc_map_at_10_max": -0.252866, + "nauc_map_at_10_std": 0.783916, + "nauc_map_at_10_diff1": 0.820196, + "nauc_map_at_20_max": -0.242291, + "nauc_map_at_20_std": 0.743905, + "nauc_map_at_20_diff1": 0.776783, + "nauc_map_at_100_max": -0.203687, + "nauc_map_at_100_std": 0.725679, + "nauc_map_at_100_diff1": 0.765643, + "nauc_map_at_1000_max": -0.203687, + "nauc_map_at_1000_std": 0.725679, + "nauc_map_at_1000_diff1": 0.765643, + "nauc_recall_at_1_max": -0.200563, + "nauc_recall_at_1_std": 1.0, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": -0.200563, + "nauc_recall_at_3_std": 1.0, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": -0.325284, + "nauc_recall_at_5_std": 0.475092, + "nauc_recall_at_5_diff1": 0.598068, + "nauc_recall_at_10_max": -0.387644, + "nauc_recall_at_10_std": 0.231419, + "nauc_recall_at_10_diff1": 0.344781, + "nauc_recall_at_20_max": -0.263289, + "nauc_recall_at_20_std": 0.204909, + "nauc_recall_at_20_diff1": 0.260946, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.200563, + "nauc_precision_at_1_std": 1.0, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": -0.200563, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": -0.325284, + "nauc_precision_at_5_std": 0.475092, + "nauc_precision_at_5_diff1": 0.598068, + "nauc_precision_at_10_max": -0.387644, + "nauc_precision_at_10_std": 0.231419, + "nauc_precision_at_10_diff1": 0.344781, + "nauc_precision_at_20_max": -0.263289, + "nauc_precision_at_20_std": 0.204909, + "nauc_precision_at_20_diff1": 0.260946, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": -0.200563, + "nauc_mrr_at_1_std": 1.0, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": -0.21741, + "nauc_mrr_at_3_std": 0.775039, + "nauc_mrr_at_3_diff1": 0.87121, + "nauc_mrr_at_5_max": -0.21741, + "nauc_mrr_at_5_std": 0.775039, + "nauc_mrr_at_5_diff1": 0.87121, + "nauc_mrr_at_10_max": -0.231758, + "nauc_mrr_at_10_std": 0.714525, + "nauc_mrr_at_10_diff1": 0.801508, + "nauc_mrr_at_20_max": -0.229987, + "nauc_mrr_at_20_std": 0.661645, + "nauc_mrr_at_20_diff1": 0.697537, + "nauc_mrr_at_100_max": -0.248784, + "nauc_mrr_at_100_std": 0.673981, + "nauc_mrr_at_100_diff1": 0.721955, + "nauc_mrr_at_1000_max": -0.248784, + "nauc_mrr_at_1000_std": 0.673981, + "nauc_mrr_at_1000_diff1": 0.721955, + "main_score": 0.04, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_16384": [ + { + "ndcg_at_1": 0.0, + "ndcg_at_3": 0.0, + "ndcg_at_5": 0.01635, + "ndcg_at_10": 0.03645, + "ndcg_at_20": 0.06653, + "ndcg_at_100": 0.20295, + "ndcg_at_1000": 0.20295, + "map_at_1": 0.0, + "map_at_3": 0.0, + "map_at_5": 0.009, + "map_at_10": 0.01769, + "map_at_20": 0.02581, + "map_at_100": 0.04171, + "map_at_1000": 0.04171, + "recall_at_1": 0.0, + "recall_at_3": 0.0, + "recall_at_5": 0.04, + "recall_at_10": 0.1, + "recall_at_20": 0.22, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.0, + "precision_at_3": 0.0, + "precision_at_5": 0.008, + "precision_at_10": 0.01, + "precision_at_20": 0.011, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.0, + "mrr_at_3": 0.006667, + "mrr_at_5": 0.006667, + "mrr_at_10": 0.0125, + "mrr_at_20": 0.020604, + "mrr_at_100": 0.036177, + "mrr_at_1000": 0.036177, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_5_max": -0.187882, + "nauc_ndcg_at_5_std": -0.205796, + "nauc_ndcg_at_5_diff1": -0.205796, + "nauc_ndcg_at_10_max": -0.253534, + "nauc_ndcg_at_10_std": -0.113682, + "nauc_ndcg_at_10_diff1": -0.113682, + "nauc_ndcg_at_20_max": -0.185988, + "nauc_ndcg_at_20_std": -0.130507, + "nauc_ndcg_at_20_diff1": -0.130507, + "nauc_ndcg_at_100_max": -0.099703, + "nauc_ndcg_at_100_std": -0.072626, + "nauc_ndcg_at_100_diff1": -0.072626, + "nauc_ndcg_at_1000_max": -0.099703, + "nauc_ndcg_at_1000_std": -0.072626, + "nauc_ndcg_at_1000_diff1": -0.072626, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_5_max": -0.180753, + "nauc_map_at_5_std": -0.205796, + "nauc_map_at_5_diff1": -0.205796, + "nauc_map_at_10_max": -0.24718, + "nauc_map_at_10_std": -0.132615, + "nauc_map_at_10_diff1": -0.132615, + "nauc_map_at_20_max": -0.200246, + "nauc_map_at_20_std": -0.142417, + "nauc_map_at_20_diff1": -0.142417, + "nauc_map_at_100_max": -0.143897, + "nauc_map_at_100_std": -0.102263, + "nauc_map_at_100_diff1": -0.102263, + "nauc_map_at_1000_max": -0.143897, + "nauc_map_at_1000_std": -0.102263, + "nauc_map_at_1000_diff1": -0.102263, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": -0.194526, + "nauc_recall_at_5_std": -0.205796, + "nauc_recall_at_5_diff1": -0.205796, + "nauc_recall_at_10_max": -0.257955, + "nauc_recall_at_10_std": -0.097773, + "nauc_recall_at_10_diff1": -0.097773, + "nauc_recall_at_20_max": -0.174597, + "nauc_recall_at_20_std": -0.122044, + "nauc_recall_at_20_diff1": -0.122044, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_3_max": NaN, + "nauc_precision_at_3_std": NaN, + "nauc_precision_at_3_diff1": NaN, + "nauc_precision_at_5_max": -0.194526, + "nauc_precision_at_5_std": -0.205796, + "nauc_precision_at_5_diff1": -0.205796, + "nauc_precision_at_10_max": -0.257955, + "nauc_precision_at_10_std": -0.097773, + "nauc_precision_at_10_diff1": -0.097773, + "nauc_precision_at_20_max": -0.174597, + "nauc_precision_at_20_std": -0.122044, + "nauc_precision_at_20_diff1": -0.122044, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_3_max": 0.661926, + "nauc_mrr_at_3_std": 0.661926, + "nauc_mrr_at_3_diff1": 0.661926, + "nauc_mrr_at_5_max": 0.661926, + "nauc_mrr_at_5_std": 0.661926, + "nauc_mrr_at_5_diff1": 0.661926, + "nauc_mrr_at_10_max": 0.178642, + "nauc_mrr_at_10_std": 0.459082, + "nauc_mrr_at_10_diff1": 0.459082, + "nauc_mrr_at_20_max": 0.045358, + "nauc_mrr_at_20_std": 0.196593, + "nauc_mrr_at_20_diff1": 0.196593, + "nauc_mrr_at_100_max": 0.00639, + "nauc_mrr_at_100_std": 0.171183, + "nauc_mrr_at_100_diff1": 0.171183, + "nauc_mrr_at_1000_max": 0.00639, + "nauc_mrr_at_1000_std": 0.171183, + "nauc_mrr_at_1000_diff1": 0.171183, + "main_score": 0.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_32768": [ + { + "ndcg_at_1": 0.02, + "ndcg_at_3": 0.02, + "ndcg_at_5": 0.03547, + "ndcg_at_10": 0.04891, + "ndcg_at_20": 0.07954, + "ndcg_at_100": 0.21384, + "ndcg_at_1000": 0.21384, + "map_at_1": 0.02, + "map_at_3": 0.02, + "map_at_5": 0.028, + "map_at_10": 0.03383, + "map_at_20": 0.04241, + "map_at_100": 0.05735, + "map_at_1000": 0.05735, + "recall_at_1": 0.02, + "recall_at_3": 0.02, + "recall_at_5": 0.06, + "recall_at_10": 0.1, + "recall_at_20": 0.22, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.02, + "precision_at_3": 0.00667, + "precision_at_5": 0.012, + "precision_at_10": 0.01, + "precision_at_20": 0.011, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.02, + "mrr_at_3": 0.02, + "mrr_at_5": 0.025, + "mrr_at_10": 0.03569, + "mrr_at_20": 0.044043, + "mrr_at_100": 0.057783, + "mrr_at_1000": 0.057783, + "nauc_ndcg_at_1_max": 1.0, + "nauc_ndcg_at_1_std": 1.0, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 1.0, + "nauc_ndcg_at_3_std": 1.0, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.409942, + "nauc_ndcg_at_5_std": 0.385364, + "nauc_ndcg_at_5_diff1": 0.368978, + "nauc_ndcg_at_10_max": 0.197404, + "nauc_ndcg_at_10_std": 0.260832, + "nauc_ndcg_at_10_diff1": 0.130366, + "nauc_ndcg_at_20_max": 0.183289, + "nauc_ndcg_at_20_std": 0.12851, + "nauc_ndcg_at_20_diff1": 0.08209, + "nauc_ndcg_at_100_max": 0.351569, + "nauc_ndcg_at_100_std": 0.293078, + "nauc_ndcg_at_100_diff1": 0.227396, + "nauc_ndcg_at_1000_max": 0.351569, + "nauc_ndcg_at_1000_std": 0.293078, + "nauc_ndcg_at_1000_diff1": 0.227396, + "nauc_map_at_1_max": 1.0, + "nauc_map_at_1_std": 1.0, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 1.0, + "nauc_map_at_3_std": 1.0, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.613515, + "nauc_map_at_5_std": 0.597417, + "nauc_map_at_5_diff1": 0.586684, + "nauc_map_at_10_max": 0.445688, + "nauc_map_at_10_std": 0.477449, + "nauc_map_at_10_diff1": 0.398238, + "nauc_map_at_20_max": 0.410958, + "nauc_map_at_20_std": 0.390133, + "nauc_map_at_20_diff1": 0.345811, + "nauc_map_at_100_max": 0.456096, + "nauc_map_at_100_std": 0.424518, + "nauc_map_at_100_diff1": 0.36953, + "nauc_map_at_1000_max": 0.456096, + "nauc_map_at_1000_std": 0.424518, + "nauc_map_at_1000_diff1": 0.36953, + "nauc_recall_at_1_max": 1.0, + "nauc_recall_at_1_std": 1.0, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": 1.0, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 0.098202, + "nauc_recall_at_5_std": 0.060639, + "nauc_recall_at_5_diff1": 0.035596, + "nauc_recall_at_10_max": -0.087792, + "nauc_recall_at_10_std": 0.018138, + "nauc_recall_at_10_diff1": -0.176603, + "nauc_recall_at_20_max": 0.00087, + "nauc_recall_at_20_std": -0.09311, + "nauc_recall_at_20_diff1": -0.133791, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 1.0, + "nauc_precision_at_1_std": 1.0, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 0.098202, + "nauc_precision_at_5_std": 0.060639, + "nauc_precision_at_5_diff1": 0.035596, + "nauc_precision_at_10_max": -0.087792, + "nauc_precision_at_10_std": 0.018138, + "nauc_precision_at_10_diff1": -0.176603, + "nauc_precision_at_20_max": 0.00087, + "nauc_precision_at_20_std": -0.09311, + "nauc_precision_at_20_diff1": -0.133791, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 1.0, + "nauc_mrr_at_1_std": 1.0, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 1.0, + "nauc_mrr_at_3_std": 1.0, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.932385, + "nauc_mrr_at_5_std": 0.819694, + "nauc_mrr_at_5_diff1": 0.716984, + "nauc_mrr_at_10_max": 0.518928, + "nauc_mrr_at_10_std": 0.554608, + "nauc_mrr_at_10_diff1": 0.515269, + "nauc_mrr_at_20_max": 0.454974, + "nauc_mrr_at_20_std": 0.486908, + "nauc_mrr_at_20_diff1": 0.46832, + "nauc_mrr_at_100_max": 0.466002, + "nauc_mrr_at_100_std": 0.502053, + "nauc_mrr_at_100_diff1": 0.480473, + "nauc_mrr_at_1000_max": 0.466002, + "nauc_mrr_at_1000_std": 0.502053, + "nauc_mrr_at_1000_diff1": 0.480473, + "main_score": 0.02, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 44.00036978721619, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBPasskeyRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBPasskeyRetrieval.json new file mode 100644 index 0000000000..d897153719 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBPasskeyRetrieval.json @@ -0,0 +1,1201 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBPasskeyRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test_256": [ + { + "ndcg_at_1": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "map_at_1": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "map_at_10": 1.0, + "map_at_20": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "recall_at_1": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 1.0, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "mrr_at_10": 1.0, + "mrr_at_20": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_1000_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "main_score": 1.0, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_512": [ + { + "ndcg_at_1": 0.98, + "ndcg_at_3": 0.99262, + "ndcg_at_5": 0.99262, + "ndcg_at_10": 0.99262, + "ndcg_at_20": 0.99262, + "ndcg_at_100": 0.99262, + "ndcg_at_1000": 0.99262, + "map_at_1": 0.98, + "map_at_3": 0.99, + "map_at_5": 0.99, + "map_at_10": 0.99, + "map_at_20": 0.99, + "map_at_100": 0.99, + "map_at_1000": 0.99, + "recall_at_1": 0.98, + "recall_at_3": 1.0, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.98, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.98, + "mrr_at_3": 0.99, + "mrr_at_5": 0.99, + "mrr_at_10": 0.99, + "mrr_at_20": 0.99, + "mrr_at_100": 0.99, + "mrr_at_1000": 0.99, + "nauc_ndcg_at_1_max": 0.722222, + "nauc_ndcg_at_1_std": -1.739963, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.722222, + "nauc_ndcg_at_3_std": -1.739963, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.722222, + "nauc_ndcg_at_5_std": -1.739963, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.722222, + "nauc_ndcg_at_10_std": -1.739963, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.722222, + "nauc_ndcg_at_20_std": -1.739963, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.722222, + "nauc_ndcg_at_100_std": -1.739963, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.722222, + "nauc_ndcg_at_1000_std": -1.739963, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_map_at_1_max": 0.722222, + "nauc_map_at_1_std": -1.739963, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.722222, + "nauc_map_at_3_std": -1.739963, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.722222, + "nauc_map_at_5_std": -1.739963, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.722222, + "nauc_map_at_10_std": -1.739963, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.722222, + "nauc_map_at_20_std": -1.739963, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.722222, + "nauc_map_at_100_std": -1.739963, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_1000_max": 0.722222, + "nauc_map_at_1000_std": -1.739963, + "nauc_map_at_1000_diff1": 1.0, + "nauc_recall_at_1_max": 0.722222, + "nauc_recall_at_1_std": -1.739963, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.722222, + "nauc_precision_at_1_std": -1.739963, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.722222, + "nauc_mrr_at_1_std": -1.739963, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.722222, + "nauc_mrr_at_3_std": -1.739963, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.722222, + "nauc_mrr_at_5_std": -1.739963, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.722222, + "nauc_mrr_at_10_std": -1.739963, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.722222, + "nauc_mrr_at_20_std": -1.739963, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.722222, + "nauc_mrr_at_100_std": -1.739963, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.722222, + "nauc_mrr_at_1000_std": -1.739963, + "nauc_mrr_at_1000_diff1": 1.0, + "main_score": 0.98, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_1024": [ + { + "ndcg_at_1": 0.62, + "ndcg_at_3": 0.62, + "ndcg_at_5": 0.62, + "ndcg_at_10": 0.62, + "ndcg_at_20": 0.62, + "ndcg_at_100": 0.68115, + "ndcg_at_1000": 0.68115, + "map_at_1": 0.62, + "map_at_3": 0.62, + "map_at_5": 0.62, + "map_at_10": 0.62, + "map_at_20": 0.62, + "map_at_100": 0.62536, + "map_at_1000": 0.62536, + "recall_at_1": 0.62, + "recall_at_3": 0.62, + "recall_at_5": 0.62, + "recall_at_10": 0.62, + "recall_at_20": 0.62, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.62, + "precision_at_3": 0.20667, + "precision_at_5": 0.124, + "precision_at_10": 0.062, + "precision_at_20": 0.031, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.62, + "mrr_at_3": 0.62, + "mrr_at_5": 0.62, + "mrr_at_10": 0.62, + "mrr_at_20": 0.62, + "mrr_at_100": 0.625268, + "mrr_at_1000": 0.625268, + "nauc_ndcg_at_1_max": 0.987078, + "nauc_ndcg_at_1_std": 0.785958, + "nauc_ndcg_at_1_diff1": 0.95865, + "nauc_ndcg_at_3_max": 0.987078, + "nauc_ndcg_at_3_std": 0.785958, + "nauc_ndcg_at_3_diff1": 0.95865, + "nauc_ndcg_at_5_max": 0.987078, + "nauc_ndcg_at_5_std": 0.785958, + "nauc_ndcg_at_5_diff1": 0.95865, + "nauc_ndcg_at_10_max": 0.987078, + "nauc_ndcg_at_10_std": 0.785958, + "nauc_ndcg_at_10_diff1": 0.95865, + "nauc_ndcg_at_20_max": 0.987078, + "nauc_ndcg_at_20_std": 0.785958, + "nauc_ndcg_at_20_diff1": 0.95865, + "nauc_ndcg_at_100_max": 0.985044, + "nauc_ndcg_at_100_std": 0.784658, + "nauc_ndcg_at_100_diff1": 0.956011, + "nauc_ndcg_at_1000_max": 0.985044, + "nauc_ndcg_at_1000_std": 0.784658, + "nauc_ndcg_at_1000_diff1": 0.956011, + "nauc_map_at_1_max": 0.987078, + "nauc_map_at_1_std": 0.785958, + "nauc_map_at_1_diff1": 0.95865, + "nauc_map_at_3_max": 0.987078, + "nauc_map_at_3_std": 0.785958, + "nauc_map_at_3_diff1": 0.95865, + "nauc_map_at_5_max": 0.987078, + "nauc_map_at_5_std": 0.785958, + "nauc_map_at_5_diff1": 0.95865, + "nauc_map_at_10_max": 0.987078, + "nauc_map_at_10_std": 0.785958, + "nauc_map_at_10_diff1": 0.95865, + "nauc_map_at_20_max": 0.987078, + "nauc_map_at_20_std": 0.785958, + "nauc_map_at_20_diff1": 0.95865, + "nauc_map_at_100_max": 0.986294, + "nauc_map_at_100_std": 0.785403, + "nauc_map_at_100_diff1": 0.957676, + "nauc_map_at_1000_max": 0.986294, + "nauc_map_at_1000_std": 0.785403, + "nauc_map_at_1000_diff1": 0.957676, + "nauc_recall_at_1_max": 0.987078, + "nauc_recall_at_1_std": 0.785958, + "nauc_recall_at_1_diff1": 0.95865, + "nauc_recall_at_3_max": 0.987078, + "nauc_recall_at_3_std": 0.785958, + "nauc_recall_at_3_diff1": 0.95865, + "nauc_recall_at_5_max": 0.987078, + "nauc_recall_at_5_std": 0.785958, + "nauc_recall_at_5_diff1": 0.95865, + "nauc_recall_at_10_max": 0.987078, + "nauc_recall_at_10_std": 0.785958, + "nauc_recall_at_10_diff1": 0.95865, + "nauc_recall_at_20_max": 0.987078, + "nauc_recall_at_20_std": 0.785958, + "nauc_recall_at_20_diff1": 0.95865, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.987078, + "nauc_precision_at_1_std": 0.785958, + "nauc_precision_at_1_diff1": 0.95865, + "nauc_precision_at_3_max": 0.987078, + "nauc_precision_at_3_std": 0.785958, + "nauc_precision_at_3_diff1": 0.95865, + "nauc_precision_at_5_max": 0.987078, + "nauc_precision_at_5_std": 0.785958, + "nauc_precision_at_5_diff1": 0.95865, + "nauc_precision_at_10_max": 0.987078, + "nauc_precision_at_10_std": 0.785958, + "nauc_precision_at_10_diff1": 0.95865, + "nauc_precision_at_20_max": 0.987078, + "nauc_precision_at_20_std": 0.785958, + "nauc_precision_at_20_diff1": 0.95865, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.987078, + "nauc_mrr_at_1_std": 0.785958, + "nauc_mrr_at_1_diff1": 0.95865, + "nauc_mrr_at_3_max": 0.987078, + "nauc_mrr_at_3_std": 0.785958, + "nauc_mrr_at_3_diff1": 0.95865, + "nauc_mrr_at_5_max": 0.987078, + "nauc_mrr_at_5_std": 0.785958, + "nauc_mrr_at_5_diff1": 0.95865, + "nauc_mrr_at_10_max": 0.987078, + "nauc_mrr_at_10_std": 0.785958, + "nauc_mrr_at_10_diff1": 0.95865, + "nauc_mrr_at_20_max": 0.987078, + "nauc_mrr_at_20_std": 0.785958, + "nauc_mrr_at_20_diff1": 0.95865, + "nauc_mrr_at_100_max": 0.986093, + "nauc_mrr_at_100_std": 0.784939, + "nauc_mrr_at_100_diff1": 0.95779, + "nauc_mrr_at_1000_max": 0.986093, + "nauc_mrr_at_1000_std": 0.784939, + "nauc_mrr_at_1000_diff1": 0.95779, + "main_score": 0.62, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_2048": [ + { + "ndcg_at_1": 0.22, + "ndcg_at_3": 0.22, + "ndcg_at_5": 0.22, + "ndcg_at_10": 0.22, + "ndcg_at_20": 0.22, + "ndcg_at_100": 0.3547, + "ndcg_at_1000": 0.3547, + "map_at_1": 0.22, + "map_at_3": 0.22, + "map_at_5": 0.22, + "map_at_10": 0.22, + "map_at_20": 0.22, + "map_at_100": 0.23513, + "map_at_1000": 0.23513, + "recall_at_1": 0.22, + "recall_at_3": 0.22, + "recall_at_5": 0.22, + "recall_at_10": 0.22, + "recall_at_20": 0.22, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.22, + "precision_at_3": 0.07333, + "precision_at_5": 0.044, + "precision_at_10": 0.022, + "precision_at_20": 0.011, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.22, + "mrr_at_3": 0.22, + "mrr_at_5": 0.22, + "mrr_at_10": 0.22, + "mrr_at_20": 0.22, + "mrr_at_100": 0.235179, + "mrr_at_1000": 0.235179, + "nauc_ndcg_at_1_max": 0.875253, + "nauc_ndcg_at_1_std": 0.638437, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.875253, + "nauc_ndcg_at_3_std": 0.638437, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.875253, + "nauc_ndcg_at_5_std": 0.638437, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.875253, + "nauc_ndcg_at_10_std": 0.638437, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.875253, + "nauc_ndcg_at_20_std": 0.638437, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.857083, + "nauc_ndcg_at_100_std": 0.6303, + "nauc_ndcg_at_100_diff1": 0.97607, + "nauc_ndcg_at_1000_max": 0.857083, + "nauc_ndcg_at_1000_std": 0.6303, + "nauc_ndcg_at_1000_diff1": 0.97607, + "nauc_map_at_1_max": 0.875253, + "nauc_map_at_1_std": 0.638437, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.875253, + "nauc_map_at_3_std": 0.638437, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.875253, + "nauc_map_at_5_std": 0.638437, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 0.875253, + "nauc_map_at_10_std": 0.638437, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.875253, + "nauc_map_at_20_std": 0.638437, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_100_max": 0.86828, + "nauc_map_at_100_std": 0.634848, + "nauc_map_at_100_diff1": 0.990495, + "nauc_map_at_1000_max": 0.86828, + "nauc_map_at_1000_std": 0.634848, + "nauc_map_at_1000_diff1": 0.990495, + "nauc_recall_at_1_max": 0.875253, + "nauc_recall_at_1_std": 0.638437, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 0.875253, + "nauc_recall_at_3_std": 0.638437, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 0.875253, + "nauc_recall_at_5_std": 0.638437, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 0.875253, + "nauc_recall_at_10_std": 0.638437, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": 0.875253, + "nauc_recall_at_20_std": 0.638437, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.875253, + "nauc_precision_at_1_std": 0.638437, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 0.875253, + "nauc_precision_at_3_std": 0.638437, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 0.875253, + "nauc_precision_at_5_std": 0.638437, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 0.875253, + "nauc_precision_at_10_std": 0.638437, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 0.875253, + "nauc_precision_at_20_std": 0.638437, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.875253, + "nauc_mrr_at_1_std": 0.638437, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.875253, + "nauc_mrr_at_3_std": 0.638437, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 0.875253, + "nauc_mrr_at_5_std": 0.638437, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.875253, + "nauc_mrr_at_10_std": 0.638437, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.875253, + "nauc_mrr_at_20_std": 0.638437, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_100_max": 0.863676, + "nauc_mrr_at_100_std": 0.627329, + "nauc_mrr_at_100_diff1": 0.98893, + "nauc_mrr_at_1000_max": 0.863676, + "nauc_mrr_at_1000_std": 0.627329, + "nauc_mrr_at_1000_diff1": 0.98893, + "main_score": 0.22, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_4096": [ + { + "ndcg_at_1": 0.14, + "ndcg_at_3": 0.14, + "ndcg_at_5": 0.14, + "ndcg_at_10": 0.14, + "ndcg_at_20": 0.14992, + "ndcg_at_100": 0.2913, + "ndcg_at_1000": 0.2913, + "map_at_1": 0.14, + "map_at_3": 0.14, + "map_at_5": 0.14, + "map_at_10": 0.14, + "map_at_20": 0.14261, + "map_at_100": 0.15827, + "map_at_1000": 0.15827, + "recall_at_1": 0.14, + "recall_at_3": 0.14, + "recall_at_5": 0.14, + "recall_at_10": 0.14, + "recall_at_20": 0.18, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.14, + "precision_at_3": 0.04667, + "precision_at_5": 0.028, + "precision_at_10": 0.014, + "precision_at_20": 0.009, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.14, + "mrr_at_3": 0.14, + "mrr_at_5": 0.14, + "mrr_at_10": 0.14, + "mrr_at_20": 0.14225, + "mrr_at_100": 0.158757, + "mrr_at_1000": 0.158757, + "nauc_ndcg_at_1_max": 1.0, + "nauc_ndcg_at_1_std": 0.6436, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 1.0, + "nauc_ndcg_at_3_std": 0.6436, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 1.0, + "nauc_ndcg_at_5_std": 0.6436, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_10_max": 1.0, + "nauc_ndcg_at_10_std": 0.6436, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.960584, + "nauc_ndcg_at_20_std": 0.59705, + "nauc_ndcg_at_20_diff1": 0.926009, + "nauc_ndcg_at_100_max": 0.946663, + "nauc_ndcg_at_100_std": 0.592072, + "nauc_ndcg_at_100_diff1": 0.938396, + "nauc_ndcg_at_1000_max": 0.946663, + "nauc_ndcg_at_1000_std": 0.592072, + "nauc_ndcg_at_1000_diff1": 0.938396, + "nauc_map_at_1_max": 1.0, + "nauc_map_at_1_std": 0.6436, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 1.0, + "nauc_map_at_3_std": 0.6436, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 1.0, + "nauc_map_at_5_std": 0.6436, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_10_max": 1.0, + "nauc_map_at_10_std": 0.6436, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_20_max": 0.989511, + "nauc_map_at_20_std": 0.630747, + "nauc_map_at_20_diff1": 0.979498, + "nauc_map_at_100_max": 0.976504, + "nauc_map_at_100_std": 0.619834, + "nauc_map_at_100_diff1": 0.970728, + "nauc_map_at_1000_max": 0.976504, + "nauc_map_at_1000_std": 0.619834, + "nauc_map_at_1000_diff1": 0.970728, + "nauc_recall_at_1_max": 1.0, + "nauc_recall_at_1_std": 0.6436, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": 0.6436, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": 0.6436, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": 0.6436, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_20_max": 0.860015, + "nauc_recall_at_20_std": 0.481318, + "nauc_recall_at_20_diff1": 0.742518, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 1.0, + "nauc_precision_at_1_std": 0.6436, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 0.6436, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 0.6436, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 0.6436, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 0.860015, + "nauc_precision_at_20_std": 0.481318, + "nauc_precision_at_20_diff1": 0.742518, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 1.0, + "nauc_mrr_at_1_std": 0.6436, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 1.0, + "nauc_mrr_at_3_std": 0.6436, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 1.0, + "nauc_mrr_at_5_std": 0.6436, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 1.0, + "nauc_mrr_at_10_std": 0.6436, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_20_max": 0.980759, + "nauc_mrr_at_20_std": 0.630509, + "nauc_mrr_at_20_diff1": 0.979816, + "nauc_mrr_at_100_max": 0.97038, + "nauc_mrr_at_100_std": 0.621888, + "nauc_mrr_at_100_diff1": 0.965303, + "nauc_mrr_at_1000_max": 0.97038, + "nauc_mrr_at_1000_std": 0.621888, + "nauc_mrr_at_1000_diff1": 0.965303, + "main_score": 0.14, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_8192": [ + { + "ndcg_at_1": 0.04, + "ndcg_at_3": 0.04, + "ndcg_at_5": 0.04861, + "ndcg_at_10": 0.06176, + "ndcg_at_20": 0.0876, + "ndcg_at_100": 0.2272, + "ndcg_at_1000": 0.2272, + "map_at_1": 0.04, + "map_at_3": 0.04, + "map_at_5": 0.045, + "map_at_10": 0.05056, + "map_at_20": 0.05792, + "map_at_100": 0.07408, + "map_at_1000": 0.07408, + "recall_at_1": 0.04, + "recall_at_3": 0.04, + "recall_at_5": 0.06, + "recall_at_10": 0.1, + "recall_at_20": 0.2, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.04, + "precision_at_3": 0.01333, + "precision_at_5": 0.012, + "precision_at_10": 0.01, + "precision_at_20": 0.01, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.04, + "mrr_at_3": 0.04, + "mrr_at_5": 0.04, + "mrr_at_10": 0.04869, + "mrr_at_20": 0.055932, + "mrr_at_100": 0.071384, + "mrr_at_1000": 0.071384, + "nauc_ndcg_at_1_max": 1.0, + "nauc_ndcg_at_1_std": 1.0, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 1.0, + "nauc_ndcg_at_3_std": 1.0, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.766385, + "nauc_ndcg_at_5_std": 0.720984, + "nauc_ndcg_at_5_diff1": 0.720984, + "nauc_ndcg_at_10_max": 0.598099, + "nauc_ndcg_at_10_std": 0.608318, + "nauc_ndcg_at_10_diff1": 0.529247, + "nauc_ndcg_at_20_max": 0.567782, + "nauc_ndcg_at_20_std": 0.531299, + "nauc_ndcg_at_20_diff1": 0.563992, + "nauc_ndcg_at_100_max": 0.645974, + "nauc_ndcg_at_100_std": 0.645694, + "nauc_ndcg_at_100_diff1": 0.619507, + "nauc_ndcg_at_1000_max": 0.645974, + "nauc_ndcg_at_1000_std": 0.645694, + "nauc_ndcg_at_1000_diff1": 0.619507, + "nauc_map_at_1_max": 1.0, + "nauc_map_at_1_std": 1.0, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 1.0, + "nauc_map_at_3_std": 1.0, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_5_max": 0.853501, + "nauc_map_at_5_std": 0.825031, + "nauc_map_at_5_diff1": 0.825031, + "nauc_map_at_10_max": 0.752946, + "nauc_map_at_10_std": 0.749895, + "nauc_map_at_10_diff1": 0.711328, + "nauc_map_at_20_max": 0.732694, + "nauc_map_at_20_std": 0.710663, + "nauc_map_at_20_diff1": 0.712749, + "nauc_map_at_100_max": 0.73839, + "nauc_map_at_100_std": 0.729309, + "nauc_map_at_100_diff1": 0.712369, + "nauc_map_at_1000_max": 0.73839, + "nauc_map_at_1000_std": 0.729309, + "nauc_map_at_1000_diff1": 0.712369, + "nauc_recall_at_1_max": 1.0, + "nauc_recall_at_1_std": 1.0, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": 1.0, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_5_max": 0.560504, + "nauc_recall_at_5_std": 0.475092, + "nauc_recall_at_5_diff1": 0.475092, + "nauc_recall_at_10_max": 0.337376, + "nauc_recall_at_10_std": 0.376281, + "nauc_recall_at_10_diff1": 0.221733, + "nauc_recall_at_20_max": 0.372102, + "nauc_recall_at_20_std": 0.316855, + "nauc_recall_at_20_diff1": 0.400161, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 1.0, + "nauc_precision_at_1_std": 1.0, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_5_max": 0.560504, + "nauc_precision_at_5_std": 0.475092, + "nauc_precision_at_5_diff1": 0.475092, + "nauc_precision_at_10_max": 0.337376, + "nauc_precision_at_10_std": 0.376281, + "nauc_precision_at_10_diff1": 0.221733, + "nauc_precision_at_20_max": 0.372102, + "nauc_precision_at_20_std": 0.316855, + "nauc_precision_at_20_diff1": 0.400161, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 1.0, + "nauc_mrr_at_1_std": 1.0, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 1.0, + "nauc_mrr_at_3_std": 1.0, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_5_max": 1.0, + "nauc_mrr_at_5_std": 1.0, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_10_max": 0.9061, + "nauc_mrr_at_10_std": 0.787548, + "nauc_mrr_at_10_diff1": 0.873532, + "nauc_mrr_at_20_max": 0.765183, + "nauc_mrr_at_20_std": 0.700313, + "nauc_mrr_at_20_diff1": 0.74019, + "nauc_mrr_at_100_max": 0.771052, + "nauc_mrr_at_100_std": 0.700982, + "nauc_mrr_at_100_diff1": 0.745436, + "nauc_mrr_at_1000_max": 0.771052, + "nauc_mrr_at_1000_std": 0.700982, + "nauc_mrr_at_1000_diff1": 0.745436, + "main_score": 0.04, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_16384": [ + { + "ndcg_at_1": 0.04, + "ndcg_at_3": 0.04, + "ndcg_at_5": 0.04, + "ndcg_at_10": 0.05245, + "ndcg_at_20": 0.08346, + "ndcg_at_100": 0.22331, + "ndcg_at_1000": 0.22331, + "map_at_1": 0.04, + "map_at_3": 0.04, + "map_at_5": 0.04, + "map_at_10": 0.04486, + "map_at_20": 0.05373, + "map_at_100": 0.07004, + "map_at_1000": 0.07004, + "recall_at_1": 0.04, + "recall_at_3": 0.04, + "recall_at_5": 0.04, + "recall_at_10": 0.08, + "recall_at_20": 0.2, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.04, + "precision_at_3": 0.01333, + "precision_at_5": 0.008, + "precision_at_10": 0.008, + "precision_at_20": 0.01, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.04, + "mrr_at_3": 0.04, + "mrr_at_5": 0.045, + "mrr_at_10": 0.056524, + "mrr_at_20": 0.061397, + "mrr_at_100": 0.07608, + "mrr_at_1000": 0.07608, + "nauc_ndcg_at_1_max": 1.0, + "nauc_ndcg_at_1_std": 1.0, + "nauc_ndcg_at_1_diff1": 0.549235, + "nauc_ndcg_at_3_max": 1.0, + "nauc_ndcg_at_3_std": 1.0, + "nauc_ndcg_at_3_diff1": 0.549235, + "nauc_ndcg_at_5_max": 1.0, + "nauc_ndcg_at_5_std": 1.0, + "nauc_ndcg_at_5_diff1": 0.549235, + "nauc_ndcg_at_10_max": 0.731006, + "nauc_ndcg_at_10_std": 0.728724, + "nauc_ndcg_at_10_diff1": 0.42834, + "nauc_ndcg_at_20_max": 0.502764, + "nauc_ndcg_at_20_std": 0.499569, + "nauc_ndcg_at_20_diff1": 0.368146, + "nauc_ndcg_at_100_max": 0.605276, + "nauc_ndcg_at_100_std": 0.648179, + "nauc_ndcg_at_100_diff1": 0.38635, + "nauc_ndcg_at_1000_max": 0.605276, + "nauc_ndcg_at_1000_std": 0.648179, + "nauc_ndcg_at_1000_diff1": 0.38635, + "nauc_map_at_1_max": 1.0, + "nauc_map_at_1_std": 1.0, + "nauc_map_at_1_diff1": 0.549235, + "nauc_map_at_3_max": 1.0, + "nauc_map_at_3_std": 1.0, + "nauc_map_at_3_diff1": 0.549235, + "nauc_map_at_5_max": 1.0, + "nauc_map_at_5_std": 1.0, + "nauc_map_at_5_diff1": 0.549235, + "nauc_map_at_10_max": 0.878049, + "nauc_map_at_10_std": 0.875465, + "nauc_map_at_10_diff1": 0.487455, + "nauc_map_at_20_max": 0.763777, + "nauc_map_at_20_std": 0.762499, + "nauc_map_at_20_diff1": 0.454389, + "nauc_map_at_100_max": 0.754017, + "nauc_map_at_100_std": 0.775099, + "nauc_map_at_100_diff1": 0.444153, + "nauc_map_at_1000_max": 0.754017, + "nauc_map_at_1000_std": 0.775099, + "nauc_map_at_1000_diff1": 0.444153, + "nauc_recall_at_1_max": 1.0, + "nauc_recall_at_1_std": 1.0, + "nauc_recall_at_1_diff1": 0.549235, + "nauc_recall_at_3_max": 1.0, + "nauc_recall_at_3_std": 1.0, + "nauc_recall_at_3_diff1": 0.549235, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": 1.0, + "nauc_recall_at_5_diff1": 0.549235, + "nauc_recall_at_10_max": 0.43091, + "nauc_recall_at_10_std": 0.43091, + "nauc_recall_at_10_diff1": 0.3152, + "nauc_recall_at_20_max": 0.189506, + "nauc_recall_at_20_std": 0.183177, + "nauc_recall_at_20_diff1": 0.270197, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 1.0, + "nauc_precision_at_1_std": 1.0, + "nauc_precision_at_1_diff1": 0.549235, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_3_diff1": 0.549235, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_precision_at_5_diff1": 0.549235, + "nauc_precision_at_10_max": 0.43091, + "nauc_precision_at_10_std": 0.43091, + "nauc_precision_at_10_diff1": 0.3152, + "nauc_precision_at_20_max": 0.189506, + "nauc_precision_at_20_std": 0.183177, + "nauc_precision_at_20_diff1": 0.270197, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 1.0, + "nauc_mrr_at_1_std": 1.0, + "nauc_mrr_at_1_diff1": 0.549235, + "nauc_mrr_at_3_max": 1.0, + "nauc_mrr_at_3_std": 1.0, + "nauc_mrr_at_3_diff1": 0.549235, + "nauc_mrr_at_5_max": 0.881048, + "nauc_mrr_at_5_std": 0.833378, + "nauc_mrr_at_5_diff1": 0.480368, + "nauc_mrr_at_10_max": 0.707464, + "nauc_mrr_at_10_std": 0.629014, + "nauc_mrr_at_10_diff1": 0.327693, + "nauc_mrr_at_20_max": 0.6734, + "nauc_mrr_at_20_std": 0.597308, + "nauc_mrr_at_20_diff1": 0.310223, + "nauc_mrr_at_100_max": 0.707687, + "nauc_mrr_at_100_std": 0.626357, + "nauc_mrr_at_100_diff1": 0.332012, + "nauc_mrr_at_1000_max": 0.707687, + "nauc_mrr_at_1000_std": 0.626357, + "nauc_mrr_at_1000_diff1": 0.332012, + "main_score": 0.04, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test_32768": [ + { + "ndcg_at_1": 0.02, + "ndcg_at_3": 0.04262, + "ndcg_at_5": 0.05123, + "ndcg_at_10": 0.07133, + "ndcg_at_20": 0.10117, + "ndcg_at_100": 0.23044, + "ndcg_at_1000": 0.23044, + "map_at_1": 0.02, + "map_at_3": 0.03667, + "map_at_5": 0.04167, + "map_at_10": 0.05036, + "map_at_20": 0.05826, + "map_at_100": 0.07334, + "map_at_1000": 0.07334, + "recall_at_1": 0.02, + "recall_at_3": 0.06, + "recall_at_5": 0.08, + "recall_at_10": 0.14, + "recall_at_20": 0.26, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.02, + "precision_at_3": 0.02, + "precision_at_5": 0.016, + "precision_at_10": 0.014, + "precision_at_20": 0.013, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.02, + "mrr_at_3": 0.026667, + "mrr_at_5": 0.026667, + "mrr_at_10": 0.037079, + "mrr_at_20": 0.039256, + "mrr_at_100": 0.056215, + "mrr_at_1000": 0.056215, + "nauc_ndcg_at_1_max": 1.0, + "nauc_ndcg_at_1_std": 1.0, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.590534, + "nauc_ndcg_at_3_std": 0.16426, + "nauc_ndcg_at_3_diff1": 0.16426, + "nauc_ndcg_at_5_max": 0.407253, + "nauc_ndcg_at_5_std": 0.066857, + "nauc_ndcg_at_5_diff1": 0.066857, + "nauc_ndcg_at_10_max": 0.23015, + "nauc_ndcg_at_10_std": -0.042341, + "nauc_ndcg_at_10_diff1": -0.042341, + "nauc_ndcg_at_20_max": 0.142312, + "nauc_ndcg_at_20_std": -0.03832, + "nauc_ndcg_at_20_diff1": -0.03832, + "nauc_ndcg_at_100_max": 0.240123, + "nauc_ndcg_at_100_std": 0.039992, + "nauc_ndcg_at_100_diff1": 0.039992, + "nauc_ndcg_at_1000_max": 0.240123, + "nauc_ndcg_at_1000_std": 0.039992, + "nauc_ndcg_at_1000_diff1": 0.039992, + "nauc_map_at_1_max": 1.0, + "nauc_map_at_1_std": 1.0, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_3_max": 0.668073, + "nauc_map_at_3_std": 0.284216, + "nauc_map_at_3_diff1": 0.284216, + "nauc_map_at_5_max": 0.527953, + "nauc_map_at_5_std": 0.200301, + "nauc_map_at_5_diff1": 0.200301, + "nauc_map_at_10_max": 0.405519, + "nauc_map_at_10_std": 0.11656, + "nauc_map_at_10_diff1": 0.11656, + "nauc_map_at_20_max": 0.355028, + "nauc_map_at_20_std": 0.104023, + "nauc_map_at_20_diff1": 0.104023, + "nauc_map_at_100_max": 0.362833, + "nauc_map_at_100_std": 0.121316, + "nauc_map_at_100_diff1": 0.121316, + "nauc_map_at_1000_max": 0.362833, + "nauc_map_at_1000_std": 0.121316, + "nauc_map_at_1000_diff1": 0.121316, + "nauc_recall_at_1_max": 1.0, + "nauc_recall_at_1_std": 1.0, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_3_max": 0.447813, + "nauc_recall_at_3_std": -0.049817, + "nauc_recall_at_3_diff1": -0.049817, + "nauc_recall_at_5_max": 0.210961, + "nauc_recall_at_5_std": -0.141132, + "nauc_recall_at_5_diff1": -0.141132, + "nauc_recall_at_10_max": 0.002715, + "nauc_recall_at_10_std": -0.237812, + "nauc_recall_at_10_diff1": -0.237812, + "nauc_recall_at_20_max": -0.07924, + "nauc_recall_at_20_std": -0.164031, + "nauc_recall_at_20_diff1": -0.164031, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 1.0, + "nauc_precision_at_1_std": 1.0, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_3_max": 0.447813, + "nauc_precision_at_3_std": -0.049817, + "nauc_precision_at_3_diff1": -0.049817, + "nauc_precision_at_5_max": 0.210961, + "nauc_precision_at_5_std": -0.141132, + "nauc_precision_at_5_diff1": -0.141132, + "nauc_precision_at_10_max": 0.002715, + "nauc_precision_at_10_std": -0.237812, + "nauc_precision_at_10_diff1": -0.237812, + "nauc_precision_at_20_max": -0.07924, + "nauc_precision_at_20_std": -0.164031, + "nauc_precision_at_20_diff1": -0.164031, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 1.0, + "nauc_mrr_at_1_std": 1.0, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_3_max": 0.915482, + "nauc_mrr_at_3_std": 0.830963, + "nauc_mrr_at_3_diff1": 0.830963, + "nauc_mrr_at_5_max": 0.915482, + "nauc_mrr_at_5_std": 0.830963, + "nauc_mrr_at_5_diff1": 0.830963, + "nauc_mrr_at_10_max": 0.727862, + "nauc_mrr_at_10_std": 0.675843, + "nauc_mrr_at_10_diff1": 0.675843, + "nauc_mrr_at_20_max": 0.687224, + "nauc_mrr_at_20_std": 0.632718, + "nauc_mrr_at_20_diff1": 0.632718, + "nauc_mrr_at_100_max": 0.701097, + "nauc_mrr_at_100_std": 0.628826, + "nauc_mrr_at_100_diff1": 0.628826, + "nauc_mrr_at_1000_max": 0.701097, + "nauc_mrr_at_1000_std": 0.628826, + "nauc_mrr_at_1000_diff1": 0.628826, + "main_score": 0.02, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 42.152791023254395, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBQMSumRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBQMSumRetrieval.json new file mode 100644 index 0000000000..5684c6539e --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBQMSumRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBQMSumRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.13949, + "ndcg_at_3": 0.21099, + "ndcg_at_5": 0.23202, + "ndcg_at_10": 0.262, + "ndcg_at_20": 0.28791, + "ndcg_at_100": 0.36307, + "ndcg_at_1000": 0.37303, + "map_at_1": 0.13949, + "map_at_3": 0.19319, + "map_at_5": 0.20485, + "map_at_10": 0.2173, + "map_at_20": 0.22436, + "map_at_100": 0.23363, + "map_at_1000": 0.23421, + "recall_at_1": 0.13949, + "recall_at_3": 0.26261, + "recall_at_5": 0.31369, + "recall_at_10": 0.40602, + "recall_at_20": 0.50884, + "recall_at_100": 0.93124, + "recall_at_1000": 1.0, + "precision_at_1": 0.13949, + "precision_at_3": 0.08754, + "precision_at_5": 0.06274, + "precision_at_10": 0.0406, + "precision_at_20": 0.02544, + "precision_at_100": 0.00931, + "precision_at_1000": 0.001, + "mrr_at_1": 0.139489, + "mrr_at_3": 0.193189, + "mrr_at_5": 0.204846, + "mrr_at_10": 0.217303, + "mrr_at_20": 0.224363, + "mrr_at_100": 0.233629, + "mrr_at_1000": 0.234212, + "nauc_ndcg_at_1_max": 0.086677, + "nauc_ndcg_at_1_std": -0.288895, + "nauc_ndcg_at_1_diff1": 0.43576, + "nauc_ndcg_at_3_max": 0.066158, + "nauc_ndcg_at_3_std": -0.31929, + "nauc_ndcg_at_3_diff1": 0.362359, + "nauc_ndcg_at_5_max": 0.050971, + "nauc_ndcg_at_5_std": -0.327863, + "nauc_ndcg_at_5_diff1": 0.342879, + "nauc_ndcg_at_10_max": 0.044503, + "nauc_ndcg_at_10_std": -0.339871, + "nauc_ndcg_at_10_diff1": 0.332315, + "nauc_ndcg_at_20_max": 0.02903, + "nauc_ndcg_at_20_std": -0.34297, + "nauc_ndcg_at_20_diff1": 0.330452, + "nauc_ndcg_at_100_max": 0.044628, + "nauc_ndcg_at_100_std": -0.32411, + "nauc_ndcg_at_100_diff1": 0.333634, + "nauc_ndcg_at_1000_max": 0.044242, + "nauc_ndcg_at_1000_std": -0.33051, + "nauc_ndcg_at_1000_diff1": 0.343774, + "nauc_map_at_1_max": 0.086677, + "nauc_map_at_1_std": -0.288895, + "nauc_map_at_1_diff1": 0.43576, + "nauc_map_at_3_max": 0.070867, + "nauc_map_at_3_std": -0.312631, + "nauc_map_at_3_diff1": 0.375345, + "nauc_map_at_5_max": 0.06208, + "nauc_map_at_5_std": -0.317666, + "nauc_map_at_5_diff1": 0.363482, + "nauc_map_at_10_max": 0.059131, + "nauc_map_at_10_std": -0.323225, + "nauc_map_at_10_diff1": 0.358427, + "nauc_map_at_20_max": 0.054605, + "nauc_map_at_20_std": -0.324041, + "nauc_map_at_20_diff1": 0.358401, + "nauc_map_at_100_max": 0.055218, + "nauc_map_at_100_std": -0.322608, + "nauc_map_at_100_diff1": 0.358448, + "nauc_map_at_1000_max": 0.055315, + "nauc_map_at_1000_std": -0.322823, + "nauc_map_at_1000_diff1": 0.35899, + "nauc_recall_at_1_max": 0.086677, + "nauc_recall_at_1_std": -0.288895, + "nauc_recall_at_1_diff1": 0.43576, + "nauc_recall_at_3_max": 0.05473, + "nauc_recall_at_3_std": -0.335358, + "nauc_recall_at_3_diff1": 0.331976, + "nauc_recall_at_5_max": 0.0235, + "nauc_recall_at_5_std": -0.352357, + "nauc_recall_at_5_diff1": 0.294453, + "nauc_recall_at_10_max": 0.008108, + "nauc_recall_at_10_std": -0.381292, + "nauc_recall_at_10_diff1": 0.270395, + "nauc_recall_at_20_max": -0.044868, + "nauc_recall_at_20_std": -0.393627, + "nauc_recall_at_20_diff1": 0.259077, + "nauc_recall_at_100_max": 0.065884, + "nauc_recall_at_100_std": -0.158524, + "nauc_recall_at_100_diff1": 0.089768, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.086677, + "nauc_precision_at_1_std": -0.288895, + "nauc_precision_at_1_diff1": 0.43576, + "nauc_precision_at_3_max": 0.05473, + "nauc_precision_at_3_std": -0.335358, + "nauc_precision_at_3_diff1": 0.331976, + "nauc_precision_at_5_max": 0.0235, + "nauc_precision_at_5_std": -0.352357, + "nauc_precision_at_5_diff1": 0.294453, + "nauc_precision_at_10_max": 0.008108, + "nauc_precision_at_10_std": -0.381292, + "nauc_precision_at_10_diff1": 0.270395, + "nauc_precision_at_20_max": -0.044868, + "nauc_precision_at_20_std": -0.393627, + "nauc_precision_at_20_diff1": 0.259077, + "nauc_precision_at_100_max": 0.065884, + "nauc_precision_at_100_std": -0.158524, + "nauc_precision_at_100_diff1": 0.089768, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.086677, + "nauc_mrr_at_1_std": -0.288895, + "nauc_mrr_at_1_diff1": 0.43576, + "nauc_mrr_at_3_max": 0.070867, + "nauc_mrr_at_3_std": -0.312631, + "nauc_mrr_at_3_diff1": 0.375345, + "nauc_mrr_at_5_max": 0.06208, + "nauc_mrr_at_5_std": -0.317666, + "nauc_mrr_at_5_diff1": 0.363482, + "nauc_mrr_at_10_max": 0.059131, + "nauc_mrr_at_10_std": -0.323225, + "nauc_mrr_at_10_diff1": 0.358427, + "nauc_mrr_at_20_max": 0.054605, + "nauc_mrr_at_20_std": -0.324041, + "nauc_mrr_at_20_diff1": 0.358401, + "nauc_mrr_at_100_max": 0.055218, + "nauc_mrr_at_100_std": -0.322608, + "nauc_mrr_at_100_diff1": 0.358448, + "nauc_mrr_at_1000_max": 0.055315, + "nauc_mrr_at_1000_std": -0.322823, + "nauc_mrr_at_1000_diff1": 0.35899, + "main_score": 0.262, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 17.63444948196411, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBSummScreenFDRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBSummScreenFDRetrieval.json new file mode 100644 index 0000000000..7db983cd2c --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBSummScreenFDRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBSummScreenFDRetrieval", + "mteb_version": "1.38.9", + "scores": { + "validation": [ + { + "ndcg_at_1": 0.6131, + "ndcg_at_3": 0.72016, + "ndcg_at_5": 0.74463, + "ndcg_at_10": 0.77406, + "ndcg_at_20": 0.77954, + "ndcg_at_100": 0.78472, + "ndcg_at_1000": 0.7851, + "map_at_1": 0.6131, + "map_at_3": 0.69345, + "map_at_5": 0.70699, + "map_at_10": 0.7189, + "map_at_20": 0.72051, + "map_at_100": 0.72129, + "map_at_1000": 0.7213, + "recall_at_1": 0.6131, + "recall_at_3": 0.79762, + "recall_at_5": 0.85714, + "recall_at_10": 0.9494, + "recall_at_20": 0.97024, + "recall_at_100": 0.99702, + "recall_at_1000": 1.0, + "precision_at_1": 0.6131, + "precision_at_3": 0.26587, + "precision_at_5": 0.17143, + "precision_at_10": 0.09494, + "precision_at_20": 0.04851, + "precision_at_100": 0.00997, + "precision_at_1000": 0.001, + "mrr_at_1": 0.613095, + "mrr_at_3": 0.693452, + "mrr_at_5": 0.706994, + "mrr_at_10": 0.718896, + "mrr_at_20": 0.720512, + "mrr_at_100": 0.721291, + "mrr_at_1000": 0.721305, + "nauc_ndcg_at_1_max": 0.432259, + "nauc_ndcg_at_1_std": -0.199492, + "nauc_ndcg_at_1_diff1": 0.746668, + "nauc_ndcg_at_3_max": 0.427443, + "nauc_ndcg_at_3_std": -0.169794, + "nauc_ndcg_at_3_diff1": 0.701194, + "nauc_ndcg_at_5_max": 0.449628, + "nauc_ndcg_at_5_std": -0.151392, + "nauc_ndcg_at_5_diff1": 0.70863, + "nauc_ndcg_at_10_max": 0.449905, + "nauc_ndcg_at_10_std": -0.154771, + "nauc_ndcg_at_10_diff1": 0.711975, + "nauc_ndcg_at_20_max": 0.452623, + "nauc_ndcg_at_20_std": -0.154624, + "nauc_ndcg_at_20_diff1": 0.716914, + "nauc_ndcg_at_100_max": 0.446339, + "nauc_ndcg_at_100_std": -0.168661, + "nauc_ndcg_at_100_diff1": 0.716451, + "nauc_ndcg_at_1000_max": 0.445161, + "nauc_ndcg_at_1000_std": -0.168653, + "nauc_ndcg_at_1000_diff1": 0.715849, + "nauc_map_at_1_max": 0.432259, + "nauc_map_at_1_std": -0.199492, + "nauc_map_at_1_diff1": 0.746668, + "nauc_map_at_3_max": 0.429887, + "nauc_map_at_3_std": -0.180603, + "nauc_map_at_3_diff1": 0.712166, + "nauc_map_at_5_max": 0.440863, + "nauc_map_at_5_std": -0.171539, + "nauc_map_at_5_diff1": 0.715828, + "nauc_map_at_10_max": 0.441277, + "nauc_map_at_10_std": -0.173727, + "nauc_map_at_10_diff1": 0.716678, + "nauc_map_at_20_max": 0.441967, + "nauc_map_at_20_std": -0.173883, + "nauc_map_at_20_diff1": 0.718016, + "nauc_map_at_100_max": 0.441283, + "nauc_map_at_100_std": -0.175484, + "nauc_map_at_100_diff1": 0.717947, + "nauc_map_at_1000_max": 0.44125, + "nauc_map_at_1000_std": -0.175484, + "nauc_map_at_1000_diff1": 0.71793, + "nauc_recall_at_1_max": 0.432259, + "nauc_recall_at_1_std": -0.199492, + "nauc_recall_at_1_diff1": 0.746668, + "nauc_recall_at_3_max": 0.41726, + "nauc_recall_at_3_std": -0.126116, + "nauc_recall_at_3_diff1": 0.659343, + "nauc_recall_at_5_max": 0.499768, + "nauc_recall_at_5_std": -0.042433, + "nauc_recall_at_5_diff1": 0.676345, + "nauc_recall_at_10_max": 0.570807, + "nauc_recall_at_10_std": 0.106491, + "nauc_recall_at_10_diff1": 0.675308, + "nauc_recall_at_20_max": 0.713366, + "nauc_recall_at_20_std": 0.295344, + "nauc_recall_at_20_diff1": 0.758389, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": -0.172032, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.432259, + "nauc_precision_at_1_std": -0.199492, + "nauc_precision_at_1_diff1": 0.746668, + "nauc_precision_at_3_max": 0.41726, + "nauc_precision_at_3_std": -0.126116, + "nauc_precision_at_3_diff1": 0.659343, + "nauc_precision_at_5_max": 0.499768, + "nauc_precision_at_5_std": -0.042433, + "nauc_precision_at_5_diff1": 0.676345, + "nauc_precision_at_10_max": 0.570807, + "nauc_precision_at_10_std": 0.106491, + "nauc_precision_at_10_diff1": 0.675308, + "nauc_precision_at_20_max": 0.713366, + "nauc_precision_at_20_std": 0.295344, + "nauc_precision_at_20_diff1": 0.758389, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": -0.172032, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.432259, + "nauc_mrr_at_1_std": -0.199492, + "nauc_mrr_at_1_diff1": 0.746668, + "nauc_mrr_at_3_max": 0.429887, + "nauc_mrr_at_3_std": -0.180603, + "nauc_mrr_at_3_diff1": 0.712166, + "nauc_mrr_at_5_max": 0.440863, + "nauc_mrr_at_5_std": -0.171539, + "nauc_mrr_at_5_diff1": 0.715828, + "nauc_mrr_at_10_max": 0.441277, + "nauc_mrr_at_10_std": -0.173727, + "nauc_mrr_at_10_diff1": 0.716678, + "nauc_mrr_at_20_max": 0.441967, + "nauc_mrr_at_20_std": -0.173883, + "nauc_mrr_at_20_diff1": 0.718016, + "nauc_mrr_at_100_max": 0.441283, + "nauc_mrr_at_100_std": -0.175484, + "nauc_mrr_at_100_diff1": 0.717947, + "nauc_mrr_at_1000_max": 0.44125, + "nauc_mrr_at_1000_std": -0.175484, + "nauc_mrr_at_1000_diff1": 0.71793, + "main_score": 0.77406, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 16.360193252563477, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBWikimQARetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBWikimQARetrieval.json new file mode 100644 index 0000000000..87818ecba1 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LEMBWikimQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "task_name": "LEMBWikimQARetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.49333, + "ndcg_at_3": 0.54917, + "ndcg_at_5": 0.56036, + "ndcg_at_10": 0.57598, + "ndcg_at_20": 0.58648, + "ndcg_at_100": 0.62024, + "ndcg_at_1000": 0.63484, + "map_at_1": 0.49333, + "map_at_3": 0.53611, + "map_at_5": 0.54244, + "map_at_10": 0.54857, + "map_at_20": 0.55122, + "map_at_100": 0.55555, + "map_at_1000": 0.55625, + "recall_at_1": 0.49333, + "recall_at_3": 0.58667, + "recall_at_5": 0.61333, + "recall_at_10": 0.66333, + "recall_at_20": 0.70667, + "recall_at_100": 0.89333, + "recall_at_1000": 1.0, + "precision_at_1": 0.49333, + "precision_at_3": 0.19556, + "precision_at_5": 0.12267, + "precision_at_10": 0.06633, + "precision_at_20": 0.03533, + "precision_at_100": 0.00893, + "precision_at_1000": 0.001, + "mrr_at_1": 0.493333, + "mrr_at_3": 0.536111, + "mrr_at_5": 0.542444, + "mrr_at_10": 0.54853, + "mrr_at_20": 0.551201, + "mrr_at_100": 0.555531, + "mrr_at_1000": 0.556233, + "nauc_ndcg_at_1_max": 0.741749, + "nauc_ndcg_at_1_std": 0.080354, + "nauc_ndcg_at_1_diff1": 0.826951, + "nauc_ndcg_at_3_max": 0.724619, + "nauc_ndcg_at_3_std": 0.081945, + "nauc_ndcg_at_3_diff1": 0.766924, + "nauc_ndcg_at_5_max": 0.717392, + "nauc_ndcg_at_5_std": 0.093934, + "nauc_ndcg_at_5_diff1": 0.75952, + "nauc_ndcg_at_10_max": 0.7133, + "nauc_ndcg_at_10_std": 0.077775, + "nauc_ndcg_at_10_diff1": 0.758099, + "nauc_ndcg_at_20_max": 0.705051, + "nauc_ndcg_at_20_std": 0.071724, + "nauc_ndcg_at_20_diff1": 0.759514, + "nauc_ndcg_at_100_max": 0.704007, + "nauc_ndcg_at_100_std": 0.093565, + "nauc_ndcg_at_100_diff1": 0.759139, + "nauc_ndcg_at_1000_max": 0.713879, + "nauc_ndcg_at_1000_std": 0.083346, + "nauc_ndcg_at_1000_diff1": 0.768272, + "nauc_map_at_1_max": 0.741749, + "nauc_map_at_1_std": 0.080354, + "nauc_map_at_1_diff1": 0.826951, + "nauc_map_at_3_max": 0.729071, + "nauc_map_at_3_std": 0.079542, + "nauc_map_at_3_diff1": 0.78179, + "nauc_map_at_5_max": 0.725134, + "nauc_map_at_5_std": 0.086211, + "nauc_map_at_5_diff1": 0.777632, + "nauc_map_at_10_max": 0.723895, + "nauc_map_at_10_std": 0.07985, + "nauc_map_at_10_diff1": 0.777615, + "nauc_map_at_20_max": 0.722122, + "nauc_map_at_20_std": 0.078181, + "nauc_map_at_20_diff1": 0.778109, + "nauc_map_at_100_max": 0.721853, + "nauc_map_at_100_std": 0.080836, + "nauc_map_at_100_diff1": 0.778166, + "nauc_map_at_1000_max": 0.722251, + "nauc_map_at_1000_std": 0.080567, + "nauc_map_at_1000_diff1": 0.77857, + "nauc_recall_at_1_max": 0.741749, + "nauc_recall_at_1_std": 0.080354, + "nauc_recall_at_1_diff1": 0.826951, + "nauc_recall_at_3_max": 0.711015, + "nauc_recall_at_3_std": 0.090128, + "nauc_recall_at_3_diff1": 0.721799, + "nauc_recall_at_5_max": 0.691994, + "nauc_recall_at_5_std": 0.120684, + "nauc_recall_at_5_diff1": 0.702035, + "nauc_recall_at_10_max": 0.67315, + "nauc_recall_at_10_std": 0.065411, + "nauc_recall_at_10_diff1": 0.688992, + "nauc_recall_at_20_max": 0.625854, + "nauc_recall_at_20_std": 0.035892, + "nauc_recall_at_20_diff1": 0.688783, + "nauc_recall_at_100_max": 0.515312, + "nauc_recall_at_100_std": 0.305057, + "nauc_recall_at_100_diff1": 0.587844, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.741749, + "nauc_precision_at_1_std": 0.080354, + "nauc_precision_at_1_diff1": 0.826951, + "nauc_precision_at_3_max": 0.711015, + "nauc_precision_at_3_std": 0.090128, + "nauc_precision_at_3_diff1": 0.721799, + "nauc_precision_at_5_max": 0.691994, + "nauc_precision_at_5_std": 0.120684, + "nauc_precision_at_5_diff1": 0.702035, + "nauc_precision_at_10_max": 0.67315, + "nauc_precision_at_10_std": 0.065411, + "nauc_precision_at_10_diff1": 0.688992, + "nauc_precision_at_20_max": 0.625854, + "nauc_precision_at_20_std": 0.035892, + "nauc_precision_at_20_diff1": 0.688783, + "nauc_precision_at_100_max": 0.515312, + "nauc_precision_at_100_std": 0.305057, + "nauc_precision_at_100_diff1": 0.587844, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.741749, + "nauc_mrr_at_1_std": 0.080354, + "nauc_mrr_at_1_diff1": 0.826951, + "nauc_mrr_at_3_max": 0.729071, + "nauc_mrr_at_3_std": 0.079542, + "nauc_mrr_at_3_diff1": 0.78179, + "nauc_mrr_at_5_max": 0.725134, + "nauc_mrr_at_5_std": 0.086211, + "nauc_mrr_at_5_diff1": 0.777632, + "nauc_mrr_at_10_max": 0.723931, + "nauc_mrr_at_10_std": 0.079957, + "nauc_mrr_at_10_diff1": 0.777629, + "nauc_mrr_at_20_max": 0.722129, + "nauc_mrr_at_20_std": 0.078327, + "nauc_mrr_at_20_diff1": 0.77816, + "nauc_mrr_at_100_max": 0.721853, + "nauc_mrr_at_100_std": 0.081, + "nauc_mrr_at_100_diff1": 0.77821, + "nauc_mrr_at_1000_max": 0.722249, + "nauc_mrr_at_1000_std": 0.080728, + "nauc_mrr_at_1000_diff1": 0.778613, + "main_score": 0.57598, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 14.75680160522461, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LegalBenchConsumerContractsQA.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LegalBenchConsumerContractsQA.json new file mode 100644 index 0000000000..62f41f5e97 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LegalBenchConsumerContractsQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b23590301ec94e8087e2850b21d43d4956b1cca9", + "task_name": "LegalBenchConsumerContractsQA", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.59848, + "ndcg_at_3": 0.7117, + "ndcg_at_5": 0.74136, + "ndcg_at_10": 0.76739, + "ndcg_at_20": 0.77527, + "ndcg_at_100": 0.78054, + "ndcg_at_1000": 0.78054, + "map_at_1": 0.59848, + "map_at_3": 0.68519, + "map_at_5": 0.70135, + "map_at_10": 0.71203, + "map_at_20": 0.71432, + "map_at_100": 0.71508, + "map_at_1000": 0.71508, + "recall_at_1": 0.59848, + "recall_at_3": 0.78788, + "recall_at_5": 0.86111, + "recall_at_10": 0.94192, + "recall_at_20": 0.97222, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.59848, + "precision_at_3": 0.26263, + "precision_at_5": 0.17222, + "precision_at_10": 0.09419, + "precision_at_20": 0.04861, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.603535, + "mrr_at_3": 0.68729, + "mrr_at_5": 0.703451, + "mrr_at_10": 0.714073, + "mrr_at_20": 0.71636, + "mrr_at_100": 0.717128, + "mrr_at_1000": 0.717128, + "nauc_ndcg_at_1_max": 0.535296, + "nauc_ndcg_at_1_std": 0.170861, + "nauc_ndcg_at_1_diff1": 0.646335, + "nauc_ndcg_at_3_max": 0.534446, + "nauc_ndcg_at_3_std": 0.14068, + "nauc_ndcg_at_3_diff1": 0.609859, + "nauc_ndcg_at_5_max": 0.531711, + "nauc_ndcg_at_5_std": 0.165734, + "nauc_ndcg_at_5_diff1": 0.596083, + "nauc_ndcg_at_10_max": 0.547957, + "nauc_ndcg_at_10_std": 0.183341, + "nauc_ndcg_at_10_diff1": 0.614689, + "nauc_ndcg_at_20_max": 0.5405, + "nauc_ndcg_at_20_std": 0.177287, + "nauc_ndcg_at_20_diff1": 0.613725, + "nauc_ndcg_at_100_max": 0.53909, + "nauc_ndcg_at_100_std": 0.165959, + "nauc_ndcg_at_100_diff1": 0.617362, + "nauc_ndcg_at_1000_max": 0.53909, + "nauc_ndcg_at_1000_std": 0.165959, + "nauc_ndcg_at_1000_diff1": 0.617362, + "nauc_map_at_1_max": 0.535296, + "nauc_map_at_1_std": 0.170861, + "nauc_map_at_1_diff1": 0.646335, + "nauc_map_at_3_max": 0.53533, + "nauc_map_at_3_std": 0.14826, + "nauc_map_at_3_diff1": 0.618428, + "nauc_map_at_5_max": 0.534573, + "nauc_map_at_5_std": 0.161537, + "nauc_map_at_5_diff1": 0.612196, + "nauc_map_at_10_max": 0.540058, + "nauc_map_at_10_std": 0.167014, + "nauc_map_at_10_diff1": 0.619207, + "nauc_map_at_20_max": 0.538373, + "nauc_map_at_20_std": 0.16528, + "nauc_map_at_20_diff1": 0.61922, + "nauc_map_at_100_max": 0.53817, + "nauc_map_at_100_std": 0.163714, + "nauc_map_at_100_diff1": 0.619884, + "nauc_map_at_1000_max": 0.53817, + "nauc_map_at_1000_std": 0.163714, + "nauc_map_at_1000_diff1": 0.619884, + "nauc_recall_at_1_max": 0.535296, + "nauc_recall_at_1_std": 0.170861, + "nauc_recall_at_1_diff1": 0.646335, + "nauc_recall_at_3_max": 0.530692, + "nauc_recall_at_3_std": 0.112156, + "nauc_recall_at_3_diff1": 0.577991, + "nauc_recall_at_5_max": 0.51396, + "nauc_recall_at_5_std": 0.194428, + "nauc_recall_at_5_diff1": 0.507988, + "nauc_recall_at_10_max": 0.662154, + "nauc_recall_at_10_std": 0.421306, + "nauc_recall_at_10_diff1": 0.585096, + "nauc_recall_at_20_max": 0.587059, + "nauc_recall_at_20_std": 0.536419, + "nauc_recall_at_20_diff1": 0.515356, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.535296, + "nauc_precision_at_1_std": 0.170861, + "nauc_precision_at_1_diff1": 0.646335, + "nauc_precision_at_3_max": 0.530692, + "nauc_precision_at_3_std": 0.112156, + "nauc_precision_at_3_diff1": 0.577991, + "nauc_precision_at_5_max": 0.51396, + "nauc_precision_at_5_std": 0.194428, + "nauc_precision_at_5_diff1": 0.507988, + "nauc_precision_at_10_max": 0.662154, + "nauc_precision_at_10_std": 0.421306, + "nauc_precision_at_10_diff1": 0.585096, + "nauc_precision_at_20_max": 0.587059, + "nauc_precision_at_20_std": 0.536419, + "nauc_precision_at_20_diff1": 0.515356, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.537734, + "nauc_mrr_at_1_std": 0.159269, + "nauc_mrr_at_1_diff1": 0.634945, + "nauc_mrr_at_3_max": 0.537338, + "nauc_mrr_at_3_std": 0.138218, + "nauc_mrr_at_3_diff1": 0.61132, + "nauc_mrr_at_5_max": 0.536689, + "nauc_mrr_at_5_std": 0.150988, + "nauc_mrr_at_5_diff1": 0.604675, + "nauc_mrr_at_10_max": 0.542303, + "nauc_mrr_at_10_std": 0.156149, + "nauc_mrr_at_10_diff1": 0.611352, + "nauc_mrr_at_20_max": 0.540634, + "nauc_mrr_at_20_std": 0.154302, + "nauc_mrr_at_20_diff1": 0.611289, + "nauc_mrr_at_100_max": 0.540438, + "nauc_mrr_at_100_std": 0.152696, + "nauc_mrr_at_100_diff1": 0.611929, + "nauc_mrr_at_1000_max": 0.540438, + "nauc_mrr_at_1000_std": 0.152696, + "nauc_mrr_at_1000_diff1": 0.611929, + "main_score": 0.76739, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.8909847736358643, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LegalBenchCorporateLobbying.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LegalBenchCorporateLobbying.json new file mode 100644 index 0000000000..5238d86561 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LegalBenchCorporateLobbying.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f69691c650464e62546d7f2a4536f8f87c891e38", + "task_name": "LegalBenchCorporateLobbying", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.85294, + "ndcg_at_3": 0.92517, + "ndcg_at_5": 0.92771, + "ndcg_at_10": 0.9296, + "ndcg_at_20": 0.93115, + "ndcg_at_100": 0.93176, + "ndcg_at_1000": 0.93262, + "map_at_1": 0.85294, + "map_at_3": 0.90833, + "map_at_5": 0.9098, + "map_at_10": 0.91059, + "map_at_20": 0.91104, + "map_at_100": 0.91115, + "map_at_1000": 0.9112, + "recall_at_1": 0.85294, + "recall_at_3": 0.97353, + "recall_at_5": 0.97941, + "recall_at_10": 0.98529, + "recall_at_20": 0.99118, + "recall_at_100": 0.99412, + "recall_at_1000": 1.0, + "precision_at_1": 0.85294, + "precision_at_3": 0.32451, + "precision_at_5": 0.19588, + "precision_at_10": 0.09853, + "precision_at_20": 0.04956, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "mrr_at_1": 0.852941, + "mrr_at_3": 0.908333, + "mrr_at_5": 0.909804, + "mrr_at_10": 0.910588, + "mrr_at_20": 0.911043, + "mrr_at_100": 0.911148, + "mrr_at_1000": 0.911201, + "nauc_ndcg_at_1_max": 0.413555, + "nauc_ndcg_at_1_std": -0.213282, + "nauc_ndcg_at_1_diff1": 0.937731, + "nauc_ndcg_at_3_max": 0.514416, + "nauc_ndcg_at_3_std": -0.132381, + "nauc_ndcg_at_3_diff1": 0.923971, + "nauc_ndcg_at_5_max": 0.499213, + "nauc_ndcg_at_5_std": -0.12142, + "nauc_ndcg_at_5_diff1": 0.923585, + "nauc_ndcg_at_10_max": 0.492118, + "nauc_ndcg_at_10_std": -0.142789, + "nauc_ndcg_at_10_diff1": 0.925728, + "nauc_ndcg_at_20_max": 0.480312, + "nauc_ndcg_at_20_std": -0.160997, + "nauc_ndcg_at_20_diff1": 0.92704, + "nauc_ndcg_at_100_max": 0.475543, + "nauc_ndcg_at_100_std": -0.167559, + "nauc_ndcg_at_100_diff1": 0.92637, + "nauc_ndcg_at_1000_max": 0.476167, + "nauc_ndcg_at_1000_std": -0.165317, + "nauc_ndcg_at_1000_diff1": 0.928274, + "nauc_map_at_1_max": 0.413555, + "nauc_map_at_1_std": -0.213282, + "nauc_map_at_1_diff1": 0.937731, + "nauc_map_at_3_max": 0.483181, + "nauc_map_at_3_std": -0.164138, + "nauc_map_at_3_diff1": 0.928974, + "nauc_map_at_5_max": 0.47556, + "nauc_map_at_5_std": -0.15956, + "nauc_map_at_5_diff1": 0.928879, + "nauc_map_at_10_max": 0.473327, + "nauc_map_at_10_std": -0.166449, + "nauc_map_at_10_diff1": 0.92981, + "nauc_map_at_20_max": 0.470536, + "nauc_map_at_20_std": -0.170748, + "nauc_map_at_20_diff1": 0.930131, + "nauc_map_at_100_max": 0.469887, + "nauc_map_at_100_std": -0.171636, + "nauc_map_at_100_diff1": 0.930045, + "nauc_map_at_1000_max": 0.469881, + "nauc_map_at_1000_std": -0.17161, + "nauc_map_at_1000_diff1": 0.930123, + "nauc_recall_at_1_max": 0.413555, + "nauc_recall_at_1_std": -0.213282, + "nauc_recall_at_1_diff1": 0.937731, + "nauc_recall_at_3_max": 0.805789, + "nauc_recall_at_3_std": 0.175329, + "nauc_recall_at_3_diff1": 0.876024, + "nauc_recall_at_5_max": 0.768974, + "nauc_recall_at_5_std": 0.349807, + "nauc_recall_at_5_diff1": 0.859277, + "nauc_recall_at_10_max": 0.765733, + "nauc_recall_at_10_std": 0.218114, + "nauc_recall_at_10_diff1": 0.858543, + "nauc_recall_at_20_max": 0.609555, + "nauc_recall_at_20_std": -0.061936, + "nauc_recall_at_20_diff1": 0.851385, + "nauc_recall_at_100_max": 0.414332, + "nauc_recall_at_100_std": -0.369981, + "nauc_recall_at_100_diff1": 0.777077, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.413555, + "nauc_precision_at_1_std": -0.213282, + "nauc_precision_at_1_diff1": 0.937731, + "nauc_precision_at_3_max": 0.805789, + "nauc_precision_at_3_std": 0.175329, + "nauc_precision_at_3_diff1": 0.876024, + "nauc_precision_at_5_max": 0.768974, + "nauc_precision_at_5_std": 0.349807, + "nauc_precision_at_5_diff1": 0.859277, + "nauc_precision_at_10_max": 0.765733, + "nauc_precision_at_10_std": 0.218114, + "nauc_precision_at_10_diff1": 0.858543, + "nauc_precision_at_20_max": 0.609555, + "nauc_precision_at_20_std": -0.061936, + "nauc_precision_at_20_diff1": 0.851385, + "nauc_precision_at_100_max": 0.414332, + "nauc_precision_at_100_std": -0.369981, + "nauc_precision_at_100_diff1": 0.777077, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.413555, + "nauc_mrr_at_1_std": -0.213282, + "nauc_mrr_at_1_diff1": 0.937731, + "nauc_mrr_at_3_max": 0.483181, + "nauc_mrr_at_3_std": -0.164138, + "nauc_mrr_at_3_diff1": 0.928974, + "nauc_mrr_at_5_max": 0.47556, + "nauc_mrr_at_5_std": -0.15956, + "nauc_mrr_at_5_diff1": 0.928879, + "nauc_mrr_at_10_max": 0.473327, + "nauc_mrr_at_10_std": -0.166449, + "nauc_mrr_at_10_diff1": 0.92981, + "nauc_mrr_at_20_max": 0.470536, + "nauc_mrr_at_20_std": -0.170748, + "nauc_mrr_at_20_diff1": 0.930131, + "nauc_mrr_at_100_max": 0.469887, + "nauc_mrr_at_100_std": -0.171636, + "nauc_mrr_at_100_diff1": 0.930045, + "nauc_mrr_at_1000_max": 0.469881, + "nauc_mrr_at_1000_std": -0.17161, + "nauc_mrr_at_1000_diff1": 0.930123, + "main_score": 0.9296, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.3488571643829346, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LegalSummarization.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LegalSummarization.json new file mode 100644 index 0000000000..a9f8590a9f --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LegalSummarization.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "3bb1a05c66872889662af04c5691c14489cebd72", + "task_name": "LegalSummarization", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.53169, + "ndcg_at_3": 0.57679, + "ndcg_at_5": 0.59785, + "ndcg_at_10": 0.64048, + "ndcg_at_20": 0.66733, + "ndcg_at_100": 0.68882, + "ndcg_at_1000": 0.69476, + "map_at_1": 0.47019, + "map_at_3": 0.53915, + "map_at_5": 0.55558, + "map_at_10": 0.57896, + "map_at_20": 0.58875, + "map_at_100": 0.5936, + "map_at_1000": 0.59396, + "recall_at_1": 0.47019, + "recall_at_3": 0.59952, + "recall_at_5": 0.65666, + "recall_at_10": 0.77935, + "recall_at_20": 0.8711, + "recall_at_100": 0.96068, + "recall_at_1000": 1.0, + "precision_at_1": 0.53169, + "precision_at_3": 0.24648, + "precision_at_5": 0.16972, + "precision_at_10": 0.10669, + "precision_at_20": 0.06162, + "precision_at_100": 0.01472, + "precision_at_1000": 0.00155, + "mrr_at_1": 0.53169, + "mrr_at_3": 0.595657, + "mrr_at_5": 0.608862, + "mrr_at_10": 0.624538, + "mrr_at_20": 0.630188, + "mrr_at_100": 0.631626, + "mrr_at_1000": 0.631772, + "nauc_ndcg_at_1_max": 0.554741, + "nauc_ndcg_at_1_std": -0.218696, + "nauc_ndcg_at_1_diff1": 0.704154, + "nauc_ndcg_at_3_max": 0.588232, + "nauc_ndcg_at_3_std": -0.260465, + "nauc_ndcg_at_3_diff1": 0.646232, + "nauc_ndcg_at_5_max": 0.596498, + "nauc_ndcg_at_5_std": -0.258981, + "nauc_ndcg_at_5_diff1": 0.653194, + "nauc_ndcg_at_10_max": 0.606518, + "nauc_ndcg_at_10_std": -0.260171, + "nauc_ndcg_at_10_diff1": 0.651251, + "nauc_ndcg_at_20_max": 0.602452, + "nauc_ndcg_at_20_std": -0.259109, + "nauc_ndcg_at_20_diff1": 0.656269, + "nauc_ndcg_at_100_max": 0.598889, + "nauc_ndcg_at_100_std": -0.233745, + "nauc_ndcg_at_100_diff1": 0.656682, + "nauc_ndcg_at_1000_max": 0.598082, + "nauc_ndcg_at_1000_std": -0.234951, + "nauc_ndcg_at_1000_diff1": 0.658486, + "nauc_map_at_1_max": 0.581985, + "nauc_map_at_1_std": -0.267971, + "nauc_map_at_1_diff1": 0.735648, + "nauc_map_at_3_max": 0.581388, + "nauc_map_at_3_std": -0.276568, + "nauc_map_at_3_diff1": 0.676827, + "nauc_map_at_5_max": 0.59097, + "nauc_map_at_5_std": -0.271188, + "nauc_map_at_5_diff1": 0.677044, + "nauc_map_at_10_max": 0.598307, + "nauc_map_at_10_std": -0.263894, + "nauc_map_at_10_diff1": 0.671482, + "nauc_map_at_20_max": 0.596409, + "nauc_map_at_20_std": -0.261254, + "nauc_map_at_20_diff1": 0.671931, + "nauc_map_at_100_max": 0.597142, + "nauc_map_at_100_std": -0.255098, + "nauc_map_at_100_diff1": 0.672108, + "nauc_map_at_1000_max": 0.597165, + "nauc_map_at_1000_std": -0.255112, + "nauc_map_at_1000_diff1": 0.672119, + "nauc_recall_at_1_max": 0.581985, + "nauc_recall_at_1_std": -0.267971, + "nauc_recall_at_1_diff1": 0.735648, + "nauc_recall_at_3_max": 0.576587, + "nauc_recall_at_3_std": -0.298297, + "nauc_recall_at_3_diff1": 0.599425, + "nauc_recall_at_5_max": 0.597396, + "nauc_recall_at_5_std": -0.277178, + "nauc_recall_at_5_diff1": 0.601069, + "nauc_recall_at_10_max": 0.628689, + "nauc_recall_at_10_std": -0.297785, + "nauc_recall_at_10_diff1": 0.563412, + "nauc_recall_at_20_max": 0.61973, + "nauc_recall_at_20_std": -0.374044, + "nauc_recall_at_20_diff1": 0.560295, + "nauc_recall_at_100_max": 0.603482, + "nauc_recall_at_100_std": -0.178716, + "nauc_recall_at_100_diff1": 0.520558, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.554741, + "nauc_precision_at_1_std": -0.218696, + "nauc_precision_at_1_diff1": 0.704154, + "nauc_precision_at_3_max": 0.409272, + "nauc_precision_at_3_std": -0.118442, + "nauc_precision_at_3_diff1": 0.314404, + "nauc_precision_at_5_max": 0.342973, + "nauc_precision_at_5_std": -0.034672, + "nauc_precision_at_5_diff1": 0.192168, + "nauc_precision_at_10_max": 0.201475, + "nauc_precision_at_10_std": 0.125044, + "nauc_precision_at_10_diff1": -0.02022, + "nauc_precision_at_20_max": 0.060143, + "nauc_precision_at_20_std": 0.226692, + "nauc_precision_at_20_diff1": -0.146191, + "nauc_precision_at_100_max": -0.067337, + "nauc_precision_at_100_std": 0.342629, + "nauc_precision_at_100_diff1": -0.256601, + "nauc_precision_at_1000_max": -0.116576, + "nauc_precision_at_1000_std": 0.344221, + "nauc_precision_at_1000_diff1": -0.300458, + "nauc_mrr_at_1_max": 0.554741, + "nauc_mrr_at_1_std": -0.218696, + "nauc_mrr_at_1_diff1": 0.704154, + "nauc_mrr_at_3_max": 0.568425, + "nauc_mrr_at_3_std": -0.233956, + "nauc_mrr_at_3_diff1": 0.653289, + "nauc_mrr_at_5_max": 0.57051, + "nauc_mrr_at_5_std": -0.229198, + "nauc_mrr_at_5_diff1": 0.653472, + "nauc_mrr_at_10_max": 0.571685, + "nauc_mrr_at_10_std": -0.228868, + "nauc_mrr_at_10_diff1": 0.652839, + "nauc_mrr_at_20_max": 0.571632, + "nauc_mrr_at_20_std": -0.228444, + "nauc_mrr_at_20_diff1": 0.655974, + "nauc_mrr_at_100_max": 0.570706, + "nauc_mrr_at_100_std": -0.227821, + "nauc_mrr_at_100_diff1": 0.655813, + "nauc_mrr_at_1000_max": 0.570666, + "nauc_mrr_at_1000_std": -0.22784, + "nauc_mrr_at_1000_diff1": 0.655915, + "main_score": 0.64048, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 2.0832087993621826, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LitSearchRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LitSearchRetrieval.json new file mode 100644 index 0000000000..c27404ae4f --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/LitSearchRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "9573fb284a1026c998df47024b888a163f0f0e25", + "task_name": "LitSearchRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.35377, + "ndcg_at_3": 0.4342, + "ndcg_at_5": 0.45665, + "ndcg_at_10": 0.47932, + "ndcg_at_20": 0.49645, + "ndcg_at_100": 0.52121, + "ndcg_at_1000": 0.53753, + "map_at_1": 0.35126, + "map_at_3": 0.41505, + "map_at_5": 0.42765, + "map_at_10": 0.4373, + "map_at_20": 0.44225, + "map_at_100": 0.44586, + "map_at_1000": 0.44649, + "recall_at_1": 0.35126, + "recall_at_3": 0.49196, + "recall_at_5": 0.5464, + "recall_at_10": 0.61424, + "recall_at_20": 0.68269, + "recall_at_100": 0.81262, + "recall_at_1000": 0.93984, + "precision_at_1": 0.36013, + "precision_at_3": 0.1703, + "precision_at_5": 0.11323, + "precision_at_10": 0.06348, + "precision_at_20": 0.03559, + "precision_at_100": 0.00854, + "precision_at_1000": 0.001, + "mrr_at_1": 0.360134, + "mrr_at_3": 0.421831, + "mrr_at_5": 0.434645, + "mrr_at_10": 0.444397, + "mrr_at_20": 0.449167, + "mrr_at_100": 0.452575, + "mrr_at_1000": 0.453121, + "nauc_ndcg_at_1_max": 0.25347, + "nauc_ndcg_at_1_std": 0.027273, + "nauc_ndcg_at_1_diff1": 0.584441, + "nauc_ndcg_at_3_max": 0.263128, + "nauc_ndcg_at_3_std": 0.037284, + "nauc_ndcg_at_3_diff1": 0.492698, + "nauc_ndcg_at_5_max": 0.264287, + "nauc_ndcg_at_5_std": 0.046938, + "nauc_ndcg_at_5_diff1": 0.4869, + "nauc_ndcg_at_10_max": 0.255803, + "nauc_ndcg_at_10_std": 0.04894, + "nauc_ndcg_at_10_diff1": 0.48261, + "nauc_ndcg_at_20_max": 0.256261, + "nauc_ndcg_at_20_std": 0.052649, + "nauc_ndcg_at_20_diff1": 0.482194, + "nauc_ndcg_at_100_max": 0.258388, + "nauc_ndcg_at_100_std": 0.062681, + "nauc_ndcg_at_100_diff1": 0.479439, + "nauc_ndcg_at_1000_max": 0.260758, + "nauc_ndcg_at_1000_std": 0.060897, + "nauc_ndcg_at_1000_diff1": 0.48716, + "nauc_map_at_1_max": 0.251664, + "nauc_map_at_1_std": 0.02053, + "nauc_map_at_1_diff1": 0.587493, + "nauc_map_at_3_max": 0.266255, + "nauc_map_at_3_std": 0.036407, + "nauc_map_at_3_diff1": 0.512887, + "nauc_map_at_5_max": 0.267213, + "nauc_map_at_5_std": 0.041413, + "nauc_map_at_5_diff1": 0.510031, + "nauc_map_at_10_max": 0.263096, + "nauc_map_at_10_std": 0.041877, + "nauc_map_at_10_diff1": 0.508398, + "nauc_map_at_20_max": 0.263582, + "nauc_map_at_20_std": 0.043463, + "nauc_map_at_20_diff1": 0.508257, + "nauc_map_at_100_max": 0.264042, + "nauc_map_at_100_std": 0.045019, + "nauc_map_at_100_diff1": 0.507861, + "nauc_map_at_1000_max": 0.264214, + "nauc_map_at_1000_std": 0.044972, + "nauc_map_at_1000_diff1": 0.508181, + "nauc_recall_at_1_max": 0.251664, + "nauc_recall_at_1_std": 0.02053, + "nauc_recall_at_1_diff1": 0.587493, + "nauc_recall_at_3_max": 0.268814, + "nauc_recall_at_3_std": 0.045494, + "nauc_recall_at_3_diff1": 0.432412, + "nauc_recall_at_5_max": 0.271516, + "nauc_recall_at_5_std": 0.0712, + "nauc_recall_at_5_diff1": 0.412918, + "nauc_recall_at_10_max": 0.246092, + "nauc_recall_at_10_std": 0.080204, + "nauc_recall_at_10_diff1": 0.38927, + "nauc_recall_at_20_max": 0.246465, + "nauc_recall_at_20_std": 0.10679, + "nauc_recall_at_20_diff1": 0.37604, + "nauc_recall_at_100_max": 0.250379, + "nauc_recall_at_100_std": 0.212561, + "nauc_recall_at_100_diff1": 0.296328, + "nauc_recall_at_1000_max": 0.292729, + "nauc_recall_at_1000_std": 0.396623, + "nauc_recall_at_1000_diff1": 0.242255, + "nauc_precision_at_1_max": 0.270776, + "nauc_precision_at_1_std": 0.043058, + "nauc_precision_at_1_diff1": 0.58238, + "nauc_precision_at_3_max": 0.292037, + "nauc_precision_at_3_std": 0.071085, + "nauc_precision_at_3_diff1": 0.392945, + "nauc_precision_at_5_max": 0.299152, + "nauc_precision_at_5_std": 0.096191, + "nauc_precision_at_5_diff1": 0.373621, + "nauc_precision_at_10_max": 0.279691, + "nauc_precision_at_10_std": 0.109195, + "nauc_precision_at_10_diff1": 0.351749, + "nauc_precision_at_20_max": 0.286969, + "nauc_precision_at_20_std": 0.148789, + "nauc_precision_at_20_diff1": 0.300303, + "nauc_precision_at_100_max": 0.319761, + "nauc_precision_at_100_std": 0.26467, + "nauc_precision_at_100_diff1": 0.170115, + "nauc_precision_at_1000_max": 0.318691, + "nauc_precision_at_1000_std": 0.340676, + "nauc_precision_at_1000_diff1": -0.013939, + "nauc_mrr_at_1_max": 0.270776, + "nauc_mrr_at_1_std": 0.043058, + "nauc_mrr_at_1_diff1": 0.58238, + "nauc_mrr_at_3_max": 0.281557, + "nauc_mrr_at_3_std": 0.052934, + "nauc_mrr_at_3_diff1": 0.515247, + "nauc_mrr_at_5_max": 0.28315, + "nauc_mrr_at_5_std": 0.057832, + "nauc_mrr_at_5_diff1": 0.512281, + "nauc_mrr_at_10_max": 0.279321, + "nauc_mrr_at_10_std": 0.058666, + "nauc_mrr_at_10_diff1": 0.510932, + "nauc_mrr_at_20_max": 0.280388, + "nauc_mrr_at_20_std": 0.059993, + "nauc_mrr_at_20_diff1": 0.51086, + "nauc_mrr_at_100_max": 0.280384, + "nauc_mrr_at_100_std": 0.061182, + "nauc_mrr_at_100_diff1": 0.511046, + "nauc_mrr_at_1000_max": 0.280326, + "nauc_mrr_at_1000_std": 0.061043, + "nauc_mrr_at_1000_diff1": 0.51128, + "main_score": 0.47932, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 244.44452118873596, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/MLQuestions.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/MLQuestions.json new file mode 100644 index 0000000000..fa32e33795 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/MLQuestions.json @@ -0,0 +1,307 @@ +{ + "dataset_revision": "83b690cb666c5a8869e7f213a877bbd24a642d7c", + "task_name": "MLQuestions", + "mteb_version": "1.38.9", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.41533, + "ndcg_at_3": 0.5485, + "ndcg_at_5": 0.59378, + "ndcg_at_10": 0.62488, + "ndcg_at_20": 0.64152, + "ndcg_at_100": 0.65272, + "ndcg_at_1000": 0.65563, + "map_at_1": 0.41533, + "map_at_3": 0.51533, + "map_at_5": 0.54043, + "map_at_10": 0.55343, + "map_at_20": 0.55806, + "map_at_100": 0.55975, + "map_at_1000": 0.55986, + "recall_at_1": 0.41533, + "recall_at_3": 0.64467, + "recall_at_5": 0.75467, + "recall_at_10": 0.85, + "recall_at_20": 0.91533, + "recall_at_100": 0.974, + "recall_at_1000": 0.99667, + "precision_at_1": 0.41533, + "precision_at_3": 0.21489, + "precision_at_5": 0.15093, + "precision_at_10": 0.085, + "precision_at_20": 0.04577, + "precision_at_100": 0.00974, + "precision_at_1000": 0.001, + "mrr_at_1": 0.425333, + "mrr_at_3": 0.520444, + "mrr_at_5": 0.545611, + "mrr_at_10": 0.559062, + "mrr_at_20": 0.563466, + "mrr_at_100": 0.565211, + "mrr_at_1000": 0.565322, + "nauc_ndcg_at_1_max": 0.137328, + "nauc_ndcg_at_1_std": -0.226329, + "nauc_ndcg_at_1_diff1": 0.581948, + "nauc_ndcg_at_3_max": 0.156454, + "nauc_ndcg_at_3_std": -0.256418, + "nauc_ndcg_at_3_diff1": 0.488247, + "nauc_ndcg_at_5_max": 0.155166, + "nauc_ndcg_at_5_std": -0.266964, + "nauc_ndcg_at_5_diff1": 0.490011, + "nauc_ndcg_at_10_max": 0.160777, + "nauc_ndcg_at_10_std": -0.240965, + "nauc_ndcg_at_10_diff1": 0.484226, + "nauc_ndcg_at_20_max": 0.164855, + "nauc_ndcg_at_20_std": -0.239026, + "nauc_ndcg_at_20_diff1": 0.488984, + "nauc_ndcg_at_100_max": 0.160161, + "nauc_ndcg_at_100_std": -0.237916, + "nauc_ndcg_at_100_diff1": 0.499063, + "nauc_ndcg_at_1000_max": 0.158136, + "nauc_ndcg_at_1000_std": -0.242942, + "nauc_ndcg_at_1000_diff1": 0.503975, + "nauc_map_at_1_max": 0.137328, + "nauc_map_at_1_std": -0.226329, + "nauc_map_at_1_diff1": 0.581948, + "nauc_map_at_3_max": 0.151503, + "nauc_map_at_3_std": -0.251781, + "nauc_map_at_3_diff1": 0.514102, + "nauc_map_at_5_max": 0.150203, + "nauc_map_at_5_std": -0.257031, + "nauc_map_at_5_diff1": 0.516084, + "nauc_map_at_10_max": 0.152431, + "nauc_map_at_10_std": -0.247001, + "nauc_map_at_10_diff1": 0.514695, + "nauc_map_at_20_max": 0.153378, + "nauc_map_at_20_std": -0.24665, + "nauc_map_at_20_diff1": 0.516093, + "nauc_map_at_100_max": 0.152894, + "nauc_map_at_100_std": -0.246604, + "nauc_map_at_100_diff1": 0.517362, + "nauc_map_at_1000_max": 0.152831, + "nauc_map_at_1000_std": -0.246744, + "nauc_map_at_1000_diff1": 0.517516, + "nauc_recall_at_1_max": 0.137328, + "nauc_recall_at_1_std": -0.226329, + "nauc_recall_at_1_diff1": 0.581948, + "nauc_recall_at_3_max": 0.17279, + "nauc_recall_at_3_std": -0.270208, + "nauc_recall_at_3_diff1": 0.40216, + "nauc_recall_at_5_max": 0.176181, + "nauc_recall_at_5_std": -0.30832, + "nauc_recall_at_5_diff1": 0.381656, + "nauc_recall_at_10_max": 0.215692, + "nauc_recall_at_10_std": -0.183632, + "nauc_recall_at_10_diff1": 0.292633, + "nauc_recall_at_20_max": 0.298852, + "nauc_recall_at_20_std": -0.120581, + "nauc_recall_at_20_diff1": 0.218965, + "nauc_recall_at_100_max": 0.345663, + "nauc_recall_at_100_std": 0.203931, + "nauc_recall_at_100_diff1": 0.118796, + "nauc_recall_at_1000_max": 0.465173, + "nauc_recall_at_1000_std": 0.294304, + "nauc_recall_at_1000_diff1": 0.296545, + "nauc_precision_at_1_max": 0.137328, + "nauc_precision_at_1_std": -0.226329, + "nauc_precision_at_1_diff1": 0.581948, + "nauc_precision_at_3_max": 0.17279, + "nauc_precision_at_3_std": -0.270208, + "nauc_precision_at_3_diff1": 0.40216, + "nauc_precision_at_5_max": 0.176181, + "nauc_precision_at_5_std": -0.30832, + "nauc_precision_at_5_diff1": 0.381656, + "nauc_precision_at_10_max": 0.215692, + "nauc_precision_at_10_std": -0.183632, + "nauc_precision_at_10_diff1": 0.292633, + "nauc_precision_at_20_max": 0.298852, + "nauc_precision_at_20_std": -0.120581, + "nauc_precision_at_20_diff1": 0.218965, + "nauc_precision_at_100_max": 0.345663, + "nauc_precision_at_100_std": 0.203931, + "nauc_precision_at_100_diff1": 0.118796, + "nauc_precision_at_1000_max": 0.465173, + "nauc_precision_at_1000_std": 0.294304, + "nauc_precision_at_1000_diff1": 0.296545, + "nauc_mrr_at_1_max": 0.130742, + "nauc_mrr_at_1_std": -0.243438, + "nauc_mrr_at_1_diff1": 0.558577, + "nauc_mrr_at_3_max": 0.143858, + "nauc_mrr_at_3_std": -0.266351, + "nauc_mrr_at_3_diff1": 0.501215, + "nauc_mrr_at_5_max": 0.142686, + "nauc_mrr_at_5_std": -0.266858, + "nauc_mrr_at_5_diff1": 0.499943, + "nauc_mrr_at_10_max": 0.145736, + "nauc_mrr_at_10_std": -0.257642, + "nauc_mrr_at_10_diff1": 0.499719, + "nauc_mrr_at_20_max": 0.146417, + "nauc_mrr_at_20_std": -0.258253, + "nauc_mrr_at_20_diff1": 0.501007, + "nauc_mrr_at_100_max": 0.145959, + "nauc_mrr_at_100_std": -0.258121, + "nauc_mrr_at_100_diff1": 0.502143, + "nauc_mrr_at_1000_max": 0.145893, + "nauc_mrr_at_1000_std": -0.258266, + "nauc_mrr_at_1000_diff1": 0.502291, + "main_score": 0.62488, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ], + "test": [ + { + "ndcg_at_1": 0.43467, + "ndcg_at_3": 0.56068, + "ndcg_at_5": 0.60352, + "ndcg_at_10": 0.63788, + "ndcg_at_20": 0.6548, + "ndcg_at_100": 0.66529, + "ndcg_at_1000": 0.66748, + "map_at_1": 0.43467, + "map_at_3": 0.53, + "map_at_5": 0.55393, + "map_at_10": 0.5683, + "map_at_20": 0.57307, + "map_at_100": 0.57465, + "map_at_1000": 0.57473, + "recall_at_1": 0.43467, + "recall_at_3": 0.64933, + "recall_at_5": 0.75267, + "recall_at_10": 0.858, + "recall_at_20": 0.924, + "recall_at_100": 0.97867, + "recall_at_1000": 0.996, + "precision_at_1": 0.43467, + "precision_at_3": 0.21644, + "precision_at_5": 0.15053, + "precision_at_10": 0.0858, + "precision_at_20": 0.0462, + "precision_at_100": 0.00979, + "precision_at_1000": 0.001, + "mrr_at_1": 0.430667, + "mrr_at_3": 0.525444, + "mrr_at_5": 0.550278, + "mrr_at_10": 0.564376, + "mrr_at_20": 0.569132, + "mrr_at_100": 0.570724, + "mrr_at_1000": 0.570804, + "nauc_ndcg_at_1_max": 0.071057, + "nauc_ndcg_at_1_std": -0.300114, + "nauc_ndcg_at_1_diff1": 0.57603, + "nauc_ndcg_at_3_max": 0.138431, + "nauc_ndcg_at_3_std": -0.325572, + "nauc_ndcg_at_3_diff1": 0.51071, + "nauc_ndcg_at_5_max": 0.156534, + "nauc_ndcg_at_5_std": -0.323951, + "nauc_ndcg_at_5_diff1": 0.504675, + "nauc_ndcg_at_10_max": 0.163531, + "nauc_ndcg_at_10_std": -0.300011, + "nauc_ndcg_at_10_diff1": 0.506253, + "nauc_ndcg_at_20_max": 0.152032, + "nauc_ndcg_at_20_std": -0.305259, + "nauc_ndcg_at_20_diff1": 0.509415, + "nauc_ndcg_at_100_max": 0.141951, + "nauc_ndcg_at_100_std": -0.304379, + "nauc_ndcg_at_100_diff1": 0.515031, + "nauc_ndcg_at_1000_max": 0.137887, + "nauc_ndcg_at_1000_std": -0.305652, + "nauc_ndcg_at_1000_diff1": 0.517786, + "nauc_map_at_1_max": 0.071057, + "nauc_map_at_1_std": -0.300114, + "nauc_map_at_1_diff1": 0.57603, + "nauc_map_at_3_max": 0.120038, + "nauc_map_at_3_std": -0.31791, + "nauc_map_at_3_diff1": 0.526444, + "nauc_map_at_5_max": 0.129058, + "nauc_map_at_5_std": -0.316678, + "nauc_map_at_5_diff1": 0.524091, + "nauc_map_at_10_max": 0.130547, + "nauc_map_at_10_std": -0.307828, + "nauc_map_at_10_diff1": 0.525415, + "nauc_map_at_20_max": 0.12733, + "nauc_map_at_20_std": -0.309261, + "nauc_map_at_20_diff1": 0.52637, + "nauc_map_at_100_max": 0.126055, + "nauc_map_at_100_std": -0.308941, + "nauc_map_at_100_diff1": 0.527086, + "nauc_map_at_1000_max": 0.12592, + "nauc_map_at_1000_std": -0.30898, + "nauc_map_at_1000_diff1": 0.527158, + "nauc_recall_at_1_max": 0.071057, + "nauc_recall_at_1_std": -0.300114, + "nauc_recall_at_1_diff1": 0.57603, + "nauc_recall_at_3_max": 0.199497, + "nauc_recall_at_3_std": -0.351336, + "nauc_recall_at_3_diff1": 0.459426, + "nauc_recall_at_5_max": 0.272104, + "nauc_recall_at_5_std": -0.354019, + "nauc_recall_at_5_diff1": 0.424858, + "nauc_recall_at_10_max": 0.385401, + "nauc_recall_at_10_std": -0.229365, + "nauc_recall_at_10_diff1": 0.38458, + "nauc_recall_at_20_max": 0.41688, + "nauc_recall_at_20_std": -0.237862, + "nauc_recall_at_20_diff1": 0.330057, + "nauc_recall_at_100_max": 0.495871, + "nauc_recall_at_100_std": -0.063901, + "nauc_recall_at_100_diff1": 0.204526, + "nauc_recall_at_1000_max": 0.199502, + "nauc_recall_at_1000_std": 0.381108, + "nauc_recall_at_1000_diff1": 0.170401, + "nauc_precision_at_1_max": 0.071057, + "nauc_precision_at_1_std": -0.300114, + "nauc_precision_at_1_diff1": 0.57603, + "nauc_precision_at_3_max": 0.199497, + "nauc_precision_at_3_std": -0.351336, + "nauc_precision_at_3_diff1": 0.459426, + "nauc_precision_at_5_max": 0.272104, + "nauc_precision_at_5_std": -0.354019, + "nauc_precision_at_5_diff1": 0.424858, + "nauc_precision_at_10_max": 0.385401, + "nauc_precision_at_10_std": -0.229365, + "nauc_precision_at_10_diff1": 0.38458, + "nauc_precision_at_20_max": 0.41688, + "nauc_precision_at_20_std": -0.237862, + "nauc_precision_at_20_diff1": 0.330057, + "nauc_precision_at_100_max": 0.495871, + "nauc_precision_at_100_std": -0.063901, + "nauc_precision_at_100_diff1": 0.204526, + "nauc_precision_at_1000_max": 0.199502, + "nauc_precision_at_1000_std": 0.381108, + "nauc_precision_at_1000_diff1": 0.170401, + "nauc_mrr_at_1_max": 0.09256, + "nauc_mrr_at_1_std": -0.302186, + "nauc_mrr_at_1_diff1": 0.584202, + "nauc_mrr_at_3_max": 0.128027, + "nauc_mrr_at_3_std": -0.322632, + "nauc_mrr_at_3_diff1": 0.536539, + "nauc_mrr_at_5_max": 0.140637, + "nauc_mrr_at_5_std": -0.322391, + "nauc_mrr_at_5_diff1": 0.531352, + "nauc_mrr_at_10_max": 0.141187, + "nauc_mrr_at_10_std": -0.313283, + "nauc_mrr_at_10_diff1": 0.533498, + "nauc_mrr_at_20_max": 0.138662, + "nauc_mrr_at_20_std": -0.314366, + "nauc_mrr_at_20_diff1": 0.534342, + "nauc_mrr_at_100_max": 0.137323, + "nauc_mrr_at_100_std": -0.314228, + "nauc_mrr_at_100_diff1": 0.535137, + "nauc_mrr_at_1000_max": 0.137192, + "nauc_mrr_at_1000_std": -0.314268, + "nauc_mrr_at_1000_diff1": 0.53521, + "main_score": 0.63788, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 33.64640522003174, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/MSMARCO.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/MSMARCO.json new file mode 100644 index 0000000000..2bd624aa9e --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/MSMARCO.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c5a29a104738b98a9e76336939199e264163d4a0", + "task_name": "MSMARCO", + "mteb_version": "1.38.9", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.23539, + "ndcg_at_3": 0.34591, + "ndcg_at_5": 0.3862, + "ndcg_at_10": 0.42558, + "ndcg_at_20": 0.45052, + "ndcg_at_100": 0.48045, + "ndcg_at_1000": 0.49202, + "map_at_1": 0.22974, + "map_at_3": 0.3161, + "map_at_5": 0.33884, + "map_at_10": 0.35538, + "map_at_20": 0.36239, + "map_at_100": 0.3668, + "map_at_1000": 0.36728, + "recall_at_1": 0.22974, + "recall_at_3": 0.42561, + "recall_at_5": 0.52236, + "recall_at_10": 0.64222, + "recall_at_20": 0.73911, + "recall_at_100": 0.89561, + "recall_at_1000": 0.98301, + "precision_at_1": 0.23539, + "precision_at_3": 0.14723, + "precision_at_5": 0.1086, + "precision_at_10": 0.06715, + "precision_at_20": 0.03875, + "precision_at_100": 0.00946, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.235387, + "mrr_at_3": 0.322541, + "mrr_at_5": 0.344625, + "mrr_at_10": 0.360683, + "mrr_at_20": 0.367467, + "mrr_at_100": 0.371539, + "mrr_at_1000": 0.371964, + "nauc_ndcg_at_1_max": 0.182639, + "nauc_ndcg_at_1_std": -0.251625, + "nauc_ndcg_at_1_diff1": 0.406002, + "nauc_ndcg_at_3_max": 0.217661, + "nauc_ndcg_at_3_std": -0.282767, + "nauc_ndcg_at_3_diff1": 0.361308, + "nauc_ndcg_at_5_max": 0.231941, + "nauc_ndcg_at_5_std": -0.29378, + "nauc_ndcg_at_5_diff1": 0.354116, + "nauc_ndcg_at_10_max": 0.237344, + "nauc_ndcg_at_10_std": -0.288795, + "nauc_ndcg_at_10_diff1": 0.349753, + "nauc_ndcg_at_20_max": 0.238157, + "nauc_ndcg_at_20_std": -0.269731, + "nauc_ndcg_at_20_diff1": 0.351287, + "nauc_ndcg_at_100_max": 0.237089, + "nauc_ndcg_at_100_std": -0.248347, + "nauc_ndcg_at_100_diff1": 0.355046, + "nauc_ndcg_at_1000_max": 0.232467, + "nauc_ndcg_at_1000_std": -0.257584, + "nauc_ndcg_at_1000_diff1": 0.356621, + "nauc_map_at_1_max": 0.182096, + "nauc_map_at_1_std": -0.250611, + "nauc_map_at_1_diff1": 0.405125, + "nauc_map_at_3_max": 0.21002, + "nauc_map_at_3_std": -0.276177, + "nauc_map_at_3_diff1": 0.370588, + "nauc_map_at_5_max": 0.21805, + "nauc_map_at_5_std": -0.282352, + "nauc_map_at_5_diff1": 0.366627, + "nauc_map_at_10_max": 0.219951, + "nauc_map_at_10_std": -0.280192, + "nauc_map_at_10_diff1": 0.365259, + "nauc_map_at_20_max": 0.219876, + "nauc_map_at_20_std": -0.275251, + "nauc_map_at_20_diff1": 0.365772, + "nauc_map_at_100_max": 0.219837, + "nauc_map_at_100_std": -0.271994, + "nauc_map_at_100_diff1": 0.366247, + "nauc_map_at_1000_max": 0.219718, + "nauc_map_at_1000_std": -0.272197, + "nauc_map_at_1000_diff1": 0.366295, + "nauc_recall_at_1_max": 0.182096, + "nauc_recall_at_1_std": -0.250611, + "nauc_recall_at_1_diff1": 0.405125, + "nauc_recall_at_3_max": 0.237351, + "nauc_recall_at_3_std": -0.301939, + "nauc_recall_at_3_diff1": 0.33497, + "nauc_recall_at_5_max": 0.270074, + "nauc_recall_at_5_std": -0.328656, + "nauc_recall_at_5_diff1": 0.315773, + "nauc_recall_at_10_max": 0.293843, + "nauc_recall_at_10_std": -0.319175, + "nauc_recall_at_10_diff1": 0.295811, + "nauc_recall_at_20_max": 0.312151, + "nauc_recall_at_20_std": -0.228816, + "nauc_recall_at_20_diff1": 0.291232, + "nauc_recall_at_100_max": 0.377069, + "nauc_recall_at_100_std": 0.089923, + "nauc_recall_at_100_diff1": 0.281732, + "nauc_recall_at_1000_max": 0.489686, + "nauc_recall_at_1000_std": 0.574919, + "nauc_recall_at_1000_diff1": 0.147037, + "nauc_precision_at_1_max": 0.182639, + "nauc_precision_at_1_std": -0.251625, + "nauc_precision_at_1_diff1": 0.406002, + "nauc_precision_at_3_max": 0.235655, + "nauc_precision_at_3_std": -0.299553, + "nauc_precision_at_3_diff1": 0.332905, + "nauc_precision_at_5_max": 0.266414, + "nauc_precision_at_5_std": -0.319597, + "nauc_precision_at_5_diff1": 0.310542, + "nauc_precision_at_10_max": 0.275382, + "nauc_precision_at_10_std": -0.290382, + "nauc_precision_at_10_diff1": 0.274157, + "nauc_precision_at_20_max": 0.279374, + "nauc_precision_at_20_std": -0.185424, + "nauc_precision_at_20_diff1": 0.248148, + "nauc_precision_at_100_max": 0.241036, + "nauc_precision_at_100_std": 0.124001, + "nauc_precision_at_100_diff1": 0.137447, + "nauc_precision_at_1000_max": 0.083963, + "nauc_precision_at_1000_std": 0.191935, + "nauc_precision_at_1000_diff1": -0.049841, + "nauc_mrr_at_1_max": 0.182639, + "nauc_mrr_at_1_std": -0.251625, + "nauc_mrr_at_1_diff1": 0.406002, + "nauc_mrr_at_3_max": 0.209049, + "nauc_mrr_at_3_std": -0.272733, + "nauc_mrr_at_3_diff1": 0.370155, + "nauc_mrr_at_5_max": 0.217139, + "nauc_mrr_at_5_std": -0.278126, + "nauc_mrr_at_5_diff1": 0.365985, + "nauc_mrr_at_10_max": 0.219536, + "nauc_mrr_at_10_std": -0.275479, + "nauc_mrr_at_10_diff1": 0.363739, + "nauc_mrr_at_20_max": 0.219511, + "nauc_mrr_at_20_std": -0.270635, + "nauc_mrr_at_20_diff1": 0.364206, + "nauc_mrr_at_100_max": 0.219206, + "nauc_mrr_at_100_std": -0.268133, + "nauc_mrr_at_100_diff1": 0.364942, + "nauc_mrr_at_1000_max": 0.219063, + "nauc_mrr_at_1000_std": -0.268363, + "nauc_mrr_at_1000_diff1": 0.364985, + "main_score": 0.42558, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 14264.506400346756, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/MSMARCOHardNegatives.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/MSMARCOHardNegatives.json new file mode 100644 index 0000000000..b45f8be2b3 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/MSMARCOHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "67c0b4f7f15946e0b15cf6cf3b8993d04cb3efc6", + "task_name": "MSMARCOHardNegatives", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.74419, + "ndcg_at_3": 0.7503, + "ndcg_at_5": 0.74664, + "ndcg_at_10": 0.73482, + "ndcg_at_20": 0.72081, + "ndcg_at_100": 0.74941, + "ndcg_at_1000": 0.86374, + "map_at_1": 0.02376, + "map_at_3": 0.07029, + "map_at_5": 0.10652, + "map_at_10": 0.17195, + "map_at_20": 0.26248, + "map_at_100": 0.51347, + "map_at_1000": 0.70166, + "recall_at_1": 0.02376, + "recall_at_3": 0.07078, + "recall_at_5": 0.10914, + "recall_at_10": 0.18151, + "recall_at_20": 0.29085, + "recall_at_100": 0.66443, + "recall_at_1000": 0.98616, + "precision_at_1": 0.95349, + "precision_at_3": 0.93798, + "precision_at_5": 0.89767, + "precision_at_10": 0.83721, + "precision_at_20": 0.75698, + "precision_at_100": 0.49116, + "precision_at_1000": 0.09449, + "mrr_at_1": 0.953488, + "mrr_at_3": 0.965116, + "mrr_at_5": 0.97093, + "mrr_at_10": 0.97093, + "mrr_at_20": 0.97093, + "mrr_at_100": 0.97093, + "mrr_at_1000": 0.97093, + "nauc_ndcg_at_1_max": -0.096767, + "nauc_ndcg_at_1_std": -0.236645, + "nauc_ndcg_at_1_diff1": 0.2443, + "nauc_ndcg_at_3_max": 0.041351, + "nauc_ndcg_at_3_std": -0.05436, + "nauc_ndcg_at_3_diff1": -0.017887, + "nauc_ndcg_at_5_max": 0.025685, + "nauc_ndcg_at_5_std": -0.066032, + "nauc_ndcg_at_5_diff1": -0.083732, + "nauc_ndcg_at_10_max": 0.02779, + "nauc_ndcg_at_10_std": 0.005011, + "nauc_ndcg_at_10_diff1": -0.230147, + "nauc_ndcg_at_20_max": 0.079924, + "nauc_ndcg_at_20_std": 0.108415, + "nauc_ndcg_at_20_diff1": -0.30623, + "nauc_ndcg_at_100_max": 0.112429, + "nauc_ndcg_at_100_std": 0.14137, + "nauc_ndcg_at_100_diff1": -0.411338, + "nauc_ndcg_at_1000_max": 0.143255, + "nauc_ndcg_at_1000_std": 0.314075, + "nauc_ndcg_at_1000_diff1": -0.438642, + "nauc_map_at_1_max": -0.103914, + "nauc_map_at_1_std": -0.466885, + "nauc_map_at_1_diff1": 0.542093, + "nauc_map_at_3_max": -0.107116, + "nauc_map_at_3_std": -0.466615, + "nauc_map_at_3_diff1": 0.543784, + "nauc_map_at_5_max": -0.11775, + "nauc_map_at_5_std": -0.478083, + "nauc_map_at_5_diff1": 0.508686, + "nauc_map_at_10_max": -0.131816, + "nauc_map_at_10_std": -0.457225, + "nauc_map_at_10_diff1": 0.362473, + "nauc_map_at_20_max": -0.077808, + "nauc_map_at_20_std": -0.406155, + "nauc_map_at_20_diff1": 0.196484, + "nauc_map_at_100_max": 0.062638, + "nauc_map_at_100_std": -0.15903, + "nauc_map_at_100_diff1": -0.159878, + "nauc_map_at_1000_max": 0.155138, + "nauc_map_at_1000_std": 0.369797, + "nauc_map_at_1000_diff1": -0.489422, + "nauc_recall_at_1_max": -0.103914, + "nauc_recall_at_1_std": -0.466885, + "nauc_recall_at_1_diff1": 0.542093, + "nauc_recall_at_3_max": -0.107051, + "nauc_recall_at_3_std": -0.468863, + "nauc_recall_at_3_diff1": 0.546505, + "nauc_recall_at_5_max": -0.131756, + "nauc_recall_at_5_std": -0.497321, + "nauc_recall_at_5_diff1": 0.519816, + "nauc_recall_at_10_max": -0.152045, + "nauc_recall_at_10_std": -0.483173, + "nauc_recall_at_10_diff1": 0.371746, + "nauc_recall_at_20_max": -0.095409, + "nauc_recall_at_20_std": -0.436187, + "nauc_recall_at_20_diff1": 0.217866, + "nauc_recall_at_100_max": -0.001039, + "nauc_recall_at_100_std": -0.337583, + "nauc_recall_at_100_diff1": -0.040251, + "nauc_recall_at_1000_max": 0.347173, + "nauc_recall_at_1000_std": 0.960271, + "nauc_recall_at_1000_diff1": -1.651619, + "nauc_precision_at_1_max": 0.614293, + "nauc_precision_at_1_std": 0.614293, + "nauc_precision_at_1_diff1": -0.374744, + "nauc_precision_at_3_max": 0.477527, + "nauc_precision_at_3_std": 0.500189, + "nauc_precision_at_3_diff1": -0.303033, + "nauc_precision_at_5_max": 0.262184, + "nauc_precision_at_5_std": 0.247773, + "nauc_precision_at_5_diff1": -0.424987, + "nauc_precision_at_10_max": 0.169802, + "nauc_precision_at_10_std": 0.332126, + "nauc_precision_at_10_diff1": -0.708213, + "nauc_precision_at_20_max": 0.247037, + "nauc_precision_at_20_std": 0.483026, + "nauc_precision_at_20_diff1": -0.73236, + "nauc_precision_at_100_max": 0.134639, + "nauc_precision_at_100_std": 0.577434, + "nauc_precision_at_100_diff1": -0.523691, + "nauc_precision_at_1000_max": 0.09257, + "nauc_precision_at_1000_std": 0.546875, + "nauc_precision_at_1000_diff1": -0.415784, + "nauc_mrr_at_1_max": 0.614293, + "nauc_mrr_at_1_std": 0.614293, + "nauc_mrr_at_1_diff1": -0.374744, + "nauc_mrr_at_3_max": 0.699624, + "nauc_mrr_at_3_std": 0.699624, + "nauc_mrr_at_3_diff1": -0.309888, + "nauc_mrr_at_5_max": 0.665492, + "nauc_mrr_at_5_std": 0.665492, + "nauc_mrr_at_5_diff1": -0.33583, + "nauc_mrr_at_10_max": 0.665492, + "nauc_mrr_at_10_std": 0.665492, + "nauc_mrr_at_10_diff1": -0.33583, + "nauc_mrr_at_20_max": 0.665492, + "nauc_mrr_at_20_std": 0.665492, + "nauc_mrr_at_20_diff1": -0.33583, + "nauc_mrr_at_100_max": 0.665492, + "nauc_mrr_at_100_std": 0.665492, + "nauc_mrr_at_100_diff1": -0.33583, + "nauc_mrr_at_1000_max": 0.665492, + "nauc_mrr_at_1000_std": 0.665492, + "nauc_mrr_at_1000_diff1": -0.33583, + "main_score": 0.73482, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 13.315577745437622, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/MedicalQARetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/MedicalQARetrieval.json new file mode 100644 index 0000000000..d60e0eef3b --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/MedicalQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae763399273d8b20506b80cf6f6f9a31a6a2b238", + "task_name": "MedicalQARetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.54639, + "ndcg_at_3": 0.66865, + "ndcg_at_5": 0.6952, + "ndcg_at_10": 0.71432, + "ndcg_at_20": 0.72334, + "ndcg_at_100": 0.73431, + "ndcg_at_1000": 0.738, + "map_at_1": 0.54639, + "map_at_3": 0.6394, + "map_at_5": 0.65403, + "map_at_10": 0.66202, + "map_at_20": 0.6645, + "map_at_100": 0.66605, + "map_at_1000": 0.66619, + "recall_at_1": 0.54639, + "recall_at_3": 0.75293, + "recall_at_5": 0.81787, + "recall_at_10": 0.87646, + "recall_at_20": 0.91211, + "recall_at_100": 0.9707, + "recall_at_1000": 0.99951, + "precision_at_1": 0.54639, + "precision_at_3": 0.25098, + "precision_at_5": 0.16357, + "precision_at_10": 0.08765, + "precision_at_20": 0.04561, + "precision_at_100": 0.00971, + "precision_at_1000": 0.001, + "mrr_at_1": 0.545898, + "mrr_at_3": 0.63916, + "mrr_at_5": 0.653687, + "mrr_at_10": 0.661755, + "mrr_at_20": 0.664237, + "mrr_at_100": 0.66579, + "mrr_at_1000": 0.66593, + "nauc_ndcg_at_1_max": 0.306716, + "nauc_ndcg_at_1_std": -0.093182, + "nauc_ndcg_at_1_diff1": 0.484906, + "nauc_ndcg_at_3_max": 0.306213, + "nauc_ndcg_at_3_std": -0.124256, + "nauc_ndcg_at_3_diff1": 0.393802, + "nauc_ndcg_at_5_max": 0.298096, + "nauc_ndcg_at_5_std": -0.113463, + "nauc_ndcg_at_5_diff1": 0.375796, + "nauc_ndcg_at_10_max": 0.306107, + "nauc_ndcg_at_10_std": -0.114031, + "nauc_ndcg_at_10_diff1": 0.378823, + "nauc_ndcg_at_20_max": 0.3071, + "nauc_ndcg_at_20_std": -0.104389, + "nauc_ndcg_at_20_diff1": 0.378272, + "nauc_ndcg_at_100_max": 0.302918, + "nauc_ndcg_at_100_std": -0.093435, + "nauc_ndcg_at_100_diff1": 0.393219, + "nauc_ndcg_at_1000_max": 0.304044, + "nauc_ndcg_at_1000_std": -0.101853, + "nauc_ndcg_at_1000_diff1": 0.398854, + "nauc_map_at_1_max": 0.306716, + "nauc_map_at_1_std": -0.093182, + "nauc_map_at_1_diff1": 0.484906, + "nauc_map_at_3_max": 0.305528, + "nauc_map_at_3_std": -0.114527, + "nauc_map_at_3_diff1": 0.418245, + "nauc_map_at_5_max": 0.301281, + "nauc_map_at_5_std": -0.108813, + "nauc_map_at_5_diff1": 0.410305, + "nauc_map_at_10_max": 0.304147, + "nauc_map_at_10_std": -0.108948, + "nauc_map_at_10_diff1": 0.412221, + "nauc_map_at_20_max": 0.304309, + "nauc_map_at_20_std": -0.106642, + "nauc_map_at_20_diff1": 0.412226, + "nauc_map_at_100_max": 0.303756, + "nauc_map_at_100_std": -0.10503, + "nauc_map_at_100_diff1": 0.41402, + "nauc_map_at_1000_max": 0.303806, + "nauc_map_at_1000_std": -0.105248, + "nauc_map_at_1000_diff1": 0.414204, + "nauc_recall_at_1_max": 0.306716, + "nauc_recall_at_1_std": -0.093182, + "nauc_recall_at_1_diff1": 0.484906, + "nauc_recall_at_3_max": 0.309198, + "nauc_recall_at_3_std": -0.16091, + "nauc_recall_at_3_diff1": 0.303683, + "nauc_recall_at_5_max": 0.282432, + "nauc_recall_at_5_std": -0.132842, + "nauc_recall_at_5_diff1": 0.21351, + "nauc_recall_at_10_max": 0.324425, + "nauc_recall_at_10_std": -0.143791, + "nauc_recall_at_10_diff1": 0.163543, + "nauc_recall_at_20_max": 0.342413, + "nauc_recall_at_20_std": -0.05862, + "nauc_recall_at_20_diff1": 0.081599, + "nauc_recall_at_100_max": 0.257412, + "nauc_recall_at_100_std": 0.410821, + "nauc_recall_at_100_diff1": 0.077691, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 0.122811, + "nauc_precision_at_1_max": 0.306716, + "nauc_precision_at_1_std": -0.093182, + "nauc_precision_at_1_diff1": 0.484906, + "nauc_precision_at_3_max": 0.309198, + "nauc_precision_at_3_std": -0.16091, + "nauc_precision_at_3_diff1": 0.303683, + "nauc_precision_at_5_max": 0.282432, + "nauc_precision_at_5_std": -0.132842, + "nauc_precision_at_5_diff1": 0.21351, + "nauc_precision_at_10_max": 0.324425, + "nauc_precision_at_10_std": -0.143791, + "nauc_precision_at_10_diff1": 0.163543, + "nauc_precision_at_20_max": 0.342413, + "nauc_precision_at_20_std": -0.05862, + "nauc_precision_at_20_diff1": 0.081599, + "nauc_precision_at_100_max": 0.257412, + "nauc_precision_at_100_std": 0.410821, + "nauc_precision_at_100_diff1": 0.077691, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 0.122811, + "nauc_mrr_at_1_max": 0.308018, + "nauc_mrr_at_1_std": -0.092322, + "nauc_mrr_at_1_diff1": 0.486114, + "nauc_mrr_at_3_max": 0.306588, + "nauc_mrr_at_3_std": -0.113941, + "nauc_mrr_at_3_diff1": 0.418974, + "nauc_mrr_at_5_max": 0.302428, + "nauc_mrr_at_5_std": -0.108514, + "nauc_mrr_at_5_diff1": 0.411295, + "nauc_mrr_at_10_max": 0.305276, + "nauc_mrr_at_10_std": -0.108498, + "nauc_mrr_at_10_diff1": 0.413041, + "nauc_mrr_at_20_max": 0.305459, + "nauc_mrr_at_20_std": -0.106189, + "nauc_mrr_at_20_diff1": 0.413053, + "nauc_mrr_at_100_max": 0.304906, + "nauc_mrr_at_100_std": -0.104574, + "nauc_mrr_at_100_diff1": 0.414852, + "nauc_mrr_at_1000_max": 0.304956, + "nauc_mrr_at_1000_std": -0.104791, + "nauc_mrr_at_1000_diff1": 0.415035, + "main_score": 0.71432, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 12.289543867111206, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/NFCorpus.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/NFCorpus.json new file mode 100644 index 0000000000..8c794a35d5 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/NFCorpus.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814", + "task_name": "NFCorpus", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.46904, + "ndcg_at_3": 0.44046, + "ndcg_at_5": 0.41528, + "ndcg_at_10": 0.3771, + "ndcg_at_20": 0.35128, + "ndcg_at_100": 0.34223, + "ndcg_at_1000": 0.43157, + "map_at_1": 0.0631, + "map_at_3": 0.10617, + "map_at_5": 0.12379, + "map_at_10": 0.14343, + "map_at_20": 0.15926, + "map_at_100": 0.18049, + "map_at_1000": 0.19607, + "recall_at_1": 0.0631, + "recall_at_3": 0.11742, + "recall_at_5": 0.14844, + "recall_at_10": 0.18538, + "recall_at_20": 0.2215, + "recall_at_100": 0.33871, + "recall_at_1000": 0.66759, + "precision_at_1": 0.48916, + "precision_at_3": 0.41692, + "precision_at_5": 0.36099, + "precision_at_10": 0.27864, + "precision_at_20": 0.20635, + "precision_at_100": 0.08588, + "precision_at_1000": 0.02172, + "mrr_at_1": 0.489164, + "mrr_at_3": 0.563983, + "mrr_at_5": 0.575439, + "mrr_at_10": 0.584233, + "mrr_at_20": 0.586153, + "mrr_at_100": 0.588274, + "mrr_at_1000": 0.588813, + "nauc_ndcg_at_1_max": 0.454012, + "nauc_ndcg_at_1_std": 0.208542, + "nauc_ndcg_at_1_diff1": 0.385429, + "nauc_ndcg_at_3_max": 0.459412, + "nauc_ndcg_at_3_std": 0.249779, + "nauc_ndcg_at_3_diff1": 0.314818, + "nauc_ndcg_at_5_max": 0.446034, + "nauc_ndcg_at_5_std": 0.282246, + "nauc_ndcg_at_5_diff1": 0.281444, + "nauc_ndcg_at_10_max": 0.431678, + "nauc_ndcg_at_10_std": 0.294504, + "nauc_ndcg_at_10_diff1": 0.273425, + "nauc_ndcg_at_20_max": 0.427088, + "nauc_ndcg_at_20_std": 0.295907, + "nauc_ndcg_at_20_diff1": 0.268033, + "nauc_ndcg_at_100_max": 0.460797, + "nauc_ndcg_at_100_std": 0.241573, + "nauc_ndcg_at_100_diff1": 0.289138, + "nauc_ndcg_at_1000_max": 0.51152, + "nauc_ndcg_at_1000_std": 0.281273, + "nauc_ndcg_at_1000_diff1": 0.276374, + "nauc_map_at_1_max": 0.214719, + "nauc_map_at_1_std": -0.223747, + "nauc_map_at_1_diff1": 0.521155, + "nauc_map_at_3_max": 0.280987, + "nauc_map_at_3_std": -0.172083, + "nauc_map_at_3_diff1": 0.434463, + "nauc_map_at_5_max": 0.302495, + "nauc_map_at_5_std": -0.139439, + "nauc_map_at_5_diff1": 0.394965, + "nauc_map_at_10_max": 0.331034, + "nauc_map_at_10_std": -0.085229, + "nauc_map_at_10_diff1": 0.362688, + "nauc_map_at_20_max": 0.358085, + "nauc_map_at_20_std": -0.021894, + "nauc_map_at_20_diff1": 0.333963, + "nauc_map_at_100_max": 0.381533, + "nauc_map_at_100_std": 0.05081, + "nauc_map_at_100_diff1": 0.317559, + "nauc_map_at_1000_max": 0.382599, + "nauc_map_at_1000_std": 0.099384, + "nauc_map_at_1000_diff1": 0.303831, + "nauc_recall_at_1_max": 0.214719, + "nauc_recall_at_1_std": -0.223747, + "nauc_recall_at_1_diff1": 0.521155, + "nauc_recall_at_3_max": 0.263119, + "nauc_recall_at_3_std": -0.14261, + "nauc_recall_at_3_diff1": 0.407079, + "nauc_recall_at_5_max": 0.25428, + "nauc_recall_at_5_std": -0.128684, + "nauc_recall_at_5_diff1": 0.315512, + "nauc_recall_at_10_max": 0.261314, + "nauc_recall_at_10_std": -0.079434, + "nauc_recall_at_10_diff1": 0.274213, + "nauc_recall_at_20_max": 0.289248, + "nauc_recall_at_20_std": -0.001018, + "nauc_recall_at_20_diff1": 0.242907, + "nauc_recall_at_100_max": 0.281543, + "nauc_recall_at_100_std": 0.094521, + "nauc_recall_at_100_diff1": 0.184031, + "nauc_recall_at_1000_max": 0.233796, + "nauc_recall_at_1000_std": 0.108806, + "nauc_recall_at_1000_diff1": 0.077487, + "nauc_precision_at_1_max": 0.469482, + "nauc_precision_at_1_std": 0.208237, + "nauc_precision_at_1_diff1": 0.369117, + "nauc_precision_at_3_max": 0.449848, + "nauc_precision_at_3_std": 0.304981, + "nauc_precision_at_3_diff1": 0.195749, + "nauc_precision_at_5_max": 0.411892, + "nauc_precision_at_5_std": 0.384648, + "nauc_precision_at_5_diff1": 0.115383, + "nauc_precision_at_10_max": 0.365697, + "nauc_precision_at_10_std": 0.458168, + "nauc_precision_at_10_diff1": 0.055594, + "nauc_precision_at_20_max": 0.301281, + "nauc_precision_at_20_std": 0.52839, + "nauc_precision_at_20_diff1": -0.004133, + "nauc_precision_at_100_max": 0.140617, + "nauc_precision_at_100_std": 0.587644, + "nauc_precision_at_100_diff1": -0.067763, + "nauc_precision_at_1000_max": -0.022081, + "nauc_precision_at_1000_std": 0.533328, + "nauc_precision_at_1000_diff1": -0.121997, + "nauc_mrr_at_1_max": 0.469482, + "nauc_mrr_at_1_std": 0.208237, + "nauc_mrr_at_1_diff1": 0.369117, + "nauc_mrr_at_3_max": 0.52378, + "nauc_mrr_at_3_std": 0.264423, + "nauc_mrr_at_3_diff1": 0.370384, + "nauc_mrr_at_5_max": 0.528384, + "nauc_mrr_at_5_std": 0.258414, + "nauc_mrr_at_5_diff1": 0.361661, + "nauc_mrr_at_10_max": 0.526491, + "nauc_mrr_at_10_std": 0.268477, + "nauc_mrr_at_10_diff1": 0.362694, + "nauc_mrr_at_20_max": 0.526978, + "nauc_mrr_at_20_std": 0.269791, + "nauc_mrr_at_20_diff1": 0.363185, + "nauc_mrr_at_100_max": 0.52852, + "nauc_mrr_at_100_std": 0.271392, + "nauc_mrr_at_100_diff1": 0.363535, + "nauc_mrr_at_1000_max": 0.528091, + "nauc_mrr_at_1000_std": 0.270707, + "nauc_mrr_at_1000_diff1": 0.363509, + "main_score": 0.3771, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 21.76242160797119, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/NQ.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/NQ.json new file mode 100644 index 0000000000..8312eb2fbb --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/NQ.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31", + "task_name": "NQ", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.39658, + "ndcg_at_3": 0.51832, + "ndcg_at_5": 0.55911, + "ndcg_at_10": 0.59316, + "ndcg_at_20": 0.61195, + "ndcg_at_100": 0.6294, + "ndcg_at_1000": 0.63444, + "map_at_1": 0.35677, + "map_at_3": 0.47683, + "map_at_5": 0.50183, + "map_at_10": 0.51766, + "map_at_20": 0.52366, + "map_at_100": 0.52661, + "map_at_1000": 0.52684, + "recall_at_1": 0.35677, + "recall_at_3": 0.60636, + "recall_at_5": 0.69943, + "recall_at_10": 0.79736, + "recall_at_20": 0.86595, + "recall_at_100": 0.95119, + "recall_at_1000": 0.98827, + "precision_at_1": 0.39658, + "precision_at_3": 0.23329, + "precision_at_5": 0.16414, + "precision_at_10": 0.09479, + "precision_at_20": 0.05191, + "precision_at_100": 0.01153, + "precision_at_1000": 0.0012, + "mrr_at_1": 0.396582, + "mrr_at_3": 0.508353, + "mrr_at_5": 0.528442, + "mrr_at_10": 0.540847, + "mrr_at_20": 0.545227, + "mrr_at_100": 0.54732, + "mrr_at_1000": 0.547453, + "nauc_ndcg_at_1_max": 0.283066, + "nauc_ndcg_at_1_std": -0.065905, + "nauc_ndcg_at_1_diff1": 0.428896, + "nauc_ndcg_at_3_max": 0.328835, + "nauc_ndcg_at_3_std": -0.087535, + "nauc_ndcg_at_3_diff1": 0.380367, + "nauc_ndcg_at_5_max": 0.343844, + "nauc_ndcg_at_5_std": -0.076172, + "nauc_ndcg_at_5_diff1": 0.381501, + "nauc_ndcg_at_10_max": 0.353712, + "nauc_ndcg_at_10_std": -0.063362, + "nauc_ndcg_at_10_diff1": 0.379298, + "nauc_ndcg_at_20_max": 0.352294, + "nauc_ndcg_at_20_std": -0.058282, + "nauc_ndcg_at_20_diff1": 0.381473, + "nauc_ndcg_at_100_max": 0.346767, + "nauc_ndcg_at_100_std": -0.048825, + "nauc_ndcg_at_100_diff1": 0.386092, + "nauc_ndcg_at_1000_max": 0.341332, + "nauc_ndcg_at_1000_std": -0.055318, + "nauc_ndcg_at_1000_diff1": 0.388229, + "nauc_map_at_1_max": 0.268215, + "nauc_map_at_1_std": -0.087852, + "nauc_map_at_1_diff1": 0.432201, + "nauc_map_at_3_max": 0.31566, + "nauc_map_at_3_std": -0.092485, + "nauc_map_at_3_diff1": 0.393099, + "nauc_map_at_5_max": 0.326164, + "nauc_map_at_5_std": -0.085529, + "nauc_map_at_5_diff1": 0.394277, + "nauc_map_at_10_max": 0.330358, + "nauc_map_at_10_std": -0.079775, + "nauc_map_at_10_diff1": 0.393755, + "nauc_map_at_20_max": 0.329676, + "nauc_map_at_20_std": -0.07793, + "nauc_map_at_20_diff1": 0.394328, + "nauc_map_at_100_max": 0.328588, + "nauc_map_at_100_std": -0.076413, + "nauc_map_at_100_diff1": 0.394847, + "nauc_map_at_1000_max": 0.328383, + "nauc_map_at_1000_std": -0.076598, + "nauc_map_at_1000_diff1": 0.39493, + "nauc_recall_at_1_max": 0.268215, + "nauc_recall_at_1_std": -0.087852, + "nauc_recall_at_1_diff1": 0.432201, + "nauc_recall_at_3_max": 0.352894, + "nauc_recall_at_3_std": -0.096689, + "nauc_recall_at_3_diff1": 0.32902, + "nauc_recall_at_5_max": 0.394819, + "nauc_recall_at_5_std": -0.070782, + "nauc_recall_at_5_diff1": 0.322948, + "nauc_recall_at_10_max": 0.459577, + "nauc_recall_at_10_std": -0.016733, + "nauc_recall_at_10_diff1": 0.29698, + "nauc_recall_at_20_max": 0.502941, + "nauc_recall_at_20_std": 0.031955, + "nauc_recall_at_20_diff1": 0.285437, + "nauc_recall_at_100_max": 0.648974, + "nauc_recall_at_100_std": 0.382756, + "nauc_recall_at_100_diff1": 0.284357, + "nauc_recall_at_1000_max": 0.814024, + "nauc_recall_at_1000_std": 0.669712, + "nauc_recall_at_1000_diff1": 0.334242, + "nauc_precision_at_1_max": 0.283066, + "nauc_precision_at_1_std": -0.065905, + "nauc_precision_at_1_diff1": 0.428896, + "nauc_precision_at_3_max": 0.330093, + "nauc_precision_at_3_std": -0.032635, + "nauc_precision_at_3_diff1": 0.255846, + "nauc_precision_at_5_max": 0.314961, + "nauc_precision_at_5_std": 0.014871, + "nauc_precision_at_5_diff1": 0.194749, + "nauc_precision_at_10_max": 0.276133, + "nauc_precision_at_10_std": 0.082754, + "nauc_precision_at_10_diff1": 0.107454, + "nauc_precision_at_20_max": 0.214109, + "nauc_precision_at_20_std": 0.125255, + "nauc_precision_at_20_diff1": 0.04409, + "nauc_precision_at_100_max": 0.088036, + "nauc_precision_at_100_std": 0.205871, + "nauc_precision_at_100_diff1": -0.044257, + "nauc_precision_at_1000_max": 0.00197, + "nauc_precision_at_1000_std": 0.171575, + "nauc_precision_at_1000_diff1": -0.082531, + "nauc_mrr_at_1_max": 0.283066, + "nauc_mrr_at_1_std": -0.065905, + "nauc_mrr_at_1_diff1": 0.428896, + "nauc_mrr_at_3_max": 0.319029, + "nauc_mrr_at_3_std": -0.066019, + "nauc_mrr_at_3_diff1": 0.39265, + "nauc_mrr_at_5_max": 0.32289, + "nauc_mrr_at_5_std": -0.060721, + "nauc_mrr_at_5_diff1": 0.393389, + "nauc_mrr_at_10_max": 0.32446, + "nauc_mrr_at_10_std": -0.057777, + "nauc_mrr_at_10_diff1": 0.3928, + "nauc_mrr_at_20_max": 0.323669, + "nauc_mrr_at_20_std": -0.057429, + "nauc_mrr_at_20_diff1": 0.393871, + "nauc_mrr_at_100_max": 0.323233, + "nauc_mrr_at_100_std": -0.056762, + "nauc_mrr_at_100_diff1": 0.394445, + "nauc_mrr_at_1000_max": 0.323088, + "nauc_mrr_at_1000_std": -0.056939, + "nauc_mrr_at_1000_diff1": 0.394503, + "main_score": 0.59316, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 5394.206969261169, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/QuoraRetrieval.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/QuoraRetrieval.json new file mode 100644 index 0000000000..0b5f922c95 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/QuoraRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e4e08e0b7dbe3c8700f0daef558ff32256715259", + "task_name": "QuoraRetrieval", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.7993, + "ndcg_at_3": 0.84448, + "ndcg_at_5": 0.86167, + "ndcg_at_10": 0.87591, + "ndcg_at_20": 0.88287, + "ndcg_at_100": 0.88816, + "ndcg_at_1000": 0.88941, + "map_at_1": 0.69609, + "map_at_3": 0.80551, + "map_at_5": 0.82444, + "map_at_10": 0.83614, + "map_at_20": 0.84054, + "map_at_100": 0.8426, + "map_at_1000": 0.84279, + "recall_at_1": 0.69609, + "recall_at_3": 0.86512, + "recall_at_5": 0.91243, + "recall_at_10": 0.95336, + "recall_at_20": 0.97516, + "recall_at_100": 0.99424, + "recall_at_1000": 0.99978, + "precision_at_1": 0.7993, + "precision_at_3": 0.36863, + "precision_at_5": 0.2433, + "precision_at_10": 0.13334, + "precision_at_20": 0.07094, + "precision_at_100": 0.01522, + "precision_at_1000": 0.00157, + "mrr_at_1": 0.799, + "mrr_at_3": 0.855283, + "mrr_at_5": 0.862658, + "mrr_at_10": 0.866024, + "mrr_at_20": 0.86676, + "mrr_at_100": 0.866997, + "mrr_at_1000": 0.867007, + "nauc_ndcg_at_1_max": 0.344074, + "nauc_ndcg_at_1_std": -0.377892, + "nauc_ndcg_at_1_diff1": 0.765606, + "nauc_ndcg_at_3_max": 0.347229, + "nauc_ndcg_at_3_std": -0.436152, + "nauc_ndcg_at_3_diff1": 0.74979, + "nauc_ndcg_at_5_max": 0.347695, + "nauc_ndcg_at_5_std": -0.449427, + "nauc_ndcg_at_5_diff1": 0.755886, + "nauc_ndcg_at_10_max": 0.347265, + "nauc_ndcg_at_10_std": -0.438899, + "nauc_ndcg_at_10_diff1": 0.755895, + "nauc_ndcg_at_20_max": 0.346239, + "nauc_ndcg_at_20_std": -0.428059, + "nauc_ndcg_at_20_diff1": 0.755484, + "nauc_ndcg_at_100_max": 0.34726, + "nauc_ndcg_at_100_std": -0.41471, + "nauc_ndcg_at_100_diff1": 0.754454, + "nauc_ndcg_at_1000_max": 0.348877, + "nauc_ndcg_at_1000_std": -0.413371, + "nauc_ndcg_at_1000_diff1": 0.754289, + "nauc_map_at_1_max": 0.275548, + "nauc_map_at_1_std": -0.398203, + "nauc_map_at_1_diff1": 0.787864, + "nauc_map_at_3_max": 0.322126, + "nauc_map_at_3_std": -0.459902, + "nauc_map_at_3_diff1": 0.762948, + "nauc_map_at_5_max": 0.330564, + "nauc_map_at_5_std": -0.461119, + "nauc_map_at_5_diff1": 0.762863, + "nauc_map_at_10_max": 0.336732, + "nauc_map_at_10_std": -0.447738, + "nauc_map_at_10_diff1": 0.760207, + "nauc_map_at_20_max": 0.34012, + "nauc_map_at_20_std": -0.438919, + "nauc_map_at_20_diff1": 0.758517, + "nauc_map_at_100_max": 0.341984, + "nauc_map_at_100_std": -0.433006, + "nauc_map_at_100_diff1": 0.757525, + "nauc_map_at_1000_max": 0.342334, + "nauc_map_at_1000_std": -0.432465, + "nauc_map_at_1000_diff1": 0.757504, + "nauc_recall_at_1_max": 0.275548, + "nauc_recall_at_1_std": -0.398203, + "nauc_recall_at_1_diff1": 0.787864, + "nauc_recall_at_3_max": 0.307626, + "nauc_recall_at_3_std": -0.502995, + "nauc_recall_at_3_diff1": 0.721091, + "nauc_recall_at_5_max": 0.308521, + "nauc_recall_at_5_std": -0.554321, + "nauc_recall_at_5_diff1": 0.718366, + "nauc_recall_at_10_max": 0.312458, + "nauc_recall_at_10_std": -0.556543, + "nauc_recall_at_10_diff1": 0.704843, + "nauc_recall_at_20_max": 0.309105, + "nauc_recall_at_20_std": -0.525682, + "nauc_recall_at_20_diff1": 0.707568, + "nauc_recall_at_100_max": 0.286504, + "nauc_recall_at_100_std": -0.293732, + "nauc_recall_at_100_diff1": 0.694806, + "nauc_recall_at_1000_max": 0.388025, + "nauc_recall_at_1000_std": 0.279872, + "nauc_recall_at_1000_diff1": 0.229559, + "nauc_precision_at_1_max": 0.344074, + "nauc_precision_at_1_std": -0.377892, + "nauc_precision_at_1_diff1": 0.765606, + "nauc_precision_at_3_max": 0.06839, + "nauc_precision_at_3_std": 0.03314, + "nauc_precision_at_3_diff1": -0.139, + "nauc_precision_at_5_max": 0.010055, + "nauc_precision_at_5_std": 0.142323, + "nauc_precision_at_5_diff1": -0.280009, + "nauc_precision_at_10_max": -0.031706, + "nauc_precision_at_10_std": 0.246986, + "nauc_precision_at_10_diff1": -0.372001, + "nauc_precision_at_20_max": -0.045283, + "nauc_precision_at_20_std": 0.302653, + "nauc_precision_at_20_diff1": -0.405613, + "nauc_precision_at_100_max": -0.053236, + "nauc_precision_at_100_std": 0.351935, + "nauc_precision_at_100_diff1": -0.427876, + "nauc_precision_at_1000_max": -0.047894, + "nauc_precision_at_1000_std": 0.365056, + "nauc_precision_at_1000_diff1": -0.432135, + "nauc_mrr_at_1_max": 0.342901, + "nauc_mrr_at_1_std": -0.377969, + "nauc_mrr_at_1_diff1": 0.766321, + "nauc_mrr_at_3_max": 0.355344, + "nauc_mrr_at_3_std": -0.396042, + "nauc_mrr_at_3_diff1": 0.755272, + "nauc_mrr_at_5_max": 0.353983, + "nauc_mrr_at_5_std": -0.397702, + "nauc_mrr_at_5_diff1": 0.757895, + "nauc_mrr_at_10_max": 0.352877, + "nauc_mrr_at_10_std": -0.394684, + "nauc_mrr_at_10_diff1": 0.758037, + "nauc_mrr_at_20_max": 0.352819, + "nauc_mrr_at_20_std": -0.394102, + "nauc_mrr_at_20_diff1": 0.758249, + "nauc_mrr_at_100_max": 0.352784, + "nauc_mrr_at_100_std": -0.393944, + "nauc_mrr_at_100_diff1": 0.758298, + "nauc_mrr_at_1000_max": 0.352776, + "nauc_mrr_at_1000_std": -0.393984, + "nauc_mrr_at_1000_diff1": 0.758304, + "main_score": 0.87591, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 341.797137260437, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/SCIDOCS.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/SCIDOCS.json new file mode 100644 index 0000000000..c34ebfdd90 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/SCIDOCS.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88", + "task_name": "SCIDOCS", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.216, + "ndcg_at_3": 0.17829, + "ndcg_at_5": 0.15496, + "ndcg_at_10": 0.1909, + "ndcg_at_20": 0.21921, + "ndcg_at_100": 0.27131, + "ndcg_at_1000": 0.32853, + "map_at_1": 0.04388, + "map_at_3": 0.07991, + "map_at_5": 0.09513, + "map_at_10": 0.11292, + "map_at_20": 0.12378, + "map_at_100": 0.13366, + "map_at_1000": 0.13693, + "recall_at_1": 0.04388, + "recall_at_3": 0.10193, + "recall_at_5": 0.13803, + "recall_at_10": 0.2043, + "recall_at_20": 0.27053, + "recall_at_100": 0.44063, + "recall_at_1000": 0.72008, + "precision_at_1": 0.216, + "precision_at_3": 0.16767, + "precision_at_5": 0.1362, + "precision_at_10": 0.1008, + "precision_at_20": 0.06665, + "precision_at_100": 0.0217, + "precision_at_1000": 0.00355, + "mrr_at_1": 0.216, + "mrr_at_3": 0.287167, + "mrr_at_5": 0.304367, + "mrr_at_10": 0.319888, + "mrr_at_20": 0.32649, + "mrr_at_100": 0.331276, + "mrr_at_1000": 0.331828, + "nauc_ndcg_at_1_max": 0.270552, + "nauc_ndcg_at_1_std": 0.051502, + "nauc_ndcg_at_1_diff1": 0.236695, + "nauc_ndcg_at_3_max": 0.315218, + "nauc_ndcg_at_3_std": 0.099945, + "nauc_ndcg_at_3_diff1": 0.206575, + "nauc_ndcg_at_5_max": 0.323847, + "nauc_ndcg_at_5_std": 0.119437, + "nauc_ndcg_at_5_diff1": 0.194549, + "nauc_ndcg_at_10_max": 0.332971, + "nauc_ndcg_at_10_std": 0.144819, + "nauc_ndcg_at_10_diff1": 0.182283, + "nauc_ndcg_at_20_max": 0.332861, + "nauc_ndcg_at_20_std": 0.1667, + "nauc_ndcg_at_20_diff1": 0.168499, + "nauc_ndcg_at_100_max": 0.355971, + "nauc_ndcg_at_100_std": 0.19364, + "nauc_ndcg_at_100_diff1": 0.154985, + "nauc_ndcg_at_1000_max": 0.356195, + "nauc_ndcg_at_1000_std": 0.212462, + "nauc_ndcg_at_1000_diff1": 0.154606, + "nauc_map_at_1_max": 0.268967, + "nauc_map_at_1_std": 0.049291, + "nauc_map_at_1_diff1": 0.239503, + "nauc_map_at_3_max": 0.320119, + "nauc_map_at_3_std": 0.070216, + "nauc_map_at_3_diff1": 0.223668, + "nauc_map_at_5_max": 0.331928, + "nauc_map_at_5_std": 0.083522, + "nauc_map_at_5_diff1": 0.209426, + "nauc_map_at_10_max": 0.341021, + "nauc_map_at_10_std": 0.118858, + "nauc_map_at_10_diff1": 0.197128, + "nauc_map_at_20_max": 0.339618, + "nauc_map_at_20_std": 0.13262, + "nauc_map_at_20_diff1": 0.186528, + "nauc_map_at_100_max": 0.34735, + "nauc_map_at_100_std": 0.14313, + "nauc_map_at_100_diff1": 0.182597, + "nauc_map_at_1000_max": 0.347869, + "nauc_map_at_1000_std": 0.146859, + "nauc_map_at_1000_diff1": 0.182533, + "nauc_recall_at_1_max": 0.268967, + "nauc_recall_at_1_std": 0.049291, + "nauc_recall_at_1_diff1": 0.239503, + "nauc_recall_at_3_max": 0.323349, + "nauc_recall_at_3_std": 0.115915, + "nauc_recall_at_3_diff1": 0.188884, + "nauc_recall_at_5_max": 0.325078, + "nauc_recall_at_5_std": 0.145866, + "nauc_recall_at_5_diff1": 0.162738, + "nauc_recall_at_10_max": 0.325974, + "nauc_recall_at_10_std": 0.18254, + "nauc_recall_at_10_diff1": 0.138275, + "nauc_recall_at_20_max": 0.305885, + "nauc_recall_at_20_std": 0.213138, + "nauc_recall_at_20_diff1": 0.104283, + "nauc_recall_at_100_max": 0.331459, + "nauc_recall_at_100_std": 0.256728, + "nauc_recall_at_100_diff1": 0.059451, + "nauc_recall_at_1000_max": 0.294912, + "nauc_recall_at_1000_std": 0.328304, + "nauc_recall_at_1000_diff1": 0.01849, + "nauc_precision_at_1_max": 0.270552, + "nauc_precision_at_1_std": 0.051502, + "nauc_precision_at_1_diff1": 0.236695, + "nauc_precision_at_3_max": 0.324807, + "nauc_precision_at_3_std": 0.118666, + "nauc_precision_at_3_diff1": 0.187327, + "nauc_precision_at_5_max": 0.328418, + "nauc_precision_at_5_std": 0.147908, + "nauc_precision_at_5_diff1": 0.162749, + "nauc_precision_at_10_max": 0.329658, + "nauc_precision_at_10_std": 0.181987, + "nauc_precision_at_10_diff1": 0.138907, + "nauc_precision_at_20_max": 0.311173, + "nauc_precision_at_20_std": 0.213278, + "nauc_precision_at_20_diff1": 0.105685, + "nauc_precision_at_100_max": 0.333111, + "nauc_precision_at_100_std": 0.254339, + "nauc_precision_at_100_diff1": 0.058785, + "nauc_precision_at_1000_max": 0.292715, + "nauc_precision_at_1000_std": 0.322032, + "nauc_precision_at_1000_diff1": 0.013743, + "nauc_mrr_at_1_max": 0.270552, + "nauc_mrr_at_1_std": 0.051502, + "nauc_mrr_at_1_diff1": 0.236695, + "nauc_mrr_at_3_max": 0.296075, + "nauc_mrr_at_3_std": 0.113503, + "nauc_mrr_at_3_diff1": 0.207801, + "nauc_mrr_at_5_max": 0.29282, + "nauc_mrr_at_5_std": 0.121034, + "nauc_mrr_at_5_diff1": 0.205928, + "nauc_mrr_at_10_max": 0.294537, + "nauc_mrr_at_10_std": 0.116293, + "nauc_mrr_at_10_diff1": 0.204221, + "nauc_mrr_at_20_max": 0.297148, + "nauc_mrr_at_20_std": 0.120347, + "nauc_mrr_at_20_diff1": 0.205301, + "nauc_mrr_at_100_max": 0.297678, + "nauc_mrr_at_100_std": 0.120605, + "nauc_mrr_at_100_diff1": 0.204445, + "nauc_mrr_at_1000_max": 0.297533, + "nauc_mrr_at_1000_std": 0.120385, + "nauc_mrr_at_1000_diff1": 0.204546, + "main_score": 0.1909, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 111.82242441177368, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/SciFact.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/SciFact.json new file mode 100644 index 0000000000..b71bc17faa --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/SciFact.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "0228b52cf27578f30900b9e5271d331663a030d7", + "task_name": "SciFact", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.63, + "ndcg_at_3": 0.70277, + "ndcg_at_5": 0.7269, + "ndcg_at_10": 0.74676, + "ndcg_at_20": 0.75935, + "ndcg_at_100": 0.76817, + "ndcg_at_1000": 0.77162, + "map_at_1": 0.59328, + "map_at_3": 0.67213, + "map_at_5": 0.68984, + "map_at_10": 0.69991, + "map_at_20": 0.70389, + "map_at_100": 0.70514, + "map_at_1000": 0.70527, + "recall_at_1": 0.59328, + "recall_at_3": 0.75767, + "recall_at_5": 0.81817, + "recall_at_10": 0.87467, + "recall_at_20": 0.922, + "recall_at_100": 0.97, + "recall_at_1000": 0.99667, + "precision_at_1": 0.63, + "precision_at_3": 0.27778, + "precision_at_5": 0.18267, + "precision_at_10": 0.099, + "precision_at_20": 0.05233, + "precision_at_100": 0.011, + "precision_at_1000": 0.00113, + "mrr_at_1": 0.63, + "mrr_at_3": 0.695, + "mrr_at_5": 0.707667, + "mrr_at_10": 0.714521, + "mrr_at_20": 0.71739, + "mrr_at_100": 0.718438, + "mrr_at_1000": 0.718573, + "nauc_ndcg_at_1_max": 0.510193, + "nauc_ndcg_at_1_std": -0.029281, + "nauc_ndcg_at_1_diff1": 0.752435, + "nauc_ndcg_at_3_max": 0.533874, + "nauc_ndcg_at_3_std": -0.094588, + "nauc_ndcg_at_3_diff1": 0.70887, + "nauc_ndcg_at_5_max": 0.529655, + "nauc_ndcg_at_5_std": -0.067916, + "nauc_ndcg_at_5_diff1": 0.717421, + "nauc_ndcg_at_10_max": 0.555475, + "nauc_ndcg_at_10_std": -0.066464, + "nauc_ndcg_at_10_diff1": 0.722283, + "nauc_ndcg_at_20_max": 0.552748, + "nauc_ndcg_at_20_std": -0.059866, + "nauc_ndcg_at_20_diff1": 0.719784, + "nauc_ndcg_at_100_max": 0.547657, + "nauc_ndcg_at_100_std": -0.05679, + "nauc_ndcg_at_100_diff1": 0.719356, + "nauc_ndcg_at_1000_max": 0.544027, + "nauc_ndcg_at_1000_std": -0.060024, + "nauc_ndcg_at_1000_diff1": 0.722998, + "nauc_map_at_1_max": 0.477571, + "nauc_map_at_1_std": -0.115046, + "nauc_map_at_1_diff1": 0.752935, + "nauc_map_at_3_max": 0.518317, + "nauc_map_at_3_std": -0.11074, + "nauc_map_at_3_diff1": 0.72169, + "nauc_map_at_5_max": 0.518318, + "nauc_map_at_5_std": -0.085786, + "nauc_map_at_5_diff1": 0.725694, + "nauc_map_at_10_max": 0.532102, + "nauc_map_at_10_std": -0.081602, + "nauc_map_at_10_diff1": 0.726745, + "nauc_map_at_20_max": 0.531608, + "nauc_map_at_20_std": -0.0776, + "nauc_map_at_20_diff1": 0.726137, + "nauc_map_at_100_max": 0.531044, + "nauc_map_at_100_std": -0.078035, + "nauc_map_at_100_diff1": 0.726012, + "nauc_map_at_1000_max": 0.530947, + "nauc_map_at_1000_std": -0.078201, + "nauc_map_at_1000_diff1": 0.726197, + "nauc_recall_at_1_max": 0.477571, + "nauc_recall_at_1_std": -0.115046, + "nauc_recall_at_1_diff1": 0.752935, + "nauc_recall_at_3_max": 0.540886, + "nauc_recall_at_3_std": -0.147166, + "nauc_recall_at_3_diff1": 0.665066, + "nauc_recall_at_5_max": 0.532647, + "nauc_recall_at_5_std": -0.058377, + "nauc_recall_at_5_diff1": 0.670435, + "nauc_recall_at_10_max": 0.660795, + "nauc_recall_at_10_std": -0.069026, + "nauc_recall_at_10_diff1": 0.69264, + "nauc_recall_at_20_max": 0.694391, + "nauc_recall_at_20_std": -0.040082, + "nauc_recall_at_20_diff1": 0.660973, + "nauc_recall_at_100_max": 0.73628, + "nauc_recall_at_100_std": 0.171958, + "nauc_recall_at_100_diff1": 0.5651, + "nauc_recall_at_1000_max": 0.554155, + "nauc_recall_at_1000_std": 0.722222, + "nauc_recall_at_1000_diff1": 0.722222, + "nauc_precision_at_1_max": 0.510193, + "nauc_precision_at_1_std": -0.029281, + "nauc_precision_at_1_diff1": 0.752435, + "nauc_precision_at_3_max": 0.411117, + "nauc_precision_at_3_std": 0.089206, + "nauc_precision_at_3_diff1": 0.387084, + "nauc_precision_at_5_max": 0.315274, + "nauc_precision_at_5_std": 0.217489, + "nauc_precision_at_5_diff1": 0.259243, + "nauc_precision_at_10_max": 0.287864, + "nauc_precision_at_10_std": 0.257423, + "nauc_precision_at_10_diff1": 0.118013, + "nauc_precision_at_20_max": 0.19852, + "nauc_precision_at_20_std": 0.329063, + "nauc_precision_at_20_diff1": -0.009932, + "nauc_precision_at_100_max": 0.087723, + "nauc_precision_at_100_std": 0.417036, + "nauc_precision_at_100_diff1": -0.160492, + "nauc_precision_at_1000_max": 0.009729, + "nauc_precision_at_1000_std": 0.4535, + "nauc_precision_at_1000_diff1": -0.24251, + "nauc_mrr_at_1_max": 0.510193, + "nauc_mrr_at_1_std": -0.029281, + "nauc_mrr_at_1_diff1": 0.752435, + "nauc_mrr_at_3_max": 0.539387, + "nauc_mrr_at_3_std": -0.054434, + "nauc_mrr_at_3_diff1": 0.722933, + "nauc_mrr_at_5_max": 0.53743, + "nauc_mrr_at_5_std": -0.036818, + "nauc_mrr_at_5_diff1": 0.723288, + "nauc_mrr_at_10_max": 0.544595, + "nauc_mrr_at_10_std": -0.038582, + "nauc_mrr_at_10_diff1": 0.726847, + "nauc_mrr_at_20_max": 0.542874, + "nauc_mrr_at_20_std": -0.040145, + "nauc_mrr_at_20_diff1": 0.726599, + "nauc_mrr_at_100_max": 0.542566, + "nauc_mrr_at_100_std": -0.040567, + "nauc_mrr_at_100_diff1": 0.72678, + "nauc_mrr_at_1000_max": 0.542472, + "nauc_mrr_at_1000_std": -0.040717, + "nauc_mrr_at_1000_diff1": 0.726972, + "main_score": 0.74676, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 28.416662454605103, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/StackOverflowQA.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/StackOverflowQA.json new file mode 100644 index 0000000000..80c3dfa6e4 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/StackOverflowQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "db8f169f3894c14a00251061f957b2063eef2bd5", + "task_name": "StackOverflowQA", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.78536, + "ndcg_at_3": 0.83756, + "ndcg_at_5": 0.84991, + "ndcg_at_10": 0.85742, + "ndcg_at_20": 0.86362, + "ndcg_at_100": 0.86855, + "ndcg_at_1000": 0.87096, + "map_at_1": 0.78536, + "map_at_3": 0.82506, + "map_at_5": 0.832, + "map_at_10": 0.83513, + "map_at_20": 0.83689, + "map_at_100": 0.83755, + "map_at_1000": 0.83763, + "recall_at_1": 0.78536, + "recall_at_3": 0.87362, + "recall_at_5": 0.90321, + "recall_at_10": 0.92628, + "recall_at_20": 0.95035, + "recall_at_100": 0.97743, + "recall_at_1000": 0.99649, + "precision_at_1": 0.78536, + "precision_at_3": 0.29121, + "precision_at_5": 0.18064, + "precision_at_10": 0.09263, + "precision_at_20": 0.04752, + "precision_at_100": 0.00977, + "precision_at_1000": 0.001, + "mrr_at_1": 0.785356, + "mrr_at_3": 0.825059, + "mrr_at_5": 0.832004, + "mrr_at_10": 0.835134, + "mrr_at_20": 0.836893, + "mrr_at_100": 0.837545, + "mrr_at_1000": 0.837634, + "nauc_ndcg_at_1_max": 0.760948, + "nauc_ndcg_at_1_std": 0.050334, + "nauc_ndcg_at_1_diff1": 0.89079, + "nauc_ndcg_at_3_max": 0.758974, + "nauc_ndcg_at_3_std": 0.087105, + "nauc_ndcg_at_3_diff1": 0.868565, + "nauc_ndcg_at_5_max": 0.756068, + "nauc_ndcg_at_5_std": 0.095085, + "nauc_ndcg_at_5_diff1": 0.865168, + "nauc_ndcg_at_10_max": 0.754591, + "nauc_ndcg_at_10_std": 0.087938, + "nauc_ndcg_at_10_diff1": 0.865026, + "nauc_ndcg_at_20_max": 0.75723, + "nauc_ndcg_at_20_std": 0.089273, + "nauc_ndcg_at_20_diff1": 0.866584, + "nauc_ndcg_at_100_max": 0.758347, + "nauc_ndcg_at_100_std": 0.088007, + "nauc_ndcg_at_100_diff1": 0.869334, + "nauc_ndcg_at_1000_max": 0.758288, + "nauc_ndcg_at_1000_std": 0.085485, + "nauc_ndcg_at_1000_diff1": 0.870194, + "nauc_map_at_1_max": 0.760948, + "nauc_map_at_1_std": 0.050334, + "nauc_map_at_1_diff1": 0.89079, + "nauc_map_at_3_max": 0.760002, + "nauc_map_at_3_std": 0.075692, + "nauc_map_at_3_diff1": 0.875096, + "nauc_map_at_5_max": 0.75849, + "nauc_map_at_5_std": 0.079364, + "nauc_map_at_5_diff1": 0.873565, + "nauc_map_at_10_max": 0.758008, + "nauc_map_at_10_std": 0.076732, + "nauc_map_at_10_diff1": 0.873686, + "nauc_map_at_20_max": 0.758712, + "nauc_map_at_20_std": 0.076638, + "nauc_map_at_20_diff1": 0.874164, + "nauc_map_at_100_max": 0.758918, + "nauc_map_at_100_std": 0.076836, + "nauc_map_at_100_diff1": 0.874517, + "nauc_map_at_1000_max": 0.758924, + "nauc_map_at_1000_std": 0.076791, + "nauc_map_at_1000_diff1": 0.874545, + "nauc_recall_at_1_max": 0.760948, + "nauc_recall_at_1_std": 0.050334, + "nauc_recall_at_1_diff1": 0.89079, + "nauc_recall_at_3_max": 0.754761, + "nauc_recall_at_3_std": 0.131474, + "nauc_recall_at_3_diff1": 0.843366, + "nauc_recall_at_5_max": 0.74388, + "nauc_recall_at_5_std": 0.171196, + "nauc_recall_at_5_diff1": 0.824972, + "nauc_recall_at_10_max": 0.731708, + "nauc_recall_at_10_std": 0.151717, + "nauc_recall_at_10_diff1": 0.811562, + "nauc_recall_at_20_max": 0.746463, + "nauc_recall_at_20_std": 0.202524, + "nauc_recall_at_20_diff1": 0.801182, + "nauc_recall_at_100_max": 0.75947, + "nauc_recall_at_100_std": 0.257567, + "nauc_recall_at_100_diff1": 0.80347, + "nauc_recall_at_1000_max": 0.741648, + "nauc_recall_at_1000_std": 0.414511, + "nauc_recall_at_1000_diff1": 0.682886, + "nauc_precision_at_1_max": 0.760948, + "nauc_precision_at_1_std": 0.050334, + "nauc_precision_at_1_diff1": 0.89079, + "nauc_precision_at_3_max": 0.754761, + "nauc_precision_at_3_std": 0.131474, + "nauc_precision_at_3_diff1": 0.843366, + "nauc_precision_at_5_max": 0.74388, + "nauc_precision_at_5_std": 0.171196, + "nauc_precision_at_5_diff1": 0.824972, + "nauc_precision_at_10_max": 0.731708, + "nauc_precision_at_10_std": 0.151717, + "nauc_precision_at_10_diff1": 0.811562, + "nauc_precision_at_20_max": 0.746463, + "nauc_precision_at_20_std": 0.202524, + "nauc_precision_at_20_diff1": 0.801182, + "nauc_precision_at_100_max": 0.75947, + "nauc_precision_at_100_std": 0.257567, + "nauc_precision_at_100_diff1": 0.80347, + "nauc_precision_at_1000_max": 0.741648, + "nauc_precision_at_1000_std": 0.414511, + "nauc_precision_at_1000_diff1": 0.682886, + "nauc_mrr_at_1_max": 0.760948, + "nauc_mrr_at_1_std": 0.050334, + "nauc_mrr_at_1_diff1": 0.89079, + "nauc_mrr_at_3_max": 0.760002, + "nauc_mrr_at_3_std": 0.075692, + "nauc_mrr_at_3_diff1": 0.875096, + "nauc_mrr_at_5_max": 0.75849, + "nauc_mrr_at_5_std": 0.079364, + "nauc_mrr_at_5_diff1": 0.873565, + "nauc_mrr_at_10_max": 0.758008, + "nauc_mrr_at_10_std": 0.076732, + "nauc_mrr_at_10_diff1": 0.873686, + "nauc_mrr_at_20_max": 0.758712, + "nauc_mrr_at_20_std": 0.076638, + "nauc_mrr_at_20_diff1": 0.874164, + "nauc_mrr_at_100_max": 0.758918, + "nauc_mrr_at_100_std": 0.076836, + "nauc_mrr_at_100_diff1": 0.874517, + "nauc_mrr_at_1000_max": 0.758924, + "nauc_mrr_at_1000_std": 0.076791, + "nauc_mrr_at_1000_diff1": 0.874545, + "main_score": 0.85742, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 128.66042852401733, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/SyntheticText2SQL.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/SyntheticText2SQL.json new file mode 100644 index 0000000000..53e509de8e --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/SyntheticText2SQL.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "686b87296c3a0191b5d9415a00526c62db9fce09", + "task_name": "SyntheticText2SQL", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.15109, + "ndcg_at_3": 0.51449, + "ndcg_at_5": 0.54304, + "ndcg_at_10": 0.56305, + "ndcg_at_20": 0.57382, + "ndcg_at_100": 0.58441, + "ndcg_at_1000": 0.5878, + "map_at_1": 0.15109, + "map_at_3": 0.43437, + "map_at_5": 0.45032, + "map_at_10": 0.45871, + "map_at_20": 0.46166, + "map_at_100": 0.46319, + "map_at_1000": 0.46333, + "recall_at_1": 0.15109, + "recall_at_3": 0.74261, + "recall_at_5": 0.81149, + "recall_at_10": 0.87267, + "recall_at_20": 0.91523, + "recall_at_100": 0.97146, + "recall_at_1000": 0.99761, + "precision_at_1": 0.15109, + "precision_at_3": 0.24754, + "precision_at_5": 0.1623, + "precision_at_10": 0.08727, + "precision_at_20": 0.04576, + "precision_at_100": 0.00971, + "precision_at_1000": 0.001, + "mrr_at_1": 0.36028, + "mrr_at_3": 0.548482, + "mrr_at_5": 0.56212, + "mrr_at_10": 0.570037, + "mrr_at_20": 0.572906, + "mrr_at_100": 0.574418, + "mrr_at_1000": 0.574554, + "nauc_ndcg_at_1_max": 0.141017, + "nauc_ndcg_at_1_std": -0.081588, + "nauc_ndcg_at_1_diff1": 0.180997, + "nauc_ndcg_at_3_max": 0.358984, + "nauc_ndcg_at_3_std": -0.24462, + "nauc_ndcg_at_3_diff1": -0.5136, + "nauc_ndcg_at_5_max": 0.35076, + "nauc_ndcg_at_5_std": -0.236875, + "nauc_ndcg_at_5_diff1": -0.46692, + "nauc_ndcg_at_10_max": 0.338845, + "nauc_ndcg_at_10_std": -0.225822, + "nauc_ndcg_at_10_diff1": -0.427826, + "nauc_ndcg_at_20_max": 0.331535, + "nauc_ndcg_at_20_std": -0.21857, + "nauc_ndcg_at_20_diff1": -0.403764, + "nauc_ndcg_at_100_max": 0.323987, + "nauc_ndcg_at_100_std": -0.20664, + "nauc_ndcg_at_100_diff1": -0.377675, + "nauc_ndcg_at_1000_max": 0.319051, + "nauc_ndcg_at_1000_std": -0.207272, + "nauc_ndcg_at_1000_diff1": -0.368594, + "nauc_map_at_1_max": 0.141017, + "nauc_map_at_1_std": -0.081588, + "nauc_map_at_1_diff1": 0.180997, + "nauc_map_at_3_max": 0.316646, + "nauc_map_at_3_std": -0.21207, + "nauc_map_at_3_diff1": -0.382196, + "nauc_map_at_5_max": 0.309971, + "nauc_map_at_5_std": -0.206481, + "nauc_map_at_5_diff1": -0.350835, + "nauc_map_at_10_max": 0.304291, + "nauc_map_at_10_std": -0.201306, + "nauc_map_at_10_diff1": -0.332351, + "nauc_map_at_20_max": 0.302218, + "nauc_map_at_20_std": -0.199201, + "nauc_map_at_20_diff1": -0.325396, + "nauc_map_at_100_max": 0.301256, + "nauc_map_at_100_std": -0.197833, + "nauc_map_at_100_diff1": -0.321805, + "nauc_map_at_1000_max": 0.301083, + "nauc_map_at_1000_std": -0.197824, + "nauc_map_at_1000_diff1": -0.321452, + "nauc_recall_at_1_max": 0.141017, + "nauc_recall_at_1_std": -0.081588, + "nauc_recall_at_1_diff1": 0.180997, + "nauc_recall_at_3_max": 0.482841, + "nauc_recall_at_3_std": -0.340307, + "nauc_recall_at_3_diff1": -0.896184, + "nauc_recall_at_5_max": 0.495291, + "nauc_recall_at_5_std": -0.344263, + "nauc_recall_at_5_diff1": -0.869233, + "nauc_recall_at_10_max": 0.497564, + "nauc_recall_at_10_std": -0.336339, + "nauc_recall_at_10_diff1": -0.849969, + "nauc_recall_at_20_max": 0.504722, + "nauc_recall_at_20_std": -0.325206, + "nauc_recall_at_20_diff1": -0.834267, + "nauc_recall_at_100_max": 0.580743, + "nauc_recall_at_100_std": -0.120681, + "nauc_recall_at_100_diff1": -0.815997, + "nauc_recall_at_1000_max": 0.598608, + "nauc_recall_at_1000_std": 0.327795, + "nauc_recall_at_1000_diff1": -0.623841, + "nauc_precision_at_1_max": 0.141017, + "nauc_precision_at_1_std": -0.081588, + "nauc_precision_at_1_diff1": 0.180997, + "nauc_precision_at_3_max": 0.482841, + "nauc_precision_at_3_std": -0.340307, + "nauc_precision_at_3_diff1": -0.896184, + "nauc_precision_at_5_max": 0.495291, + "nauc_precision_at_5_std": -0.344263, + "nauc_precision_at_5_diff1": -0.869233, + "nauc_precision_at_10_max": 0.497564, + "nauc_precision_at_10_std": -0.336339, + "nauc_precision_at_10_diff1": -0.849969, + "nauc_precision_at_20_max": 0.504722, + "nauc_precision_at_20_std": -0.325206, + "nauc_precision_at_20_diff1": -0.834267, + "nauc_precision_at_100_max": 0.580743, + "nauc_precision_at_100_std": -0.120681, + "nauc_precision_at_100_diff1": -0.815997, + "nauc_precision_at_1000_max": 0.598608, + "nauc_precision_at_1000_std": 0.327795, + "nauc_precision_at_1000_diff1": -0.623841, + "nauc_mrr_at_1_max": 0.213741, + "nauc_mrr_at_1_std": -0.146452, + "nauc_mrr_at_1_diff1": -0.33545, + "nauc_mrr_at_3_max": 0.31828, + "nauc_mrr_at_3_std": -0.223236, + "nauc_mrr_at_3_diff1": -0.563428, + "nauc_mrr_at_5_max": 0.314992, + "nauc_mrr_at_5_std": -0.218213, + "nauc_mrr_at_5_diff1": -0.547529, + "nauc_mrr_at_10_max": 0.31097, + "nauc_mrr_at_10_std": -0.214911, + "nauc_mrr_at_10_diff1": -0.538081, + "nauc_mrr_at_20_max": 0.309435, + "nauc_mrr_at_20_std": -0.213291, + "nauc_mrr_at_20_diff1": -0.53482, + "nauc_mrr_at_100_max": 0.308751, + "nauc_mrr_at_100_std": -0.212357, + "nauc_mrr_at_100_diff1": -0.533091, + "nauc_mrr_at_1000_max": 0.308614, + "nauc_mrr_at_1000_std": -0.212364, + "nauc_mrr_at_1000_diff1": -0.532916, + "main_score": 0.56305, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "sql-Code" + ] + } + ] + }, + "evaluation_time": 124.03231525421143, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/TRECCOVID.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/TRECCOVID.json new file mode 100644 index 0000000000..b0bffe4ad9 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/TRECCOVID.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bb9466bac8153a0349341eb1b22e06409e78ef4e", + "task_name": "TRECCOVID", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.83, + "ndcg_at_3": 0.82899, + "ndcg_at_5": 0.81833, + "ndcg_at_10": 0.80555, + "ndcg_at_20": 0.76281, + "ndcg_at_100": 0.61602, + "ndcg_at_1000": 0.55466, + "map_at_1": 0.00229, + "map_at_3": 0.00649, + "map_at_5": 0.01047, + "map_at_10": 0.02023, + "map_at_20": 0.03595, + "map_at_100": 0.12021, + "map_at_1000": 0.30443, + "recall_at_1": 0.00229, + "recall_at_3": 0.00679, + "recall_at_5": 0.01116, + "recall_at_10": 0.02227, + "recall_at_20": 0.04042, + "recall_at_100": 0.14984, + "recall_at_1000": 0.52396, + "precision_at_1": 0.88, + "precision_at_3": 0.87333, + "precision_at_5": 0.868, + "precision_at_10": 0.86, + "precision_at_20": 0.803, + "precision_at_100": 0.637, + "precision_at_1000": 0.25036, + "mrr_at_1": 0.88, + "mrr_at_3": 0.94, + "mrr_at_5": 0.94, + "mrr_at_10": 0.94, + "mrr_at_20": 0.94, + "mrr_at_100": 0.94, + "mrr_at_1000": 0.94, + "nauc_ndcg_at_1_max": 0.376549, + "nauc_ndcg_at_1_std": 0.536395, + "nauc_ndcg_at_1_diff1": 0.115381, + "nauc_ndcg_at_3_max": 0.380501, + "nauc_ndcg_at_3_std": 0.637445, + "nauc_ndcg_at_3_diff1": -0.338478, + "nauc_ndcg_at_5_max": 0.375456, + "nauc_ndcg_at_5_std": 0.639062, + "nauc_ndcg_at_5_diff1": -0.387105, + "nauc_ndcg_at_10_max": 0.41533, + "nauc_ndcg_at_10_std": 0.696309, + "nauc_ndcg_at_10_diff1": -0.450656, + "nauc_ndcg_at_20_max": 0.446139, + "nauc_ndcg_at_20_std": 0.748536, + "nauc_ndcg_at_20_diff1": -0.54324, + "nauc_ndcg_at_100_max": 0.474741, + "nauc_ndcg_at_100_std": 0.874371, + "nauc_ndcg_at_100_diff1": -0.623003, + "nauc_ndcg_at_1000_max": 0.491573, + "nauc_ndcg_at_1000_std": 0.81848, + "nauc_ndcg_at_1000_diff1": -0.517246, + "nauc_map_at_1_max": 0.024357, + "nauc_map_at_1_std": -0.010956, + "nauc_map_at_1_diff1": 0.349616, + "nauc_map_at_3_max": 0.016383, + "nauc_map_at_3_std": 0.060231, + "nauc_map_at_3_diff1": 0.170942, + "nauc_map_at_5_max": -0.00058, + "nauc_map_at_5_std": 0.077911, + "nauc_map_at_5_diff1": 0.116699, + "nauc_map_at_10_max": 0.023196, + "nauc_map_at_10_std": 0.103372, + "nauc_map_at_10_diff1": 0.073417, + "nauc_map_at_20_max": 0.114388, + "nauc_map_at_20_std": 0.230483, + "nauc_map_at_20_diff1": -0.056752, + "nauc_map_at_100_max": 0.309816, + "nauc_map_at_100_std": 0.679712, + "nauc_map_at_100_diff1": -0.386344, + "nauc_map_at_1000_max": 0.50664, + "nauc_map_at_1000_std": 0.856615, + "nauc_map_at_1000_diff1": -0.543029, + "nauc_recall_at_1_max": 0.024357, + "nauc_recall_at_1_std": -0.010956, + "nauc_recall_at_1_diff1": 0.349616, + "nauc_recall_at_3_max": -0.050936, + "nauc_recall_at_3_std": 0.001257, + "nauc_recall_at_3_diff1": 0.153662, + "nauc_recall_at_5_max": -0.078823, + "nauc_recall_at_5_std": -0.003343, + "nauc_recall_at_5_diff1": 0.13274, + "nauc_recall_at_10_max": -0.063403, + "nauc_recall_at_10_std": -0.019033, + "nauc_recall_at_10_diff1": 0.140437, + "nauc_recall_at_20_max": 0.008106, + "nauc_recall_at_20_std": 0.085094, + "nauc_recall_at_20_diff1": 0.054392, + "nauc_recall_at_100_max": 0.166235, + "nauc_recall_at_100_std": 0.488216, + "nauc_recall_at_100_diff1": -0.218367, + "nauc_recall_at_1000_max": 0.423447, + "nauc_recall_at_1000_std": 0.683329, + "nauc_recall_at_1000_diff1": -0.415654, + "nauc_precision_at_1_max": 0.733614, + "nauc_precision_at_1_std": 0.679924, + "nauc_precision_at_1_diff1": 0.186923, + "nauc_precision_at_3_max": 0.544834, + "nauc_precision_at_3_std": 0.750489, + "nauc_precision_at_3_diff1": -0.552295, + "nauc_precision_at_5_max": 0.392115, + "nauc_precision_at_5_std": 0.698027, + "nauc_precision_at_5_diff1": -0.518538, + "nauc_precision_at_10_max": 0.490748, + "nauc_precision_at_10_std": 0.737148, + "nauc_precision_at_10_diff1": -0.574025, + "nauc_precision_at_20_max": 0.482003, + "nauc_precision_at_20_std": 0.755094, + "nauc_precision_at_20_diff1": -0.611621, + "nauc_precision_at_100_max": 0.470808, + "nauc_precision_at_100_std": 0.874671, + "nauc_precision_at_100_diff1": -0.667358, + "nauc_precision_at_1000_max": 0.441613, + "nauc_precision_at_1000_std": 0.55038, + "nauc_precision_at_1000_diff1": -0.48463, + "nauc_mrr_at_1_max": 0.733614, + "nauc_mrr_at_1_std": 0.679924, + "nauc_mrr_at_1_diff1": 0.186923, + "nauc_mrr_at_3_max": 0.733614, + "nauc_mrr_at_3_std": 0.679924, + "nauc_mrr_at_3_diff1": 0.186923, + "nauc_mrr_at_5_max": 0.733614, + "nauc_mrr_at_5_std": 0.679924, + "nauc_mrr_at_5_diff1": 0.186923, + "nauc_mrr_at_10_max": 0.733614, + "nauc_mrr_at_10_std": 0.679924, + "nauc_mrr_at_10_diff1": 0.186923, + "nauc_mrr_at_20_max": 0.733614, + "nauc_mrr_at_20_std": 0.679924, + "nauc_mrr_at_20_diff1": 0.186923, + "nauc_mrr_at_100_max": 0.733614, + "nauc_mrr_at_100_std": 0.679924, + "nauc_mrr_at_100_diff1": 0.186923, + "nauc_mrr_at_1000_max": 0.733614, + "nauc_mrr_at_1000_std": 0.679924, + "nauc_mrr_at_1000_diff1": 0.186923, + "main_score": 0.80555, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 708.0054268836975, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/Touche2020.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/Touche2020.json new file mode 100644 index 0000000000..74c41f6721 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/Touche2020.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f", + "task_name": "Touche2020", + "mteb_version": "1.38.9", + "scores": { + "test": [ + { + "ndcg_at_1": 0.46939, + "ndcg_at_3": 0.36773, + "ndcg_at_5": 0.33804, + "ndcg_at_10": 0.30417, + "ndcg_at_20": 0.3078, + "ndcg_at_100": 0.40927, + "ndcg_at_1000": 0.51879, + "map_at_1": 0.03942, + "map_at_3": 0.06967, + "map_at_5": 0.0922, + "map_at_10": 0.12651, + "map_at_20": 0.15299, + "map_at_100": 0.18786, + "map_at_1000": 0.20284, + "recall_at_1": 0.03942, + "recall_at_3": 0.08085, + "recall_at_5": 0.11535, + "recall_at_10": 0.18514, + "recall_at_20": 0.26337, + "recall_at_100": 0.49404, + "recall_at_1000": 0.82637, + "precision_at_1": 0.5102, + "precision_at_3": 0.36735, + "precision_at_5": 0.32245, + "precision_at_10": 0.2551, + "precision_at_20": 0.19184, + "precision_at_100": 0.0802, + "precision_at_1000": 0.01537, + "mrr_at_1": 0.510204, + "mrr_at_3": 0.612245, + "mrr_at_5": 0.622449, + "mrr_at_10": 0.630442, + "mrr_at_20": 0.636852, + "mrr_at_100": 0.638774, + "mrr_at_1000": 0.638774, + "nauc_ndcg_at_1_max": 0.137368, + "nauc_ndcg_at_1_std": 0.221845, + "nauc_ndcg_at_1_diff1": 0.118183, + "nauc_ndcg_at_3_max": 0.086149, + "nauc_ndcg_at_3_std": 0.165678, + "nauc_ndcg_at_3_diff1": 0.040755, + "nauc_ndcg_at_5_max": 0.099131, + "nauc_ndcg_at_5_std": 0.229039, + "nauc_ndcg_at_5_diff1": -0.035288, + "nauc_ndcg_at_10_max": 0.104266, + "nauc_ndcg_at_10_std": 0.151705, + "nauc_ndcg_at_10_diff1": 0.045074, + "nauc_ndcg_at_20_max": 0.10088, + "nauc_ndcg_at_20_std": 0.08631, + "nauc_ndcg_at_20_diff1": 0.074851, + "nauc_ndcg_at_100_max": 0.134989, + "nauc_ndcg_at_100_std": 0.312564, + "nauc_ndcg_at_100_diff1": 0.002511, + "nauc_ndcg_at_1000_max": 0.212205, + "nauc_ndcg_at_1000_std": 0.405541, + "nauc_ndcg_at_1000_diff1": -0.025047, + "nauc_map_at_1_max": 0.081443, + "nauc_map_at_1_std": -0.024712, + "nauc_map_at_1_diff1": 0.126154, + "nauc_map_at_3_max": 0.16868, + "nauc_map_at_3_std": -0.032647, + "nauc_map_at_3_diff1": 0.017425, + "nauc_map_at_5_max": 0.146128, + "nauc_map_at_5_std": 0.00868, + "nauc_map_at_5_diff1": -0.058214, + "nauc_map_at_10_max": 0.119409, + "nauc_map_at_10_std": -0.025884, + "nauc_map_at_10_diff1": -0.01049, + "nauc_map_at_20_max": 0.107629, + "nauc_map_at_20_std": -0.020201, + "nauc_map_at_20_diff1": -0.007801, + "nauc_map_at_100_max": 0.121715, + "nauc_map_at_100_std": 0.093367, + "nauc_map_at_100_diff1": -0.050567, + "nauc_map_at_1000_max": 0.13534, + "nauc_map_at_1000_std": 0.121743, + "nauc_map_at_1000_diff1": -0.058278, + "nauc_recall_at_1_max": 0.081443, + "nauc_recall_at_1_std": -0.024712, + "nauc_recall_at_1_diff1": 0.126154, + "nauc_recall_at_3_max": 0.110689, + "nauc_recall_at_3_std": -0.027773, + "nauc_recall_at_3_diff1": 0.005644, + "nauc_recall_at_5_max": 0.13955, + "nauc_recall_at_5_std": 0.058647, + "nauc_recall_at_5_diff1": -0.083321, + "nauc_recall_at_10_max": 0.09965, + "nauc_recall_at_10_std": -0.006341, + "nauc_recall_at_10_diff1": 0.053683, + "nauc_recall_at_20_max": 0.091297, + "nauc_recall_at_20_std": 0.003132, + "nauc_recall_at_20_diff1": 0.079477, + "nauc_recall_at_100_max": 0.14726, + "nauc_recall_at_100_std": 0.414648, + "nauc_recall_at_100_diff1": -0.071915, + "nauc_recall_at_1000_max": 0.299375, + "nauc_recall_at_1000_std": 0.684448, + "nauc_recall_at_1000_diff1": -0.179385, + "nauc_precision_at_1_max": 0.097247, + "nauc_precision_at_1_std": 0.189118, + "nauc_precision_at_1_diff1": 0.107633, + "nauc_precision_at_3_max": 0.057191, + "nauc_precision_at_3_std": 0.123983, + "nauc_precision_at_3_diff1": 0.003381, + "nauc_precision_at_5_max": 0.068529, + "nauc_precision_at_5_std": 0.225681, + "nauc_precision_at_5_diff1": -0.107054, + "nauc_precision_at_10_max": 0.04232, + "nauc_precision_at_10_std": 0.186568, + "nauc_precision_at_10_diff1": 0.037153, + "nauc_precision_at_20_max": 0.045171, + "nauc_precision_at_20_std": 0.204801, + "nauc_precision_at_20_diff1": 0.041197, + "nauc_precision_at_100_max": 0.117158, + "nauc_precision_at_100_std": 0.708024, + "nauc_precision_at_100_diff1": -0.187937, + "nauc_precision_at_1000_max": 0.136516, + "nauc_precision_at_1000_std": 0.315973, + "nauc_precision_at_1000_diff1": -0.221353, + "nauc_mrr_at_1_max": 0.097247, + "nauc_mrr_at_1_std": 0.189118, + "nauc_mrr_at_1_diff1": 0.107633, + "nauc_mrr_at_3_max": -0.007577, + "nauc_mrr_at_3_std": 0.223435, + "nauc_mrr_at_3_diff1": 0.104235, + "nauc_mrr_at_5_max": 0.031668, + "nauc_mrr_at_5_std": 0.268511, + "nauc_mrr_at_5_diff1": 0.116931, + "nauc_mrr_at_10_max": 0.030893, + "nauc_mrr_at_10_std": 0.261391, + "nauc_mrr_at_10_diff1": 0.12496, + "nauc_mrr_at_20_max": 0.031485, + "nauc_mrr_at_20_std": 0.258391, + "nauc_mrr_at_20_diff1": 0.131667, + "nauc_mrr_at_100_max": 0.030553, + "nauc_mrr_at_100_std": 0.253472, + "nauc_mrr_at_100_diff1": 0.127866, + "nauc_mrr_at_1000_max": 0.030553, + "nauc_mrr_at_1000_std": 0.253472, + "nauc_mrr_at_1000_diff1": 0.127866, + "main_score": 0.30417, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1419.1369605064392, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/model_meta.json b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/model_meta.json new file mode 100644 index 0000000000..b030fd4a38 --- /dev/null +++ b/results/manveertamber__cadet-embed-base-v1/8056d118be37a566f20972a5f35cda815f6bc47e/model_meta.json @@ -0,0 +1,23 @@ +{"name": "manveertamber/cadet-embed-base-v1", +"revision": "8056d118be37a566f20972a5f35cda815f6bc47e", +"release_date": "2025-05-11", +"languages": ["eng-Latn"], +"n_parameters": 109000000, +"memory_usage_mb": 418, +"max_tokens": 512, +"embed_dim": 768, +"license": "apache-2.0", +"open_weights": true, +"public_training_code": "https://github.com/manveertamber/cadet-dense-retrieval", +"public_training_data": "https://github.com/manveertamber/cadet-dense-retrieval", +"framework": ["Sentence Transformers", "PyTorch"], +"reference": "https://huggingface.co/manveertamber/cadet-embed-base-v1", +"similarity_fn_name": "cosine", +"use_instructions": true, +"training_datasets": {"MSMARCO": ["train"]}, +"adapted_from": "intfloat/e5-base-unsupervised", +"superseded_by": null, +"is_cross_encoder": false, +"modalities": ["text"], +"loader": "sentence_transformers_loader" +}