diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/AppsRetrieval.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/AppsRetrieval.json new file mode 100644 index 0000000000..2b8af52d0f --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/AppsRetrieval.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "f22508f96b7a36c2415181ed8bb76f76e04ae2d5", + "task_name": "AppsRetrieval", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.04967, + "ndcg_at_3": 0.06406, + "ndcg_at_5": 0.07172, + "ndcg_at_10": 0.08041, + "ndcg_at_20": 0.08804, + "ndcg_at_100": 0.11117, + "ndcg_at_1000": 0.15317, + "map_at_1": 0.04967, + "map_at_3": 0.06042, + "map_at_5": 0.06467, + "map_at_10": 0.06822, + "map_at_20": 0.07026, + "map_at_100": 0.07325, + "map_at_1000": 0.0745, + "recall_at_1": 0.04967, + "recall_at_3": 0.07463, + "recall_at_5": 0.09323, + "recall_at_10": 0.12032, + "recall_at_20": 0.15086, + "recall_at_100": 0.27862, + "recall_at_1000": 0.62922, + "precision_at_1": 0.04967, + "precision_at_3": 0.02488, + "precision_at_5": 0.01865, + "precision_at_10": 0.01203, + "precision_at_20": 0.00754, + "precision_at_100": 0.00279, + "precision_at_1000": 0.00063, + "mrr_at_1": 0.049668, + "mrr_at_3": 0.060425, + "mrr_at_5": 0.064675, + "mrr_at_10": 0.068219, + "mrr_at_20": 0.070263, + "mrr_at_100": 0.073254, + "mrr_at_1000": 0.074503, + "nauc_ndcg_at_1_max": 0.166536, + "nauc_ndcg_at_1_std": 0.192576, + "nauc_ndcg_at_1_diff1": 0.445726, + "nauc_ndcg_at_3_max": 0.134681, + "nauc_ndcg_at_3_std": 0.188882, + "nauc_ndcg_at_3_diff1": 0.321478, + "nauc_ndcg_at_5_max": 0.121701, + "nauc_ndcg_at_5_std": 0.197249, + "nauc_ndcg_at_5_diff1": 0.295295, + "nauc_ndcg_at_10_max": 0.107303, + "nauc_ndcg_at_10_std": 0.194609, + "nauc_ndcg_at_10_diff1": 0.269218, + "nauc_ndcg_at_20_max": 0.096718, + "nauc_ndcg_at_20_std": 0.193836, + "nauc_ndcg_at_20_diff1": 0.251734, + "nauc_ndcg_at_100_max": 0.089069, + "nauc_ndcg_at_100_std": 0.195898, + "nauc_ndcg_at_100_diff1": 0.227381, + "nauc_ndcg_at_1000_max": 0.091329, + "nauc_ndcg_at_1000_std": 0.211974, + "nauc_ndcg_at_1000_diff1": 0.216686, + "nauc_map_at_1_max": 0.166536, + "nauc_map_at_1_std": 0.192576, + "nauc_map_at_1_diff1": 0.445726, + "nauc_map_at_3_max": 0.141222, + "nauc_map_at_3_std": 0.19006, + "nauc_map_at_3_diff1": 0.3472, + "nauc_map_at_5_max": 0.1332, + "nauc_map_at_5_std": 0.19534, + "nauc_map_at_5_diff1": 0.329407, + "nauc_map_at_10_max": 0.125597, + "nauc_map_at_10_std": 0.19425, + "nauc_map_at_10_diff1": 0.315412, + "nauc_map_at_20_max": 0.12171, + "nauc_map_at_20_std": 0.193863, + "nauc_map_at_20_diff1": 0.309099, + "nauc_map_at_100_max": 0.119807, + "nauc_map_at_100_std": 0.193919, + "nauc_map_at_100_diff1": 0.302933, + "nauc_map_at_1000_max": 0.11959, + "nauc_map_at_1000_std": 0.19471, + "nauc_map_at_1000_diff1": 0.302153, + "nauc_recall_at_1_max": 0.166536, + "nauc_recall_at_1_std": 0.192576, + "nauc_recall_at_1_diff1": 0.445726, + "nauc_recall_at_3_max": 0.119325, + "nauc_recall_at_3_std": 0.185981, + "nauc_recall_at_3_diff1": 0.261015, + "nauc_recall_at_5_max": 0.097005, + "nauc_recall_at_5_std": 0.201589, + "nauc_recall_at_5_diff1": 0.224144, + "nauc_recall_at_10_max": 0.071428, + "nauc_recall_at_10_std": 0.194927, + "nauc_recall_at_10_diff1": 0.181522, + "nauc_recall_at_20_max": 0.050791, + "nauc_recall_at_20_std": 0.193509, + "nauc_recall_at_20_diff1": 0.150076, + "nauc_recall_at_100_max": 0.043903, + "nauc_recall_at_100_std": 0.201229, + "nauc_recall_at_100_diff1": 0.118423, + "nauc_recall_at_1000_max": 0.05675, + "nauc_recall_at_1000_std": 0.271575, + "nauc_recall_at_1000_diff1": 0.065535, + "nauc_precision_at_1_max": 0.166536, + "nauc_precision_at_1_std": 0.192576, + "nauc_precision_at_1_diff1": 0.445726, + "nauc_precision_at_3_max": 0.119325, + "nauc_precision_at_3_std": 0.185981, + "nauc_precision_at_3_diff1": 0.261015, + "nauc_precision_at_5_max": 0.097005, + "nauc_precision_at_5_std": 0.201589, + "nauc_precision_at_5_diff1": 0.224144, + "nauc_precision_at_10_max": 0.071428, + "nauc_precision_at_10_std": 0.194927, + "nauc_precision_at_10_diff1": 0.181522, + "nauc_precision_at_20_max": 0.050791, + "nauc_precision_at_20_std": 0.193509, + "nauc_precision_at_20_diff1": 0.150076, + "nauc_precision_at_100_max": 0.043903, + "nauc_precision_at_100_std": 0.201229, + "nauc_precision_at_100_diff1": 0.118423, + "nauc_precision_at_1000_max": 0.05675, + "nauc_precision_at_1000_std": 0.271575, + "nauc_precision_at_1000_diff1": 0.065535, + "nauc_mrr_at_1_max": 0.166536, + "nauc_mrr_at_1_std": 0.192576, + "nauc_mrr_at_1_diff1": 0.445726, + "nauc_mrr_at_3_max": 0.141222, + "nauc_mrr_at_3_std": 0.19006, + "nauc_mrr_at_3_diff1": 0.3472, + "nauc_mrr_at_5_max": 0.1332, + "nauc_mrr_at_5_std": 0.19534, + "nauc_mrr_at_5_diff1": 0.329407, + "nauc_mrr_at_10_max": 0.125597, + "nauc_mrr_at_10_std": 0.19425, + "nauc_mrr_at_10_diff1": 0.315412, + "nauc_mrr_at_20_max": 0.12171, + "nauc_mrr_at_20_std": 0.193863, + "nauc_mrr_at_20_diff1": 0.309099, + "nauc_mrr_at_100_max": 0.119807, + "nauc_mrr_at_100_std": 0.193919, + "nauc_mrr_at_100_diff1": 0.302933, + "nauc_mrr_at_1000_max": 0.11959, + "nauc_mrr_at_1000_std": 0.19471, + "nauc_mrr_at_1000_diff1": 0.302153, + "main_score": 0.08041, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 172.013081073761, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/COIRCodeSearchNetRetrieval.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/COIRCodeSearchNetRetrieval.json new file mode 100644 index 0000000000..df7b7be3b2 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/COIRCodeSearchNetRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "4adc7bc41202b5c13543c9c886a25f340634dab3", + "task_name": "COIRCodeSearchNetRetrieval", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.9697, + "ndcg_at_3": 0.97896, + "ndcg_at_5": 0.98057, + "ndcg_at_10": 0.98154, + "ndcg_at_20": 0.98217, + "ndcg_at_100": 0.9828, + "ndcg_at_1000": 0.98303, + "map_at_1": 0.9697, + "map_at_3": 0.97683, + "map_at_5": 0.97773, + "map_at_10": 0.97815, + "map_at_20": 0.97833, + "map_at_100": 0.97842, + "map_at_1000": 0.97843, + "recall_at_1": 0.9697, + "recall_at_3": 0.98505, + "recall_at_5": 0.98894, + "recall_at_10": 0.99182, + "recall_at_20": 0.9943, + "recall_at_100": 0.99765, + "recall_at_1000": 0.9994, + "precision_at_1": 0.9697, + "precision_at_3": 0.32835, + "precision_at_5": 0.19779, + "precision_at_10": 0.09918, + "precision_at_20": 0.04972, + "precision_at_100": 0.00998, + "precision_at_1000": 0.001, + "mrr_at_1": 0.969701, + "mrr_at_3": 0.976829, + "mrr_at_5": 0.977734, + "mrr_at_10": 0.978153, + "mrr_at_20": 0.97833, + "mrr_at_100": 0.978422, + "mrr_at_1000": 0.978431, + "nauc_ndcg_at_1_max": 0.880875, + "nauc_ndcg_at_1_std": -0.483928, + "nauc_ndcg_at_1_diff1": 0.996819, + "nauc_ndcg_at_3_max": 0.905432, + "nauc_ndcg_at_3_std": -0.52267, + "nauc_ndcg_at_3_diff1": 0.996941, + "nauc_ndcg_at_5_max": 0.903436, + "nauc_ndcg_at_5_std": -0.507441, + "nauc_ndcg_at_5_diff1": 0.99727, + "nauc_ndcg_at_10_max": 0.903114, + "nauc_ndcg_at_10_std": -0.489, + "nauc_ndcg_at_10_diff1": 0.997127, + "nauc_ndcg_at_20_max": 0.901591, + "nauc_ndcg_at_20_std": -0.486472, + "nauc_ndcg_at_20_diff1": 0.997148, + "nauc_ndcg_at_100_max": 0.900198, + "nauc_ndcg_at_100_std": -0.482701, + "nauc_ndcg_at_100_diff1": 0.997122, + "nauc_ndcg_at_1000_max": 0.899775, + "nauc_ndcg_at_1000_std": -0.485055, + "nauc_ndcg_at_1000_diff1": 0.997084, + "nauc_map_at_1_max": 0.880875, + "nauc_map_at_1_std": -0.483928, + "nauc_map_at_1_diff1": 0.996819, + "nauc_map_at_3_max": 0.898773, + "nauc_map_at_3_std": -0.511652, + "nauc_map_at_3_diff1": 0.996912, + "nauc_map_at_5_max": 0.897553, + "nauc_map_at_5_std": -0.503714, + "nauc_map_at_5_diff1": 0.997081, + "nauc_map_at_10_max": 0.897323, + "nauc_map_at_10_std": -0.496769, + "nauc_map_at_10_diff1": 0.997025, + "nauc_map_at_20_max": 0.896922, + "nauc_map_at_20_std": -0.496187, + "nauc_map_at_20_diff1": 0.997028, + "nauc_map_at_100_max": 0.896776, + "nauc_map_at_100_std": -0.495498, + "nauc_map_at_100_diff1": 0.99702, + "nauc_map_at_1000_max": 0.896762, + "nauc_map_at_1000_std": -0.495517, + "nauc_map_at_1000_diff1": 0.997019, + "nauc_recall_at_1_max": 0.880875, + "nauc_recall_at_1_std": -0.483928, + "nauc_recall_at_1_diff1": 0.996819, + "nauc_recall_at_3_max": 0.934525, + "nauc_recall_at_3_std": -0.571189, + "nauc_recall_at_3_diff1": 0.997069, + "nauc_recall_at_5_max": 0.936097, + "nauc_recall_at_5_std": -0.524272, + "nauc_recall_at_5_diff1": 0.998416, + "nauc_recall_at_10_max": 0.945393, + "nauc_recall_at_10_std": -0.407866, + "nauc_recall_at_10_diff1": 0.997857, + "nauc_recall_at_20_max": 0.945268, + "nauc_recall_at_20_std": -0.342985, + "nauc_recall_at_20_diff1": 0.998462, + "nauc_recall_at_100_max": 0.94978, + "nauc_recall_at_100_std": -0.024641, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.666524, + "nauc_recall_at_1000_diff1": 1.0, + "nauc_precision_at_1_max": 0.880875, + "nauc_precision_at_1_std": -0.483928, + "nauc_precision_at_1_diff1": 0.996819, + "nauc_precision_at_3_max": 0.934525, + "nauc_precision_at_3_std": -0.571189, + "nauc_precision_at_3_diff1": 0.997069, + "nauc_precision_at_5_max": 0.936097, + "nauc_precision_at_5_std": -0.524272, + "nauc_precision_at_5_diff1": 0.998416, + "nauc_precision_at_10_max": 0.945393, + "nauc_precision_at_10_std": -0.407866, + "nauc_precision_at_10_diff1": 0.997857, + "nauc_precision_at_20_max": 0.945268, + "nauc_precision_at_20_std": -0.342985, + "nauc_precision_at_20_diff1": 0.998462, + "nauc_precision_at_100_max": 0.94978, + "nauc_precision_at_100_std": -0.024641, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 0.666524, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.880875, + "nauc_mrr_at_1_std": -0.483928, + "nauc_mrr_at_1_diff1": 0.996819, + "nauc_mrr_at_3_max": 0.898773, + "nauc_mrr_at_3_std": -0.511652, + "nauc_mrr_at_3_diff1": 0.996912, + "nauc_mrr_at_5_max": 0.897553, + "nauc_mrr_at_5_std": -0.503714, + "nauc_mrr_at_5_diff1": 0.997081, + "nauc_mrr_at_10_max": 0.897323, + "nauc_mrr_at_10_std": -0.496769, + "nauc_mrr_at_10_diff1": 0.997025, + "nauc_mrr_at_20_max": 0.896922, + "nauc_mrr_at_20_std": -0.496187, + "nauc_mrr_at_20_diff1": 0.997028, + "nauc_mrr_at_100_max": 0.896776, + "nauc_mrr_at_100_std": -0.495498, + "nauc_mrr_at_100_diff1": 0.99702, + "nauc_mrr_at_1000_max": 0.896762, + "nauc_mrr_at_1000_std": -0.495517, + "nauc_mrr_at_1000_diff1": 0.997019, + "main_score": 0.98154, + "hf_subset": "python", + "languages": [ + "python-Code" + ] + }, + { + "ndcg_at_1": 0.53115, + "ndcg_at_3": 0.59276, + "ndcg_at_5": 0.60998, + "ndcg_at_10": 0.62485, + "ndcg_at_20": 0.63658, + "ndcg_at_100": 0.65106, + "ndcg_at_1000": 0.66124, + "map_at_1": 0.53115, + "map_at_3": 0.57789, + "map_at_5": 0.58742, + "map_at_10": 0.59369, + "map_at_20": 0.59693, + "map_at_100": 0.59891, + "map_at_1000": 0.59927, + "recall_at_1": 0.53115, + "recall_at_3": 0.63567, + "recall_at_5": 0.67761, + "recall_at_10": 0.72288, + "recall_at_20": 0.76907, + "recall_at_100": 0.84746, + "recall_at_1000": 0.9295, + "precision_at_1": 0.53115, + "precision_at_3": 0.21189, + "precision_at_5": 0.13552, + "precision_at_10": 0.07229, + "precision_at_20": 0.03845, + "precision_at_100": 0.00847, + "precision_at_1000": 0.00093, + "mrr_at_1": 0.531146, + "mrr_at_3": 0.577889, + "mrr_at_5": 0.587415, + "mrr_at_10": 0.593688, + "mrr_at_20": 0.596931, + "mrr_at_100": 0.598913, + "mrr_at_1000": 0.599267, + "nauc_ndcg_at_1_max": 0.598518, + "nauc_ndcg_at_1_std": -0.054296, + "nauc_ndcg_at_1_diff1": 0.782342, + "nauc_ndcg_at_3_max": 0.613996, + "nauc_ndcg_at_3_std": -0.035789, + "nauc_ndcg_at_3_diff1": 0.738191, + "nauc_ndcg_at_5_max": 0.610867, + "nauc_ndcg_at_5_std": -0.031708, + "nauc_ndcg_at_5_diff1": 0.731835, + "nauc_ndcg_at_10_max": 0.611426, + "nauc_ndcg_at_10_std": -0.025904, + "nauc_ndcg_at_10_diff1": 0.728062, + "nauc_ndcg_at_20_max": 0.612203, + "nauc_ndcg_at_20_std": -0.021254, + "nauc_ndcg_at_20_diff1": 0.726428, + "nauc_ndcg_at_100_max": 0.610401, + "nauc_ndcg_at_100_std": -0.020451, + "nauc_ndcg_at_100_diff1": 0.725879, + "nauc_ndcg_at_1000_max": 0.610361, + "nauc_ndcg_at_1000_std": -0.021249, + "nauc_ndcg_at_1000_diff1": 0.728997, + "nauc_map_at_1_max": 0.598518, + "nauc_map_at_1_std": -0.054296, + "nauc_map_at_1_diff1": 0.782342, + "nauc_map_at_3_max": 0.610236, + "nauc_map_at_3_std": -0.041066, + "nauc_map_at_3_diff1": 0.7488, + "nauc_map_at_5_max": 0.608681, + "nauc_map_at_5_std": -0.038623, + "nauc_map_at_5_diff1": 0.745541, + "nauc_map_at_10_max": 0.608855, + "nauc_map_at_10_std": -0.036339, + "nauc_map_at_10_diff1": 0.744179, + "nauc_map_at_20_max": 0.609118, + "nauc_map_at_20_std": -0.035187, + "nauc_map_at_20_diff1": 0.743896, + "nauc_map_at_100_max": 0.608923, + "nauc_map_at_100_std": -0.035042, + "nauc_map_at_100_diff1": 0.743804, + "nauc_map_at_1000_max": 0.608923, + "nauc_map_at_1000_std": -0.035038, + "nauc_map_at_1000_diff1": 0.743898, + "nauc_recall_at_1_max": 0.598518, + "nauc_recall_at_1_std": -0.054296, + "nauc_recall_at_1_diff1": 0.782342, + "nauc_recall_at_3_max": 0.625691, + "nauc_recall_at_3_std": -0.019009, + "nauc_recall_at_3_diff1": 0.705256, + "nauc_recall_at_5_max": 0.61729, + "nauc_recall_at_5_std": -0.008136, + "nauc_recall_at_5_diff1": 0.685302, + "nauc_recall_at_10_max": 0.620281, + "nauc_recall_at_10_std": 0.015787, + "nauc_recall_at_10_diff1": 0.666006, + "nauc_recall_at_20_max": 0.624968, + "nauc_recall_at_20_std": 0.046994, + "nauc_recall_at_20_diff1": 0.647161, + "nauc_recall_at_100_max": 0.611322, + "nauc_recall_at_100_std": 0.082042, + "nauc_recall_at_100_diff1": 0.610531, + "nauc_recall_at_1000_max": 0.610188, + "nauc_recall_at_1000_std": 0.155768, + "nauc_recall_at_1000_diff1": 0.586534, + "nauc_precision_at_1_max": 0.598518, + "nauc_precision_at_1_std": -0.054296, + "nauc_precision_at_1_diff1": 0.782342, + "nauc_precision_at_3_max": 0.625691, + "nauc_precision_at_3_std": -0.019009, + "nauc_precision_at_3_diff1": 0.705256, + "nauc_precision_at_5_max": 0.61729, + "nauc_precision_at_5_std": -0.008136, + "nauc_precision_at_5_diff1": 0.685302, + "nauc_precision_at_10_max": 0.620281, + "nauc_precision_at_10_std": 0.015787, + "nauc_precision_at_10_diff1": 0.666006, + "nauc_precision_at_20_max": 0.624968, + "nauc_precision_at_20_std": 0.046994, + "nauc_precision_at_20_diff1": 0.647161, + "nauc_precision_at_100_max": 0.611322, + "nauc_precision_at_100_std": 0.082042, + "nauc_precision_at_100_diff1": 0.610531, + "nauc_precision_at_1000_max": 0.610188, + "nauc_precision_at_1000_std": 0.155768, + "nauc_precision_at_1000_diff1": 0.586534, + "nauc_mrr_at_1_max": 0.598518, + "nauc_mrr_at_1_std": -0.054296, + "nauc_mrr_at_1_diff1": 0.782342, + "nauc_mrr_at_3_max": 0.610236, + "nauc_mrr_at_3_std": -0.041066, + "nauc_mrr_at_3_diff1": 0.7488, + "nauc_mrr_at_5_max": 0.608681, + "nauc_mrr_at_5_std": -0.038623, + "nauc_mrr_at_5_diff1": 0.745541, + "nauc_mrr_at_10_max": 0.608855, + "nauc_mrr_at_10_std": -0.036339, + "nauc_mrr_at_10_diff1": 0.744179, + "nauc_mrr_at_20_max": 0.609118, + "nauc_mrr_at_20_std": -0.035187, + "nauc_mrr_at_20_diff1": 0.743896, + "nauc_mrr_at_100_max": 0.608923, + "nauc_mrr_at_100_std": -0.035042, + "nauc_mrr_at_100_diff1": 0.743804, + "nauc_mrr_at_1000_max": 0.608923, + "nauc_mrr_at_1000_std": -0.035038, + "nauc_mrr_at_1000_diff1": 0.743898, + "main_score": 0.62485, + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ] + }, + { + "ndcg_at_1": 0.67693, + "ndcg_at_3": 0.74662, + "ndcg_at_5": 0.76177, + "ndcg_at_10": 0.7741, + "ndcg_at_20": 0.782, + "ndcg_at_100": 0.79195, + "ndcg_at_1000": 0.79607, + "map_at_1": 0.67693, + "map_at_3": 0.72985, + "map_at_5": 0.73828, + "map_at_10": 0.74343, + "map_at_20": 0.74564, + "map_at_100": 0.74709, + "map_at_1000": 0.74724, + "recall_at_1": 0.67693, + "recall_at_3": 0.795, + "recall_at_5": 0.83169, + "recall_at_10": 0.86949, + "recall_at_20": 0.90039, + "recall_at_100": 0.95309, + "recall_at_1000": 0.98596, + "precision_at_1": 0.67693, + "precision_at_3": 0.265, + "precision_at_5": 0.16634, + "precision_at_10": 0.08695, + "precision_at_20": 0.04502, + "precision_at_100": 0.00953, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.676927, + "mrr_at_3": 0.729849, + "mrr_at_5": 0.738277, + "mrr_at_10": 0.743431, + "mrr_at_20": 0.745645, + "mrr_at_100": 0.747088, + "mrr_at_1000": 0.747236, + "nauc_ndcg_at_1_max": 0.333898, + "nauc_ndcg_at_1_std": -0.349944, + "nauc_ndcg_at_1_diff1": 0.819798, + "nauc_ndcg_at_3_max": 0.35296, + "nauc_ndcg_at_3_std": -0.380796, + "nauc_ndcg_at_3_diff1": 0.791903, + "nauc_ndcg_at_5_max": 0.353349, + "nauc_ndcg_at_5_std": -0.372479, + "nauc_ndcg_at_5_diff1": 0.790502, + "nauc_ndcg_at_10_max": 0.361411, + "nauc_ndcg_at_10_std": -0.366766, + "nauc_ndcg_at_10_diff1": 0.78986, + "nauc_ndcg_at_20_max": 0.359609, + "nauc_ndcg_at_20_std": -0.360864, + "nauc_ndcg_at_20_diff1": 0.787957, + "nauc_ndcg_at_100_max": 0.359569, + "nauc_ndcg_at_100_std": -0.351795, + "nauc_ndcg_at_100_diff1": 0.792164, + "nauc_ndcg_at_1000_max": 0.358235, + "nauc_ndcg_at_1000_std": -0.354778, + "nauc_ndcg_at_1000_diff1": 0.793617, + "nauc_map_at_1_max": 0.333898, + "nauc_map_at_1_std": -0.349944, + "nauc_map_at_1_diff1": 0.819798, + "nauc_map_at_3_max": 0.347717, + "nauc_map_at_3_std": -0.37343, + "nauc_map_at_3_diff1": 0.799649, + "nauc_map_at_5_max": 0.347732, + "nauc_map_at_5_std": -0.369099, + "nauc_map_at_5_diff1": 0.799257, + "nauc_map_at_10_max": 0.350918, + "nauc_map_at_10_std": -0.366864, + "nauc_map_at_10_diff1": 0.799296, + "nauc_map_at_20_max": 0.350383, + "nauc_map_at_20_std": -0.365537, + "nauc_map_at_20_diff1": 0.798922, + "nauc_map_at_100_max": 0.350332, + "nauc_map_at_100_std": -0.364402, + "nauc_map_at_100_diff1": 0.799557, + "nauc_map_at_1000_max": 0.350293, + "nauc_map_at_1000_std": -0.364469, + "nauc_map_at_1000_diff1": 0.799608, + "nauc_recall_at_1_max": 0.333898, + "nauc_recall_at_1_std": -0.349944, + "nauc_recall_at_1_diff1": 0.819798, + "nauc_recall_at_3_max": 0.37164, + "nauc_recall_at_3_std": -0.406465, + "nauc_recall_at_3_diff1": 0.764266, + "nauc_recall_at_5_max": 0.376668, + "nauc_recall_at_5_std": -0.383746, + "nauc_recall_at_5_diff1": 0.753851, + "nauc_recall_at_10_max": 0.419286, + "nauc_recall_at_10_std": -0.359172, + "nauc_recall_at_10_diff1": 0.740251, + "nauc_recall_at_20_max": 0.421235, + "nauc_recall_at_20_std": -0.310311, + "nauc_recall_at_20_diff1": 0.712235, + "nauc_recall_at_100_max": 0.487996, + "nauc_recall_at_100_std": -0.070952, + "nauc_recall_at_100_diff1": 0.711162, + "nauc_recall_at_1000_max": 0.648558, + "nauc_recall_at_1000_std": 0.257471, + "nauc_recall_at_1000_diff1": 0.666596, + "nauc_precision_at_1_max": 0.333898, + "nauc_precision_at_1_std": -0.349944, + "nauc_precision_at_1_diff1": 0.819798, + "nauc_precision_at_3_max": 0.37164, + "nauc_precision_at_3_std": -0.406465, + "nauc_precision_at_3_diff1": 0.764266, + "nauc_precision_at_5_max": 0.376668, + "nauc_precision_at_5_std": -0.383746, + "nauc_precision_at_5_diff1": 0.753851, + "nauc_precision_at_10_max": 0.419286, + "nauc_precision_at_10_std": -0.359172, + "nauc_precision_at_10_diff1": 0.740251, + "nauc_precision_at_20_max": 0.421235, + "nauc_precision_at_20_std": -0.310311, + "nauc_precision_at_20_diff1": 0.712235, + "nauc_precision_at_100_max": 0.487996, + "nauc_precision_at_100_std": -0.070952, + "nauc_precision_at_100_diff1": 0.711162, + "nauc_precision_at_1000_max": 0.648558, + "nauc_precision_at_1000_std": 0.257471, + "nauc_precision_at_1000_diff1": 0.666596, + "nauc_mrr_at_1_max": 0.333898, + "nauc_mrr_at_1_std": -0.349944, + "nauc_mrr_at_1_diff1": 0.819798, + "nauc_mrr_at_3_max": 0.347717, + "nauc_mrr_at_3_std": -0.37343, + "nauc_mrr_at_3_diff1": 0.799649, + "nauc_mrr_at_5_max": 0.347732, + "nauc_mrr_at_5_std": -0.369099, + "nauc_mrr_at_5_diff1": 0.799257, + "nauc_mrr_at_10_max": 0.350918, + "nauc_mrr_at_10_std": -0.366864, + "nauc_mrr_at_10_diff1": 0.799296, + "nauc_mrr_at_20_max": 0.350383, + "nauc_mrr_at_20_std": -0.365537, + "nauc_mrr_at_20_diff1": 0.798922, + "nauc_mrr_at_100_max": 0.350332, + "nauc_mrr_at_100_std": -0.364402, + "nauc_mrr_at_100_diff1": 0.799557, + "nauc_mrr_at_1000_max": 0.350293, + "nauc_mrr_at_1000_std": -0.364469, + "nauc_mrr_at_1000_diff1": 0.799608, + "main_score": 0.7741, + "hf_subset": "go", + "languages": [ + "go-Code" + ] + }, + { + "ndcg_at_1": 0.51071, + "ndcg_at_3": 0.58415, + "ndcg_at_5": 0.60548, + "ndcg_at_10": 0.62273, + "ndcg_at_20": 0.63486, + "ndcg_at_100": 0.65118, + "ndcg_at_1000": 0.66049, + "map_at_1": 0.51071, + "map_at_3": 0.56622, + "map_at_5": 0.57811, + "map_at_10": 0.58527, + "map_at_20": 0.58865, + "map_at_100": 0.59092, + "map_at_1000": 0.59125, + "recall_at_1": 0.51071, + "recall_at_3": 0.636, + "recall_at_5": 0.68755, + "recall_at_10": 0.74068, + "recall_at_20": 0.78826, + "recall_at_100": 0.87629, + "recall_at_1000": 0.95083, + "precision_at_1": 0.51071, + "precision_at_3": 0.212, + "precision_at_5": 0.13751, + "precision_at_10": 0.07407, + "precision_at_20": 0.03941, + "precision_at_100": 0.00876, + "precision_at_1000": 0.00095, + "mrr_at_1": 0.510706, + "mrr_at_3": 0.566217, + "mrr_at_5": 0.578113, + "mrr_at_10": 0.585268, + "mrr_at_20": 0.588651, + "mrr_at_100": 0.59092, + "mrr_at_1000": 0.591251, + "nauc_ndcg_at_1_max": 0.443937, + "nauc_ndcg_at_1_std": -0.1792, + "nauc_ndcg_at_1_diff1": 0.687841, + "nauc_ndcg_at_3_max": 0.438691, + "nauc_ndcg_at_3_std": -0.172318, + "nauc_ndcg_at_3_diff1": 0.628111, + "nauc_ndcg_at_5_max": 0.444403, + "nauc_ndcg_at_5_std": -0.166047, + "nauc_ndcg_at_5_diff1": 0.627373, + "nauc_ndcg_at_10_max": 0.43813, + "nauc_ndcg_at_10_std": -0.17361, + "nauc_ndcg_at_10_diff1": 0.625897, + "nauc_ndcg_at_20_max": 0.440532, + "nauc_ndcg_at_20_std": -0.171036, + "nauc_ndcg_at_20_diff1": 0.622902, + "nauc_ndcg_at_100_max": 0.442208, + "nauc_ndcg_at_100_std": -0.156868, + "nauc_ndcg_at_100_diff1": 0.623056, + "nauc_ndcg_at_1000_max": 0.4409, + "nauc_ndcg_at_1000_std": -0.162063, + "nauc_ndcg_at_1000_diff1": 0.626743, + "nauc_map_at_1_max": 0.443937, + "nauc_map_at_1_std": -0.1792, + "nauc_map_at_1_diff1": 0.687841, + "nauc_map_at_3_max": 0.439889, + "nauc_map_at_3_std": -0.17371, + "nauc_map_at_3_diff1": 0.642674, + "nauc_map_at_5_max": 0.442651, + "nauc_map_at_5_std": -0.170684, + "nauc_map_at_5_diff1": 0.642393, + "nauc_map_at_10_max": 0.440139, + "nauc_map_at_10_std": -0.173549, + "nauc_map_at_10_diff1": 0.642218, + "nauc_map_at_20_max": 0.440731, + "nauc_map_at_20_std": -0.173, + "nauc_map_at_20_diff1": 0.641534, + "nauc_map_at_100_max": 0.440821, + "nauc_map_at_100_std": -0.171438, + "nauc_map_at_100_diff1": 0.64151, + "nauc_map_at_1000_max": 0.44078, + "nauc_map_at_1000_std": -0.17165, + "nauc_map_at_1000_diff1": 0.641654, + "nauc_recall_at_1_max": 0.443937, + "nauc_recall_at_1_std": -0.1792, + "nauc_recall_at_1_diff1": 0.687841, + "nauc_recall_at_3_max": 0.434957, + "nauc_recall_at_3_std": -0.168069, + "nauc_recall_at_3_diff1": 0.582227, + "nauc_recall_at_5_max": 0.451953, + "nauc_recall_at_5_std": -0.148756, + "nauc_recall_at_5_diff1": 0.57619, + "nauc_recall_at_10_max": 0.428812, + "nauc_recall_at_10_std": -0.176797, + "nauc_recall_at_10_diff1": 0.561039, + "nauc_recall_at_20_max": 0.441332, + "nauc_recall_at_20_std": -0.161295, + "nauc_recall_at_20_diff1": 0.533045, + "nauc_recall_at_100_max": 0.466808, + "nauc_recall_at_100_std": 0.022567, + "nauc_recall_at_100_diff1": 0.485245, + "nauc_recall_at_1000_max": 0.448673, + "nauc_recall_at_1000_std": 0.076337, + "nauc_recall_at_1000_diff1": 0.436948, + "nauc_precision_at_1_max": 0.443937, + "nauc_precision_at_1_std": -0.1792, + "nauc_precision_at_1_diff1": 0.687841, + "nauc_precision_at_3_max": 0.434957, + "nauc_precision_at_3_std": -0.168069, + "nauc_precision_at_3_diff1": 0.582227, + "nauc_precision_at_5_max": 0.451953, + "nauc_precision_at_5_std": -0.148756, + "nauc_precision_at_5_diff1": 0.57619, + "nauc_precision_at_10_max": 0.428812, + "nauc_precision_at_10_std": -0.176797, + "nauc_precision_at_10_diff1": 0.561039, + "nauc_precision_at_20_max": 0.441332, + "nauc_precision_at_20_std": -0.161295, + "nauc_precision_at_20_diff1": 0.533045, + "nauc_precision_at_100_max": 0.466808, + "nauc_precision_at_100_std": 0.022567, + "nauc_precision_at_100_diff1": 0.485245, + "nauc_precision_at_1000_max": 0.448673, + "nauc_precision_at_1000_std": 0.076337, + "nauc_precision_at_1000_diff1": 0.436948, + "nauc_mrr_at_1_max": 0.443937, + "nauc_mrr_at_1_std": -0.1792, + "nauc_mrr_at_1_diff1": 0.687841, + "nauc_mrr_at_3_max": 0.439889, + "nauc_mrr_at_3_std": -0.17371, + "nauc_mrr_at_3_diff1": 0.642674, + "nauc_mrr_at_5_max": 0.442651, + "nauc_mrr_at_5_std": -0.170684, + "nauc_mrr_at_5_diff1": 0.642393, + "nauc_mrr_at_10_max": 0.440139, + "nauc_mrr_at_10_std": -0.173549, + "nauc_mrr_at_10_diff1": 0.642218, + "nauc_mrr_at_20_max": 0.440731, + "nauc_mrr_at_20_std": -0.173, + "nauc_mrr_at_20_diff1": 0.641534, + "nauc_mrr_at_100_max": 0.440821, + "nauc_mrr_at_100_std": -0.171438, + "nauc_mrr_at_100_diff1": 0.64151, + "nauc_mrr_at_1000_max": 0.44078, + "nauc_mrr_at_1000_std": -0.17165, + "nauc_mrr_at_1000_diff1": 0.641654, + "main_score": 0.62273, + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ] + }, + { + "ndcg_at_1": 0.62529, + "ndcg_at_3": 0.68578, + "ndcg_at_5": 0.6986, + "ndcg_at_10": 0.71073, + "ndcg_at_20": 0.71888, + "ndcg_at_100": 0.73059, + "ndcg_at_1000": 0.73759, + "map_at_1": 0.62529, + "map_at_3": 0.67149, + "map_at_5": 0.6786, + "map_at_10": 0.6837, + "map_at_20": 0.68594, + "map_at_100": 0.68757, + "map_at_1000": 0.68781, + "recall_at_1": 0.62529, + "recall_at_3": 0.72688, + "recall_at_5": 0.75801, + "recall_at_10": 0.79507, + "recall_at_20": 0.82729, + "recall_at_100": 0.89028, + "recall_at_1000": 0.94651, + "precision_at_1": 0.62529, + "precision_at_3": 0.24229, + "precision_at_5": 0.1516, + "precision_at_10": 0.07951, + "precision_at_20": 0.04136, + "precision_at_100": 0.0089, + "precision_at_1000": 0.00095, + "mrr_at_1": 0.625285, + "mrr_at_3": 0.671489, + "mrr_at_5": 0.678605, + "mrr_at_10": 0.683697, + "mrr_at_20": 0.685936, + "mrr_at_100": 0.687566, + "mrr_at_1000": 0.687809, + "nauc_ndcg_at_1_max": 0.51288, + "nauc_ndcg_at_1_std": -0.27144, + "nauc_ndcg_at_1_diff1": 0.835005, + "nauc_ndcg_at_3_max": 0.531312, + "nauc_ndcg_at_3_std": -0.265753, + "nauc_ndcg_at_3_diff1": 0.797538, + "nauc_ndcg_at_5_max": 0.531342, + "nauc_ndcg_at_5_std": -0.262231, + "nauc_ndcg_at_5_diff1": 0.792575, + "nauc_ndcg_at_10_max": 0.530317, + "nauc_ndcg_at_10_std": -0.251897, + "nauc_ndcg_at_10_diff1": 0.787032, + "nauc_ndcg_at_20_max": 0.52998, + "nauc_ndcg_at_20_std": -0.24764, + "nauc_ndcg_at_20_diff1": 0.786317, + "nauc_ndcg_at_100_max": 0.532178, + "nauc_ndcg_at_100_std": -0.237723, + "nauc_ndcg_at_100_diff1": 0.786859, + "nauc_ndcg_at_1000_max": 0.533445, + "nauc_ndcg_at_1000_std": -0.237798, + "nauc_ndcg_at_1000_diff1": 0.789233, + "nauc_map_at_1_max": 0.51288, + "nauc_map_at_1_std": -0.27144, + "nauc_map_at_1_diff1": 0.835005, + "nauc_map_at_3_max": 0.527035, + "nauc_map_at_3_std": -0.267417, + "nauc_map_at_3_diff1": 0.806589, + "nauc_map_at_5_max": 0.527056, + "nauc_map_at_5_std": -0.265466, + "nauc_map_at_5_diff1": 0.804124, + "nauc_map_at_10_max": 0.526594, + "nauc_map_at_10_std": -0.261497, + "nauc_map_at_10_diff1": 0.802121, + "nauc_map_at_20_max": 0.526429, + "nauc_map_at_20_std": -0.260527, + "nauc_map_at_20_diff1": 0.801994, + "nauc_map_at_100_max": 0.526689, + "nauc_map_at_100_std": -0.259391, + "nauc_map_at_100_diff1": 0.802171, + "nauc_map_at_1000_max": 0.526719, + "nauc_map_at_1000_std": -0.259371, + "nauc_map_at_1000_diff1": 0.802238, + "nauc_recall_at_1_max": 0.51288, + "nauc_recall_at_1_std": -0.27144, + "nauc_recall_at_1_diff1": 0.835005, + "nauc_recall_at_3_max": 0.545104, + "nauc_recall_at_3_std": -0.260203, + "nauc_recall_at_3_diff1": 0.768163, + "nauc_recall_at_5_max": 0.546169, + "nauc_recall_at_5_std": -0.250196, + "nauc_recall_at_5_diff1": 0.751026, + "nauc_recall_at_10_max": 0.544234, + "nauc_recall_at_10_std": -0.207333, + "nauc_recall_at_10_diff1": 0.723218, + "nauc_recall_at_20_max": 0.54524, + "nauc_recall_at_20_std": -0.174818, + "nauc_recall_at_20_diff1": 0.70906, + "nauc_recall_at_100_max": 0.577202, + "nauc_recall_at_100_std": -0.02554, + "nauc_recall_at_100_diff1": 0.675189, + "nauc_recall_at_1000_max": 0.665971, + "nauc_recall_at_1000_std": 0.177894, + "nauc_recall_at_1000_diff1": 0.644345, + "nauc_precision_at_1_max": 0.51288, + "nauc_precision_at_1_std": -0.27144, + "nauc_precision_at_1_diff1": 0.835005, + "nauc_precision_at_3_max": 0.545104, + "nauc_precision_at_3_std": -0.260203, + "nauc_precision_at_3_diff1": 0.768163, + "nauc_precision_at_5_max": 0.546169, + "nauc_precision_at_5_std": -0.250196, + "nauc_precision_at_5_diff1": 0.751026, + "nauc_precision_at_10_max": 0.544234, + "nauc_precision_at_10_std": -0.207333, + "nauc_precision_at_10_diff1": 0.723218, + "nauc_precision_at_20_max": 0.54524, + "nauc_precision_at_20_std": -0.174818, + "nauc_precision_at_20_diff1": 0.70906, + "nauc_precision_at_100_max": 0.577202, + "nauc_precision_at_100_std": -0.02554, + "nauc_precision_at_100_diff1": 0.675189, + "nauc_precision_at_1000_max": 0.665971, + "nauc_precision_at_1000_std": 0.177894, + "nauc_precision_at_1000_diff1": 0.644345, + "nauc_mrr_at_1_max": 0.51288, + "nauc_mrr_at_1_std": -0.27144, + "nauc_mrr_at_1_diff1": 0.835005, + "nauc_mrr_at_3_max": 0.527035, + "nauc_mrr_at_3_std": -0.267417, + "nauc_mrr_at_3_diff1": 0.806589, + "nauc_mrr_at_5_max": 0.527056, + "nauc_mrr_at_5_std": -0.265466, + "nauc_mrr_at_5_diff1": 0.804124, + "nauc_mrr_at_10_max": 0.526594, + "nauc_mrr_at_10_std": -0.261497, + "nauc_mrr_at_10_diff1": 0.802121, + "nauc_mrr_at_20_max": 0.526429, + "nauc_mrr_at_20_std": -0.260527, + "nauc_mrr_at_20_diff1": 0.801994, + "nauc_mrr_at_100_max": 0.526689, + "nauc_mrr_at_100_std": -0.259391, + "nauc_mrr_at_100_diff1": 0.802171, + "nauc_mrr_at_1000_max": 0.526719, + "nauc_mrr_at_1000_std": -0.259371, + "nauc_mrr_at_1000_diff1": 0.802238, + "main_score": 0.71073, + "hf_subset": "java", + "languages": [ + "java-Code" + ] + }, + { + "ndcg_at_1": 0.63808, + "ndcg_at_3": 0.71004, + "ndcg_at_5": 0.7252, + "ndcg_at_10": 0.73997, + "ndcg_at_20": 0.74886, + "ndcg_at_100": 0.7594, + "ndcg_at_1000": 0.76501, + "map_at_1": 0.63808, + "map_at_3": 0.69287, + "map_at_5": 0.70125, + "map_at_10": 0.7074, + "map_at_20": 0.70986, + "map_at_100": 0.71132, + "map_at_1000": 0.71153, + "recall_at_1": 0.63808, + "recall_at_3": 0.75953, + "recall_at_5": 0.79642, + "recall_at_10": 0.8418, + "recall_at_20": 0.87691, + "recall_at_100": 0.93357, + "recall_at_1000": 0.97809, + "precision_at_1": 0.63808, + "precision_at_3": 0.25318, + "precision_at_5": 0.15928, + "precision_at_10": 0.08418, + "precision_at_20": 0.04385, + "precision_at_100": 0.00934, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.638076, + "mrr_at_3": 0.692867, + "mrr_at_5": 0.701255, + "mrr_at_10": 0.707403, + "mrr_at_20": 0.709855, + "mrr_at_100": 0.711324, + "mrr_at_1000": 0.711528, + "nauc_ndcg_at_1_max": 0.357618, + "nauc_ndcg_at_1_std": -0.367341, + "nauc_ndcg_at_1_diff1": 0.821693, + "nauc_ndcg_at_3_max": 0.377624, + "nauc_ndcg_at_3_std": -0.377518, + "nauc_ndcg_at_3_diff1": 0.786146, + "nauc_ndcg_at_5_max": 0.378147, + "nauc_ndcg_at_5_std": -0.371256, + "nauc_ndcg_at_5_diff1": 0.780256, + "nauc_ndcg_at_10_max": 0.382728, + "nauc_ndcg_at_10_std": -0.363445, + "nauc_ndcg_at_10_diff1": 0.780201, + "nauc_ndcg_at_20_max": 0.386213, + "nauc_ndcg_at_20_std": -0.355619, + "nauc_ndcg_at_20_diff1": 0.779369, + "nauc_ndcg_at_100_max": 0.386206, + "nauc_ndcg_at_100_std": -0.34862, + "nauc_ndcg_at_100_diff1": 0.781703, + "nauc_ndcg_at_1000_max": 0.384171, + "nauc_ndcg_at_1000_std": -0.352262, + "nauc_ndcg_at_1000_diff1": 0.784269, + "nauc_map_at_1_max": 0.357618, + "nauc_map_at_1_std": -0.367341, + "nauc_map_at_1_diff1": 0.821693, + "nauc_map_at_3_max": 0.372597, + "nauc_map_at_3_std": -0.37569, + "nauc_map_at_3_diff1": 0.795178, + "nauc_map_at_5_max": 0.372718, + "nauc_map_at_5_std": -0.372426, + "nauc_map_at_5_diff1": 0.792333, + "nauc_map_at_10_max": 0.374248, + "nauc_map_at_10_std": -0.369626, + "nauc_map_at_10_diff1": 0.792585, + "nauc_map_at_20_max": 0.37503, + "nauc_map_at_20_std": -0.367845, + "nauc_map_at_20_diff1": 0.792501, + "nauc_map_at_100_max": 0.374988, + "nauc_map_at_100_std": -0.366917, + "nauc_map_at_100_diff1": 0.792793, + "nauc_map_at_1000_max": 0.374946, + "nauc_map_at_1000_std": -0.367004, + "nauc_map_at_1000_diff1": 0.792878, + "nauc_recall_at_1_max": 0.357618, + "nauc_recall_at_1_std": -0.367341, + "nauc_recall_at_1_diff1": 0.821693, + "nauc_recall_at_3_max": 0.394796, + "nauc_recall_at_3_std": -0.383372, + "nauc_recall_at_3_diff1": 0.75524, + "nauc_recall_at_5_max": 0.398966, + "nauc_recall_at_5_std": -0.364974, + "nauc_recall_at_5_diff1": 0.732998, + "nauc_recall_at_10_max": 0.426036, + "nauc_recall_at_10_std": -0.326527, + "nauc_recall_at_10_diff1": 0.720377, + "nauc_recall_at_20_max": 0.461781, + "nauc_recall_at_20_std": -0.260309, + "nauc_recall_at_20_diff1": 0.699087, + "nauc_recall_at_100_max": 0.521263, + "nauc_recall_at_100_std": -0.075669, + "nauc_recall_at_100_diff1": 0.676323, + "nauc_recall_at_1000_max": 0.628449, + "nauc_recall_at_1000_std": 0.188727, + "nauc_recall_at_1000_diff1": 0.649675, + "nauc_precision_at_1_max": 0.357618, + "nauc_precision_at_1_std": -0.367341, + "nauc_precision_at_1_diff1": 0.821693, + "nauc_precision_at_3_max": 0.394796, + "nauc_precision_at_3_std": -0.383372, + "nauc_precision_at_3_diff1": 0.75524, + "nauc_precision_at_5_max": 0.398966, + "nauc_precision_at_5_std": -0.364974, + "nauc_precision_at_5_diff1": 0.732998, + "nauc_precision_at_10_max": 0.426036, + "nauc_precision_at_10_std": -0.326527, + "nauc_precision_at_10_diff1": 0.720377, + "nauc_precision_at_20_max": 0.461781, + "nauc_precision_at_20_std": -0.260309, + "nauc_precision_at_20_diff1": 0.699087, + "nauc_precision_at_100_max": 0.521263, + "nauc_precision_at_100_std": -0.075669, + "nauc_precision_at_100_diff1": 0.676323, + "nauc_precision_at_1000_max": 0.628449, + "nauc_precision_at_1000_std": 0.188727, + "nauc_precision_at_1000_diff1": 0.649675, + "nauc_mrr_at_1_max": 0.357618, + "nauc_mrr_at_1_std": -0.367341, + "nauc_mrr_at_1_diff1": 0.821693, + "nauc_mrr_at_3_max": 0.372597, + "nauc_mrr_at_3_std": -0.37569, + "nauc_mrr_at_3_diff1": 0.795178, + "nauc_mrr_at_5_max": 0.372718, + "nauc_mrr_at_5_std": -0.372426, + "nauc_mrr_at_5_diff1": 0.792333, + "nauc_mrr_at_10_max": 0.374248, + "nauc_mrr_at_10_std": -0.369626, + "nauc_mrr_at_10_diff1": 0.792585, + "nauc_mrr_at_20_max": 0.37503, + "nauc_mrr_at_20_std": -0.367845, + "nauc_mrr_at_20_diff1": 0.792501, + "nauc_mrr_at_100_max": 0.374988, + "nauc_mrr_at_100_std": -0.366917, + "nauc_mrr_at_100_diff1": 0.792793, + "nauc_mrr_at_1000_max": 0.374946, + "nauc_mrr_at_1000_std": -0.367004, + "nauc_mrr_at_1000_diff1": 0.792878, + "main_score": 0.73997, + "hf_subset": "php", + "languages": [ + "php-Code" + ] + } + ] + }, + "evaluation_time": 3240.4301722049713, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeEditSearchRetrieval.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeEditSearchRetrieval.json new file mode 100644 index 0000000000..887bd22833 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeEditSearchRetrieval.json @@ -0,0 +1,1922 @@ +{ + "dataset_revision": "4e51c66e0939303f6928472f13ad0848b2a3f4c0", + "task_name": "CodeEditSearchRetrieval", + "mteb_version": "1.38.57", + "scores": { + "train": [ + { + "ndcg_at_1": 0.278, + "ndcg_at_3": 0.34388, + "ndcg_at_5": 0.36195, + "ndcg_at_10": 0.38279, + "ndcg_at_20": 0.40198, + "ndcg_at_100": 0.43597, + "ndcg_at_1000": 0.46487, + "map_at_1": 0.278, + "map_at_3": 0.327, + "map_at_5": 0.337, + "map_at_10": 0.34549, + "map_at_20": 0.35075, + "map_at_100": 0.35512, + "map_at_1000": 0.35609, + "recall_at_1": 0.278, + "recall_at_3": 0.393, + "recall_at_5": 0.437, + "recall_at_10": 0.502, + "recall_at_20": 0.578, + "recall_at_100": 0.766, + "recall_at_1000": 1.0, + "precision_at_1": 0.278, + "precision_at_3": 0.131, + "precision_at_5": 0.0874, + "precision_at_10": 0.0502, + "precision_at_20": 0.0289, + "precision_at_100": 0.00766, + "precision_at_1000": 0.001, + "mrr_at_1": 0.278, + "mrr_at_3": 0.327, + "mrr_at_5": 0.337, + "mrr_at_10": 0.345488, + "mrr_at_20": 0.350747, + "mrr_at_100": 0.355119, + "mrr_at_1000": 0.35609, + "nauc_ndcg_at_1_max": 0.419963, + "nauc_ndcg_at_1_std": -0.089221, + "nauc_ndcg_at_1_diff1": 0.568897, + "nauc_ndcg_at_3_max": 0.3704, + "nauc_ndcg_at_3_std": -0.126275, + "nauc_ndcg_at_3_diff1": 0.489316, + "nauc_ndcg_at_5_max": 0.364653, + "nauc_ndcg_at_5_std": -0.112915, + "nauc_ndcg_at_5_diff1": 0.472375, + "nauc_ndcg_at_10_max": 0.352176, + "nauc_ndcg_at_10_std": -0.108901, + "nauc_ndcg_at_10_diff1": 0.453092, + "nauc_ndcg_at_20_max": 0.347903, + "nauc_ndcg_at_20_std": -0.107805, + "nauc_ndcg_at_20_diff1": 0.443168, + "nauc_ndcg_at_100_max": 0.355751, + "nauc_ndcg_at_100_std": -0.100261, + "nauc_ndcg_at_100_diff1": 0.447691, + "nauc_ndcg_at_1000_max": 0.363579, + "nauc_ndcg_at_1000_std": -0.104158, + "nauc_ndcg_at_1000_diff1": 0.466459, + "nauc_map_at_1_max": 0.419963, + "nauc_map_at_1_std": -0.089221, + "nauc_map_at_1_diff1": 0.568897, + "nauc_map_at_3_max": 0.383109, + "nauc_map_at_3_std": -0.118201, + "nauc_map_at_3_diff1": 0.509244, + "nauc_map_at_5_max": 0.379868, + "nauc_map_at_5_std": -0.110865, + "nauc_map_at_5_diff1": 0.499901, + "nauc_map_at_10_max": 0.374581, + "nauc_map_at_10_std": -0.109678, + "nauc_map_at_10_diff1": 0.491842, + "nauc_map_at_20_max": 0.373859, + "nauc_map_at_20_std": -0.109175, + "nauc_map_at_20_diff1": 0.489325, + "nauc_map_at_100_max": 0.374912, + "nauc_map_at_100_std": -0.107931, + "nauc_map_at_100_diff1": 0.489842, + "nauc_map_at_1000_max": 0.375275, + "nauc_map_at_1000_std": -0.107859, + "nauc_map_at_1000_diff1": 0.490558, + "nauc_recall_at_1_max": 0.419963, + "nauc_recall_at_1_std": -0.089221, + "nauc_recall_at_1_diff1": 0.568897, + "nauc_recall_at_3_max": 0.334838, + "nauc_recall_at_3_std": -0.148354, + "nauc_recall_at_3_diff1": 0.433722, + "nauc_recall_at_5_max": 0.321115, + "nauc_recall_at_5_std": -0.116329, + "nauc_recall_at_5_diff1": 0.39274, + "nauc_recall_at_10_max": 0.282803, + "nauc_recall_at_10_std": -0.101628, + "nauc_recall_at_10_diff1": 0.332699, + "nauc_recall_at_20_max": 0.256771, + "nauc_recall_at_20_std": -0.098095, + "nauc_recall_at_20_diff1": 0.281471, + "nauc_recall_at_100_max": 0.276732, + "nauc_recall_at_100_std": -0.037276, + "nauc_recall_at_100_diff1": 0.244204, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.419963, + "nauc_precision_at_1_std": -0.089221, + "nauc_precision_at_1_diff1": 0.568897, + "nauc_precision_at_3_max": 0.334838, + "nauc_precision_at_3_std": -0.148354, + "nauc_precision_at_3_diff1": 0.433722, + "nauc_precision_at_5_max": 0.321115, + "nauc_precision_at_5_std": -0.116329, + "nauc_precision_at_5_diff1": 0.39274, + "nauc_precision_at_10_max": 0.282803, + "nauc_precision_at_10_std": -0.101628, + "nauc_precision_at_10_diff1": 0.332699, + "nauc_precision_at_20_max": 0.256771, + "nauc_precision_at_20_std": -0.098095, + "nauc_precision_at_20_diff1": 0.281471, + "nauc_precision_at_100_max": 0.276732, + "nauc_precision_at_100_std": -0.037276, + "nauc_precision_at_100_diff1": 0.244204, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.419963, + "nauc_mrr_at_1_std": -0.089221, + "nauc_mrr_at_1_diff1": 0.568897, + "nauc_mrr_at_3_max": 0.383109, + "nauc_mrr_at_3_std": -0.118201, + "nauc_mrr_at_3_diff1": 0.509244, + "nauc_mrr_at_5_max": 0.379868, + "nauc_mrr_at_5_std": -0.110865, + "nauc_mrr_at_5_diff1": 0.499901, + "nauc_mrr_at_10_max": 0.374581, + "nauc_mrr_at_10_std": -0.109678, + "nauc_mrr_at_10_diff1": 0.491842, + "nauc_mrr_at_20_max": 0.373859, + "nauc_mrr_at_20_std": -0.109175, + "nauc_mrr_at_20_diff1": 0.489325, + "nauc_mrr_at_100_max": 0.374912, + "nauc_mrr_at_100_std": -0.107931, + "nauc_mrr_at_100_diff1": 0.489842, + "nauc_mrr_at_1000_max": 0.375275, + "nauc_mrr_at_1000_std": -0.107859, + "nauc_mrr_at_1000_diff1": 0.490558, + "main_score": 0.38279, + "hf_subset": "python", + "languages": [ + "python-Code" + ] + }, + { + "ndcg_at_1": 0.255, + "ndcg_at_3": 0.32508, + "ndcg_at_5": 0.34677, + "ndcg_at_10": 0.37007, + "ndcg_at_20": 0.39234, + "ndcg_at_100": 0.42482, + "ndcg_at_1000": 0.45372, + "map_at_1": 0.255, + "map_at_3": 0.30783, + "map_at_5": 0.31978, + "map_at_10": 0.3294, + "map_at_20": 0.33554, + "map_at_100": 0.33983, + "map_at_1000": 0.34079, + "recall_at_1": 0.255, + "recall_at_3": 0.375, + "recall_at_5": 0.428, + "recall_at_10": 0.5, + "recall_at_20": 0.588, + "recall_at_100": 0.766, + "recall_at_1000": 1.0, + "precision_at_1": 0.255, + "precision_at_3": 0.125, + "precision_at_5": 0.0856, + "precision_at_10": 0.05, + "precision_at_20": 0.0294, + "precision_at_100": 0.00766, + "precision_at_1000": 0.001, + "mrr_at_1": 0.255, + "mrr_at_3": 0.307833, + "mrr_at_5": 0.319783, + "mrr_at_10": 0.3294, + "mrr_at_20": 0.335536, + "mrr_at_100": 0.339827, + "mrr_at_1000": 0.340792, + "nauc_ndcg_at_1_max": 0.403291, + "nauc_ndcg_at_1_std": -0.018221, + "nauc_ndcg_at_1_diff1": 0.492258, + "nauc_ndcg_at_3_max": 0.37325, + "nauc_ndcg_at_3_std": -0.039442, + "nauc_ndcg_at_3_diff1": 0.435544, + "nauc_ndcg_at_5_max": 0.379399, + "nauc_ndcg_at_5_std": -0.031054, + "nauc_ndcg_at_5_diff1": 0.431271, + "nauc_ndcg_at_10_max": 0.375488, + "nauc_ndcg_at_10_std": -0.033856, + "nauc_ndcg_at_10_diff1": 0.426359, + "nauc_ndcg_at_20_max": 0.375894, + "nauc_ndcg_at_20_std": -0.017825, + "nauc_ndcg_at_20_diff1": 0.420779, + "nauc_ndcg_at_100_max": 0.379133, + "nauc_ndcg_at_100_std": -0.022626, + "nauc_ndcg_at_100_diff1": 0.426512, + "nauc_ndcg_at_1000_max": 0.378943, + "nauc_ndcg_at_1000_std": -0.023287, + "nauc_ndcg_at_1000_diff1": 0.431575, + "nauc_map_at_1_max": 0.403291, + "nauc_map_at_1_std": -0.018221, + "nauc_map_at_1_diff1": 0.492258, + "nauc_map_at_3_max": 0.381571, + "nauc_map_at_3_std": -0.033366, + "nauc_map_at_3_diff1": 0.448888, + "nauc_map_at_5_max": 0.385039, + "nauc_map_at_5_std": -0.028702, + "nauc_map_at_5_diff1": 0.446472, + "nauc_map_at_10_max": 0.383157, + "nauc_map_at_10_std": -0.030053, + "nauc_map_at_10_diff1": 0.444586, + "nauc_map_at_20_max": 0.383291, + "nauc_map_at_20_std": -0.02576, + "nauc_map_at_20_diff1": 0.443097, + "nauc_map_at_100_max": 0.383189, + "nauc_map_at_100_std": -0.026129, + "nauc_map_at_100_diff1": 0.443421, + "nauc_map_at_1000_max": 0.383178, + "nauc_map_at_1000_std": -0.02604, + "nauc_map_at_1000_diff1": 0.443552, + "nauc_recall_at_1_max": 0.403291, + "nauc_recall_at_1_std": -0.018221, + "nauc_recall_at_1_diff1": 0.492258, + "nauc_recall_at_3_max": 0.350029, + "nauc_recall_at_3_std": -0.05646, + "nauc_recall_at_3_diff1": 0.399069, + "nauc_recall_at_5_max": 0.364333, + "nauc_recall_at_5_std": -0.036701, + "nauc_recall_at_5_diff1": 0.389198, + "nauc_recall_at_10_max": 0.353555, + "nauc_recall_at_10_std": -0.044232, + "nauc_recall_at_10_diff1": 0.372774, + "nauc_recall_at_20_max": 0.353357, + "nauc_recall_at_20_std": 0.021952, + "nauc_recall_at_20_diff1": 0.345016, + "nauc_recall_at_100_max": 0.381887, + "nauc_recall_at_100_std": -0.002907, + "nauc_recall_at_100_diff1": 0.366397, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.403291, + "nauc_precision_at_1_std": -0.018221, + "nauc_precision_at_1_diff1": 0.492258, + "nauc_precision_at_3_max": 0.350029, + "nauc_precision_at_3_std": -0.05646, + "nauc_precision_at_3_diff1": 0.399069, + "nauc_precision_at_5_max": 0.364333, + "nauc_precision_at_5_std": -0.036701, + "nauc_precision_at_5_diff1": 0.389198, + "nauc_precision_at_10_max": 0.353555, + "nauc_precision_at_10_std": -0.044232, + "nauc_precision_at_10_diff1": 0.372774, + "nauc_precision_at_20_max": 0.353357, + "nauc_precision_at_20_std": 0.021952, + "nauc_precision_at_20_diff1": 0.345016, + "nauc_precision_at_100_max": 0.381887, + "nauc_precision_at_100_std": -0.002907, + "nauc_precision_at_100_diff1": 0.366397, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.403291, + "nauc_mrr_at_1_std": -0.018221, + "nauc_mrr_at_1_diff1": 0.492258, + "nauc_mrr_at_3_max": 0.381571, + "nauc_mrr_at_3_std": -0.033366, + "nauc_mrr_at_3_diff1": 0.448888, + "nauc_mrr_at_5_max": 0.385039, + "nauc_mrr_at_5_std": -0.028702, + "nauc_mrr_at_5_diff1": 0.446472, + "nauc_mrr_at_10_max": 0.383157, + "nauc_mrr_at_10_std": -0.030053, + "nauc_mrr_at_10_diff1": 0.444586, + "nauc_mrr_at_20_max": 0.383291, + "nauc_mrr_at_20_std": -0.02576, + "nauc_mrr_at_20_diff1": 0.443097, + "nauc_mrr_at_100_max": 0.383189, + "nauc_mrr_at_100_std": -0.026129, + "nauc_mrr_at_100_diff1": 0.443421, + "nauc_mrr_at_1000_max": 0.383178, + "nauc_mrr_at_1000_std": -0.02604, + "nauc_mrr_at_1000_diff1": 0.443552, + "main_score": 0.37007, + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ] + }, + { + "ndcg_at_1": 0.302, + "ndcg_at_3": 0.36238, + "ndcg_at_5": 0.37782, + "ndcg_at_10": 0.40021, + "ndcg_at_20": 0.4201, + "ndcg_at_100": 0.45469, + "ndcg_at_1000": 0.48176, + "map_at_1": 0.302, + "map_at_3": 0.34733, + "map_at_5": 0.35578, + "map_at_10": 0.36488, + "map_at_20": 0.37029, + "map_at_100": 0.37488, + "map_at_1000": 0.37582, + "recall_at_1": 0.302, + "recall_at_3": 0.406, + "recall_at_5": 0.444, + "recall_at_10": 0.514, + "recall_at_20": 0.593, + "recall_at_100": 0.782, + "recall_at_1000": 1.0, + "precision_at_1": 0.302, + "precision_at_3": 0.13533, + "precision_at_5": 0.0888, + "precision_at_10": 0.0514, + "precision_at_20": 0.02965, + "precision_at_100": 0.00782, + "precision_at_1000": 0.001, + "mrr_at_1": 0.302, + "mrr_at_3": 0.347333, + "mrr_at_5": 0.355783, + "mrr_at_10": 0.364877, + "mrr_at_20": 0.370287, + "mrr_at_100": 0.37488, + "mrr_at_1000": 0.37582, + "nauc_ndcg_at_1_max": 0.437402, + "nauc_ndcg_at_1_std": 0.00489, + "nauc_ndcg_at_1_diff1": 0.505472, + "nauc_ndcg_at_3_max": 0.442499, + "nauc_ndcg_at_3_std": 0.017254, + "nauc_ndcg_at_3_diff1": 0.43424, + "nauc_ndcg_at_5_max": 0.437827, + "nauc_ndcg_at_5_std": 0.020786, + "nauc_ndcg_at_5_diff1": 0.424386, + "nauc_ndcg_at_10_max": 0.43197, + "nauc_ndcg_at_10_std": 0.034395, + "nauc_ndcg_at_10_diff1": 0.40486, + "nauc_ndcg_at_20_max": 0.429718, + "nauc_ndcg_at_20_std": 0.040126, + "nauc_ndcg_at_20_diff1": 0.404048, + "nauc_ndcg_at_100_max": 0.42877, + "nauc_ndcg_at_100_std": 0.037908, + "nauc_ndcg_at_100_diff1": 0.410326, + "nauc_ndcg_at_1000_max": 0.434446, + "nauc_ndcg_at_1000_std": 0.031096, + "nauc_ndcg_at_1000_diff1": 0.421679, + "nauc_map_at_1_max": 0.437402, + "nauc_map_at_1_std": 0.00489, + "nauc_map_at_1_diff1": 0.505472, + "nauc_map_at_3_max": 0.443238, + "nauc_map_at_3_std": 0.015127, + "nauc_map_at_3_diff1": 0.451714, + "nauc_map_at_5_max": 0.440866, + "nauc_map_at_5_std": 0.017173, + "nauc_map_at_5_diff1": 0.446364, + "nauc_map_at_10_max": 0.438789, + "nauc_map_at_10_std": 0.023044, + "nauc_map_at_10_diff1": 0.438279, + "nauc_map_at_20_max": 0.438184, + "nauc_map_at_20_std": 0.024547, + "nauc_map_at_20_diff1": 0.438133, + "nauc_map_at_100_max": 0.438042, + "nauc_map_at_100_std": 0.024033, + "nauc_map_at_100_diff1": 0.438896, + "nauc_map_at_1000_max": 0.438344, + "nauc_map_at_1000_std": 0.023808, + "nauc_map_at_1000_diff1": 0.439365, + "nauc_recall_at_1_max": 0.437402, + "nauc_recall_at_1_std": 0.00489, + "nauc_recall_at_1_diff1": 0.505472, + "nauc_recall_at_3_max": 0.439634, + "nauc_recall_at_3_std": 0.022919, + "nauc_recall_at_3_diff1": 0.384792, + "nauc_recall_at_5_max": 0.427444, + "nauc_recall_at_5_std": 0.031136, + "nauc_recall_at_5_diff1": 0.360118, + "nauc_recall_at_10_max": 0.407113, + "nauc_recall_at_10_std": 0.072733, + "nauc_recall_at_10_diff1": 0.297494, + "nauc_recall_at_20_max": 0.395592, + "nauc_recall_at_20_std": 0.100533, + "nauc_recall_at_20_diff1": 0.285344, + "nauc_recall_at_100_max": 0.370495, + "nauc_recall_at_100_std": 0.119985, + "nauc_recall_at_100_diff1": 0.278711, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.437402, + "nauc_precision_at_1_std": 0.00489, + "nauc_precision_at_1_diff1": 0.505472, + "nauc_precision_at_3_max": 0.439634, + "nauc_precision_at_3_std": 0.022919, + "nauc_precision_at_3_diff1": 0.384792, + "nauc_precision_at_5_max": 0.427444, + "nauc_precision_at_5_std": 0.031136, + "nauc_precision_at_5_diff1": 0.360118, + "nauc_precision_at_10_max": 0.407113, + "nauc_precision_at_10_std": 0.072733, + "nauc_precision_at_10_diff1": 0.297494, + "nauc_precision_at_20_max": 0.395592, + "nauc_precision_at_20_std": 0.100533, + "nauc_precision_at_20_diff1": 0.285344, + "nauc_precision_at_100_max": 0.370495, + "nauc_precision_at_100_std": 0.119985, + "nauc_precision_at_100_diff1": 0.278711, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.437402, + "nauc_mrr_at_1_std": 0.00489, + "nauc_mrr_at_1_diff1": 0.505472, + "nauc_mrr_at_3_max": 0.443238, + "nauc_mrr_at_3_std": 0.015127, + "nauc_mrr_at_3_diff1": 0.451714, + "nauc_mrr_at_5_max": 0.440866, + "nauc_mrr_at_5_std": 0.017173, + "nauc_mrr_at_5_diff1": 0.446364, + "nauc_mrr_at_10_max": 0.438789, + "nauc_mrr_at_10_std": 0.023044, + "nauc_mrr_at_10_diff1": 0.438279, + "nauc_mrr_at_20_max": 0.438184, + "nauc_mrr_at_20_std": 0.024547, + "nauc_mrr_at_20_diff1": 0.438133, + "nauc_mrr_at_100_max": 0.438042, + "nauc_mrr_at_100_std": 0.024033, + "nauc_mrr_at_100_diff1": 0.438896, + "nauc_mrr_at_1000_max": 0.438344, + "nauc_mrr_at_1000_std": 0.023808, + "nauc_mrr_at_1000_diff1": 0.439365, + "main_score": 0.40021, + "hf_subset": "typescript", + "languages": [ + "typescript-Code" + ] + }, + { + "ndcg_at_1": 0.298, + "ndcg_at_3": 0.3599, + "ndcg_at_5": 0.3831, + "ndcg_at_10": 0.40373, + "ndcg_at_20": 0.42395, + "ndcg_at_100": 0.45386, + "ndcg_at_1000": 0.48293, + "map_at_1": 0.298, + "map_at_3": 0.34467, + "map_at_5": 0.35762, + "map_at_10": 0.36609, + "map_at_20": 0.37163, + "map_at_100": 0.37571, + "map_at_1000": 0.37672, + "recall_at_1": 0.298, + "recall_at_3": 0.404, + "recall_at_5": 0.46, + "recall_at_10": 0.524, + "recall_at_20": 0.604, + "recall_at_100": 0.766, + "recall_at_1000": 1.0, + "precision_at_1": 0.298, + "precision_at_3": 0.13467, + "precision_at_5": 0.092, + "precision_at_10": 0.0524, + "precision_at_20": 0.0302, + "precision_at_100": 0.00766, + "precision_at_1000": 0.001, + "mrr_at_1": 0.298, + "mrr_at_3": 0.344667, + "mrr_at_5": 0.357617, + "mrr_at_10": 0.366086, + "mrr_at_20": 0.371633, + "mrr_at_100": 0.375713, + "mrr_at_1000": 0.376724, + "nauc_ndcg_at_1_max": 0.393396, + "nauc_ndcg_at_1_std": -0.076359, + "nauc_ndcg_at_1_diff1": 0.537286, + "nauc_ndcg_at_3_max": 0.366324, + "nauc_ndcg_at_3_std": -0.075748, + "nauc_ndcg_at_3_diff1": 0.469691, + "nauc_ndcg_at_5_max": 0.365758, + "nauc_ndcg_at_5_std": -0.078007, + "nauc_ndcg_at_5_diff1": 0.457504, + "nauc_ndcg_at_10_max": 0.359725, + "nauc_ndcg_at_10_std": -0.081071, + "nauc_ndcg_at_10_diff1": 0.449804, + "nauc_ndcg_at_20_max": 0.359846, + "nauc_ndcg_at_20_std": -0.066318, + "nauc_ndcg_at_20_diff1": 0.444616, + "nauc_ndcg_at_100_max": 0.362732, + "nauc_ndcg_at_100_std": -0.06166, + "nauc_ndcg_at_100_diff1": 0.445125, + "nauc_ndcg_at_1000_max": 0.366388, + "nauc_ndcg_at_1000_std": -0.067715, + "nauc_ndcg_at_1000_diff1": 0.457541, + "nauc_map_at_1_max": 0.393396, + "nauc_map_at_1_std": -0.076359, + "nauc_map_at_1_diff1": 0.537286, + "nauc_map_at_3_max": 0.37306, + "nauc_map_at_3_std": -0.077153, + "nauc_map_at_3_diff1": 0.485934, + "nauc_map_at_5_max": 0.373057, + "nauc_map_at_5_std": -0.078116, + "nauc_map_at_5_diff1": 0.479257, + "nauc_map_at_10_max": 0.370929, + "nauc_map_at_10_std": -0.078724, + "nauc_map_at_10_diff1": 0.476057, + "nauc_map_at_20_max": 0.370796, + "nauc_map_at_20_std": -0.074845, + "nauc_map_at_20_diff1": 0.474618, + "nauc_map_at_100_max": 0.371039, + "nauc_map_at_100_std": -0.074189, + "nauc_map_at_100_diff1": 0.474751, + "nauc_map_at_1000_max": 0.371268, + "nauc_map_at_1000_std": -0.074204, + "nauc_map_at_1000_diff1": 0.475117, + "nauc_recall_at_1_max": 0.393396, + "nauc_recall_at_1_std": -0.076359, + "nauc_recall_at_1_diff1": 0.537286, + "nauc_recall_at_3_max": 0.347266, + "nauc_recall_at_3_std": -0.071294, + "nauc_recall_at_3_diff1": 0.423955, + "nauc_recall_at_5_max": 0.344381, + "nauc_recall_at_5_std": -0.077876, + "nauc_recall_at_5_diff1": 0.393603, + "nauc_recall_at_10_max": 0.322856, + "nauc_recall_at_10_std": -0.090975, + "nauc_recall_at_10_diff1": 0.367641, + "nauc_recall_at_20_max": 0.321836, + "nauc_recall_at_20_std": -0.027454, + "nauc_recall_at_20_diff1": 0.339243, + "nauc_recall_at_100_max": 0.331333, + "nauc_recall_at_100_std": 0.02742, + "nauc_recall_at_100_diff1": 0.296368, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.393396, + "nauc_precision_at_1_std": -0.076359, + "nauc_precision_at_1_diff1": 0.537286, + "nauc_precision_at_3_max": 0.347266, + "nauc_precision_at_3_std": -0.071294, + "nauc_precision_at_3_diff1": 0.423955, + "nauc_precision_at_5_max": 0.344381, + "nauc_precision_at_5_std": -0.077876, + "nauc_precision_at_5_diff1": 0.393603, + "nauc_precision_at_10_max": 0.322856, + "nauc_precision_at_10_std": -0.090975, + "nauc_precision_at_10_diff1": 0.367641, + "nauc_precision_at_20_max": 0.321836, + "nauc_precision_at_20_std": -0.027454, + "nauc_precision_at_20_diff1": 0.339243, + "nauc_precision_at_100_max": 0.331333, + "nauc_precision_at_100_std": 0.02742, + "nauc_precision_at_100_diff1": 0.296368, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.393396, + "nauc_mrr_at_1_std": -0.076359, + "nauc_mrr_at_1_diff1": 0.537286, + "nauc_mrr_at_3_max": 0.37306, + "nauc_mrr_at_3_std": -0.077153, + "nauc_mrr_at_3_diff1": 0.485934, + "nauc_mrr_at_5_max": 0.373057, + "nauc_mrr_at_5_std": -0.078116, + "nauc_mrr_at_5_diff1": 0.479257, + "nauc_mrr_at_10_max": 0.370929, + "nauc_mrr_at_10_std": -0.078724, + "nauc_mrr_at_10_diff1": 0.476057, + "nauc_mrr_at_20_max": 0.370796, + "nauc_mrr_at_20_std": -0.074845, + "nauc_mrr_at_20_diff1": 0.474618, + "nauc_mrr_at_100_max": 0.371039, + "nauc_mrr_at_100_std": -0.074189, + "nauc_mrr_at_100_diff1": 0.474751, + "nauc_mrr_at_1000_max": 0.371268, + "nauc_mrr_at_1000_std": -0.074204, + "nauc_mrr_at_1000_diff1": 0.475117, + "main_score": 0.40373, + "hf_subset": "go", + "languages": [ + "go-Code" + ] + }, + { + "ndcg_at_1": 0.274, + "ndcg_at_3": 0.33872, + "ndcg_at_5": 0.36411, + "ndcg_at_10": 0.38491, + "ndcg_at_20": 0.40087, + "ndcg_at_100": 0.43296, + "ndcg_at_1000": 0.46366, + "map_at_1": 0.274, + "map_at_3": 0.32183, + "map_at_5": 0.33583, + "map_at_10": 0.34448, + "map_at_20": 0.34888, + "map_at_100": 0.35326, + "map_at_1000": 0.3543, + "recall_at_1": 0.274, + "recall_at_3": 0.388, + "recall_at_5": 0.45, + "recall_at_10": 0.514, + "recall_at_20": 0.577, + "recall_at_100": 0.751, + "recall_at_1000": 1.0, + "precision_at_1": 0.274, + "precision_at_3": 0.12933, + "precision_at_5": 0.09, + "precision_at_10": 0.0514, + "precision_at_20": 0.02885, + "precision_at_100": 0.00751, + "precision_at_1000": 0.001, + "mrr_at_1": 0.274, + "mrr_at_3": 0.321833, + "mrr_at_5": 0.335833, + "mrr_at_10": 0.344479, + "mrr_at_20": 0.348875, + "mrr_at_100": 0.353264, + "mrr_at_1000": 0.354298, + "nauc_ndcg_at_1_max": 0.419907, + "nauc_ndcg_at_1_std": -0.120403, + "nauc_ndcg_at_1_diff1": 0.579384, + "nauc_ndcg_at_3_max": 0.379483, + "nauc_ndcg_at_3_std": -0.115595, + "nauc_ndcg_at_3_diff1": 0.492085, + "nauc_ndcg_at_5_max": 0.368127, + "nauc_ndcg_at_5_std": -0.117211, + "nauc_ndcg_at_5_diff1": 0.476957, + "nauc_ndcg_at_10_max": 0.358739, + "nauc_ndcg_at_10_std": -0.119321, + "nauc_ndcg_at_10_diff1": 0.47106, + "nauc_ndcg_at_20_max": 0.354125, + "nauc_ndcg_at_20_std": -0.107443, + "nauc_ndcg_at_20_diff1": 0.460171, + "nauc_ndcg_at_100_max": 0.357032, + "nauc_ndcg_at_100_std": -0.099394, + "nauc_ndcg_at_100_diff1": 0.465237, + "nauc_ndcg_at_1000_max": 0.365904, + "nauc_ndcg_at_1000_std": -0.108309, + "nauc_ndcg_at_1000_diff1": 0.479282, + "nauc_map_at_1_max": 0.419907, + "nauc_map_at_1_std": -0.120403, + "nauc_map_at_1_diff1": 0.579384, + "nauc_map_at_3_max": 0.389102, + "nauc_map_at_3_std": -0.117382, + "nauc_map_at_3_diff1": 0.513334, + "nauc_map_at_5_max": 0.383008, + "nauc_map_at_5_std": -0.117831, + "nauc_map_at_5_diff1": 0.504868, + "nauc_map_at_10_max": 0.378772, + "nauc_map_at_10_std": -0.118793, + "nauc_map_at_10_diff1": 0.502255, + "nauc_map_at_20_max": 0.377467, + "nauc_map_at_20_std": -0.115824, + "nauc_map_at_20_diff1": 0.499495, + "nauc_map_at_100_max": 0.377581, + "nauc_map_at_100_std": -0.11509, + "nauc_map_at_100_diff1": 0.500375, + "nauc_map_at_1000_max": 0.377953, + "nauc_map_at_1000_std": -0.115227, + "nauc_map_at_1000_diff1": 0.500779, + "nauc_recall_at_1_max": 0.419907, + "nauc_recall_at_1_std": -0.120403, + "nauc_recall_at_1_diff1": 0.579384, + "nauc_recall_at_3_max": 0.352892, + "nauc_recall_at_3_std": -0.11041, + "nauc_recall_at_3_diff1": 0.43316, + "nauc_recall_at_5_max": 0.324781, + "nauc_recall_at_5_std": -0.116005, + "nauc_recall_at_5_diff1": 0.396915, + "nauc_recall_at_10_max": 0.296551, + "nauc_recall_at_10_std": -0.122057, + "nauc_recall_at_10_diff1": 0.377671, + "nauc_recall_at_20_max": 0.274291, + "nauc_recall_at_20_std": -0.070969, + "nauc_recall_at_20_diff1": 0.325669, + "nauc_recall_at_100_max": 0.26932, + "nauc_recall_at_100_std": 0.012541, + "nauc_recall_at_100_diff1": 0.305416, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.419907, + "nauc_precision_at_1_std": -0.120403, + "nauc_precision_at_1_diff1": 0.579384, + "nauc_precision_at_3_max": 0.352892, + "nauc_precision_at_3_std": -0.11041, + "nauc_precision_at_3_diff1": 0.43316, + "nauc_precision_at_5_max": 0.324781, + "nauc_precision_at_5_std": -0.116005, + "nauc_precision_at_5_diff1": 0.396915, + "nauc_precision_at_10_max": 0.296551, + "nauc_precision_at_10_std": -0.122057, + "nauc_precision_at_10_diff1": 0.377671, + "nauc_precision_at_20_max": 0.274291, + "nauc_precision_at_20_std": -0.070969, + "nauc_precision_at_20_diff1": 0.325669, + "nauc_precision_at_100_max": 0.26932, + "nauc_precision_at_100_std": 0.012541, + "nauc_precision_at_100_diff1": 0.305416, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.419907, + "nauc_mrr_at_1_std": -0.120403, + "nauc_mrr_at_1_diff1": 0.579384, + "nauc_mrr_at_3_max": 0.389102, + "nauc_mrr_at_3_std": -0.117382, + "nauc_mrr_at_3_diff1": 0.513334, + "nauc_mrr_at_5_max": 0.383008, + "nauc_mrr_at_5_std": -0.117831, + "nauc_mrr_at_5_diff1": 0.504868, + "nauc_mrr_at_10_max": 0.378772, + "nauc_mrr_at_10_std": -0.118793, + "nauc_mrr_at_10_diff1": 0.502255, + "nauc_mrr_at_20_max": 0.377467, + "nauc_mrr_at_20_std": -0.115824, + "nauc_mrr_at_20_diff1": 0.499495, + "nauc_mrr_at_100_max": 0.377581, + "nauc_mrr_at_100_std": -0.11509, + "nauc_mrr_at_100_diff1": 0.500375, + "nauc_mrr_at_1000_max": 0.377953, + "nauc_mrr_at_1000_std": -0.115227, + "nauc_mrr_at_1000_diff1": 0.500779, + "main_score": 0.38491, + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ] + }, + { + "ndcg_at_1": 0.263, + "ndcg_at_3": 0.3264, + "ndcg_at_5": 0.34616, + "ndcg_at_10": 0.36753, + "ndcg_at_20": 0.38902, + "ndcg_at_100": 0.42043, + "ndcg_at_1000": 0.45182, + "map_at_1": 0.263, + "map_at_3": 0.31067, + "map_at_5": 0.32162, + "map_at_10": 0.33065, + "map_at_20": 0.33654, + "map_at_100": 0.34074, + "map_at_1000": 0.34172, + "recall_at_1": 0.263, + "recall_at_3": 0.372, + "recall_at_5": 0.42, + "recall_at_10": 0.485, + "recall_at_20": 0.57, + "recall_at_100": 0.741, + "recall_at_1000": 1.0, + "precision_at_1": 0.263, + "precision_at_3": 0.124, + "precision_at_5": 0.084, + "precision_at_10": 0.0485, + "precision_at_20": 0.0285, + "precision_at_100": 0.00741, + "precision_at_1000": 0.001, + "mrr_at_1": 0.263, + "mrr_at_3": 0.310667, + "mrr_at_5": 0.321617, + "mrr_at_10": 0.330649, + "mrr_at_20": 0.336542, + "mrr_at_100": 0.340739, + "mrr_at_1000": 0.34172, + "nauc_ndcg_at_1_max": 0.441832, + "nauc_ndcg_at_1_std": 0.000518, + "nauc_ndcg_at_1_diff1": 0.553913, + "nauc_ndcg_at_3_max": 0.398602, + "nauc_ndcg_at_3_std": -0.00592, + "nauc_ndcg_at_3_diff1": 0.489201, + "nauc_ndcg_at_5_max": 0.399427, + "nauc_ndcg_at_5_std": -0.001075, + "nauc_ndcg_at_5_diff1": 0.469746, + "nauc_ndcg_at_10_max": 0.392637, + "nauc_ndcg_at_10_std": 0.010139, + "nauc_ndcg_at_10_diff1": 0.454187, + "nauc_ndcg_at_20_max": 0.387024, + "nauc_ndcg_at_20_std": 0.008697, + "nauc_ndcg_at_20_diff1": 0.456262, + "nauc_ndcg_at_100_max": 0.392422, + "nauc_ndcg_at_100_std": 0.008187, + "nauc_ndcg_at_100_diff1": 0.457127, + "nauc_ndcg_at_1000_max": 0.397183, + "nauc_ndcg_at_1000_std": 0.007855, + "nauc_ndcg_at_1000_diff1": 0.468199, + "nauc_map_at_1_max": 0.441832, + "nauc_map_at_1_std": 0.000518, + "nauc_map_at_1_diff1": 0.553913, + "nauc_map_at_3_max": 0.408574, + "nauc_map_at_3_std": -0.004203, + "nauc_map_at_3_diff1": 0.50353, + "nauc_map_at_5_max": 0.408832, + "nauc_map_at_5_std": -0.001872, + "nauc_map_at_5_diff1": 0.49237, + "nauc_map_at_10_max": 0.406546, + "nauc_map_at_10_std": 0.003481, + "nauc_map_at_10_diff1": 0.486063, + "nauc_map_at_20_max": 0.404817, + "nauc_map_at_20_std": 0.002933, + "nauc_map_at_20_diff1": 0.486566, + "nauc_map_at_100_max": 0.405495, + "nauc_map_at_100_std": 0.003275, + "nauc_map_at_100_diff1": 0.486371, + "nauc_map_at_1000_max": 0.405647, + "nauc_map_at_1000_std": 0.003386, + "nauc_map_at_1000_diff1": 0.486654, + "nauc_recall_at_1_max": 0.441832, + "nauc_recall_at_1_std": 0.000518, + "nauc_recall_at_1_diff1": 0.553913, + "nauc_recall_at_3_max": 0.371246, + "nauc_recall_at_3_std": -0.010721, + "nauc_recall_at_3_diff1": 0.450127, + "nauc_recall_at_5_max": 0.374078, + "nauc_recall_at_5_std": 0.002145, + "nauc_recall_at_5_diff1": 0.405729, + "nauc_recall_at_10_max": 0.350479, + "nauc_recall_at_10_std": 0.032517, + "nauc_recall_at_10_diff1": 0.356869, + "nauc_recall_at_20_max": 0.325993, + "nauc_recall_at_20_std": 0.02939, + "nauc_recall_at_20_diff1": 0.359003, + "nauc_recall_at_100_max": 0.342785, + "nauc_recall_at_100_std": 0.023975, + "nauc_recall_at_100_diff1": 0.334086, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.441832, + "nauc_precision_at_1_std": 0.000518, + "nauc_precision_at_1_diff1": 0.553913, + "nauc_precision_at_3_max": 0.371246, + "nauc_precision_at_3_std": -0.010721, + "nauc_precision_at_3_diff1": 0.450127, + "nauc_precision_at_5_max": 0.374078, + "nauc_precision_at_5_std": 0.002145, + "nauc_precision_at_5_diff1": 0.405729, + "nauc_precision_at_10_max": 0.350479, + "nauc_precision_at_10_std": 0.032517, + "nauc_precision_at_10_diff1": 0.356869, + "nauc_precision_at_20_max": 0.325993, + "nauc_precision_at_20_std": 0.02939, + "nauc_precision_at_20_diff1": 0.359003, + "nauc_precision_at_100_max": 0.342785, + "nauc_precision_at_100_std": 0.023975, + "nauc_precision_at_100_diff1": 0.334086, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.441832, + "nauc_mrr_at_1_std": 0.000518, + "nauc_mrr_at_1_diff1": 0.553913, + "nauc_mrr_at_3_max": 0.408574, + "nauc_mrr_at_3_std": -0.004203, + "nauc_mrr_at_3_diff1": 0.50353, + "nauc_mrr_at_5_max": 0.408832, + "nauc_mrr_at_5_std": -0.001872, + "nauc_mrr_at_5_diff1": 0.49237, + "nauc_mrr_at_10_max": 0.406546, + "nauc_mrr_at_10_std": 0.003481, + "nauc_mrr_at_10_diff1": 0.486063, + "nauc_mrr_at_20_max": 0.404817, + "nauc_mrr_at_20_std": 0.002933, + "nauc_mrr_at_20_diff1": 0.486566, + "nauc_mrr_at_100_max": 0.405495, + "nauc_mrr_at_100_std": 0.003275, + "nauc_mrr_at_100_diff1": 0.486371, + "nauc_mrr_at_1000_max": 0.405647, + "nauc_mrr_at_1000_std": 0.003386, + "nauc_mrr_at_1000_diff1": 0.486654, + "main_score": 0.36753, + "hf_subset": "java", + "languages": [ + "java-Code" + ] + }, + { + "ndcg_at_1": 0.237, + "ndcg_at_3": 0.28912, + "ndcg_at_5": 0.31107, + "ndcg_at_10": 0.33899, + "ndcg_at_20": 0.36063, + "ndcg_at_100": 0.39798, + "ndcg_at_1000": 0.4277, + "map_at_1": 0.237, + "map_at_3": 0.27667, + "map_at_5": 0.28892, + "map_at_10": 0.30051, + "map_at_20": 0.30638, + "map_at_100": 0.31132, + "map_at_1000": 0.31225, + "recall_at_1": 0.237, + "recall_at_3": 0.325, + "recall_at_5": 0.378, + "recall_at_10": 0.464, + "recall_at_20": 0.55, + "recall_at_100": 0.755, + "recall_at_1000": 1.0, + "precision_at_1": 0.237, + "precision_at_3": 0.10833, + "precision_at_5": 0.0756, + "precision_at_10": 0.0464, + "precision_at_20": 0.0275, + "precision_at_100": 0.00755, + "precision_at_1000": 0.001, + "mrr_at_1": 0.237, + "mrr_at_3": 0.276667, + "mrr_at_5": 0.288917, + "mrr_at_10": 0.300506, + "mrr_at_20": 0.306379, + "mrr_at_100": 0.311313, + "mrr_at_1000": 0.31224, + "nauc_ndcg_at_1_max": 0.39929, + "nauc_ndcg_at_1_std": -0.066865, + "nauc_ndcg_at_1_diff1": 0.528967, + "nauc_ndcg_at_3_max": 0.391372, + "nauc_ndcg_at_3_std": -0.062073, + "nauc_ndcg_at_3_diff1": 0.461804, + "nauc_ndcg_at_5_max": 0.385904, + "nauc_ndcg_at_5_std": -0.052349, + "nauc_ndcg_at_5_diff1": 0.44526, + "nauc_ndcg_at_10_max": 0.386027, + "nauc_ndcg_at_10_std": -0.041755, + "nauc_ndcg_at_10_diff1": 0.422548, + "nauc_ndcg_at_20_max": 0.388226, + "nauc_ndcg_at_20_std": -0.043639, + "nauc_ndcg_at_20_diff1": 0.423933, + "nauc_ndcg_at_100_max": 0.388434, + "nauc_ndcg_at_100_std": -0.037763, + "nauc_ndcg_at_100_diff1": 0.427412, + "nauc_ndcg_at_1000_max": 0.389117, + "nauc_ndcg_at_1000_std": -0.044921, + "nauc_ndcg_at_1000_diff1": 0.439884, + "nauc_map_at_1_max": 0.39929, + "nauc_map_at_1_std": -0.066865, + "nauc_map_at_1_diff1": 0.528967, + "nauc_map_at_3_max": 0.393566, + "nauc_map_at_3_std": -0.062773, + "nauc_map_at_3_diff1": 0.476704, + "nauc_map_at_5_max": 0.390948, + "nauc_map_at_5_std": -0.057196, + "nauc_map_at_5_diff1": 0.467532, + "nauc_map_at_10_max": 0.390448, + "nauc_map_at_10_std": -0.052658, + "nauc_map_at_10_diff1": 0.45757, + "nauc_map_at_20_max": 0.390958, + "nauc_map_at_20_std": -0.053193, + "nauc_map_at_20_diff1": 0.458143, + "nauc_map_at_100_max": 0.390726, + "nauc_map_at_100_std": -0.052367, + "nauc_map_at_100_diff1": 0.45865, + "nauc_map_at_1000_max": 0.390801, + "nauc_map_at_1000_std": -0.052553, + "nauc_map_at_1000_diff1": 0.459042, + "nauc_recall_at_1_max": 0.39929, + "nauc_recall_at_1_std": -0.066865, + "nauc_recall_at_1_diff1": 0.528967, + "nauc_recall_at_3_max": 0.385262, + "nauc_recall_at_3_std": -0.060324, + "nauc_recall_at_3_diff1": 0.42149, + "nauc_recall_at_5_max": 0.371013, + "nauc_recall_at_5_std": -0.038352, + "nauc_recall_at_5_diff1": 0.383515, + "nauc_recall_at_10_max": 0.374047, + "nauc_recall_at_10_std": -0.007375, + "nauc_recall_at_10_diff1": 0.318156, + "nauc_recall_at_20_max": 0.382806, + "nauc_recall_at_20_std": -0.012659, + "nauc_recall_at_20_diff1": 0.316206, + "nauc_recall_at_100_max": 0.386015, + "nauc_recall_at_100_std": 0.043773, + "nauc_recall_at_100_diff1": 0.291848, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.39929, + "nauc_precision_at_1_std": -0.066865, + "nauc_precision_at_1_diff1": 0.528967, + "nauc_precision_at_3_max": 0.385262, + "nauc_precision_at_3_std": -0.060324, + "nauc_precision_at_3_diff1": 0.42149, + "nauc_precision_at_5_max": 0.371013, + "nauc_precision_at_5_std": -0.038352, + "nauc_precision_at_5_diff1": 0.383515, + "nauc_precision_at_10_max": 0.374047, + "nauc_precision_at_10_std": -0.007375, + "nauc_precision_at_10_diff1": 0.318156, + "nauc_precision_at_20_max": 0.382806, + "nauc_precision_at_20_std": -0.012659, + "nauc_precision_at_20_diff1": 0.316206, + "nauc_precision_at_100_max": 0.386015, + "nauc_precision_at_100_std": 0.043773, + "nauc_precision_at_100_diff1": 0.291848, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.39929, + "nauc_mrr_at_1_std": -0.066865, + "nauc_mrr_at_1_diff1": 0.528967, + "nauc_mrr_at_3_max": 0.393566, + "nauc_mrr_at_3_std": -0.062773, + "nauc_mrr_at_3_diff1": 0.476704, + "nauc_mrr_at_5_max": 0.390948, + "nauc_mrr_at_5_std": -0.057196, + "nauc_mrr_at_5_diff1": 0.467532, + "nauc_mrr_at_10_max": 0.390448, + "nauc_mrr_at_10_std": -0.052658, + "nauc_mrr_at_10_diff1": 0.45757, + "nauc_mrr_at_20_max": 0.390933, + "nauc_mrr_at_20_std": -0.053196, + "nauc_mrr_at_20_diff1": 0.458154, + "nauc_mrr_at_100_max": 0.390701, + "nauc_mrr_at_100_std": -0.05237, + "nauc_mrr_at_100_diff1": 0.458661, + "nauc_mrr_at_1000_max": 0.390777, + "nauc_mrr_at_1000_std": -0.052556, + "nauc_mrr_at_1000_diff1": 0.459053, + "main_score": 0.33899, + "hf_subset": "php", + "languages": [ + "php-Code" + ] + }, + { + "ndcg_at_1": 0.202, + "ndcg_at_3": 0.26275, + "ndcg_at_5": 0.27768, + "ndcg_at_10": 0.29888, + "ndcg_at_20": 0.31829, + "ndcg_at_100": 0.35023, + "ndcg_at_1000": 0.39069, + "map_at_1": 0.202, + "map_at_3": 0.2475, + "map_at_5": 0.25585, + "map_at_10": 0.2647, + "map_at_20": 0.26999, + "map_at_100": 0.27401, + "map_at_1000": 0.27523, + "recall_at_1": 0.202, + "recall_at_3": 0.307, + "recall_at_5": 0.343, + "recall_at_10": 0.408, + "recall_at_20": 0.485, + "recall_at_100": 0.663, + "recall_at_1000": 1.0, + "precision_at_1": 0.202, + "precision_at_3": 0.10233, + "precision_at_5": 0.0686, + "precision_at_10": 0.0408, + "precision_at_20": 0.02425, + "precision_at_100": 0.00663, + "precision_at_1000": 0.001, + "mrr_at_1": 0.202, + "mrr_at_3": 0.247167, + "mrr_at_5": 0.255717, + "mrr_at_10": 0.264562, + "mrr_at_20": 0.269861, + "mrr_at_100": 0.273881, + "mrr_at_1000": 0.275093, + "nauc_ndcg_at_1_max": 0.445297, + "nauc_ndcg_at_1_std": 0.00893, + "nauc_ndcg_at_1_diff1": 0.567837, + "nauc_ndcg_at_3_max": 0.423827, + "nauc_ndcg_at_3_std": 0.026419, + "nauc_ndcg_at_3_diff1": 0.458748, + "nauc_ndcg_at_5_max": 0.418488, + "nauc_ndcg_at_5_std": 0.02881, + "nauc_ndcg_at_5_diff1": 0.44102, + "nauc_ndcg_at_10_max": 0.411162, + "nauc_ndcg_at_10_std": 0.031704, + "nauc_ndcg_at_10_diff1": 0.426376, + "nauc_ndcg_at_20_max": 0.409307, + "nauc_ndcg_at_20_std": 0.030459, + "nauc_ndcg_at_20_diff1": 0.422439, + "nauc_ndcg_at_100_max": 0.412059, + "nauc_ndcg_at_100_std": 0.036018, + "nauc_ndcg_at_100_diff1": 0.421662, + "nauc_ndcg_at_1000_max": 0.41579, + "nauc_ndcg_at_1000_std": 0.031645, + "nauc_ndcg_at_1000_diff1": 0.438563, + "nauc_map_at_1_max": 0.445297, + "nauc_map_at_1_std": 0.00893, + "nauc_map_at_1_diff1": 0.567837, + "nauc_map_at_3_max": 0.430149, + "nauc_map_at_3_std": 0.021969, + "nauc_map_at_3_diff1": 0.484731, + "nauc_map_at_5_max": 0.4268, + "nauc_map_at_5_std": 0.022985, + "nauc_map_at_5_diff1": 0.474706, + "nauc_map_at_10_max": 0.42378, + "nauc_map_at_10_std": 0.024327, + "nauc_map_at_10_diff1": 0.468198, + "nauc_map_at_20_max": 0.42323, + "nauc_map_at_20_std": 0.02399, + "nauc_map_at_20_diff1": 0.466986, + "nauc_map_at_100_max": 0.42334, + "nauc_map_at_100_std": 0.024954, + "nauc_map_at_100_diff1": 0.466298, + "nauc_map_at_1000_max": 0.423458, + "nauc_map_at_1000_std": 0.024948, + "nauc_map_at_1000_diff1": 0.466763, + "nauc_recall_at_1_max": 0.445297, + "nauc_recall_at_1_std": 0.00893, + "nauc_recall_at_1_diff1": 0.567837, + "nauc_recall_at_3_max": 0.40667, + "nauc_recall_at_3_std": 0.038298, + "nauc_recall_at_3_diff1": 0.389969, + "nauc_recall_at_5_max": 0.396011, + "nauc_recall_at_5_std": 0.044822, + "nauc_recall_at_5_diff1": 0.350633, + "nauc_recall_at_10_max": 0.375312, + "nauc_recall_at_10_std": 0.052104, + "nauc_recall_at_10_diff1": 0.312292, + "nauc_recall_at_20_max": 0.36788, + "nauc_recall_at_20_std": 0.047915, + "nauc_recall_at_20_diff1": 0.296068, + "nauc_recall_at_100_max": 0.380175, + "nauc_recall_at_100_std": 0.082767, + "nauc_recall_at_100_diff1": 0.272427, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.445297, + "nauc_precision_at_1_std": 0.00893, + "nauc_precision_at_1_diff1": 0.567837, + "nauc_precision_at_3_max": 0.40667, + "nauc_precision_at_3_std": 0.038298, + "nauc_precision_at_3_diff1": 0.389969, + "nauc_precision_at_5_max": 0.396011, + "nauc_precision_at_5_std": 0.044822, + "nauc_precision_at_5_diff1": 0.350633, + "nauc_precision_at_10_max": 0.375312, + "nauc_precision_at_10_std": 0.052104, + "nauc_precision_at_10_diff1": 0.312292, + "nauc_precision_at_20_max": 0.36788, + "nauc_precision_at_20_std": 0.047915, + "nauc_precision_at_20_diff1": 0.296068, + "nauc_precision_at_100_max": 0.380175, + "nauc_precision_at_100_std": 0.082767, + "nauc_precision_at_100_diff1": 0.272427, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.445297, + "nauc_mrr_at_1_std": 0.00893, + "nauc_mrr_at_1_diff1": 0.567837, + "nauc_mrr_at_3_max": 0.429942, + "nauc_mrr_at_3_std": 0.022414, + "nauc_mrr_at_3_diff1": 0.48345, + "nauc_mrr_at_5_max": 0.426903, + "nauc_mrr_at_5_std": 0.023348, + "nauc_mrr_at_5_diff1": 0.474478, + "nauc_mrr_at_10_max": 0.423882, + "nauc_mrr_at_10_std": 0.024689, + "nauc_mrr_at_10_diff1": 0.46797, + "nauc_mrr_at_20_max": 0.423332, + "nauc_mrr_at_20_std": 0.024353, + "nauc_mrr_at_20_diff1": 0.466758, + "nauc_mrr_at_100_max": 0.423443, + "nauc_mrr_at_100_std": 0.025319, + "nauc_mrr_at_100_diff1": 0.46607, + "nauc_mrr_at_1000_max": 0.423561, + "nauc_mrr_at_1000_std": 0.025314, + "nauc_mrr_at_1000_diff1": 0.466536, + "main_score": 0.29888, + "hf_subset": "c", + "languages": [ + "c-Code" + ] + }, + { + "ndcg_at_1": 0.242, + "ndcg_at_3": 0.30603, + "ndcg_at_5": 0.32682, + "ndcg_at_10": 0.34726, + "ndcg_at_20": 0.36283, + "ndcg_at_100": 0.39582, + "ndcg_at_1000": 0.43168, + "map_at_1": 0.242, + "map_at_3": 0.29017, + "map_at_5": 0.30182, + "map_at_10": 0.31029, + "map_at_20": 0.31464, + "map_at_100": 0.31886, + "map_at_1000": 0.32005, + "recall_at_1": 0.242, + "recall_at_3": 0.352, + "recall_at_5": 0.402, + "recall_at_10": 0.465, + "recall_at_20": 0.526, + "recall_at_100": 0.709, + "recall_at_1000": 1.0, + "precision_at_1": 0.242, + "precision_at_3": 0.11733, + "precision_at_5": 0.0804, + "precision_at_10": 0.0465, + "precision_at_20": 0.0263, + "precision_at_100": 0.00709, + "precision_at_1000": 0.001, + "mrr_at_1": 0.242, + "mrr_at_3": 0.290333, + "mrr_at_5": 0.301733, + "mrr_at_10": 0.310203, + "mrr_at_20": 0.314556, + "mrr_at_100": 0.318775, + "mrr_at_1000": 0.319971, + "nauc_ndcg_at_1_max": 0.471694, + "nauc_ndcg_at_1_std": 0.038833, + "nauc_ndcg_at_1_diff1": 0.529278, + "nauc_ndcg_at_3_max": 0.446027, + "nauc_ndcg_at_3_std": 0.010784, + "nauc_ndcg_at_3_diff1": 0.456033, + "nauc_ndcg_at_5_max": 0.443631, + "nauc_ndcg_at_5_std": 0.017631, + "nauc_ndcg_at_5_diff1": 0.443372, + "nauc_ndcg_at_10_max": 0.448391, + "nauc_ndcg_at_10_std": 0.031945, + "nauc_ndcg_at_10_diff1": 0.429419, + "nauc_ndcg_at_20_max": 0.441683, + "nauc_ndcg_at_20_std": 0.032027, + "nauc_ndcg_at_20_diff1": 0.429839, + "nauc_ndcg_at_100_max": 0.439281, + "nauc_ndcg_at_100_std": 0.037836, + "nauc_ndcg_at_100_diff1": 0.427232, + "nauc_ndcg_at_1000_max": 0.445094, + "nauc_ndcg_at_1000_std": 0.030844, + "nauc_ndcg_at_1000_diff1": 0.441836, + "nauc_map_at_1_max": 0.471694, + "nauc_map_at_1_std": 0.038833, + "nauc_map_at_1_diff1": 0.529278, + "nauc_map_at_3_max": 0.452097, + "nauc_map_at_3_std": 0.017897, + "nauc_map_at_3_diff1": 0.472432, + "nauc_map_at_5_max": 0.450763, + "nauc_map_at_5_std": 0.021317, + "nauc_map_at_5_diff1": 0.465173, + "nauc_map_at_10_max": 0.453196, + "nauc_map_at_10_std": 0.027893, + "nauc_map_at_10_diff1": 0.459477, + "nauc_map_at_20_max": 0.45132, + "nauc_map_at_20_std": 0.027915, + "nauc_map_at_20_diff1": 0.459827, + "nauc_map_at_100_max": 0.450745, + "nauc_map_at_100_std": 0.028285, + "nauc_map_at_100_diff1": 0.45939, + "nauc_map_at_1000_max": 0.450982, + "nauc_map_at_1000_std": 0.028027, + "nauc_map_at_1000_diff1": 0.45992, + "nauc_recall_at_1_max": 0.471694, + "nauc_recall_at_1_std": 0.038833, + "nauc_recall_at_1_diff1": 0.529278, + "nauc_recall_at_3_max": 0.429503, + "nauc_recall_at_3_std": -0.008761, + "nauc_recall_at_3_diff1": 0.411729, + "nauc_recall_at_5_max": 0.424162, + "nauc_recall_at_5_std": 0.008941, + "nauc_recall_at_5_diff1": 0.383706, + "nauc_recall_at_10_max": 0.435678, + "nauc_recall_at_10_std": 0.048281, + "nauc_recall_at_10_diff1": 0.340664, + "nauc_recall_at_20_max": 0.40922, + "nauc_recall_at_20_std": 0.049199, + "nauc_recall_at_20_diff1": 0.337589, + "nauc_recall_at_100_max": 0.387162, + "nauc_recall_at_100_std": 0.102421, + "nauc_recall_at_100_diff1": 0.292269, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.471694, + "nauc_precision_at_1_std": 0.038833, + "nauc_precision_at_1_diff1": 0.529278, + "nauc_precision_at_3_max": 0.429503, + "nauc_precision_at_3_std": -0.008761, + "nauc_precision_at_3_diff1": 0.411729, + "nauc_precision_at_5_max": 0.424162, + "nauc_precision_at_5_std": 0.008941, + "nauc_precision_at_5_diff1": 0.383706, + "nauc_precision_at_10_max": 0.435678, + "nauc_precision_at_10_std": 0.048281, + "nauc_precision_at_10_diff1": 0.340664, + "nauc_precision_at_20_max": 0.40922, + "nauc_precision_at_20_std": 0.049199, + "nauc_precision_at_20_diff1": 0.337589, + "nauc_precision_at_100_max": 0.387162, + "nauc_precision_at_100_std": 0.102421, + "nauc_precision_at_100_diff1": 0.292269, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.469813, + "nauc_mrr_at_1_std": 0.040714, + "nauc_mrr_at_1_diff1": 0.529278, + "nauc_mrr_at_3_max": 0.450178, + "nauc_mrr_at_3_std": 0.018669, + "nauc_mrr_at_3_diff1": 0.472337, + "nauc_mrr_at_5_max": 0.449004, + "nauc_mrr_at_5_std": 0.02196, + "nauc_mrr_at_5_diff1": 0.465553, + "nauc_mrr_at_10_max": 0.451432, + "nauc_mrr_at_10_std": 0.028538, + "nauc_mrr_at_10_diff1": 0.459858, + "nauc_mrr_at_20_max": 0.449551, + "nauc_mrr_at_20_std": 0.028562, + "nauc_mrr_at_20_diff1": 0.460209, + "nauc_mrr_at_100_max": 0.448966, + "nauc_mrr_at_100_std": 0.028936, + "nauc_mrr_at_100_diff1": 0.459775, + "nauc_mrr_at_1000_max": 0.449198, + "nauc_mrr_at_1000_std": 0.028679, + "nauc_mrr_at_1000_diff1": 0.460306, + "main_score": 0.34726, + "hf_subset": "c++", + "languages": [ + "c++-Code" + ] + }, + { + "ndcg_at_1": 0.273, + "ndcg_at_3": 0.33595, + "ndcg_at_5": 0.35114, + "ndcg_at_10": 0.37149, + "ndcg_at_20": 0.38618, + "ndcg_at_100": 0.42021, + "ndcg_at_1000": 0.45377, + "map_at_1": 0.273, + "map_at_3": 0.321, + "map_at_5": 0.3294, + "map_at_10": 0.33779, + "map_at_20": 0.34197, + "map_at_100": 0.34629, + "map_at_1000": 0.34735, + "recall_at_1": 0.273, + "recall_at_3": 0.379, + "recall_at_5": 0.416, + "recall_at_10": 0.479, + "recall_at_20": 0.536, + "recall_at_100": 0.725, + "recall_at_1000": 1.0, + "precision_at_1": 0.273, + "precision_at_3": 0.12633, + "precision_at_5": 0.0832, + "precision_at_10": 0.0479, + "precision_at_20": 0.0268, + "precision_at_100": 0.00725, + "precision_at_1000": 0.001, + "mrr_at_1": 0.274, + "mrr_at_3": 0.3215, + "mrr_at_5": 0.3299, + "mrr_at_10": 0.338292, + "mrr_at_20": 0.342466, + "mrr_at_100": 0.346787, + "mrr_at_1000": 0.347852, + "nauc_ndcg_at_1_max": 0.540252, + "nauc_ndcg_at_1_std": -0.054167, + "nauc_ndcg_at_1_diff1": 0.60256, + "nauc_ndcg_at_3_max": 0.50739, + "nauc_ndcg_at_3_std": -0.065121, + "nauc_ndcg_at_3_diff1": 0.533949, + "nauc_ndcg_at_5_max": 0.505453, + "nauc_ndcg_at_5_std": -0.057671, + "nauc_ndcg_at_5_diff1": 0.521045, + "nauc_ndcg_at_10_max": 0.498332, + "nauc_ndcg_at_10_std": -0.054565, + "nauc_ndcg_at_10_diff1": 0.509207, + "nauc_ndcg_at_20_max": 0.496978, + "nauc_ndcg_at_20_std": -0.042932, + "nauc_ndcg_at_20_diff1": 0.505329, + "nauc_ndcg_at_100_max": 0.492867, + "nauc_ndcg_at_100_std": -0.041028, + "nauc_ndcg_at_100_diff1": 0.507118, + "nauc_ndcg_at_1000_max": 0.502093, + "nauc_ndcg_at_1000_std": -0.046889, + "nauc_ndcg_at_1000_diff1": 0.519536, + "nauc_map_at_1_max": 0.540252, + "nauc_map_at_1_std": -0.054167, + "nauc_map_at_1_diff1": 0.60256, + "nauc_map_at_3_max": 0.515276, + "nauc_map_at_3_std": -0.063615, + "nauc_map_at_3_diff1": 0.55016, + "nauc_map_at_5_max": 0.514325, + "nauc_map_at_5_std": -0.058928, + "nauc_map_at_5_diff1": 0.542897, + "nauc_map_at_10_max": 0.511891, + "nauc_map_at_10_std": -0.057044, + "nauc_map_at_10_diff1": 0.538486, + "nauc_map_at_20_max": 0.511434, + "nauc_map_at_20_std": -0.053718, + "nauc_map_at_20_diff1": 0.537334, + "nauc_map_at_100_max": 0.510934, + "nauc_map_at_100_std": -0.053436, + "nauc_map_at_100_diff1": 0.537491, + "nauc_map_at_1000_max": 0.51127, + "nauc_map_at_1000_std": -0.053486, + "nauc_map_at_1000_diff1": 0.53788, + "nauc_recall_at_1_max": 0.540252, + "nauc_recall_at_1_std": -0.054167, + "nauc_recall_at_1_diff1": 0.60256, + "nauc_recall_at_3_max": 0.485489, + "nauc_recall_at_3_std": -0.068877, + "nauc_recall_at_3_diff1": 0.489028, + "nauc_recall_at_5_max": 0.480457, + "nauc_recall_at_5_std": -0.05337, + "nauc_recall_at_5_diff1": 0.45892, + "nauc_recall_at_10_max": 0.455704, + "nauc_recall_at_10_std": -0.046909, + "nauc_recall_at_10_diff1": 0.419318, + "nauc_recall_at_20_max": 0.449363, + "nauc_recall_at_20_std": -0.000949, + "nauc_recall_at_20_diff1": 0.401079, + "nauc_recall_at_100_max": 0.401929, + "nauc_recall_at_100_std": 0.028107, + "nauc_recall_at_100_diff1": 0.378465, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.540252, + "nauc_precision_at_1_std": -0.054167, + "nauc_precision_at_1_diff1": 0.60256, + "nauc_precision_at_3_max": 0.485489, + "nauc_precision_at_3_std": -0.068877, + "nauc_precision_at_3_diff1": 0.489028, + "nauc_precision_at_5_max": 0.480457, + "nauc_precision_at_5_std": -0.05337, + "nauc_precision_at_5_diff1": 0.45892, + "nauc_precision_at_10_max": 0.455704, + "nauc_precision_at_10_std": -0.046909, + "nauc_precision_at_10_diff1": 0.419318, + "nauc_precision_at_20_max": 0.449363, + "nauc_precision_at_20_std": -0.000949, + "nauc_precision_at_20_diff1": 0.401079, + "nauc_precision_at_100_max": 0.401929, + "nauc_precision_at_100_std": 0.028107, + "nauc_precision_at_100_diff1": 0.378465, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.537657, + "nauc_mrr_at_1_std": -0.050594, + "nauc_mrr_at_1_diff1": 0.598512, + "nauc_mrr_at_3_max": 0.514049, + "nauc_mrr_at_3_std": -0.061889, + "nauc_mrr_at_3_diff1": 0.548256, + "nauc_mrr_at_5_max": 0.513101, + "nauc_mrr_at_5_std": -0.057209, + "nauc_mrr_at_5_diff1": 0.541002, + "nauc_mrr_at_10_max": 0.510665, + "nauc_mrr_at_10_std": -0.055322, + "nauc_mrr_at_10_diff1": 0.536589, + "nauc_mrr_at_20_max": 0.510205, + "nauc_mrr_at_20_std": -0.051993, + "nauc_mrr_at_20_diff1": 0.535433, + "nauc_mrr_at_100_max": 0.509697, + "nauc_mrr_at_100_std": -0.051701, + "nauc_mrr_at_100_diff1": 0.535578, + "nauc_mrr_at_1000_max": 0.51003, + "nauc_mrr_at_1000_std": -0.051746, + "nauc_mrr_at_1000_diff1": 0.535962, + "main_score": 0.37149, + "hf_subset": "rust", + "languages": [ + "rust-Code" + ] + }, + { + "ndcg_at_1": 0.28, + "ndcg_at_3": 0.35339, + "ndcg_at_5": 0.37625, + "ndcg_at_10": 0.39581, + "ndcg_at_20": 0.41179, + "ndcg_at_100": 0.44661, + "ndcg_at_1000": 0.47425, + "map_at_1": 0.28, + "map_at_3": 0.33583, + "map_at_5": 0.34863, + "map_at_10": 0.35661, + "map_at_20": 0.36102, + "map_at_100": 0.36573, + "map_at_1000": 0.36659, + "recall_at_1": 0.28, + "recall_at_3": 0.404, + "recall_at_5": 0.459, + "recall_at_10": 0.52, + "recall_at_20": 0.583, + "recall_at_100": 0.772, + "recall_at_1000": 1.0, + "precision_at_1": 0.28, + "precision_at_3": 0.13467, + "precision_at_5": 0.0918, + "precision_at_10": 0.052, + "precision_at_20": 0.02915, + "precision_at_100": 0.00772, + "precision_at_1000": 0.001, + "mrr_at_1": 0.28, + "mrr_at_3": 0.335833, + "mrr_at_5": 0.348633, + "mrr_at_10": 0.356614, + "mrr_at_20": 0.361023, + "mrr_at_100": 0.36573, + "mrr_at_1000": 0.366592, + "nauc_ndcg_at_1_max": 0.451598, + "nauc_ndcg_at_1_std": -0.035108, + "nauc_ndcg_at_1_diff1": 0.559568, + "nauc_ndcg_at_3_max": 0.442794, + "nauc_ndcg_at_3_std": -0.021435, + "nauc_ndcg_at_3_diff1": 0.467057, + "nauc_ndcg_at_5_max": 0.447429, + "nauc_ndcg_at_5_std": -0.00919, + "nauc_ndcg_at_5_diff1": 0.462687, + "nauc_ndcg_at_10_max": 0.444868, + "nauc_ndcg_at_10_std": -0.006602, + "nauc_ndcg_at_10_diff1": 0.458494, + "nauc_ndcg_at_20_max": 0.441828, + "nauc_ndcg_at_20_std": 0.00541, + "nauc_ndcg_at_20_diff1": 0.457592, + "nauc_ndcg_at_100_max": 0.44662, + "nauc_ndcg_at_100_std": 0.003575, + "nauc_ndcg_at_100_diff1": 0.463733, + "nauc_ndcg_at_1000_max": 0.446457, + "nauc_ndcg_at_1000_std": -0.004699, + "nauc_ndcg_at_1000_diff1": 0.470728, + "nauc_map_at_1_max": 0.451598, + "nauc_map_at_1_std": -0.035108, + "nauc_map_at_1_diff1": 0.559568, + "nauc_map_at_3_max": 0.445972, + "nauc_map_at_3_std": -0.025863, + "nauc_map_at_3_diff1": 0.488951, + "nauc_map_at_5_max": 0.448869, + "nauc_map_at_5_std": -0.018906, + "nauc_map_at_5_diff1": 0.486829, + "nauc_map_at_10_max": 0.447741, + "nauc_map_at_10_std": -0.017738, + "nauc_map_at_10_diff1": 0.485139, + "nauc_map_at_20_max": 0.447233, + "nauc_map_at_20_std": -0.014564, + "nauc_map_at_20_diff1": 0.485301, + "nauc_map_at_100_max": 0.448161, + "nauc_map_at_100_std": -0.015053, + "nauc_map_at_100_diff1": 0.486088, + "nauc_map_at_1000_max": 0.448087, + "nauc_map_at_1000_std": -0.015203, + "nauc_map_at_1000_diff1": 0.486238, + "nauc_recall_at_1_max": 0.451598, + "nauc_recall_at_1_std": -0.035108, + "nauc_recall_at_1_diff1": 0.559568, + "nauc_recall_at_3_max": 0.433551, + "nauc_recall_at_3_std": -0.008682, + "nauc_recall_at_3_diff1": 0.40629, + "nauc_recall_at_5_max": 0.443228, + "nauc_recall_at_5_std": 0.020173, + "nauc_recall_at_5_diff1": 0.393844, + "nauc_recall_at_10_max": 0.43552, + "nauc_recall_at_10_std": 0.02848, + "nauc_recall_at_10_diff1": 0.378686, + "nauc_recall_at_20_max": 0.419689, + "nauc_recall_at_20_std": 0.081357, + "nauc_recall_at_20_diff1": 0.367002, + "nauc_recall_at_100_max": 0.440759, + "nauc_recall_at_100_std": 0.110402, + "nauc_recall_at_100_diff1": 0.372984, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.451598, + "nauc_precision_at_1_std": -0.035108, + "nauc_precision_at_1_diff1": 0.559568, + "nauc_precision_at_3_max": 0.433551, + "nauc_precision_at_3_std": -0.008682, + "nauc_precision_at_3_diff1": 0.40629, + "nauc_precision_at_5_max": 0.443228, + "nauc_precision_at_5_std": 0.020173, + "nauc_precision_at_5_diff1": 0.393844, + "nauc_precision_at_10_max": 0.43552, + "nauc_precision_at_10_std": 0.02848, + "nauc_precision_at_10_diff1": 0.378686, + "nauc_precision_at_20_max": 0.419689, + "nauc_precision_at_20_std": 0.081357, + "nauc_precision_at_20_diff1": 0.367002, + "nauc_precision_at_100_max": 0.440759, + "nauc_precision_at_100_std": 0.110402, + "nauc_precision_at_100_diff1": 0.372984, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.451598, + "nauc_mrr_at_1_std": -0.035108, + "nauc_mrr_at_1_diff1": 0.559568, + "nauc_mrr_at_3_max": 0.445972, + "nauc_mrr_at_3_std": -0.025863, + "nauc_mrr_at_3_diff1": 0.488951, + "nauc_mrr_at_5_max": 0.448869, + "nauc_mrr_at_5_std": -0.018906, + "nauc_mrr_at_5_diff1": 0.486829, + "nauc_mrr_at_10_max": 0.447741, + "nauc_mrr_at_10_std": -0.017738, + "nauc_mrr_at_10_diff1": 0.485139, + "nauc_mrr_at_20_max": 0.447233, + "nauc_mrr_at_20_std": -0.014564, + "nauc_mrr_at_20_diff1": 0.485301, + "nauc_mrr_at_100_max": 0.448161, + "nauc_mrr_at_100_std": -0.015053, + "nauc_mrr_at_100_diff1": 0.486088, + "nauc_mrr_at_1000_max": 0.448087, + "nauc_mrr_at_1000_std": -0.015203, + "nauc_mrr_at_1000_diff1": 0.486238, + "main_score": 0.39581, + "hf_subset": "swift", + "languages": [ + "swift-Code" + ] + }, + { + "ndcg_at_1": 0.184, + "ndcg_at_3": 0.21721, + "ndcg_at_5": 0.23352, + "ndcg_at_10": 0.24941, + "ndcg_at_20": 0.26702, + "ndcg_at_100": 0.31427, + "ndcg_at_1000": 0.35751, + "map_at_1": 0.184, + "map_at_3": 0.209, + "map_at_5": 0.21795, + "map_at_10": 0.22453, + "map_at_20": 0.22932, + "map_at_100": 0.23584, + "map_at_1000": 0.23739, + "recall_at_1": 0.184, + "recall_at_3": 0.241, + "recall_at_5": 0.281, + "recall_at_10": 0.33, + "recall_at_20": 0.4, + "recall_at_100": 0.655, + "recall_at_1000": 1.0, + "precision_at_1": 0.184, + "precision_at_3": 0.08033, + "precision_at_5": 0.0562, + "precision_at_10": 0.033, + "precision_at_20": 0.02, + "precision_at_100": 0.00655, + "precision_at_1000": 0.001, + "mrr_at_1": 0.184, + "mrr_at_3": 0.209, + "mrr_at_5": 0.21795, + "mrr_at_10": 0.224531, + "mrr_at_20": 0.229323, + "mrr_at_100": 0.235842, + "mrr_at_1000": 0.237393, + "nauc_ndcg_at_1_max": 0.037658, + "nauc_ndcg_at_1_std": -0.36494, + "nauc_ndcg_at_1_diff1": 0.628813, + "nauc_ndcg_at_3_max": -0.008506, + "nauc_ndcg_at_3_std": -0.393418, + "nauc_ndcg_at_3_diff1": 0.571384, + "nauc_ndcg_at_5_max": -0.026258, + "nauc_ndcg_at_5_std": -0.404046, + "nauc_ndcg_at_5_diff1": 0.554396, + "nauc_ndcg_at_10_max": -0.025053, + "nauc_ndcg_at_10_std": -0.398233, + "nauc_ndcg_at_10_diff1": 0.544172, + "nauc_ndcg_at_20_max": -0.026246, + "nauc_ndcg_at_20_std": -0.393267, + "nauc_ndcg_at_20_diff1": 0.531348, + "nauc_ndcg_at_100_max": -0.018779, + "nauc_ndcg_at_100_std": -0.376712, + "nauc_ndcg_at_100_diff1": 0.510772, + "nauc_ndcg_at_1000_max": -0.010515, + "nauc_ndcg_at_1000_std": -0.379871, + "nauc_ndcg_at_1000_diff1": 0.538287, + "nauc_map_at_1_max": 0.037658, + "nauc_map_at_1_std": -0.36494, + "nauc_map_at_1_diff1": 0.628813, + "nauc_map_at_3_max": 0.00125, + "nauc_map_at_3_std": -0.387338, + "nauc_map_at_3_diff1": 0.583923, + "nauc_map_at_5_max": -0.008936, + "nauc_map_at_5_std": -0.393548, + "nauc_map_at_5_diff1": 0.574218, + "nauc_map_at_10_max": -0.008598, + "nauc_map_at_10_std": -0.390896, + "nauc_map_at_10_diff1": 0.569399, + "nauc_map_at_20_max": -0.009195, + "nauc_map_at_20_std": -0.389822, + "nauc_map_at_20_diff1": 0.566026, + "nauc_map_at_100_max": -0.008382, + "nauc_map_at_100_std": -0.387547, + "nauc_map_at_100_diff1": 0.563026, + "nauc_map_at_1000_max": -0.007334, + "nauc_map_at_1000_std": -0.386888, + "nauc_map_at_1000_diff1": 0.563896, + "nauc_recall_at_1_max": 0.037658, + "nauc_recall_at_1_std": -0.36494, + "nauc_recall_at_1_diff1": 0.628813, + "nauc_recall_at_3_max": -0.03436, + "nauc_recall_at_3_std": -0.409556, + "nauc_recall_at_3_diff1": 0.537989, + "nauc_recall_at_5_max": -0.072432, + "nauc_recall_at_5_std": -0.431912, + "nauc_recall_at_5_diff1": 0.501781, + "nauc_recall_at_10_max": -0.066937, + "nauc_recall_at_10_std": -0.415795, + "nauc_recall_at_10_diff1": 0.477367, + "nauc_recall_at_20_max": -0.068381, + "nauc_recall_at_20_std": -0.396177, + "nauc_recall_at_20_diff1": 0.433321, + "nauc_recall_at_100_max": -0.032422, + "nauc_recall_at_100_std": -0.301452, + "nauc_recall_at_100_diff1": 0.292921, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.037658, + "nauc_precision_at_1_std": -0.36494, + "nauc_precision_at_1_diff1": 0.628813, + "nauc_precision_at_3_max": -0.03436, + "nauc_precision_at_3_std": -0.409556, + "nauc_precision_at_3_diff1": 0.537989, + "nauc_precision_at_5_max": -0.072432, + "nauc_precision_at_5_std": -0.431912, + "nauc_precision_at_5_diff1": 0.501781, + "nauc_precision_at_10_max": -0.066937, + "nauc_precision_at_10_std": -0.415795, + "nauc_precision_at_10_diff1": 0.477367, + "nauc_precision_at_20_max": -0.068381, + "nauc_precision_at_20_std": -0.396177, + "nauc_precision_at_20_diff1": 0.433321, + "nauc_precision_at_100_max": -0.032422, + "nauc_precision_at_100_std": -0.301452, + "nauc_precision_at_100_diff1": 0.292921, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.037658, + "nauc_mrr_at_1_std": -0.36494, + "nauc_mrr_at_1_diff1": 0.628813, + "nauc_mrr_at_3_max": 0.00125, + "nauc_mrr_at_3_std": -0.387338, + "nauc_mrr_at_3_diff1": 0.583923, + "nauc_mrr_at_5_max": -0.008936, + "nauc_mrr_at_5_std": -0.393548, + "nauc_mrr_at_5_diff1": 0.574218, + "nauc_mrr_at_10_max": -0.008598, + "nauc_mrr_at_10_std": -0.390896, + "nauc_mrr_at_10_diff1": 0.569399, + "nauc_mrr_at_20_max": -0.009195, + "nauc_mrr_at_20_std": -0.389822, + "nauc_mrr_at_20_diff1": 0.566026, + "nauc_mrr_at_100_max": -0.008382, + "nauc_mrr_at_100_std": -0.387547, + "nauc_mrr_at_100_diff1": 0.563026, + "nauc_mrr_at_1000_max": -0.007334, + "nauc_mrr_at_1000_std": -0.386888, + "nauc_mrr_at_1000_diff1": 0.563896, + "main_score": 0.24941, + "hf_subset": "scala", + "languages": [ + "scala-Code" + ] + }, + { + "ndcg_at_1": 0.177, + "ndcg_at_3": 0.2418, + "ndcg_at_5": 0.26116, + "ndcg_at_10": 0.2832, + "ndcg_at_20": 0.30647, + "ndcg_at_100": 0.342, + "ndcg_at_1000": 0.37982, + "map_at_1": 0.177, + "map_at_3": 0.22583, + "map_at_5": 0.23658, + "map_at_10": 0.2457, + "map_at_20": 0.25209, + "map_at_100": 0.25664, + "map_at_1000": 0.25786, + "recall_at_1": 0.177, + "recall_at_3": 0.288, + "recall_at_5": 0.335, + "recall_at_10": 0.403, + "recall_at_20": 0.495, + "recall_at_100": 0.692, + "recall_at_1000": 1.0, + "precision_at_1": 0.177, + "precision_at_3": 0.096, + "precision_at_5": 0.067, + "precision_at_10": 0.0403, + "precision_at_20": 0.02475, + "precision_at_100": 0.00692, + "precision_at_1000": 0.001, + "mrr_at_1": 0.177, + "mrr_at_3": 0.225833, + "mrr_at_5": 0.236583, + "mrr_at_10": 0.245699, + "mrr_at_20": 0.252086, + "mrr_at_100": 0.256635, + "mrr_at_1000": 0.257864, + "nauc_ndcg_at_1_max": 0.177417, + "nauc_ndcg_at_1_std": -0.052005, + "nauc_ndcg_at_1_diff1": 0.364215, + "nauc_ndcg_at_3_max": 0.15817, + "nauc_ndcg_at_3_std": -0.033733, + "nauc_ndcg_at_3_diff1": 0.259118, + "nauc_ndcg_at_5_max": 0.16372, + "nauc_ndcg_at_5_std": -0.026832, + "nauc_ndcg_at_5_diff1": 0.255703, + "nauc_ndcg_at_10_max": 0.160554, + "nauc_ndcg_at_10_std": -0.021755, + "nauc_ndcg_at_10_diff1": 0.238498, + "nauc_ndcg_at_20_max": 0.157112, + "nauc_ndcg_at_20_std": -0.011167, + "nauc_ndcg_at_20_diff1": 0.224544, + "nauc_ndcg_at_100_max": 0.163887, + "nauc_ndcg_at_100_std": 0.000369, + "nauc_ndcg_at_100_diff1": 0.215483, + "nauc_ndcg_at_1000_max": 0.16292, + "nauc_ndcg_at_1000_std": -0.014944, + "nauc_ndcg_at_1000_diff1": 0.240979, + "nauc_map_at_1_max": 0.177417, + "nauc_map_at_1_std": -0.052005, + "nauc_map_at_1_diff1": 0.364215, + "nauc_map_at_3_max": 0.161771, + "nauc_map_at_3_std": -0.038392, + "nauc_map_at_3_diff1": 0.281373, + "nauc_map_at_5_max": 0.164791, + "nauc_map_at_5_std": -0.034645, + "nauc_map_at_5_diff1": 0.278974, + "nauc_map_at_10_max": 0.163494, + "nauc_map_at_10_std": -0.032206, + "nauc_map_at_10_diff1": 0.271475, + "nauc_map_at_20_max": 0.162541, + "nauc_map_at_20_std": -0.029172, + "nauc_map_at_20_diff1": 0.267569, + "nauc_map_at_100_max": 0.163294, + "nauc_map_at_100_std": -0.027781, + "nauc_map_at_100_diff1": 0.266749, + "nauc_map_at_1000_max": 0.163369, + "nauc_map_at_1000_std": -0.028228, + "nauc_map_at_1000_diff1": 0.267609, + "nauc_recall_at_1_max": 0.177417, + "nauc_recall_at_1_std": -0.052005, + "nauc_recall_at_1_diff1": 0.364215, + "nauc_recall_at_3_max": 0.149243, + "nauc_recall_at_3_std": -0.021674, + "nauc_recall_at_3_diff1": 0.202876, + "nauc_recall_at_5_max": 0.162129, + "nauc_recall_at_5_std": -0.00614, + "nauc_recall_at_5_diff1": 0.198234, + "nauc_recall_at_10_max": 0.153249, + "nauc_recall_at_10_std": 0.006018, + "nauc_recall_at_10_diff1": 0.153209, + "nauc_recall_at_20_max": 0.140621, + "nauc_recall_at_20_std": 0.044015, + "nauc_recall_at_20_diff1": 0.101154, + "nauc_recall_at_100_max": 0.180079, + "nauc_recall_at_100_std": 0.133857, + "nauc_recall_at_100_diff1": 0.001851, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.177417, + "nauc_precision_at_1_std": -0.052005, + "nauc_precision_at_1_diff1": 0.364215, + "nauc_precision_at_3_max": 0.149243, + "nauc_precision_at_3_std": -0.021674, + "nauc_precision_at_3_diff1": 0.202876, + "nauc_precision_at_5_max": 0.162129, + "nauc_precision_at_5_std": -0.00614, + "nauc_precision_at_5_diff1": 0.198234, + "nauc_precision_at_10_max": 0.153249, + "nauc_precision_at_10_std": 0.006018, + "nauc_precision_at_10_diff1": 0.153209, + "nauc_precision_at_20_max": 0.140621, + "nauc_precision_at_20_std": 0.044015, + "nauc_precision_at_20_diff1": 0.101154, + "nauc_precision_at_100_max": 0.180079, + "nauc_precision_at_100_std": 0.133857, + "nauc_precision_at_100_diff1": 0.001851, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.177417, + "nauc_mrr_at_1_std": -0.052005, + "nauc_mrr_at_1_diff1": 0.364215, + "nauc_mrr_at_3_max": 0.161771, + "nauc_mrr_at_3_std": -0.038392, + "nauc_mrr_at_3_diff1": 0.281373, + "nauc_mrr_at_5_max": 0.164791, + "nauc_mrr_at_5_std": -0.034645, + "nauc_mrr_at_5_diff1": 0.278974, + "nauc_mrr_at_10_max": 0.163494, + "nauc_mrr_at_10_std": -0.032206, + "nauc_mrr_at_10_diff1": 0.271475, + "nauc_mrr_at_20_max": 0.162541, + "nauc_mrr_at_20_std": -0.029172, + "nauc_mrr_at_20_diff1": 0.267569, + "nauc_mrr_at_100_max": 0.163294, + "nauc_mrr_at_100_std": -0.027781, + "nauc_mrr_at_100_diff1": 0.266749, + "nauc_mrr_at_1000_max": 0.163369, + "nauc_mrr_at_1000_std": -0.028228, + "nauc_mrr_at_1000_diff1": 0.267609, + "main_score": 0.2832, + "hf_subset": "shell", + "languages": [ + "shell-Code" + ] + } + ] + }, + "evaluation_time": 109.69983267784119, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeFeedbackMT.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeFeedbackMT.json new file mode 100644 index 0000000000..f716b3baf1 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeFeedbackMT.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b0f12fa0c0dd67f59c95a5c33d02aeeb4c398c5f", + "task_name": "CodeFeedbackMT", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.34405, + "ndcg_at_3": 0.40427, + "ndcg_at_5": 0.42182, + "ndcg_at_10": 0.44014, + "ndcg_at_20": 0.45321, + "ndcg_at_100": 0.47414, + "ndcg_at_1000": 0.49282, + "map_at_1": 0.34405, + "map_at_3": 0.38958, + "map_at_5": 0.39931, + "map_at_10": 0.40688, + "map_at_20": 0.41047, + "map_at_100": 0.41327, + "map_at_1000": 0.41389, + "recall_at_1": 0.34405, + "recall_at_3": 0.44671, + "recall_at_5": 0.48934, + "recall_at_10": 0.54598, + "recall_at_20": 0.59765, + "recall_at_100": 0.71191, + "recall_at_1000": 0.86367, + "precision_at_1": 0.34405, + "precision_at_3": 0.1489, + "precision_at_5": 0.09787, + "precision_at_10": 0.0546, + "precision_at_20": 0.02988, + "precision_at_100": 0.00712, + "precision_at_1000": 0.00086, + "mrr_at_1": 0.344054, + "mrr_at_3": 0.389583, + "mrr_at_5": 0.399315, + "mrr_at_10": 0.406878, + "mrr_at_20": 0.410471, + "mrr_at_100": 0.413267, + "mrr_at_1000": 0.413888, + "nauc_ndcg_at_1_max": 0.301928, + "nauc_ndcg_at_1_std": -0.144984, + "nauc_ndcg_at_1_diff1": 0.680006, + "nauc_ndcg_at_3_max": 0.299374, + "nauc_ndcg_at_3_std": -0.140953, + "nauc_ndcg_at_3_diff1": 0.622338, + "nauc_ndcg_at_5_max": 0.297675, + "nauc_ndcg_at_5_std": -0.13568, + "nauc_ndcg_at_5_diff1": 0.609409, + "nauc_ndcg_at_10_max": 0.292337, + "nauc_ndcg_at_10_std": -0.135441, + "nauc_ndcg_at_10_diff1": 0.600893, + "nauc_ndcg_at_20_max": 0.29099, + "nauc_ndcg_at_20_std": -0.130397, + "nauc_ndcg_at_20_diff1": 0.59495, + "nauc_ndcg_at_100_max": 0.289759, + "nauc_ndcg_at_100_std": -0.122167, + "nauc_ndcg_at_100_diff1": 0.591474, + "nauc_ndcg_at_1000_max": 0.292494, + "nauc_ndcg_at_1000_std": -0.11858, + "nauc_ndcg_at_1000_diff1": 0.594321, + "nauc_map_at_1_max": 0.301928, + "nauc_map_at_1_std": -0.144984, + "nauc_map_at_1_diff1": 0.680006, + "nauc_map_at_3_max": 0.300359, + "nauc_map_at_3_std": -0.141943, + "nauc_map_at_3_diff1": 0.635878, + "nauc_map_at_5_max": 0.299433, + "nauc_map_at_5_std": -0.139106, + "nauc_map_at_5_diff1": 0.628789, + "nauc_map_at_10_max": 0.29729, + "nauc_map_at_10_std": -0.138976, + "nauc_map_at_10_diff1": 0.625331, + "nauc_map_at_20_max": 0.296914, + "nauc_map_at_20_std": -0.137688, + "nauc_map_at_20_diff1": 0.623859, + "nauc_map_at_100_max": 0.296805, + "nauc_map_at_100_std": -0.136704, + "nauc_map_at_100_diff1": 0.623574, + "nauc_map_at_1000_max": 0.296908, + "nauc_map_at_1000_std": -0.136565, + "nauc_map_at_1000_diff1": 0.623677, + "nauc_recall_at_1_max": 0.301928, + "nauc_recall_at_1_std": -0.144984, + "nauc_recall_at_1_diff1": 0.680006, + "nauc_recall_at_3_max": 0.296388, + "nauc_recall_at_3_std": -0.138094, + "nauc_recall_at_3_diff1": 0.583508, + "nauc_recall_at_5_max": 0.292141, + "nauc_recall_at_5_std": -0.12483, + "nauc_recall_at_5_diff1": 0.551221, + "nauc_recall_at_10_max": 0.274501, + "nauc_recall_at_10_std": -0.123786, + "nauc_recall_at_10_diff1": 0.521277, + "nauc_recall_at_20_max": 0.267922, + "nauc_recall_at_20_std": -0.100901, + "nauc_recall_at_20_diff1": 0.490695, + "nauc_recall_at_100_max": 0.252954, + "nauc_recall_at_100_std": -0.0331, + "nauc_recall_at_100_diff1": 0.438119, + "nauc_recall_at_1000_max": 0.271255, + "nauc_recall_at_1000_std": 0.108852, + "nauc_recall_at_1000_diff1": 0.359331, + "nauc_precision_at_1_max": 0.301928, + "nauc_precision_at_1_std": -0.144984, + "nauc_precision_at_1_diff1": 0.680006, + "nauc_precision_at_3_max": 0.296388, + "nauc_precision_at_3_std": -0.138094, + "nauc_precision_at_3_diff1": 0.583508, + "nauc_precision_at_5_max": 0.292141, + "nauc_precision_at_5_std": -0.12483, + "nauc_precision_at_5_diff1": 0.551221, + "nauc_precision_at_10_max": 0.274501, + "nauc_precision_at_10_std": -0.123786, + "nauc_precision_at_10_diff1": 0.521277, + "nauc_precision_at_20_max": 0.267922, + "nauc_precision_at_20_std": -0.100901, + "nauc_precision_at_20_diff1": 0.490695, + "nauc_precision_at_100_max": 0.252954, + "nauc_precision_at_100_std": -0.0331, + "nauc_precision_at_100_diff1": 0.438119, + "nauc_precision_at_1000_max": 0.271255, + "nauc_precision_at_1000_std": 0.108852, + "nauc_precision_at_1000_diff1": 0.359331, + "nauc_mrr_at_1_max": 0.301928, + "nauc_mrr_at_1_std": -0.144984, + "nauc_mrr_at_1_diff1": 0.680006, + "nauc_mrr_at_3_max": 0.300359, + "nauc_mrr_at_3_std": -0.141943, + "nauc_mrr_at_3_diff1": 0.635878, + "nauc_mrr_at_5_max": 0.299433, + "nauc_mrr_at_5_std": -0.139106, + "nauc_mrr_at_5_diff1": 0.628789, + "nauc_mrr_at_10_max": 0.29729, + "nauc_mrr_at_10_std": -0.138976, + "nauc_mrr_at_10_diff1": 0.625331, + "nauc_mrr_at_20_max": 0.296914, + "nauc_mrr_at_20_std": -0.137688, + "nauc_mrr_at_20_diff1": 0.623859, + "nauc_mrr_at_100_max": 0.296805, + "nauc_mrr_at_100_std": -0.136704, + "nauc_mrr_at_100_diff1": 0.623574, + "nauc_mrr_at_1000_max": 0.296908, + "nauc_mrr_at_1000_std": -0.136565, + "nauc_mrr_at_1000_diff1": 0.623677, + "main_score": 0.44014, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1007.3014879226685, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeFeedbackST.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeFeedbackST.json new file mode 100644 index 0000000000..f4fd48ecb5 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeFeedbackST.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "d213819e87aab9010628da8b73ab4eb337c89340", + "task_name": "CodeFeedbackST", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.43056, + "ndcg_at_3": 0.52834, + "ndcg_at_5": 0.55545, + "ndcg_at_10": 0.57786, + "ndcg_at_20": 0.59306, + "ndcg_at_100": 0.61287, + "ndcg_at_1000": 0.62274, + "map_at_1": 0.43056, + "map_at_3": 0.50431, + "map_at_5": 0.51936, + "map_at_10": 0.52874, + "map_at_20": 0.53294, + "map_at_100": 0.5357, + "map_at_1000": 0.53606, + "recall_at_1": 0.43056, + "recall_at_3": 0.59787, + "recall_at_5": 0.66364, + "recall_at_10": 0.73222, + "recall_at_20": 0.79218, + "recall_at_100": 0.89874, + "recall_at_1000": 0.97655, + "precision_at_1": 0.43056, + "precision_at_3": 0.19929, + "precision_at_5": 0.13273, + "precision_at_10": 0.07322, + "precision_at_20": 0.03961, + "precision_at_100": 0.00899, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.429662, + "mrr_at_3": 0.503913, + "mrr_at_5": 0.518937, + "mrr_at_10": 0.528312, + "mrr_at_20": 0.532514, + "mrr_at_100": 0.535267, + "mrr_at_1000": 0.535634, + "nauc_ndcg_at_1_max": 0.266947, + "nauc_ndcg_at_1_std": -0.202067, + "nauc_ndcg_at_1_diff1": 0.691954, + "nauc_ndcg_at_3_max": 0.273061, + "nauc_ndcg_at_3_std": -0.208904, + "nauc_ndcg_at_3_diff1": 0.62183, + "nauc_ndcg_at_5_max": 0.276679, + "nauc_ndcg_at_5_std": -0.203555, + "nauc_ndcg_at_5_diff1": 0.611183, + "nauc_ndcg_at_10_max": 0.276048, + "nauc_ndcg_at_10_std": -0.198523, + "nauc_ndcg_at_10_diff1": 0.607176, + "nauc_ndcg_at_20_max": 0.274979, + "nauc_ndcg_at_20_std": -0.195474, + "nauc_ndcg_at_20_diff1": 0.608151, + "nauc_ndcg_at_100_max": 0.275378, + "nauc_ndcg_at_100_std": -0.187391, + "nauc_ndcg_at_100_diff1": 0.612647, + "nauc_ndcg_at_1000_max": 0.276318, + "nauc_ndcg_at_1000_std": -0.189157, + "nauc_ndcg_at_1000_diff1": 0.618822, + "nauc_map_at_1_max": 0.266947, + "nauc_map_at_1_std": -0.202067, + "nauc_map_at_1_diff1": 0.691954, + "nauc_map_at_3_max": 0.271556, + "nauc_map_at_3_std": -0.207327, + "nauc_map_at_3_diff1": 0.639314, + "nauc_map_at_5_max": 0.273467, + "nauc_map_at_5_std": -0.204508, + "nauc_map_at_5_diff1": 0.634029, + "nauc_map_at_10_max": 0.273212, + "nauc_map_at_10_std": -0.202491, + "nauc_map_at_10_diff1": 0.632711, + "nauc_map_at_20_max": 0.272945, + "nauc_map_at_20_std": -0.20177, + "nauc_map_at_20_diff1": 0.633216, + "nauc_map_at_100_max": 0.272994, + "nauc_map_at_100_std": -0.200818, + "nauc_map_at_100_diff1": 0.633909, + "nauc_map_at_1000_max": 0.273049, + "nauc_map_at_1000_std": -0.200815, + "nauc_map_at_1000_diff1": 0.634115, + "nauc_recall_at_1_max": 0.266947, + "nauc_recall_at_1_std": -0.202067, + "nauc_recall_at_1_diff1": 0.691954, + "nauc_recall_at_3_max": 0.27773, + "nauc_recall_at_3_std": -0.213749, + "nauc_recall_at_3_diff1": 0.567468, + "nauc_recall_at_5_max": 0.288278, + "nauc_recall_at_5_std": -0.199172, + "nauc_recall_at_5_diff1": 0.531113, + "nauc_recall_at_10_max": 0.28753, + "nauc_recall_at_10_std": -0.178916, + "nauc_recall_at_10_diff1": 0.501632, + "nauc_recall_at_20_max": 0.283339, + "nauc_recall_at_20_std": -0.156036, + "nauc_recall_at_20_diff1": 0.483523, + "nauc_recall_at_100_max": 0.296213, + "nauc_recall_at_100_std": -0.002201, + "nauc_recall_at_100_diff1": 0.43838, + "nauc_recall_at_1000_max": 0.425492, + "nauc_recall_at_1000_std": 0.393871, + "nauc_recall_at_1000_diff1": 0.425064, + "nauc_precision_at_1_max": 0.266947, + "nauc_precision_at_1_std": -0.202067, + "nauc_precision_at_1_diff1": 0.691954, + "nauc_precision_at_3_max": 0.27773, + "nauc_precision_at_3_std": -0.213749, + "nauc_precision_at_3_diff1": 0.567468, + "nauc_precision_at_5_max": 0.288278, + "nauc_precision_at_5_std": -0.199172, + "nauc_precision_at_5_diff1": 0.531113, + "nauc_precision_at_10_max": 0.28753, + "nauc_precision_at_10_std": -0.178916, + "nauc_precision_at_10_diff1": 0.501632, + "nauc_precision_at_20_max": 0.283339, + "nauc_precision_at_20_std": -0.156036, + "nauc_precision_at_20_diff1": 0.483523, + "nauc_precision_at_100_max": 0.296213, + "nauc_precision_at_100_std": -0.002201, + "nauc_precision_at_100_diff1": 0.43838, + "nauc_precision_at_1000_max": 0.425492, + "nauc_precision_at_1000_std": 0.393871, + "nauc_precision_at_1000_diff1": 0.425064, + "nauc_mrr_at_1_max": 0.26695, + "nauc_mrr_at_1_std": -0.199238, + "nauc_mrr_at_1_diff1": 0.694234, + "nauc_mrr_at_3_max": 0.2719, + "nauc_mrr_at_3_std": -0.206052, + "nauc_mrr_at_3_diff1": 0.640131, + "nauc_mrr_at_5_max": 0.273533, + "nauc_mrr_at_5_std": -0.203174, + "nauc_mrr_at_5_diff1": 0.634938, + "nauc_mrr_at_10_max": 0.273261, + "nauc_mrr_at_10_std": -0.201203, + "nauc_mrr_at_10_diff1": 0.633766, + "nauc_mrr_at_20_max": 0.272984, + "nauc_mrr_at_20_std": -0.200484, + "nauc_mrr_at_20_diff1": 0.634276, + "nauc_mrr_at_100_max": 0.27304, + "nauc_mrr_at_100_std": -0.199511, + "nauc_mrr_at_100_diff1": 0.634974, + "nauc_mrr_at_1000_max": 0.273095, + "nauc_mrr_at_1000_std": -0.199507, + "nauc_mrr_at_1000_diff1": 0.635181, + "main_score": 0.57786, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1997.7772071361542, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeSearchNetCCRetrieval.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeSearchNetCCRetrieval.json new file mode 100644 index 0000000000..f0a7be3ad7 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeSearchNetCCRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "6e1effa2c03723c5fde48ee912b5ee08d4f211e8", + "task_name": "CodeSearchNetCCRetrieval", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.27584, + "ndcg_at_3": 0.34474, + "ndcg_at_5": 0.36402, + "ndcg_at_10": 0.38361, + "ndcg_at_20": 0.39927, + "ndcg_at_100": 0.42381, + "ndcg_at_1000": 0.44351, + "map_at_1": 0.27584, + "map_at_3": 0.328, + "map_at_5": 0.3387, + "map_at_10": 0.34676, + "map_at_20": 0.35109, + "map_at_100": 0.35442, + "map_at_1000": 0.35506, + "recall_at_1": 0.27584, + "recall_at_3": 0.39308, + "recall_at_5": 0.43994, + "recall_at_10": 0.5006, + "recall_at_20": 0.56234, + "recall_at_100": 0.6956, + "recall_at_1000": 0.85635, + "precision_at_1": 0.27584, + "precision_at_3": 0.13103, + "precision_at_5": 0.08799, + "precision_at_10": 0.05006, + "precision_at_20": 0.02812, + "precision_at_100": 0.00696, + "precision_at_1000": 0.00086, + "mrr_at_1": 0.275774, + "mrr_at_3": 0.327971, + "mrr_at_5": 0.338662, + "mrr_at_10": 0.346722, + "mrr_at_20": 0.351053, + "mrr_at_100": 0.354385, + "mrr_at_1000": 0.355028, + "nauc_ndcg_at_1_max": 0.162813, + "nauc_ndcg_at_1_std": -0.149933, + "nauc_ndcg_at_1_diff1": 0.538561, + "nauc_ndcg_at_3_max": 0.147748, + "nauc_ndcg_at_3_std": -0.158358, + "nauc_ndcg_at_3_diff1": 0.485704, + "nauc_ndcg_at_5_max": 0.142874, + "nauc_ndcg_at_5_std": -0.157834, + "nauc_ndcg_at_5_diff1": 0.479532, + "nauc_ndcg_at_10_max": 0.136636, + "nauc_ndcg_at_10_std": -0.15497, + "nauc_ndcg_at_10_diff1": 0.470066, + "nauc_ndcg_at_20_max": 0.136144, + "nauc_ndcg_at_20_std": -0.149403, + "nauc_ndcg_at_20_diff1": 0.468601, + "nauc_ndcg_at_100_max": 0.136495, + "nauc_ndcg_at_100_std": -0.140199, + "nauc_ndcg_at_100_diff1": 0.463324, + "nauc_ndcg_at_1000_max": 0.137323, + "nauc_ndcg_at_1000_std": -0.14074, + "nauc_ndcg_at_1000_diff1": 0.466776, + "nauc_map_at_1_max": 0.162813, + "nauc_map_at_1_std": -0.149933, + "nauc_map_at_1_diff1": 0.538561, + "nauc_map_at_3_max": 0.151456, + "nauc_map_at_3_std": -0.156868, + "nauc_map_at_3_diff1": 0.497916, + "nauc_map_at_5_max": 0.148786, + "nauc_map_at_5_std": -0.156571, + "nauc_map_at_5_diff1": 0.494504, + "nauc_map_at_10_max": 0.146268, + "nauc_map_at_10_std": -0.155327, + "nauc_map_at_10_diff1": 0.490668, + "nauc_map_at_20_max": 0.146167, + "nauc_map_at_20_std": -0.153817, + "nauc_map_at_20_diff1": 0.490424, + "nauc_map_at_100_max": 0.146217, + "nauc_map_at_100_std": -0.152549, + "nauc_map_at_100_diff1": 0.489699, + "nauc_map_at_1000_max": 0.14625, + "nauc_map_at_1000_std": -0.152537, + "nauc_map_at_1000_diff1": 0.48982, + "nauc_recall_at_1_max": 0.162813, + "nauc_recall_at_1_std": -0.149933, + "nauc_recall_at_1_diff1": 0.538561, + "nauc_recall_at_3_max": 0.137432, + "nauc_recall_at_3_std": -0.162288, + "nauc_recall_at_3_diff1": 0.452022, + "nauc_recall_at_5_max": 0.125595, + "nauc_recall_at_5_std": -0.161087, + "nauc_recall_at_5_diff1": 0.437013, + "nauc_recall_at_10_max": 0.105896, + "nauc_recall_at_10_std": -0.152662, + "nauc_recall_at_10_diff1": 0.406909, + "nauc_recall_at_20_max": 0.101902, + "nauc_recall_at_20_std": -0.12963, + "nauc_recall_at_20_diff1": 0.396168, + "nauc_recall_at_100_max": 0.096088, + "nauc_recall_at_100_std": -0.06358, + "nauc_recall_at_100_diff1": 0.344428, + "nauc_recall_at_1000_max": 0.076518, + "nauc_recall_at_1000_std": -0.011067, + "nauc_recall_at_1000_diff1": 0.304837, + "nauc_precision_at_1_max": 0.162813, + "nauc_precision_at_1_std": -0.149933, + "nauc_precision_at_1_diff1": 0.538561, + "nauc_precision_at_3_max": 0.137432, + "nauc_precision_at_3_std": -0.162288, + "nauc_precision_at_3_diff1": 0.452022, + "nauc_precision_at_5_max": 0.125595, + "nauc_precision_at_5_std": -0.161087, + "nauc_precision_at_5_diff1": 0.437013, + "nauc_precision_at_10_max": 0.105896, + "nauc_precision_at_10_std": -0.152662, + "nauc_precision_at_10_diff1": 0.406909, + "nauc_precision_at_20_max": 0.101902, + "nauc_precision_at_20_std": -0.12963, + "nauc_precision_at_20_diff1": 0.396168, + "nauc_precision_at_100_max": 0.096088, + "nauc_precision_at_100_std": -0.06358, + "nauc_precision_at_100_diff1": 0.344428, + "nauc_precision_at_1000_max": 0.076518, + "nauc_precision_at_1000_std": -0.011067, + "nauc_precision_at_1000_diff1": 0.304837, + "nauc_mrr_at_1_max": 0.16248, + "nauc_mrr_at_1_std": -0.149985, + "nauc_mrr_at_1_diff1": 0.538825, + "nauc_mrr_at_3_max": 0.151296, + "nauc_mrr_at_3_std": -0.156893, + "nauc_mrr_at_3_diff1": 0.498041, + "nauc_mrr_at_5_max": 0.148625, + "nauc_mrr_at_5_std": -0.156597, + "nauc_mrr_at_5_diff1": 0.494628, + "nauc_mrr_at_10_max": 0.146096, + "nauc_mrr_at_10_std": -0.155354, + "nauc_mrr_at_10_diff1": 0.49076, + "nauc_mrr_at_20_max": 0.146004, + "nauc_mrr_at_20_std": -0.153842, + "nauc_mrr_at_20_diff1": 0.490547, + "nauc_mrr_at_100_max": 0.146054, + "nauc_mrr_at_100_std": -0.152574, + "nauc_mrr_at_100_diff1": 0.489822, + "nauc_mrr_at_1000_max": 0.146087, + "nauc_mrr_at_1000_std": -0.152562, + "nauc_mrr_at_1000_diff1": 0.489944, + "main_score": 0.38361, + "hf_subset": "python", + "languages": [ + "python-Code" + ] + }, + { + "ndcg_at_1": 0.31814, + "ndcg_at_3": 0.39595, + "ndcg_at_5": 0.41696, + "ndcg_at_10": 0.43607, + "ndcg_at_20": 0.45353, + "ndcg_at_100": 0.4771, + "ndcg_at_1000": 0.49596, + "map_at_1": 0.31814, + "map_at_3": 0.37739, + "map_at_5": 0.38897, + "map_at_10": 0.39684, + "map_at_20": 0.40164, + "map_at_100": 0.4048, + "map_at_1000": 0.40541, + "recall_at_1": 0.31814, + "recall_at_3": 0.44941, + "recall_at_5": 0.50076, + "recall_at_10": 0.56001, + "recall_at_20": 0.62899, + "recall_at_100": 0.75752, + "recall_at_1000": 0.91127, + "precision_at_1": 0.31814, + "precision_at_3": 0.1498, + "precision_at_5": 0.10015, + "precision_at_10": 0.056, + "precision_at_20": 0.03145, + "precision_at_100": 0.00758, + "precision_at_1000": 0.00091, + "mrr_at_1": 0.317837, + "mrr_at_3": 0.377241, + "mrr_at_5": 0.388818, + "mrr_at_10": 0.396699, + "mrr_at_20": 0.401499, + "mrr_at_100": 0.404659, + "mrr_at_1000": 0.405274, + "nauc_ndcg_at_1_max": 0.290557, + "nauc_ndcg_at_1_std": -0.047047, + "nauc_ndcg_at_1_diff1": 0.554357, + "nauc_ndcg_at_3_max": 0.294872, + "nauc_ndcg_at_3_std": -0.05224, + "nauc_ndcg_at_3_diff1": 0.509338, + "nauc_ndcg_at_5_max": 0.288176, + "nauc_ndcg_at_5_std": -0.052221, + "nauc_ndcg_at_5_diff1": 0.49979, + "nauc_ndcg_at_10_max": 0.280751, + "nauc_ndcg_at_10_std": -0.057299, + "nauc_ndcg_at_10_diff1": 0.496874, + "nauc_ndcg_at_20_max": 0.279435, + "nauc_ndcg_at_20_std": -0.052256, + "nauc_ndcg_at_20_diff1": 0.493232, + "nauc_ndcg_at_100_max": 0.279248, + "nauc_ndcg_at_100_std": -0.045558, + "nauc_ndcg_at_100_diff1": 0.490225, + "nauc_ndcg_at_1000_max": 0.280594, + "nauc_ndcg_at_1000_std": -0.045169, + "nauc_ndcg_at_1000_diff1": 0.494366, + "nauc_map_at_1_max": 0.290557, + "nauc_map_at_1_std": -0.047047, + "nauc_map_at_1_diff1": 0.554357, + "nauc_map_at_3_max": 0.293897, + "nauc_map_at_3_std": -0.051457, + "nauc_map_at_3_diff1": 0.519352, + "nauc_map_at_5_max": 0.2904, + "nauc_map_at_5_std": -0.051311, + "nauc_map_at_5_diff1": 0.514015, + "nauc_map_at_10_max": 0.287449, + "nauc_map_at_10_std": -0.053289, + "nauc_map_at_10_diff1": 0.513016, + "nauc_map_at_20_max": 0.287043, + "nauc_map_at_20_std": -0.051981, + "nauc_map_at_20_diff1": 0.512148, + "nauc_map_at_100_max": 0.286971, + "nauc_map_at_100_std": -0.05099, + "nauc_map_at_100_diff1": 0.511721, + "nauc_map_at_1000_max": 0.286955, + "nauc_map_at_1000_std": -0.050983, + "nauc_map_at_1000_diff1": 0.511818, + "nauc_recall_at_1_max": 0.290557, + "nauc_recall_at_1_std": -0.047047, + "nauc_recall_at_1_diff1": 0.554357, + "nauc_recall_at_3_max": 0.29764, + "nauc_recall_at_3_std": -0.054294, + "nauc_recall_at_3_diff1": 0.480982, + "nauc_recall_at_5_max": 0.280447, + "nauc_recall_at_5_std": -0.054835, + "nauc_recall_at_5_diff1": 0.457525, + "nauc_recall_at_10_max": 0.25551, + "nauc_recall_at_10_std": -0.071988, + "nauc_recall_at_10_diff1": 0.444877, + "nauc_recall_at_20_max": 0.247903, + "nauc_recall_at_20_std": -0.050387, + "nauc_recall_at_20_diff1": 0.423207, + "nauc_recall_at_100_max": 0.236842, + "nauc_recall_at_100_std": 0.001209, + "nauc_recall_at_100_diff1": 0.377081, + "nauc_recall_at_1000_max": 0.224578, + "nauc_recall_at_1000_std": 0.074268, + "nauc_recall_at_1000_diff1": 0.341775, + "nauc_precision_at_1_max": 0.290557, + "nauc_precision_at_1_std": -0.047047, + "nauc_precision_at_1_diff1": 0.554357, + "nauc_precision_at_3_max": 0.29764, + "nauc_precision_at_3_std": -0.054294, + "nauc_precision_at_3_diff1": 0.480982, + "nauc_precision_at_5_max": 0.280447, + "nauc_precision_at_5_std": -0.054835, + "nauc_precision_at_5_diff1": 0.457525, + "nauc_precision_at_10_max": 0.25551, + "nauc_precision_at_10_std": -0.071988, + "nauc_precision_at_10_diff1": 0.444877, + "nauc_precision_at_20_max": 0.247903, + "nauc_precision_at_20_std": -0.050387, + "nauc_precision_at_20_diff1": 0.423207, + "nauc_precision_at_100_max": 0.236842, + "nauc_precision_at_100_std": 0.001209, + "nauc_precision_at_100_diff1": 0.377081, + "nauc_precision_at_1000_max": 0.224578, + "nauc_precision_at_1000_std": 0.074268, + "nauc_precision_at_1000_diff1": 0.341775, + "nauc_mrr_at_1_max": 0.29123, + "nauc_mrr_at_1_std": -0.046425, + "nauc_mrr_at_1_diff1": 0.555311, + "nauc_mrr_at_3_max": 0.294229, + "nauc_mrr_at_3_std": -0.05115, + "nauc_mrr_at_3_diff1": 0.51982, + "nauc_mrr_at_5_max": 0.290734, + "nauc_mrr_at_5_std": -0.051003, + "nauc_mrr_at_5_diff1": 0.514485, + "nauc_mrr_at_10_max": 0.28776, + "nauc_mrr_at_10_std": -0.052983, + "nauc_mrr_at_10_diff1": 0.513498, + "nauc_mrr_at_20_max": 0.28736, + "nauc_mrr_at_20_std": -0.051671, + "nauc_mrr_at_20_diff1": 0.512635, + "nauc_mrr_at_100_max": 0.28729, + "nauc_mrr_at_100_std": -0.050678, + "nauc_mrr_at_100_diff1": 0.51221, + "nauc_mrr_at_1000_max": 0.287275, + "nauc_mrr_at_1000_std": -0.05067, + "nauc_mrr_at_1000_diff1": 0.512308, + "main_score": 0.43607, + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ] + }, + { + "ndcg_at_1": 0.29266, + "ndcg_at_3": 0.36361, + "ndcg_at_5": 0.38473, + "ndcg_at_10": 0.4042, + "ndcg_at_20": 0.41905, + "ndcg_at_100": 0.44294, + "ndcg_at_1000": 0.46146, + "map_at_1": 0.29266, + "map_at_3": 0.34636, + "map_at_5": 0.35806, + "map_at_10": 0.3661, + "map_at_20": 0.37017, + "map_at_100": 0.37335, + "map_at_1000": 0.37395, + "recall_at_1": 0.29266, + "recall_at_3": 0.41344, + "recall_at_5": 0.46479, + "recall_at_10": 0.52499, + "recall_at_20": 0.58372, + "recall_at_100": 0.7146, + "recall_at_1000": 0.8658, + "precision_at_1": 0.29266, + "precision_at_3": 0.13781, + "precision_at_5": 0.09296, + "precision_at_10": 0.0525, + "precision_at_20": 0.02919, + "precision_at_100": 0.00715, + "precision_at_1000": 0.00087, + "mrr_at_1": 0.292539, + "mrr_at_3": 0.346261, + "mrr_at_5": 0.357989, + "mrr_at_10": 0.365999, + "mrr_at_20": 0.370067, + "mrr_at_100": 0.373242, + "mrr_at_1000": 0.373846, + "nauc_ndcg_at_1_max": 0.192045, + "nauc_ndcg_at_1_std": -0.142118, + "nauc_ndcg_at_1_diff1": 0.509008, + "nauc_ndcg_at_3_max": 0.182334, + "nauc_ndcg_at_3_std": -0.142611, + "nauc_ndcg_at_3_diff1": 0.451577, + "nauc_ndcg_at_5_max": 0.181969, + "nauc_ndcg_at_5_std": -0.138746, + "nauc_ndcg_at_5_diff1": 0.442551, + "nauc_ndcg_at_10_max": 0.178054, + "nauc_ndcg_at_10_std": -0.136652, + "nauc_ndcg_at_10_diff1": 0.435679, + "nauc_ndcg_at_20_max": 0.175477, + "nauc_ndcg_at_20_std": -0.133157, + "nauc_ndcg_at_20_diff1": 0.433141, + "nauc_ndcg_at_100_max": 0.178227, + "nauc_ndcg_at_100_std": -0.12191, + "nauc_ndcg_at_100_diff1": 0.429782, + "nauc_ndcg_at_1000_max": 0.178335, + "nauc_ndcg_at_1000_std": -0.122313, + "nauc_ndcg_at_1000_diff1": 0.433759, + "nauc_map_at_1_max": 0.192045, + "nauc_map_at_1_std": -0.142118, + "nauc_map_at_1_diff1": 0.509008, + "nauc_map_at_3_max": 0.184943, + "nauc_map_at_3_std": -0.142326, + "nauc_map_at_3_diff1": 0.464745, + "nauc_map_at_5_max": 0.184806, + "nauc_map_at_5_std": -0.140172, + "nauc_map_at_5_diff1": 0.459772, + "nauc_map_at_10_max": 0.183262, + "nauc_map_at_10_std": -0.139338, + "nauc_map_at_10_diff1": 0.457026, + "nauc_map_at_20_max": 0.182553, + "nauc_map_at_20_std": -0.138452, + "nauc_map_at_20_diff1": 0.456415, + "nauc_map_at_100_max": 0.182849, + "nauc_map_at_100_std": -0.137114, + "nauc_map_at_100_diff1": 0.455989, + "nauc_map_at_1000_max": 0.182834, + "nauc_map_at_1000_std": -0.137128, + "nauc_map_at_1000_diff1": 0.456111, + "nauc_recall_at_1_max": 0.192045, + "nauc_recall_at_1_std": -0.142118, + "nauc_recall_at_1_diff1": 0.509008, + "nauc_recall_at_3_max": 0.174885, + "nauc_recall_at_3_std": -0.143478, + "nauc_recall_at_3_diff1": 0.41479, + "nauc_recall_at_5_max": 0.173654, + "nauc_recall_at_5_std": -0.134206, + "nauc_recall_at_5_diff1": 0.39245, + "nauc_recall_at_10_max": 0.160777, + "nauc_recall_at_10_std": -0.127327, + "nauc_recall_at_10_diff1": 0.368936, + "nauc_recall_at_20_max": 0.149095, + "nauc_recall_at_20_std": -0.111572, + "nauc_recall_at_20_diff1": 0.35399, + "nauc_recall_at_100_max": 0.162946, + "nauc_recall_at_100_std": -0.023174, + "nauc_recall_at_100_diff1": 0.309329, + "nauc_recall_at_1000_max": 0.154742, + "nauc_recall_at_1000_std": 0.05602, + "nauc_recall_at_1000_diff1": 0.281775, + "nauc_precision_at_1_max": 0.192045, + "nauc_precision_at_1_std": -0.142118, + "nauc_precision_at_1_diff1": 0.509008, + "nauc_precision_at_3_max": 0.174885, + "nauc_precision_at_3_std": -0.143478, + "nauc_precision_at_3_diff1": 0.41479, + "nauc_precision_at_5_max": 0.173654, + "nauc_precision_at_5_std": -0.134206, + "nauc_precision_at_5_diff1": 0.39245, + "nauc_precision_at_10_max": 0.160777, + "nauc_precision_at_10_std": -0.127327, + "nauc_precision_at_10_diff1": 0.368936, + "nauc_precision_at_20_max": 0.149095, + "nauc_precision_at_20_std": -0.111572, + "nauc_precision_at_20_diff1": 0.35399, + "nauc_precision_at_100_max": 0.162946, + "nauc_precision_at_100_std": -0.023174, + "nauc_precision_at_100_diff1": 0.309329, + "nauc_precision_at_1000_max": 0.154742, + "nauc_precision_at_1000_std": 0.05602, + "nauc_precision_at_1000_diff1": 0.281775, + "nauc_mrr_at_1_max": 0.191853, + "nauc_mrr_at_1_std": -0.142258, + "nauc_mrr_at_1_diff1": 0.509473, + "nauc_mrr_at_3_max": 0.184626, + "nauc_mrr_at_3_std": -0.142635, + "nauc_mrr_at_3_diff1": 0.465093, + "nauc_mrr_at_5_max": 0.184744, + "nauc_mrr_at_5_std": -0.140221, + "nauc_mrr_at_5_diff1": 0.460138, + "nauc_mrr_at_10_max": 0.183036, + "nauc_mrr_at_10_std": -0.13961, + "nauc_mrr_at_10_diff1": 0.457368, + "nauc_mrr_at_20_max": 0.182344, + "nauc_mrr_at_20_std": -0.138711, + "nauc_mrr_at_20_diff1": 0.45677, + "nauc_mrr_at_100_max": 0.182623, + "nauc_mrr_at_100_std": -0.13739, + "nauc_mrr_at_100_diff1": 0.456331, + "nauc_mrr_at_1000_max": 0.182608, + "nauc_mrr_at_1000_std": -0.137405, + "nauc_mrr_at_1000_diff1": 0.456453, + "main_score": 0.4042, + "hf_subset": "go", + "languages": [ + "go-Code" + ] + }, + { + "ndcg_at_1": 0.38224, + "ndcg_at_3": 0.4607, + "ndcg_at_5": 0.47759, + "ndcg_at_10": 0.49917, + "ndcg_at_20": 0.51725, + "ndcg_at_100": 0.53644, + "ndcg_at_1000": 0.55153, + "map_at_1": 0.38224, + "map_at_3": 0.44052, + "map_at_5": 0.44984, + "map_at_10": 0.45877, + "map_at_20": 0.46376, + "map_at_100": 0.46627, + "map_at_1000": 0.46677, + "recall_at_1": 0.38224, + "recall_at_3": 0.51943, + "recall_at_5": 0.56067, + "recall_at_10": 0.62728, + "recall_at_20": 0.69865, + "recall_at_100": 0.80412, + "recall_at_1000": 0.92704, + "precision_at_1": 0.38224, + "precision_at_3": 0.17314, + "precision_at_5": 0.11213, + "precision_at_10": 0.06273, + "precision_at_20": 0.03493, + "precision_at_100": 0.00804, + "precision_at_1000": 0.00093, + "mrr_at_1": 0.382236, + "mrr_at_3": 0.440523, + "mrr_at_5": 0.449841, + "mrr_at_10": 0.458775, + "mrr_at_20": 0.463758, + "mrr_at_100": 0.466272, + "mrr_at_1000": 0.466772, + "nauc_ndcg_at_1_max": 0.368681, + "nauc_ndcg_at_1_std": -0.222928, + "nauc_ndcg_at_1_diff1": 0.677098, + "nauc_ndcg_at_3_max": 0.346279, + "nauc_ndcg_at_3_std": -0.261805, + "nauc_ndcg_at_3_diff1": 0.621628, + "nauc_ndcg_at_5_max": 0.338019, + "nauc_ndcg_at_5_std": -0.256641, + "nauc_ndcg_at_5_diff1": 0.607906, + "nauc_ndcg_at_10_max": 0.333905, + "nauc_ndcg_at_10_std": -0.257738, + "nauc_ndcg_at_10_diff1": 0.60631, + "nauc_ndcg_at_20_max": 0.334436, + "nauc_ndcg_at_20_std": -0.255412, + "nauc_ndcg_at_20_diff1": 0.602379, + "nauc_ndcg_at_100_max": 0.336506, + "nauc_ndcg_at_100_std": -0.251341, + "nauc_ndcg_at_100_diff1": 0.604128, + "nauc_ndcg_at_1000_max": 0.338877, + "nauc_ndcg_at_1000_std": -0.248388, + "nauc_ndcg_at_1000_diff1": 0.611939, + "nauc_map_at_1_max": 0.368681, + "nauc_map_at_1_std": -0.222928, + "nauc_map_at_1_diff1": 0.677098, + "nauc_map_at_3_max": 0.352253, + "nauc_map_at_3_std": -0.252062, + "nauc_map_at_3_diff1": 0.635946, + "nauc_map_at_5_max": 0.347579, + "nauc_map_at_5_std": -0.249526, + "nauc_map_at_5_diff1": 0.628705, + "nauc_map_at_10_max": 0.346002, + "nauc_map_at_10_std": -0.250109, + "nauc_map_at_10_diff1": 0.628081, + "nauc_map_at_20_max": 0.346505, + "nauc_map_at_20_std": -0.249224, + "nauc_map_at_20_diff1": 0.627079, + "nauc_map_at_100_max": 0.34672, + "nauc_map_at_100_std": -0.24889, + "nauc_map_at_100_diff1": 0.627385, + "nauc_map_at_1000_max": 0.346755, + "nauc_map_at_1000_std": -0.248826, + "nauc_map_at_1000_diff1": 0.627584, + "nauc_recall_at_1_max": 0.368681, + "nauc_recall_at_1_std": -0.222928, + "nauc_recall_at_1_diff1": 0.677098, + "nauc_recall_at_3_max": 0.328326, + "nauc_recall_at_3_std": -0.29084, + "nauc_recall_at_3_diff1": 0.578784, + "nauc_recall_at_5_max": 0.307423, + "nauc_recall_at_5_std": -0.277311, + "nauc_recall_at_5_diff1": 0.541355, + "nauc_recall_at_10_max": 0.290436, + "nauc_recall_at_10_std": -0.281804, + "nauc_recall_at_10_diff1": 0.530662, + "nauc_recall_at_20_max": 0.283787, + "nauc_recall_at_20_std": -0.276193, + "nauc_recall_at_20_diff1": 0.501211, + "nauc_recall_at_100_max": 0.28386, + "nauc_recall_at_100_std": -0.242597, + "nauc_recall_at_100_diff1": 0.475681, + "nauc_recall_at_1000_max": 0.292263, + "nauc_recall_at_1000_std": -0.12795, + "nauc_recall_at_1000_diff1": 0.554992, + "nauc_precision_at_1_max": 0.368681, + "nauc_precision_at_1_std": -0.222928, + "nauc_precision_at_1_diff1": 0.677098, + "nauc_precision_at_3_max": 0.328326, + "nauc_precision_at_3_std": -0.29084, + "nauc_precision_at_3_diff1": 0.578784, + "nauc_precision_at_5_max": 0.307423, + "nauc_precision_at_5_std": -0.277311, + "nauc_precision_at_5_diff1": 0.541355, + "nauc_precision_at_10_max": 0.290436, + "nauc_precision_at_10_std": -0.281804, + "nauc_precision_at_10_diff1": 0.530662, + "nauc_precision_at_20_max": 0.283787, + "nauc_precision_at_20_std": -0.276193, + "nauc_precision_at_20_diff1": 0.501211, + "nauc_precision_at_100_max": 0.28386, + "nauc_precision_at_100_std": -0.242597, + "nauc_precision_at_100_diff1": 0.475681, + "nauc_precision_at_1000_max": 0.292263, + "nauc_precision_at_1000_std": -0.12795, + "nauc_precision_at_1000_diff1": 0.554992, + "nauc_mrr_at_1_max": 0.368681, + "nauc_mrr_at_1_std": -0.222928, + "nauc_mrr_at_1_diff1": 0.677098, + "nauc_mrr_at_3_max": 0.352253, + "nauc_mrr_at_3_std": -0.252062, + "nauc_mrr_at_3_diff1": 0.635946, + "nauc_mrr_at_5_max": 0.347579, + "nauc_mrr_at_5_std": -0.249526, + "nauc_mrr_at_5_diff1": 0.628705, + "nauc_mrr_at_10_max": 0.346002, + "nauc_mrr_at_10_std": -0.250109, + "nauc_mrr_at_10_diff1": 0.628081, + "nauc_mrr_at_20_max": 0.346505, + "nauc_mrr_at_20_std": -0.249224, + "nauc_mrr_at_20_diff1": 0.627079, + "nauc_mrr_at_100_max": 0.346721, + "nauc_mrr_at_100_std": -0.248889, + "nauc_mrr_at_100_diff1": 0.627385, + "nauc_mrr_at_1000_max": 0.346756, + "nauc_mrr_at_1000_std": -0.248825, + "nauc_mrr_at_1000_diff1": 0.627584, + "main_score": 0.49917, + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ] + }, + { + "ndcg_at_1": 0.37736, + "ndcg_at_3": 0.45585, + "ndcg_at_5": 0.4758, + "ndcg_at_10": 0.49389, + "ndcg_at_20": 0.50693, + "ndcg_at_100": 0.52769, + "ndcg_at_1000": 0.54223, + "map_at_1": 0.37736, + "map_at_3": 0.43682, + "map_at_5": 0.44793, + "map_at_10": 0.45544, + "map_at_20": 0.45906, + "map_at_100": 0.46185, + "map_at_1000": 0.46234, + "recall_at_1": 0.37736, + "recall_at_3": 0.51082, + "recall_at_5": 0.55911, + "recall_at_10": 0.61479, + "recall_at_20": 0.66609, + "recall_at_100": 0.7791, + "recall_at_1000": 0.89667, + "precision_at_1": 0.37736, + "precision_at_3": 0.17027, + "precision_at_5": 0.11182, + "precision_at_10": 0.06148, + "precision_at_20": 0.0333, + "precision_at_100": 0.00779, + "precision_at_1000": 0.0009, + "mrr_at_1": 0.377545, + "mrr_at_3": 0.436909, + "mrr_at_5": 0.448013, + "mrr_at_10": 0.455507, + "mrr_at_20": 0.45914, + "mrr_at_100": 0.461931, + "mrr_at_1000": 0.46242, + "nauc_ndcg_at_1_max": 0.209272, + "nauc_ndcg_at_1_std": -0.180282, + "nauc_ndcg_at_1_diff1": 0.597495, + "nauc_ndcg_at_3_max": 0.196217, + "nauc_ndcg_at_3_std": -0.190347, + "nauc_ndcg_at_3_diff1": 0.551387, + "nauc_ndcg_at_5_max": 0.192851, + "nauc_ndcg_at_5_std": -0.189791, + "nauc_ndcg_at_5_diff1": 0.547323, + "nauc_ndcg_at_10_max": 0.190877, + "nauc_ndcg_at_10_std": -0.186622, + "nauc_ndcg_at_10_diff1": 0.544164, + "nauc_ndcg_at_20_max": 0.189047, + "nauc_ndcg_at_20_std": -0.184173, + "nauc_ndcg_at_20_diff1": 0.541743, + "nauc_ndcg_at_100_max": 0.190588, + "nauc_ndcg_at_100_std": -0.175457, + "nauc_ndcg_at_100_diff1": 0.541151, + "nauc_ndcg_at_1000_max": 0.191846, + "nauc_ndcg_at_1000_std": -0.173548, + "nauc_ndcg_at_1000_diff1": 0.543036, + "nauc_map_at_1_max": 0.209272, + "nauc_map_at_1_std": -0.180282, + "nauc_map_at_1_diff1": 0.597495, + "nauc_map_at_3_max": 0.200119, + "nauc_map_at_3_std": -0.18775, + "nauc_map_at_3_diff1": 0.562743, + "nauc_map_at_5_max": 0.198336, + "nauc_map_at_5_std": -0.187371, + "nauc_map_at_5_diff1": 0.560661, + "nauc_map_at_10_max": 0.197663, + "nauc_map_at_10_std": -0.186044, + "nauc_map_at_10_diff1": 0.559454, + "nauc_map_at_20_max": 0.197173, + "nauc_map_at_20_std": -0.185383, + "nauc_map_at_20_diff1": 0.558896, + "nauc_map_at_100_max": 0.197265, + "nauc_map_at_100_std": -0.184336, + "nauc_map_at_100_diff1": 0.558855, + "nauc_map_at_1000_max": 0.197255, + "nauc_map_at_1000_std": -0.184296, + "nauc_map_at_1000_diff1": 0.558916, + "nauc_recall_at_1_max": 0.209272, + "nauc_recall_at_1_std": -0.180282, + "nauc_recall_at_1_diff1": 0.597495, + "nauc_recall_at_3_max": 0.184417, + "nauc_recall_at_3_std": -0.198061, + "nauc_recall_at_3_diff1": 0.517873, + "nauc_recall_at_5_max": 0.175172, + "nauc_recall_at_5_std": -0.197253, + "nauc_recall_at_5_diff1": 0.505729, + "nauc_recall_at_10_max": 0.166708, + "nauc_recall_at_10_std": -0.187208, + "nauc_recall_at_10_diff1": 0.492177, + "nauc_recall_at_20_max": 0.156073, + "nauc_recall_at_20_std": -0.175937, + "nauc_recall_at_20_diff1": 0.475608, + "nauc_recall_at_100_max": 0.160246, + "nauc_recall_at_100_std": -0.097831, + "nauc_recall_at_100_diff1": 0.446892, + "nauc_recall_at_1000_max": 0.172112, + "nauc_recall_at_1000_std": 0.024595, + "nauc_recall_at_1000_diff1": 0.404514, + "nauc_precision_at_1_max": 0.209272, + "nauc_precision_at_1_std": -0.180282, + "nauc_precision_at_1_diff1": 0.597495, + "nauc_precision_at_3_max": 0.184417, + "nauc_precision_at_3_std": -0.198061, + "nauc_precision_at_3_diff1": 0.517873, + "nauc_precision_at_5_max": 0.175172, + "nauc_precision_at_5_std": -0.197253, + "nauc_precision_at_5_diff1": 0.505729, + "nauc_precision_at_10_max": 0.166708, + "nauc_precision_at_10_std": -0.187208, + "nauc_precision_at_10_diff1": 0.492177, + "nauc_precision_at_20_max": 0.156073, + "nauc_precision_at_20_std": -0.175937, + "nauc_precision_at_20_diff1": 0.475608, + "nauc_precision_at_100_max": 0.160246, + "nauc_precision_at_100_std": -0.097831, + "nauc_precision_at_100_diff1": 0.446892, + "nauc_precision_at_1000_max": 0.172112, + "nauc_precision_at_1000_std": 0.024595, + "nauc_precision_at_1000_diff1": 0.404514, + "nauc_mrr_at_1_max": 0.209925, + "nauc_mrr_at_1_std": -0.180365, + "nauc_mrr_at_1_diff1": 0.596935, + "nauc_mrr_at_3_max": 0.20045, + "nauc_mrr_at_3_std": -0.187792, + "nauc_mrr_at_3_diff1": 0.56246, + "nauc_mrr_at_5_max": 0.198699, + "nauc_mrr_at_5_std": -0.187331, + "nauc_mrr_at_5_diff1": 0.560369, + "nauc_mrr_at_10_max": 0.198022, + "nauc_mrr_at_10_std": -0.186072, + "nauc_mrr_at_10_diff1": 0.559145, + "nauc_mrr_at_20_max": 0.197522, + "nauc_mrr_at_20_std": -0.185412, + "nauc_mrr_at_20_diff1": 0.558611, + "nauc_mrr_at_100_max": 0.197615, + "nauc_mrr_at_100_std": -0.184367, + "nauc_mrr_at_100_diff1": 0.558569, + "nauc_mrr_at_1000_max": 0.197606, + "nauc_mrr_at_1000_std": -0.184327, + "nauc_mrr_at_1000_diff1": 0.558629, + "main_score": 0.49389, + "hf_subset": "java", + "languages": [ + "java-Code" + ] + }, + { + "ndcg_at_1": 0.24055, + "ndcg_at_3": 0.30417, + "ndcg_at_5": 0.32452, + "ndcg_at_10": 0.34575, + "ndcg_at_20": 0.361, + "ndcg_at_100": 0.38643, + "ndcg_at_1000": 0.40703, + "map_at_1": 0.24055, + "map_at_3": 0.28853, + "map_at_5": 0.29983, + "map_at_10": 0.30863, + "map_at_20": 0.3128, + "map_at_100": 0.31621, + "map_at_1000": 0.31689, + "recall_at_1": 0.24055, + "recall_at_3": 0.34944, + "recall_at_5": 0.39882, + "recall_at_10": 0.46425, + "recall_at_20": 0.52469, + "recall_at_100": 0.66341, + "recall_at_1000": 0.83067, + "precision_at_1": 0.24055, + "precision_at_3": 0.11648, + "precision_at_5": 0.07976, + "precision_at_10": 0.04643, + "precision_at_20": 0.02623, + "precision_at_100": 0.00663, + "precision_at_1000": 0.00083, + "mrr_at_1": 0.240402, + "mrr_at_3": 0.288473, + "mrr_at_5": 0.299748, + "mrr_at_10": 0.308565, + "mrr_at_20": 0.312725, + "mrr_at_100": 0.316135, + "mrr_at_1000": 0.316822, + "nauc_ndcg_at_1_max": 0.165473, + "nauc_ndcg_at_1_std": -0.136026, + "nauc_ndcg_at_1_diff1": 0.514168, + "nauc_ndcg_at_3_max": 0.148716, + "nauc_ndcg_at_3_std": -0.141357, + "nauc_ndcg_at_3_diff1": 0.452096, + "nauc_ndcg_at_5_max": 0.144303, + "nauc_ndcg_at_5_std": -0.138147, + "nauc_ndcg_at_5_diff1": 0.439343, + "nauc_ndcg_at_10_max": 0.139299, + "nauc_ndcg_at_10_std": -0.133911, + "nauc_ndcg_at_10_diff1": 0.431128, + "nauc_ndcg_at_20_max": 0.136593, + "nauc_ndcg_at_20_std": -0.127572, + "nauc_ndcg_at_20_diff1": 0.428395, + "nauc_ndcg_at_100_max": 0.135639, + "nauc_ndcg_at_100_std": -0.118435, + "nauc_ndcg_at_100_diff1": 0.424875, + "nauc_ndcg_at_1000_max": 0.136763, + "nauc_ndcg_at_1000_std": -0.116519, + "nauc_ndcg_at_1000_diff1": 0.427778, + "nauc_map_at_1_max": 0.165473, + "nauc_map_at_1_std": -0.136026, + "nauc_map_at_1_diff1": 0.514168, + "nauc_map_at_3_max": 0.152702, + "nauc_map_at_3_std": -0.1405, + "nauc_map_at_3_diff1": 0.465443, + "nauc_map_at_5_max": 0.150122, + "nauc_map_at_5_std": -0.138803, + "nauc_map_at_5_diff1": 0.458182, + "nauc_map_at_10_max": 0.147873, + "nauc_map_at_10_std": -0.13716, + "nauc_map_at_10_diff1": 0.454683, + "nauc_map_at_20_max": 0.147121, + "nauc_map_at_20_std": -0.135386, + "nauc_map_at_20_diff1": 0.453967, + "nauc_map_at_100_max": 0.146999, + "nauc_map_at_100_std": -0.134265, + "nauc_map_at_100_diff1": 0.453543, + "nauc_map_at_1000_max": 0.147088, + "nauc_map_at_1000_std": -0.134149, + "nauc_map_at_1000_diff1": 0.453656, + "nauc_recall_at_1_max": 0.165473, + "nauc_recall_at_1_std": -0.136026, + "nauc_recall_at_1_diff1": 0.514168, + "nauc_recall_at_3_max": 0.137861, + "nauc_recall_at_3_std": -0.143542, + "nauc_recall_at_3_diff1": 0.416267, + "nauc_recall_at_5_max": 0.128235, + "nauc_recall_at_5_std": -0.135703, + "nauc_recall_at_5_diff1": 0.387781, + "nauc_recall_at_10_max": 0.113955, + "nauc_recall_at_10_std": -0.122474, + "nauc_recall_at_10_diff1": 0.362974, + "nauc_recall_at_20_max": 0.102634, + "nauc_recall_at_20_std": -0.09761, + "nauc_recall_at_20_diff1": 0.34987, + "nauc_recall_at_100_max": 0.089617, + "nauc_recall_at_100_std": -0.032492, + "nauc_recall_at_100_diff1": 0.311441, + "nauc_recall_at_1000_max": 0.069269, + "nauc_recall_at_1000_std": 0.052529, + "nauc_recall_at_1000_diff1": 0.274866, + "nauc_precision_at_1_max": 0.165473, + "nauc_precision_at_1_std": -0.136026, + "nauc_precision_at_1_diff1": 0.514168, + "nauc_precision_at_3_max": 0.137861, + "nauc_precision_at_3_std": -0.143542, + "nauc_precision_at_3_diff1": 0.416267, + "nauc_precision_at_5_max": 0.128235, + "nauc_precision_at_5_std": -0.135703, + "nauc_precision_at_5_diff1": 0.387781, + "nauc_precision_at_10_max": 0.113955, + "nauc_precision_at_10_std": -0.122474, + "nauc_precision_at_10_diff1": 0.362974, + "nauc_precision_at_20_max": 0.102634, + "nauc_precision_at_20_std": -0.09761, + "nauc_precision_at_20_diff1": 0.34987, + "nauc_precision_at_100_max": 0.089617, + "nauc_precision_at_100_std": -0.032492, + "nauc_precision_at_100_diff1": 0.311441, + "nauc_precision_at_1000_max": 0.069269, + "nauc_precision_at_1000_std": 0.052529, + "nauc_precision_at_1000_diff1": 0.274866, + "nauc_mrr_at_1_max": 0.165513, + "nauc_mrr_at_1_std": -0.136324, + "nauc_mrr_at_1_diff1": 0.514758, + "nauc_mrr_at_3_max": 0.152855, + "nauc_mrr_at_3_std": -0.140565, + "nauc_mrr_at_3_diff1": 0.465746, + "nauc_mrr_at_5_max": 0.150179, + "nauc_mrr_at_5_std": -0.138866, + "nauc_mrr_at_5_diff1": 0.458402, + "nauc_mrr_at_10_max": 0.147903, + "nauc_mrr_at_10_std": -0.137236, + "nauc_mrr_at_10_diff1": 0.454922, + "nauc_mrr_at_20_max": 0.147153, + "nauc_mrr_at_20_std": -0.135461, + "nauc_mrr_at_20_diff1": 0.454194, + "nauc_mrr_at_100_max": 0.147032, + "nauc_mrr_at_100_std": -0.134342, + "nauc_mrr_at_100_diff1": 0.453768, + "nauc_mrr_at_1000_max": 0.14712, + "nauc_mrr_at_1000_std": -0.134226, + "nauc_mrr_at_1000_diff1": 0.453885, + "main_score": 0.34575, + "hf_subset": "php", + "languages": [ + "php-Code" + ] + } + ] + }, + "evaluation_time": 3261.282051086426, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeSearchNetRetrieval.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeSearchNetRetrieval.json new file mode 100644 index 0000000000..7a9b688355 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeSearchNetRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "fdc6a9e39575768c27eb8a2a5f702bf846eb4759", + "task_name": "CodeSearchNetRetrieval", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.79, + "ndcg_at_3": 0.86405, + "ndcg_at_5": 0.87369, + "ndcg_at_10": 0.88049, + "ndcg_at_20": 0.88327, + "ndcg_at_100": 0.88748, + "ndcg_at_1000": 0.8885, + "map_at_1": 0.79, + "map_at_3": 0.84667, + "map_at_5": 0.85212, + "map_at_10": 0.85493, + "map_at_20": 0.85569, + "map_at_100": 0.85626, + "map_at_1000": 0.8563, + "recall_at_1": 0.79, + "recall_at_3": 0.914, + "recall_at_5": 0.937, + "recall_at_10": 0.958, + "recall_at_20": 0.969, + "recall_at_100": 0.992, + "recall_at_1000": 1.0, + "precision_at_1": 0.79, + "precision_at_3": 0.30467, + "precision_at_5": 0.1874, + "precision_at_10": 0.0958, + "precision_at_20": 0.04845, + "precision_at_100": 0.00992, + "precision_at_1000": 0.001, + "mrr_at_1": 0.79, + "mrr_at_3": 0.846667, + "mrr_at_5": 0.852117, + "mrr_at_10": 0.854929, + "mrr_at_20": 0.855693, + "mrr_at_100": 0.856258, + "mrr_at_1000": 0.856298, + "nauc_ndcg_at_1_max": 0.491916, + "nauc_ndcg_at_1_std": -0.201755, + "nauc_ndcg_at_1_diff1": 0.829545, + "nauc_ndcg_at_3_max": 0.586605, + "nauc_ndcg_at_3_std": -0.204438, + "nauc_ndcg_at_3_diff1": 0.805435, + "nauc_ndcg_at_5_max": 0.583722, + "nauc_ndcg_at_5_std": -0.213011, + "nauc_ndcg_at_5_diff1": 0.805832, + "nauc_ndcg_at_10_max": 0.571243, + "nauc_ndcg_at_10_std": -0.19381, + "nauc_ndcg_at_10_diff1": 0.806652, + "nauc_ndcg_at_20_max": 0.572817, + "nauc_ndcg_at_20_std": -0.192943, + "nauc_ndcg_at_20_diff1": 0.813066, + "nauc_ndcg_at_100_max": 0.563605, + "nauc_ndcg_at_100_std": -0.192434, + "nauc_ndcg_at_100_diff1": 0.813248, + "nauc_ndcg_at_1000_max": 0.559996, + "nauc_ndcg_at_1000_std": -0.196894, + "nauc_ndcg_at_1000_diff1": 0.812886, + "nauc_map_at_1_max": 0.491916, + "nauc_map_at_1_std": -0.201755, + "nauc_map_at_1_diff1": 0.829545, + "nauc_map_at_3_max": 0.556432, + "nauc_map_at_3_std": -0.204467, + "nauc_map_at_3_diff1": 0.811404, + "nauc_map_at_5_max": 0.553617, + "nauc_map_at_5_std": -0.208649, + "nauc_map_at_5_diff1": 0.811894, + "nauc_map_at_10_max": 0.549009, + "nauc_map_at_10_std": -0.201903, + "nauc_map_at_10_diff1": 0.812439, + "nauc_map_at_20_max": 0.549183, + "nauc_map_at_20_std": -0.201822, + "nauc_map_at_20_diff1": 0.814046, + "nauc_map_at_100_max": 0.548325, + "nauc_map_at_100_std": -0.201389, + "nauc_map_at_100_diff1": 0.814112, + "nauc_map_at_1000_max": 0.548218, + "nauc_map_at_1000_std": -0.201497, + "nauc_map_at_1000_diff1": 0.814103, + "nauc_recall_at_1_max": 0.491916, + "nauc_recall_at_1_std": -0.201755, + "nauc_recall_at_1_diff1": 0.829545, + "nauc_recall_at_3_max": 0.733155, + "nauc_recall_at_3_std": -0.203673, + "nauc_recall_at_3_diff1": 0.777653, + "nauc_recall_at_5_max": 0.775688, + "nauc_recall_at_5_std": -0.242304, + "nauc_recall_at_5_diff1": 0.76873, + "nauc_recall_at_10_max": 0.762727, + "nauc_recall_at_10_std": -0.09815, + "nauc_recall_at_10_diff1": 0.75458, + "nauc_recall_at_20_max": 0.85508, + "nauc_recall_at_20_std": -0.049441, + "nauc_recall_at_20_diff1": 0.821993, + "nauc_recall_at_100_max": 0.944269, + "nauc_recall_at_100_std": 0.295226, + "nauc_recall_at_100_diff1": 0.852358, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.491916, + "nauc_precision_at_1_std": -0.201755, + "nauc_precision_at_1_diff1": 0.829545, + "nauc_precision_at_3_max": 0.733155, + "nauc_precision_at_3_std": -0.203673, + "nauc_precision_at_3_diff1": 0.777653, + "nauc_precision_at_5_max": 0.775688, + "nauc_precision_at_5_std": -0.242304, + "nauc_precision_at_5_diff1": 0.76873, + "nauc_precision_at_10_max": 0.762727, + "nauc_precision_at_10_std": -0.09815, + "nauc_precision_at_10_diff1": 0.75458, + "nauc_precision_at_20_max": 0.85508, + "nauc_precision_at_20_std": -0.049441, + "nauc_precision_at_20_diff1": 0.821993, + "nauc_precision_at_100_max": 0.944269, + "nauc_precision_at_100_std": 0.295226, + "nauc_precision_at_100_diff1": 0.852358, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.491916, + "nauc_mrr_at_1_std": -0.201755, + "nauc_mrr_at_1_diff1": 0.829545, + "nauc_mrr_at_3_max": 0.556432, + "nauc_mrr_at_3_std": -0.204467, + "nauc_mrr_at_3_diff1": 0.811404, + "nauc_mrr_at_5_max": 0.553617, + "nauc_mrr_at_5_std": -0.208649, + "nauc_mrr_at_5_diff1": 0.811894, + "nauc_mrr_at_10_max": 0.549009, + "nauc_mrr_at_10_std": -0.201903, + "nauc_mrr_at_10_diff1": 0.812439, + "nauc_mrr_at_20_max": 0.549183, + "nauc_mrr_at_20_std": -0.201822, + "nauc_mrr_at_20_diff1": 0.814046, + "nauc_mrr_at_100_max": 0.548325, + "nauc_mrr_at_100_std": -0.201389, + "nauc_mrr_at_100_diff1": 0.814112, + "nauc_mrr_at_1000_max": 0.548218, + "nauc_mrr_at_1000_std": -0.201497, + "nauc_mrr_at_1000_diff1": 0.814103, + "main_score": 0.88049, + "hf_subset": "python", + "languages": [ + "python-Code" + ] + }, + { + "ndcg_at_1": 0.675, + "ndcg_at_3": 0.74452, + "ndcg_at_5": 0.75855, + "ndcg_at_10": 0.76952, + "ndcg_at_20": 0.7754, + "ndcg_at_100": 0.78518, + "ndcg_at_1000": 0.79345, + "map_at_1": 0.675, + "map_at_3": 0.72833, + "map_at_5": 0.73613, + "map_at_10": 0.74064, + "map_at_20": 0.7423, + "map_at_100": 0.7437, + "map_at_1000": 0.74399, + "recall_at_1": 0.675, + "recall_at_3": 0.791, + "recall_at_5": 0.825, + "recall_at_10": 0.859, + "recall_at_20": 0.882, + "recall_at_100": 0.934, + "recall_at_1000": 1.0, + "precision_at_1": 0.675, + "precision_at_3": 0.26367, + "precision_at_5": 0.165, + "precision_at_10": 0.0859, + "precision_at_20": 0.0441, + "precision_at_100": 0.00934, + "precision_at_1000": 0.001, + "mrr_at_1": 0.675, + "mrr_at_3": 0.728333, + "mrr_at_5": 0.736133, + "mrr_at_10": 0.740641, + "mrr_at_20": 0.742297, + "mrr_at_100": 0.743699, + "mrr_at_1000": 0.743995, + "nauc_ndcg_at_1_max": 0.738193, + "nauc_ndcg_at_1_std": 0.07856, + "nauc_ndcg_at_1_diff1": 0.852427, + "nauc_ndcg_at_3_max": 0.776233, + "nauc_ndcg_at_3_std": 0.107263, + "nauc_ndcg_at_3_diff1": 0.819321, + "nauc_ndcg_at_5_max": 0.777893, + "nauc_ndcg_at_5_std": 0.119711, + "nauc_ndcg_at_5_diff1": 0.817412, + "nauc_ndcg_at_10_max": 0.781065, + "nauc_ndcg_at_10_std": 0.135692, + "nauc_ndcg_at_10_diff1": 0.822785, + "nauc_ndcg_at_20_max": 0.779419, + "nauc_ndcg_at_20_std": 0.138919, + "nauc_ndcg_at_20_diff1": 0.821706, + "nauc_ndcg_at_100_max": 0.778496, + "nauc_ndcg_at_100_std": 0.14232, + "nauc_ndcg_at_100_diff1": 0.822686, + "nauc_ndcg_at_1000_max": 0.774437, + "nauc_ndcg_at_1000_std": 0.128223, + "nauc_ndcg_at_1000_diff1": 0.825352, + "nauc_map_at_1_max": 0.738193, + "nauc_map_at_1_std": 0.07856, + "nauc_map_at_1_diff1": 0.852427, + "nauc_map_at_3_max": 0.767093, + "nauc_map_at_3_std": 0.099808, + "nauc_map_at_3_diff1": 0.828103, + "nauc_map_at_5_max": 0.767653, + "nauc_map_at_5_std": 0.105767, + "nauc_map_at_5_diff1": 0.827466, + "nauc_map_at_10_max": 0.768802, + "nauc_map_at_10_std": 0.112097, + "nauc_map_at_10_diff1": 0.829738, + "nauc_map_at_20_max": 0.768253, + "nauc_map_at_20_std": 0.112567, + "nauc_map_at_20_diff1": 0.82952, + "nauc_map_at_100_max": 0.768128, + "nauc_map_at_100_std": 0.112785, + "nauc_map_at_100_diff1": 0.829675, + "nauc_map_at_1000_max": 0.768014, + "nauc_map_at_1000_std": 0.112523, + "nauc_map_at_1000_diff1": 0.829756, + "nauc_recall_at_1_max": 0.738193, + "nauc_recall_at_1_std": 0.07856, + "nauc_recall_at_1_diff1": 0.852427, + "nauc_recall_at_3_max": 0.807764, + "nauc_recall_at_3_std": 0.133307, + "nauc_recall_at_3_diff1": 0.788542, + "nauc_recall_at_5_max": 0.818601, + "nauc_recall_at_5_std": 0.178318, + "nauc_recall_at_5_diff1": 0.776699, + "nauc_recall_at_10_max": 0.84098, + "nauc_recall_at_10_std": 0.260114, + "nauc_recall_at_10_diff1": 0.79203, + "nauc_recall_at_20_max": 0.841044, + "nauc_recall_at_20_std": 0.306149, + "nauc_recall_at_20_diff1": 0.779304, + "nauc_recall_at_100_max": 0.871021, + "nauc_recall_at_100_std": 0.485329, + "nauc_recall_at_100_diff1": 0.762683, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.738193, + "nauc_precision_at_1_std": 0.07856, + "nauc_precision_at_1_diff1": 0.852427, + "nauc_precision_at_3_max": 0.807764, + "nauc_precision_at_3_std": 0.133307, + "nauc_precision_at_3_diff1": 0.788542, + "nauc_precision_at_5_max": 0.818601, + "nauc_precision_at_5_std": 0.178318, + "nauc_precision_at_5_diff1": 0.776699, + "nauc_precision_at_10_max": 0.84098, + "nauc_precision_at_10_std": 0.260114, + "nauc_precision_at_10_diff1": 0.79203, + "nauc_precision_at_20_max": 0.841044, + "nauc_precision_at_20_std": 0.306149, + "nauc_precision_at_20_diff1": 0.779304, + "nauc_precision_at_100_max": 0.871021, + "nauc_precision_at_100_std": 0.485329, + "nauc_precision_at_100_diff1": 0.762683, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.738193, + "nauc_mrr_at_1_std": 0.07856, + "nauc_mrr_at_1_diff1": 0.852427, + "nauc_mrr_at_3_max": 0.767093, + "nauc_mrr_at_3_std": 0.099808, + "nauc_mrr_at_3_diff1": 0.828103, + "nauc_mrr_at_5_max": 0.767653, + "nauc_mrr_at_5_std": 0.105767, + "nauc_mrr_at_5_diff1": 0.827466, + "nauc_mrr_at_10_max": 0.768802, + "nauc_mrr_at_10_std": 0.112097, + "nauc_mrr_at_10_diff1": 0.829738, + "nauc_mrr_at_20_max": 0.768253, + "nauc_mrr_at_20_std": 0.112567, + "nauc_mrr_at_20_diff1": 0.82952, + "nauc_mrr_at_100_max": 0.768128, + "nauc_mrr_at_100_std": 0.112785, + "nauc_mrr_at_100_diff1": 0.829675, + "nauc_mrr_at_1000_max": 0.768014, + "nauc_mrr_at_1000_std": 0.112523, + "nauc_mrr_at_1000_diff1": 0.829756, + "main_score": 0.76952, + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ] + }, + { + "ndcg_at_1": 0.891, + "ndcg_at_3": 0.93562, + "ndcg_at_5": 0.94268, + "ndcg_at_10": 0.94528, + "ndcg_at_20": 0.94584, + "ndcg_at_100": 0.94695, + "ndcg_at_1000": 0.94732, + "map_at_1": 0.891, + "map_at_3": 0.92567, + "map_at_5": 0.92962, + "map_at_10": 0.9307, + "map_at_20": 0.93089, + "map_at_100": 0.93104, + "map_at_1000": 0.93105, + "recall_at_1": 0.891, + "recall_at_3": 0.964, + "recall_at_5": 0.981, + "recall_at_10": 0.989, + "recall_at_20": 0.991, + "recall_at_100": 0.997, + "recall_at_1000": 1.0, + "precision_at_1": 0.891, + "precision_at_3": 0.32133, + "precision_at_5": 0.1962, + "precision_at_10": 0.0989, + "precision_at_20": 0.04955, + "precision_at_100": 0.00997, + "precision_at_1000": 0.001, + "mrr_at_1": 0.891, + "mrr_at_3": 0.925667, + "mrr_at_5": 0.929617, + "mrr_at_10": 0.930704, + "mrr_at_20": 0.930886, + "mrr_at_100": 0.931036, + "mrr_at_1000": 0.931049, + "nauc_ndcg_at_1_max": 0.539185, + "nauc_ndcg_at_1_std": -0.170122, + "nauc_ndcg_at_1_diff1": 0.915114, + "nauc_ndcg_at_3_max": 0.622581, + "nauc_ndcg_at_3_std": -0.113321, + "nauc_ndcg_at_3_diff1": 0.909071, + "nauc_ndcg_at_5_max": 0.614036, + "nauc_ndcg_at_5_std": -0.084795, + "nauc_ndcg_at_5_diff1": 0.913739, + "nauc_ndcg_at_10_max": 0.606474, + "nauc_ndcg_at_10_std": -0.089033, + "nauc_ndcg_at_10_diff1": 0.914942, + "nauc_ndcg_at_20_max": 0.608469, + "nauc_ndcg_at_20_std": -0.085437, + "nauc_ndcg_at_20_diff1": 0.914737, + "nauc_ndcg_at_100_max": 0.600249, + "nauc_ndcg_at_100_std": -0.108224, + "nauc_ndcg_at_100_diff1": 0.914786, + "nauc_ndcg_at_1000_max": 0.597383, + "nauc_ndcg_at_1000_std": -0.11326, + "nauc_ndcg_at_1000_diff1": 0.914175, + "nauc_map_at_1_max": 0.539185, + "nauc_map_at_1_std": -0.170122, + "nauc_map_at_1_diff1": 0.915114, + "nauc_map_at_3_max": 0.597267, + "nauc_map_at_3_std": -0.133318, + "nauc_map_at_3_diff1": 0.911186, + "nauc_map_at_5_max": 0.592062, + "nauc_map_at_5_std": -0.122924, + "nauc_map_at_5_diff1": 0.913615, + "nauc_map_at_10_max": 0.588906, + "nauc_map_at_10_std": -0.126027, + "nauc_map_at_10_diff1": 0.913971, + "nauc_map_at_20_max": 0.589369, + "nauc_map_at_20_std": -0.125207, + "nauc_map_at_20_diff1": 0.913916, + "nauc_map_at_100_max": 0.588467, + "nauc_map_at_100_std": -0.127678, + "nauc_map_at_100_diff1": 0.913886, + "nauc_map_at_1000_max": 0.588389, + "nauc_map_at_1000_std": -0.127839, + "nauc_map_at_1000_diff1": 0.91387, + "nauc_recall_at_1_max": 0.539185, + "nauc_recall_at_1_std": -0.170122, + "nauc_recall_at_1_diff1": 0.915114, + "nauc_recall_at_3_max": 0.768765, + "nauc_recall_at_3_std": 0.004863, + "nauc_recall_at_3_diff1": 0.896579, + "nauc_recall_at_5_max": 0.836036, + "nauc_recall_at_5_std": 0.338936, + "nauc_recall_at_5_diff1": 0.916335, + "nauc_recall_at_10_max": 0.893515, + "nauc_recall_at_10_std": 0.618751, + "nauc_recall_at_10_diff1": 0.937611, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 0.852993, + "nauc_recall_at_20_diff1": 0.938272, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.558979, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.539185, + "nauc_precision_at_1_std": -0.170122, + "nauc_precision_at_1_diff1": 0.915114, + "nauc_precision_at_3_max": 0.768765, + "nauc_precision_at_3_std": 0.004863, + "nauc_precision_at_3_diff1": 0.896579, + "nauc_precision_at_5_max": 0.836036, + "nauc_precision_at_5_std": 0.338936, + "nauc_precision_at_5_diff1": 0.916335, + "nauc_precision_at_10_max": 0.893515, + "nauc_precision_at_10_std": 0.618751, + "nauc_precision_at_10_diff1": 0.937611, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 0.852993, + "nauc_precision_at_20_diff1": 0.938272, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.558979, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.539185, + "nauc_mrr_at_1_std": -0.170122, + "nauc_mrr_at_1_diff1": 0.915114, + "nauc_mrr_at_3_max": 0.597267, + "nauc_mrr_at_3_std": -0.133318, + "nauc_mrr_at_3_diff1": 0.911186, + "nauc_mrr_at_5_max": 0.592062, + "nauc_mrr_at_5_std": -0.122924, + "nauc_mrr_at_5_diff1": 0.913615, + "nauc_mrr_at_10_max": 0.588906, + "nauc_mrr_at_10_std": -0.126027, + "nauc_mrr_at_10_diff1": 0.913971, + "nauc_mrr_at_20_max": 0.589369, + "nauc_mrr_at_20_std": -0.125207, + "nauc_mrr_at_20_diff1": 0.913916, + "nauc_mrr_at_100_max": 0.588467, + "nauc_mrr_at_100_std": -0.127678, + "nauc_mrr_at_100_diff1": 0.913886, + "nauc_mrr_at_1000_max": 0.588389, + "nauc_mrr_at_1000_std": -0.127839, + "nauc_mrr_at_1000_diff1": 0.91387, + "main_score": 0.94528, + "hf_subset": "go", + "languages": [ + "go-Code" + ] + }, + { + "ndcg_at_1": 0.712, + "ndcg_at_3": 0.78387, + "ndcg_at_5": 0.7945, + "ndcg_at_10": 0.80581, + "ndcg_at_20": 0.81097, + "ndcg_at_100": 0.81771, + "ndcg_at_1000": 0.82363, + "map_at_1": 0.712, + "map_at_3": 0.7665, + "map_at_5": 0.77235, + "map_at_10": 0.77702, + "map_at_20": 0.77849, + "map_at_100": 0.77938, + "map_at_1000": 0.77958, + "recall_at_1": 0.712, + "recall_at_3": 0.834, + "recall_at_5": 0.86, + "recall_at_10": 0.895, + "recall_at_20": 0.915, + "recall_at_100": 0.952, + "recall_at_1000": 1.0, + "precision_at_1": 0.712, + "precision_at_3": 0.278, + "precision_at_5": 0.172, + "precision_at_10": 0.0895, + "precision_at_20": 0.04575, + "precision_at_100": 0.00952, + "precision_at_1000": 0.001, + "mrr_at_1": 0.712, + "mrr_at_3": 0.7665, + "mrr_at_5": 0.77235, + "mrr_at_10": 0.777023, + "mrr_at_20": 0.778488, + "mrr_at_100": 0.779377, + "mrr_at_1000": 0.779578, + "nauc_ndcg_at_1_max": 0.643172, + "nauc_ndcg_at_1_std": 0.012197, + "nauc_ndcg_at_1_diff1": 0.812499, + "nauc_ndcg_at_3_max": 0.71863, + "nauc_ndcg_at_3_std": 0.095734, + "nauc_ndcg_at_3_diff1": 0.776373, + "nauc_ndcg_at_5_max": 0.722325, + "nauc_ndcg_at_5_std": 0.126112, + "nauc_ndcg_at_5_diff1": 0.77218, + "nauc_ndcg_at_10_max": 0.7298, + "nauc_ndcg_at_10_std": 0.147872, + "nauc_ndcg_at_10_diff1": 0.777584, + "nauc_ndcg_at_20_max": 0.725895, + "nauc_ndcg_at_20_std": 0.153365, + "nauc_ndcg_at_20_diff1": 0.777065, + "nauc_ndcg_at_100_max": 0.722439, + "nauc_ndcg_at_100_std": 0.148258, + "nauc_ndcg_at_100_diff1": 0.780702, + "nauc_ndcg_at_1000_max": 0.714345, + "nauc_ndcg_at_1000_std": 0.1257, + "nauc_ndcg_at_1000_diff1": 0.78254, + "nauc_map_at_1_max": 0.643172, + "nauc_map_at_1_std": 0.012197, + "nauc_map_at_1_diff1": 0.812499, + "nauc_map_at_3_max": 0.698711, + "nauc_map_at_3_std": 0.073982, + "nauc_map_at_3_diff1": 0.785568, + "nauc_map_at_5_max": 0.700218, + "nauc_map_at_5_std": 0.089476, + "nauc_map_at_5_diff1": 0.783622, + "nauc_map_at_10_max": 0.702426, + "nauc_map_at_10_std": 0.096519, + "nauc_map_at_10_diff1": 0.785947, + "nauc_map_at_20_max": 0.701279, + "nauc_map_at_20_std": 0.09745, + "nauc_map_at_20_diff1": 0.785924, + "nauc_map_at_100_max": 0.700941, + "nauc_map_at_100_std": 0.097034, + "nauc_map_at_100_diff1": 0.786428, + "nauc_map_at_1000_max": 0.70072, + "nauc_map_at_1000_std": 0.096421, + "nauc_map_at_1000_diff1": 0.786538, + "nauc_recall_at_1_max": 0.643172, + "nauc_recall_at_1_std": 0.012197, + "nauc_recall_at_1_diff1": 0.812499, + "nauc_recall_at_3_max": 0.794124, + "nauc_recall_at_3_std": 0.177993, + "nauc_recall_at_3_diff1": 0.741726, + "nauc_recall_at_5_max": 0.818415, + "nauc_recall_at_5_std": 0.290117, + "nauc_recall_at_5_diff1": 0.722104, + "nauc_recall_at_10_max": 0.886412, + "nauc_recall_at_10_std": 0.453586, + "nauc_recall_at_10_diff1": 0.734737, + "nauc_recall_at_20_max": 0.891234, + "nauc_recall_at_20_std": 0.567221, + "nauc_recall_at_20_diff1": 0.72008, + "nauc_recall_at_100_max": 0.944182, + "nauc_recall_at_100_std": 0.765445, + "nauc_recall_at_100_diff1": 0.741412, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.643172, + "nauc_precision_at_1_std": 0.012197, + "nauc_precision_at_1_diff1": 0.812499, + "nauc_precision_at_3_max": 0.794124, + "nauc_precision_at_3_std": 0.177993, + "nauc_precision_at_3_diff1": 0.741726, + "nauc_precision_at_5_max": 0.818415, + "nauc_precision_at_5_std": 0.290117, + "nauc_precision_at_5_diff1": 0.722104, + "nauc_precision_at_10_max": 0.886412, + "nauc_precision_at_10_std": 0.453586, + "nauc_precision_at_10_diff1": 0.734737, + "nauc_precision_at_20_max": 0.891234, + "nauc_precision_at_20_std": 0.567221, + "nauc_precision_at_20_diff1": 0.72008, + "nauc_precision_at_100_max": 0.944182, + "nauc_precision_at_100_std": 0.765445, + "nauc_precision_at_100_diff1": 0.741412, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.643172, + "nauc_mrr_at_1_std": 0.012197, + "nauc_mrr_at_1_diff1": 0.812499, + "nauc_mrr_at_3_max": 0.698711, + "nauc_mrr_at_3_std": 0.073982, + "nauc_mrr_at_3_diff1": 0.785568, + "nauc_mrr_at_5_max": 0.700218, + "nauc_mrr_at_5_std": 0.089476, + "nauc_mrr_at_5_diff1": 0.783622, + "nauc_mrr_at_10_max": 0.702426, + "nauc_mrr_at_10_std": 0.096519, + "nauc_mrr_at_10_diff1": 0.785947, + "nauc_mrr_at_20_max": 0.701279, + "nauc_mrr_at_20_std": 0.09745, + "nauc_mrr_at_20_diff1": 0.785924, + "nauc_mrr_at_100_max": 0.700941, + "nauc_mrr_at_100_std": 0.097034, + "nauc_mrr_at_100_diff1": 0.786428, + "nauc_mrr_at_1000_max": 0.70072, + "nauc_mrr_at_1000_std": 0.096421, + "nauc_mrr_at_1000_diff1": 0.786538, + "main_score": 0.80581, + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ] + }, + { + "ndcg_at_1": 0.836, + "ndcg_at_3": 0.89661, + "ndcg_at_5": 0.90155, + "ndcg_at_10": 0.90514, + "ndcg_at_20": 0.90772, + "ndcg_at_100": 0.91012, + "ndcg_at_1000": 0.9124, + "map_at_1": 0.836, + "map_at_3": 0.88283, + "map_at_5": 0.88558, + "map_at_10": 0.88708, + "map_at_20": 0.88781, + "map_at_100": 0.88815, + "map_at_1000": 0.88823, + "recall_at_1": 0.836, + "recall_at_3": 0.936, + "recall_at_5": 0.948, + "recall_at_10": 0.959, + "recall_at_20": 0.969, + "recall_at_100": 0.982, + "recall_at_1000": 1.0, + "precision_at_1": 0.836, + "precision_at_3": 0.312, + "precision_at_5": 0.1896, + "precision_at_10": 0.0959, + "precision_at_20": 0.04845, + "precision_at_100": 0.00982, + "precision_at_1000": 0.001, + "mrr_at_1": 0.836, + "mrr_at_3": 0.882833, + "mrr_at_5": 0.885583, + "mrr_at_10": 0.887081, + "mrr_at_20": 0.887811, + "mrr_at_100": 0.888146, + "mrr_at_1000": 0.888229, + "nauc_ndcg_at_1_max": 0.630447, + "nauc_ndcg_at_1_std": 0.230949, + "nauc_ndcg_at_1_diff1": 0.8219, + "nauc_ndcg_at_3_max": 0.757589, + "nauc_ndcg_at_3_std": 0.379179, + "nauc_ndcg_at_3_diff1": 0.805119, + "nauc_ndcg_at_5_max": 0.752825, + "nauc_ndcg_at_5_std": 0.385409, + "nauc_ndcg_at_5_diff1": 0.802989, + "nauc_ndcg_at_10_max": 0.74747, + "nauc_ndcg_at_10_std": 0.39179, + "nauc_ndcg_at_10_diff1": 0.80566, + "nauc_ndcg_at_20_max": 0.741411, + "nauc_ndcg_at_20_std": 0.387385, + "nauc_ndcg_at_20_diff1": 0.80911, + "nauc_ndcg_at_100_max": 0.734245, + "nauc_ndcg_at_100_std": 0.375096, + "nauc_ndcg_at_100_diff1": 0.810003, + "nauc_ndcg_at_1000_max": 0.727077, + "nauc_ndcg_at_1000_std": 0.358241, + "nauc_ndcg_at_1000_diff1": 0.809622, + "nauc_map_at_1_max": 0.630447, + "nauc_map_at_1_std": 0.230949, + "nauc_map_at_1_diff1": 0.8219, + "nauc_map_at_3_max": 0.719938, + "nauc_map_at_3_std": 0.330798, + "nauc_map_at_3_diff1": 0.809752, + "nauc_map_at_5_max": 0.716614, + "nauc_map_at_5_std": 0.332314, + "nauc_map_at_5_diff1": 0.808834, + "nauc_map_at_10_max": 0.714262, + "nauc_map_at_10_std": 0.334138, + "nauc_map_at_10_diff1": 0.809678, + "nauc_map_at_20_max": 0.712638, + "nauc_map_at_20_std": 0.332744, + "nauc_map_at_20_diff1": 0.810629, + "nauc_map_at_100_max": 0.711744, + "nauc_map_at_100_std": 0.331257, + "nauc_map_at_100_diff1": 0.810739, + "nauc_map_at_1000_max": 0.711523, + "nauc_map_at_1000_std": 0.330746, + "nauc_map_at_1000_diff1": 0.810735, + "nauc_recall_at_1_max": 0.630447, + "nauc_recall_at_1_std": 0.230949, + "nauc_recall_at_1_diff1": 0.8219, + "nauc_recall_at_3_max": 0.946057, + "nauc_recall_at_3_std": 0.625029, + "nauc_recall_at_3_diff1": 0.782198, + "nauc_recall_at_5_max": 0.969206, + "nauc_recall_at_5_std": 0.711853, + "nauc_recall_at_5_diff1": 0.767534, + "nauc_recall_at_10_max": 0.990037, + "nauc_recall_at_10_std": 0.838719, + "nauc_recall_at_10_diff1": 0.778882, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 0.932607, + "nauc_recall_at_20_diff1": 0.805955, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 1.0, + "nauc_recall_at_100_diff1": 0.826097, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.630447, + "nauc_precision_at_1_std": 0.230949, + "nauc_precision_at_1_diff1": 0.8219, + "nauc_precision_at_3_max": 0.946057, + "nauc_precision_at_3_std": 0.625029, + "nauc_precision_at_3_diff1": 0.782198, + "nauc_precision_at_5_max": 0.969206, + "nauc_precision_at_5_std": 0.711853, + "nauc_precision_at_5_diff1": 0.767534, + "nauc_precision_at_10_max": 0.990037, + "nauc_precision_at_10_std": 0.838719, + "nauc_precision_at_10_diff1": 0.778882, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 0.932607, + "nauc_precision_at_20_diff1": 0.805955, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 0.826097, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.630447, + "nauc_mrr_at_1_std": 0.230949, + "nauc_mrr_at_1_diff1": 0.8219, + "nauc_mrr_at_3_max": 0.719938, + "nauc_mrr_at_3_std": 0.330798, + "nauc_mrr_at_3_diff1": 0.809752, + "nauc_mrr_at_5_max": 0.716614, + "nauc_mrr_at_5_std": 0.332314, + "nauc_mrr_at_5_diff1": 0.808834, + "nauc_mrr_at_10_max": 0.714262, + "nauc_mrr_at_10_std": 0.334138, + "nauc_mrr_at_10_diff1": 0.809678, + "nauc_mrr_at_20_max": 0.712638, + "nauc_mrr_at_20_std": 0.332744, + "nauc_mrr_at_20_diff1": 0.810629, + "nauc_mrr_at_100_max": 0.711744, + "nauc_mrr_at_100_std": 0.331257, + "nauc_mrr_at_100_diff1": 0.810739, + "nauc_mrr_at_1000_max": 0.711523, + "nauc_mrr_at_1000_std": 0.330746, + "nauc_mrr_at_1000_diff1": 0.810735, + "main_score": 0.90514, + "hf_subset": "java", + "languages": [ + "java-Code" + ] + }, + { + "ndcg_at_1": 0.808, + "ndcg_at_3": 0.86871, + "ndcg_at_5": 0.87827, + "ndcg_at_10": 0.88695, + "ndcg_at_20": 0.88872, + "ndcg_at_100": 0.89154, + "ndcg_at_1000": 0.89387, + "map_at_1": 0.808, + "map_at_3": 0.85467, + "map_at_5": 0.86002, + "map_at_10": 0.86357, + "map_at_20": 0.86406, + "map_at_100": 0.86446, + "map_at_1000": 0.86454, + "recall_at_1": 0.808, + "recall_at_3": 0.909, + "recall_at_5": 0.932, + "recall_at_10": 0.959, + "recall_at_20": 0.966, + "recall_at_100": 0.981, + "recall_at_1000": 1.0, + "precision_at_1": 0.808, + "precision_at_3": 0.303, + "precision_at_5": 0.1864, + "precision_at_10": 0.0959, + "precision_at_20": 0.0483, + "precision_at_100": 0.00981, + "precision_at_1000": 0.001, + "mrr_at_1": 0.808, + "mrr_at_3": 0.854667, + "mrr_at_5": 0.860017, + "mrr_at_10": 0.863574, + "mrr_at_20": 0.864056, + "mrr_at_100": 0.86446, + "mrr_at_1000": 0.864539, + "nauc_ndcg_at_1_max": 0.434232, + "nauc_ndcg_at_1_std": -0.168979, + "nauc_ndcg_at_1_diff1": 0.903643, + "nauc_ndcg_at_3_max": 0.584633, + "nauc_ndcg_at_3_std": -0.054904, + "nauc_ndcg_at_3_diff1": 0.890846, + "nauc_ndcg_at_5_max": 0.5985, + "nauc_ndcg_at_5_std": -0.010241, + "nauc_ndcg_at_5_diff1": 0.893348, + "nauc_ndcg_at_10_max": 0.584729, + "nauc_ndcg_at_10_std": 0.004294, + "nauc_ndcg_at_10_diff1": 0.902244, + "nauc_ndcg_at_20_max": 0.57814, + "nauc_ndcg_at_20_std": 0.001136, + "nauc_ndcg_at_20_diff1": 0.903483, + "nauc_ndcg_at_100_max": 0.566947, + "nauc_ndcg_at_100_std": -0.013944, + "nauc_ndcg_at_100_diff1": 0.9022, + "nauc_ndcg_at_1000_max": 0.557155, + "nauc_ndcg_at_1000_std": -0.0364, + "nauc_ndcg_at_1000_diff1": 0.900331, + "nauc_map_at_1_max": 0.434232, + "nauc_map_at_1_std": -0.168979, + "nauc_map_at_1_diff1": 0.903643, + "nauc_map_at_3_max": 0.539621, + "nauc_map_at_3_std": -0.090006, + "nauc_map_at_3_diff1": 0.894181, + "nauc_map_at_5_max": 0.544815, + "nauc_map_at_5_std": -0.06985, + "nauc_map_at_5_diff1": 0.895567, + "nauc_map_at_10_max": 0.538771, + "nauc_map_at_10_std": -0.067081, + "nauc_map_at_10_diff1": 0.898693, + "nauc_map_at_20_max": 0.537109, + "nauc_map_at_20_std": -0.068115, + "nauc_map_at_20_diff1": 0.898942, + "nauc_map_at_100_max": 0.535707, + "nauc_map_at_100_std": -0.069658, + "nauc_map_at_100_diff1": 0.898819, + "nauc_map_at_1000_max": 0.535433, + "nauc_map_at_1000_std": -0.07028, + "nauc_map_at_1000_diff1": 0.898773, + "nauc_recall_at_1_max": 0.434232, + "nauc_recall_at_1_std": -0.168979, + "nauc_recall_at_1_diff1": 0.903643, + "nauc_recall_at_3_max": 0.781841, + "nauc_recall_at_3_std": 0.099517, + "nauc_recall_at_3_diff1": 0.876556, + "nauc_recall_at_5_max": 0.904405, + "nauc_recall_at_5_std": 0.339574, + "nauc_recall_at_5_diff1": 0.881893, + "nauc_recall_at_10_max": 0.990435, + "nauc_recall_at_10_std": 0.698367, + "nauc_recall_at_10_diff1": 0.946995, + "nauc_recall_at_20_max": 0.992311, + "nauc_recall_at_20_std": 0.803674, + "nauc_recall_at_20_diff1": 0.972126, + "nauc_recall_at_100_max": 0.99312, + "nauc_recall_at_100_std": 0.961915, + "nauc_recall_at_100_diff1": 0.98538, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.434232, + "nauc_precision_at_1_std": -0.168979, + "nauc_precision_at_1_diff1": 0.903643, + "nauc_precision_at_3_max": 0.781841, + "nauc_precision_at_3_std": 0.099517, + "nauc_precision_at_3_diff1": 0.876556, + "nauc_precision_at_5_max": 0.904405, + "nauc_precision_at_5_std": 0.339574, + "nauc_precision_at_5_diff1": 0.881893, + "nauc_precision_at_10_max": 0.990435, + "nauc_precision_at_10_std": 0.698367, + "nauc_precision_at_10_diff1": 0.946995, + "nauc_precision_at_20_max": 0.992311, + "nauc_precision_at_20_std": 0.803674, + "nauc_precision_at_20_diff1": 0.972126, + "nauc_precision_at_100_max": 0.99312, + "nauc_precision_at_100_std": 0.961915, + "nauc_precision_at_100_diff1": 0.98538, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.434232, + "nauc_mrr_at_1_std": -0.168979, + "nauc_mrr_at_1_diff1": 0.903643, + "nauc_mrr_at_3_max": 0.539621, + "nauc_mrr_at_3_std": -0.090006, + "nauc_mrr_at_3_diff1": 0.894181, + "nauc_mrr_at_5_max": 0.544815, + "nauc_mrr_at_5_std": -0.06985, + "nauc_mrr_at_5_diff1": 0.895567, + "nauc_mrr_at_10_max": 0.538771, + "nauc_mrr_at_10_std": -0.067081, + "nauc_mrr_at_10_diff1": 0.898693, + "nauc_mrr_at_20_max": 0.537109, + "nauc_mrr_at_20_std": -0.068115, + "nauc_mrr_at_20_diff1": 0.898942, + "nauc_mrr_at_100_max": 0.535707, + "nauc_mrr_at_100_std": -0.069658, + "nauc_mrr_at_100_diff1": 0.898819, + "nauc_mrr_at_1000_max": 0.535433, + "nauc_mrr_at_1000_std": -0.07028, + "nauc_mrr_at_1000_diff1": 0.898773, + "main_score": 0.88695, + "hf_subset": "php", + "languages": [ + "php-Code" + ] + } + ] + }, + "evaluation_time": 67.19848847389221, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeTransOceanContest.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeTransOceanContest.json new file mode 100644 index 0000000000..b5a21808b6 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeTransOceanContest.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "20da4eb20a4b17300c0986ee148c90867a7f2a4d", + "task_name": "CodeTransOceanContest", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.48416, + "ndcg_at_3": 0.56723, + "ndcg_at_5": 0.58922, + "ndcg_at_10": 0.60684, + "ndcg_at_20": 0.62423, + "ndcg_at_100": 0.6408, + "ndcg_at_1000": 0.6543, + "map_at_1": 0.48416, + "map_at_3": 0.546, + "map_at_5": 0.55799, + "map_at_10": 0.5653, + "map_at_20": 0.57019, + "map_at_100": 0.57237, + "map_at_1000": 0.57291, + "recall_at_1": 0.48416, + "recall_at_3": 0.62896, + "recall_at_5": 0.68326, + "recall_at_10": 0.73756, + "recall_at_20": 0.80543, + "recall_at_100": 0.89593, + "recall_at_1000": 1.0, + "precision_at_1": 0.48416, + "precision_at_3": 0.20965, + "precision_at_5": 0.13665, + "precision_at_10": 0.07376, + "precision_at_20": 0.04027, + "precision_at_100": 0.00896, + "precision_at_1000": 0.001, + "mrr_at_1": 0.484163, + "mrr_at_3": 0.546003, + "mrr_at_5": 0.557994, + "mrr_at_10": 0.5653, + "mrr_at_20": 0.570192, + "mrr_at_100": 0.572372, + "mrr_at_1000": 0.572907, + "nauc_ndcg_at_1_max": 0.462667, + "nauc_ndcg_at_1_std": -0.059197, + "nauc_ndcg_at_1_diff1": 0.743938, + "nauc_ndcg_at_3_max": 0.417628, + "nauc_ndcg_at_3_std": -0.107877, + "nauc_ndcg_at_3_diff1": 0.693777, + "nauc_ndcg_at_5_max": 0.443063, + "nauc_ndcg_at_5_std": -0.099159, + "nauc_ndcg_at_5_diff1": 0.690082, + "nauc_ndcg_at_10_max": 0.421137, + "nauc_ndcg_at_10_std": -0.114053, + "nauc_ndcg_at_10_diff1": 0.688116, + "nauc_ndcg_at_20_max": 0.426036, + "nauc_ndcg_at_20_std": -0.109014, + "nauc_ndcg_at_20_diff1": 0.690603, + "nauc_ndcg_at_100_max": 0.42185, + "nauc_ndcg_at_100_std": -0.11073, + "nauc_ndcg_at_100_diff1": 0.681004, + "nauc_ndcg_at_1000_max": 0.429157, + "nauc_ndcg_at_1000_std": -0.102954, + "nauc_ndcg_at_1000_diff1": 0.694365, + "nauc_map_at_1_max": 0.462667, + "nauc_map_at_1_std": -0.059197, + "nauc_map_at_1_diff1": 0.743938, + "nauc_map_at_3_max": 0.43115, + "nauc_map_at_3_std": -0.093621, + "nauc_map_at_3_diff1": 0.707041, + "nauc_map_at_5_max": 0.444298, + "nauc_map_at_5_std": -0.08908, + "nauc_map_at_5_diff1": 0.70529, + "nauc_map_at_10_max": 0.435869, + "nauc_map_at_10_std": -0.094489, + "nauc_map_at_10_diff1": 0.704945, + "nauc_map_at_20_max": 0.437787, + "nauc_map_at_20_std": -0.09241, + "nauc_map_at_20_diff1": 0.705836, + "nauc_map_at_100_max": 0.436918, + "nauc_map_at_100_std": -0.093114, + "nauc_map_at_100_diff1": 0.704653, + "nauc_map_at_1000_max": 0.43714, + "nauc_map_at_1000_std": -0.092834, + "nauc_map_at_1000_diff1": 0.705124, + "nauc_recall_at_1_max": 0.462667, + "nauc_recall_at_1_std": -0.059197, + "nauc_recall_at_1_diff1": 0.743938, + "nauc_recall_at_3_max": 0.373635, + "nauc_recall_at_3_std": -0.154118, + "nauc_recall_at_3_diff1": 0.651327, + "nauc_recall_at_5_max": 0.443195, + "nauc_recall_at_5_std": -0.13296, + "nauc_recall_at_5_diff1": 0.636505, + "nauc_recall_at_10_max": 0.356078, + "nauc_recall_at_10_std": -0.19869, + "nauc_recall_at_10_diff1": 0.618856, + "nauc_recall_at_20_max": 0.361913, + "nauc_recall_at_20_std": -0.198348, + "nauc_recall_at_20_diff1": 0.614243, + "nauc_recall_at_100_max": 0.275754, + "nauc_recall_at_100_std": -0.261555, + "nauc_recall_at_100_diff1": 0.428006, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.462667, + "nauc_precision_at_1_std": -0.059197, + "nauc_precision_at_1_diff1": 0.743938, + "nauc_precision_at_3_max": 0.373635, + "nauc_precision_at_3_std": -0.154118, + "nauc_precision_at_3_diff1": 0.651327, + "nauc_precision_at_5_max": 0.443195, + "nauc_precision_at_5_std": -0.13296, + "nauc_precision_at_5_diff1": 0.636505, + "nauc_precision_at_10_max": 0.356078, + "nauc_precision_at_10_std": -0.19869, + "nauc_precision_at_10_diff1": 0.618856, + "nauc_precision_at_20_max": 0.361913, + "nauc_precision_at_20_std": -0.198348, + "nauc_precision_at_20_diff1": 0.614243, + "nauc_precision_at_100_max": 0.275754, + "nauc_precision_at_100_std": -0.261555, + "nauc_precision_at_100_diff1": 0.428006, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.462667, + "nauc_mrr_at_1_std": -0.059197, + "nauc_mrr_at_1_diff1": 0.743938, + "nauc_mrr_at_3_max": 0.43115, + "nauc_mrr_at_3_std": -0.093621, + "nauc_mrr_at_3_diff1": 0.707041, + "nauc_mrr_at_5_max": 0.444298, + "nauc_mrr_at_5_std": -0.08908, + "nauc_mrr_at_5_diff1": 0.70529, + "nauc_mrr_at_10_max": 0.435869, + "nauc_mrr_at_10_std": -0.094489, + "nauc_mrr_at_10_diff1": 0.704945, + "nauc_mrr_at_20_max": 0.437787, + "nauc_mrr_at_20_std": -0.09241, + "nauc_mrr_at_20_diff1": 0.705836, + "nauc_mrr_at_100_max": 0.436918, + "nauc_mrr_at_100_std": -0.093114, + "nauc_mrr_at_100_diff1": 0.704653, + "nauc_mrr_at_1000_max": 0.43714, + "nauc_mrr_at_1000_std": -0.092834, + "nauc_mrr_at_1000_diff1": 0.705124, + "main_score": 0.60684, + "hf_subset": "default", + "languages": [ + "python-Code", + "c++-Code" + ] + } + ] + }, + "evaluation_time": 13.947719097137451, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeTransOceanDL.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeTransOceanDL.json new file mode 100644 index 0000000000..188c2cf4ad --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CodeTransOceanDL.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "281562cb8a1265ab5c0824bfa6ddcd9b0a15618f", + "task_name": "CodeTransOceanDL", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.08333, + "ndcg_at_3": 0.13347, + "ndcg_at_5": 0.20042, + "ndcg_at_10": 0.35163, + "ndcg_at_20": 0.3987, + "ndcg_at_100": 0.40092, + "ndcg_at_1000": 0.40092, + "map_at_1": 0.08333, + "map_at_3": 0.11852, + "map_at_5": 0.15602, + "map_at_10": 0.2165, + "map_at_20": 0.23131, + "map_at_100": 0.23166, + "map_at_1000": 0.23166, + "recall_at_1": 0.08333, + "recall_at_3": 0.17778, + "recall_at_5": 0.33889, + "recall_at_10": 0.81667, + "recall_at_20": 0.98889, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.08333, + "precision_at_3": 0.05926, + "precision_at_5": 0.06778, + "precision_at_10": 0.08167, + "precision_at_20": 0.04944, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.05, + "mrr_at_3": 0.116667, + "mrr_at_5": 0.144167, + "mrr_at_10": 0.202033, + "mrr_at_20": 0.219962, + "mrr_at_100": 0.220315, + "mrr_at_1000": 0.220315, + "nauc_ndcg_at_1_max": -0.385654, + "nauc_ndcg_at_1_std": -3.6e-05, + "nauc_ndcg_at_1_diff1": -0.204132, + "nauc_ndcg_at_3_max": -0.377686, + "nauc_ndcg_at_3_std": 0.064217, + "nauc_ndcg_at_3_diff1": -0.15218, + "nauc_ndcg_at_5_max": -0.290205, + "nauc_ndcg_at_5_std": 0.091485, + "nauc_ndcg_at_5_diff1": -0.072769, + "nauc_ndcg_at_10_max": -0.239605, + "nauc_ndcg_at_10_std": 0.098665, + "nauc_ndcg_at_10_diff1": -0.157257, + "nauc_ndcg_at_20_max": -0.289822, + "nauc_ndcg_at_20_std": 0.037175, + "nauc_ndcg_at_20_diff1": -0.119418, + "nauc_ndcg_at_100_max": -0.309314, + "nauc_ndcg_at_100_std": 0.055142, + "nauc_ndcg_at_100_diff1": -0.123581, + "nauc_ndcg_at_1000_max": -0.309314, + "nauc_ndcg_at_1000_std": 0.055142, + "nauc_ndcg_at_1000_diff1": -0.123581, + "nauc_map_at_1_max": -0.385654, + "nauc_map_at_1_std": -3.6e-05, + "nauc_map_at_1_diff1": -0.204132, + "nauc_map_at_3_max": -0.377907, + "nauc_map_at_3_std": 0.047966, + "nauc_map_at_3_diff1": -0.161976, + "nauc_map_at_5_max": -0.318075, + "nauc_map_at_5_std": 0.064468, + "nauc_map_at_5_diff1": -0.107969, + "nauc_map_at_10_max": -0.299561, + "nauc_map_at_10_std": 0.066417, + "nauc_map_at_10_diff1": -0.142502, + "nauc_map_at_20_max": -0.318442, + "nauc_map_at_20_std": 0.047865, + "nauc_map_at_20_diff1": -0.130959, + "nauc_map_at_100_max": -0.321056, + "nauc_map_at_100_std": 0.050243, + "nauc_map_at_100_diff1": -0.131544, + "nauc_map_at_1000_max": -0.321056, + "nauc_map_at_1000_std": 0.050243, + "nauc_map_at_1000_diff1": -0.131544, + "nauc_recall_at_1_max": -0.385654, + "nauc_recall_at_1_std": -3.6e-05, + "nauc_recall_at_1_diff1": -0.204132, + "nauc_recall_at_3_max": -0.377598, + "nauc_recall_at_3_std": 0.099425, + "nauc_recall_at_3_diff1": -0.131681, + "nauc_recall_at_5_max": -0.237335, + "nauc_recall_at_5_std": 0.14371, + "nauc_recall_at_5_diff1": -0.006326, + "nauc_recall_at_10_max": -0.008573, + "nauc_recall_at_10_std": 0.212526, + "nauc_recall_at_10_diff1": -0.253362, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": -1.151727, + "nauc_recall_at_20_diff1": 0.152894, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.385654, + "nauc_precision_at_1_std": -3.6e-05, + "nauc_precision_at_1_diff1": -0.204132, + "nauc_precision_at_3_max": -0.377598, + "nauc_precision_at_3_std": 0.099425, + "nauc_precision_at_3_diff1": -0.131681, + "nauc_precision_at_5_max": -0.237335, + "nauc_precision_at_5_std": 0.14371, + "nauc_precision_at_5_diff1": -0.006326, + "nauc_precision_at_10_max": -0.008573, + "nauc_precision_at_10_std": 0.212526, + "nauc_precision_at_10_diff1": -0.253362, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": -1.151727, + "nauc_precision_at_20_diff1": 0.152894, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": -0.368721, + "nauc_mrr_at_1_std": 0.066691, + "nauc_mrr_at_1_diff1": -0.170408, + "nauc_mrr_at_3_max": -0.268066, + "nauc_mrr_at_3_std": 0.062695, + "nauc_mrr_at_3_diff1": 0.020956, + "nauc_mrr_at_5_max": -0.274146, + "nauc_mrr_at_5_std": 0.081719, + "nauc_mrr_at_5_diff1": 0.034459, + "nauc_mrr_at_10_max": -0.269092, + "nauc_mrr_at_10_std": 0.10124, + "nauc_mrr_at_10_diff1": 0.061346, + "nauc_mrr_at_20_max": -0.271881, + "nauc_mrr_at_20_std": 0.077642, + "nauc_mrr_at_20_diff1": 0.04109, + "nauc_mrr_at_100_max": -0.274778, + "nauc_mrr_at_100_std": 0.080443, + "nauc_mrr_at_100_diff1": 0.04081, + "nauc_mrr_at_1000_max": -0.274778, + "nauc_mrr_at_1000_std": 0.080443, + "nauc_mrr_at_1000_diff1": 0.04081, + "main_score": 0.35163, + "hf_subset": "default", + "languages": [ + "python-Code" + ] + } + ] + }, + "evaluation_time": 12.345524072647095, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CosQA.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CosQA.json new file mode 100644 index 0000000000..e85efae9c1 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/CosQA.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "bc5efb7e9d437246ce393ed19d772e08e4a79535", + "task_name": "CosQA", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.114, + "ndcg_at_3": 0.17502, + "ndcg_at_5": 0.20918, + "ndcg_at_10": 0.25563, + "ndcg_at_20": 0.28632, + "ndcg_at_100": 0.33197, + "ndcg_at_1000": 0.35068, + "map_at_1": 0.114, + "map_at_3": 0.15833, + "map_at_5": 0.17703, + "map_at_10": 0.1965, + "map_at_20": 0.20511, + "map_at_100": 0.21145, + "map_at_1000": 0.21215, + "recall_at_1": 0.114, + "recall_at_3": 0.224, + "recall_at_5": 0.308, + "recall_at_10": 0.45, + "recall_at_20": 0.57, + "recall_at_100": 0.816, + "recall_at_1000": 0.964, + "precision_at_1": 0.114, + "precision_at_3": 0.07467, + "precision_at_5": 0.0616, + "precision_at_10": 0.045, + "precision_at_20": 0.0285, + "precision_at_100": 0.00816, + "precision_at_1000": 0.00096, + "mrr_at_1": 0.108, + "mrr_at_3": 0.151333, + "mrr_at_5": 0.169433, + "mrr_at_10": 0.188523, + "mrr_at_20": 0.196894, + "mrr_at_100": 0.203606, + "mrr_at_1000": 0.204309, + "nauc_ndcg_at_1_max": 0.172931, + "nauc_ndcg_at_1_std": -0.080523, + "nauc_ndcg_at_1_diff1": 0.339403, + "nauc_ndcg_at_3_max": 0.168177, + "nauc_ndcg_at_3_std": -0.071712, + "nauc_ndcg_at_3_diff1": 0.191576, + "nauc_ndcg_at_5_max": 0.208909, + "nauc_ndcg_at_5_std": -0.0289, + "nauc_ndcg_at_5_diff1": 0.140592, + "nauc_ndcg_at_10_max": 0.211547, + "nauc_ndcg_at_10_std": -0.031469, + "nauc_ndcg_at_10_diff1": 0.108224, + "nauc_ndcg_at_20_max": 0.246486, + "nauc_ndcg_at_20_std": 0.008339, + "nauc_ndcg_at_20_diff1": 0.088257, + "nauc_ndcg_at_100_max": 0.233615, + "nauc_ndcg_at_100_std": 0.011137, + "nauc_ndcg_at_100_diff1": 0.109208, + "nauc_ndcg_at_1000_max": 0.225461, + "nauc_ndcg_at_1000_std": 0.002046, + "nauc_ndcg_at_1000_diff1": 0.122326, + "nauc_map_at_1_max": 0.172931, + "nauc_map_at_1_std": -0.080523, + "nauc_map_at_1_diff1": 0.339403, + "nauc_map_at_3_max": 0.168866, + "nauc_map_at_3_std": -0.071028, + "nauc_map_at_3_diff1": 0.223306, + "nauc_map_at_5_max": 0.194378, + "nauc_map_at_5_std": -0.044222, + "nauc_map_at_5_diff1": 0.190906, + "nauc_map_at_10_max": 0.195972, + "nauc_map_at_10_std": -0.045005, + "nauc_map_at_10_diff1": 0.173901, + "nauc_map_at_20_max": 0.206452, + "nauc_map_at_20_std": -0.032995, + "nauc_map_at_20_diff1": 0.168086, + "nauc_map_at_100_max": 0.204823, + "nauc_map_at_100_std": -0.032494, + "nauc_map_at_100_diff1": 0.171563, + "nauc_map_at_1000_max": 0.204656, + "nauc_map_at_1000_std": -0.032558, + "nauc_map_at_1000_diff1": 0.172248, + "nauc_recall_at_1_max": 0.172931, + "nauc_recall_at_1_std": -0.080523, + "nauc_recall_at_1_diff1": 0.339403, + "nauc_recall_at_3_max": 0.166699, + "nauc_recall_at_3_std": -0.074026, + "nauc_recall_at_3_diff1": 0.11905, + "nauc_recall_at_5_max": 0.242354, + "nauc_recall_at_5_std": 0.00562, + "nauc_recall_at_5_diff1": 0.031342, + "nauc_recall_at_10_max": 0.246953, + "nauc_recall_at_10_std": -0.002247, + "nauc_recall_at_10_diff1": -0.036278, + "nauc_recall_at_20_max": 0.362452, + "nauc_recall_at_20_std": 0.128374, + "nauc_recall_at_20_diff1": -0.112757, + "nauc_recall_at_100_max": 0.353527, + "nauc_recall_at_100_std": 0.241948, + "nauc_recall_at_100_diff1": -0.123343, + "nauc_recall_at_1000_max": 0.353045, + "nauc_recall_at_1000_std": 0.553999, + "nauc_recall_at_1000_diff1": -0.378359, + "nauc_precision_at_1_max": 0.172931, + "nauc_precision_at_1_std": -0.080523, + "nauc_precision_at_1_diff1": 0.339403, + "nauc_precision_at_3_max": 0.166699, + "nauc_precision_at_3_std": -0.074026, + "nauc_precision_at_3_diff1": 0.11905, + "nauc_precision_at_5_max": 0.242354, + "nauc_precision_at_5_std": 0.00562, + "nauc_precision_at_5_diff1": 0.031342, + "nauc_precision_at_10_max": 0.246953, + "nauc_precision_at_10_std": -0.002247, + "nauc_precision_at_10_diff1": -0.036278, + "nauc_precision_at_20_max": 0.362452, + "nauc_precision_at_20_std": 0.128374, + "nauc_precision_at_20_diff1": -0.112757, + "nauc_precision_at_100_max": 0.353527, + "nauc_precision_at_100_std": 0.241948, + "nauc_precision_at_100_diff1": -0.123343, + "nauc_precision_at_1000_max": 0.353045, + "nauc_precision_at_1000_std": 0.553999, + "nauc_precision_at_1000_diff1": -0.378359, + "nauc_mrr_at_1_max": 0.152331, + "nauc_mrr_at_1_std": -0.101907, + "nauc_mrr_at_1_diff1": 0.33977, + "nauc_mrr_at_3_max": 0.152872, + "nauc_mrr_at_3_std": -0.093646, + "nauc_mrr_at_3_diff1": 0.237201, + "nauc_mrr_at_5_max": 0.165568, + "nauc_mrr_at_5_std": -0.073254, + "nauc_mrr_at_5_diff1": 0.205149, + "nauc_mrr_at_10_max": 0.168902, + "nauc_mrr_at_10_std": -0.07186, + "nauc_mrr_at_10_diff1": 0.189812, + "nauc_mrr_at_20_max": 0.180509, + "nauc_mrr_at_20_std": -0.062478, + "nauc_mrr_at_20_diff1": 0.184194, + "nauc_mrr_at_100_max": 0.178425, + "nauc_mrr_at_100_std": -0.061767, + "nauc_mrr_at_100_diff1": 0.186776, + "nauc_mrr_at_1000_max": 0.178178, + "nauc_mrr_at_1000_std": -0.061913, + "nauc_mrr_at_1000_diff1": 0.187513, + "main_score": 0.25563, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 57.257691621780396, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/DS1000Retrieval.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/DS1000Retrieval.json new file mode 100644 index 0000000000..206510e710 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/DS1000Retrieval.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "25cd4dc8172e799235d83c66439b6b7b8e6583ec", + "task_name": "DS1000Retrieval", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.13263, + "ndcg_at_3": 0.25107, + "ndcg_at_5": 0.2906, + "ndcg_at_10": 0.32416, + "ndcg_at_20": 0.3458, + "ndcg_at_100": 0.37821, + "ndcg_at_1000": 0.40133, + "map_at_1": 0.13263, + "map_at_3": 0.22256, + "map_at_5": 0.24458, + "map_at_10": 0.25876, + "map_at_20": 0.2647, + "map_at_100": 0.2691, + "map_at_1000": 0.26988, + "recall_at_1": 0.13263, + "recall_at_3": 0.33333, + "recall_at_5": 0.42893, + "recall_at_10": 0.53103, + "recall_at_20": 0.61662, + "recall_at_100": 0.79279, + "recall_at_1000": 0.98048, + "precision_at_1": 0.13263, + "precision_at_3": 0.11111, + "precision_at_5": 0.08579, + "precision_at_10": 0.0531, + "precision_at_20": 0.03083, + "precision_at_100": 0.00793, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.139139, + "mrr_at_3": 0.227311, + "mrr_at_5": 0.248657, + "mrr_at_10": 0.262805, + "mrr_at_20": 0.268683, + "mrr_at_100": 0.273105, + "mrr_at_1000": 0.273873, + "nauc_ndcg_at_1_max": 0.290107, + "nauc_ndcg_at_1_std": 0.22427, + "nauc_ndcg_at_1_diff1": -0.104152, + "nauc_ndcg_at_3_max": 0.374288, + "nauc_ndcg_at_3_std": 0.254716, + "nauc_ndcg_at_3_diff1": -0.10799, + "nauc_ndcg_at_5_max": 0.411664, + "nauc_ndcg_at_5_std": 0.272854, + "nauc_ndcg_at_5_diff1": -0.118625, + "nauc_ndcg_at_10_max": 0.41825, + "nauc_ndcg_at_10_std": 0.27724, + "nauc_ndcg_at_10_diff1": -0.111009, + "nauc_ndcg_at_20_max": 0.419725, + "nauc_ndcg_at_20_std": 0.283315, + "nauc_ndcg_at_20_diff1": -0.126011, + "nauc_ndcg_at_100_max": 0.426486, + "nauc_ndcg_at_100_std": 0.304806, + "nauc_ndcg_at_100_diff1": -0.127789, + "nauc_ndcg_at_1000_max": 0.410259, + "nauc_ndcg_at_1000_std": 0.288767, + "nauc_ndcg_at_1000_diff1": -0.125191, + "nauc_map_at_1_max": 0.290107, + "nauc_map_at_1_std": 0.22427, + "nauc_map_at_1_diff1": -0.104152, + "nauc_map_at_3_max": 0.35839, + "nauc_map_at_3_std": 0.248863, + "nauc_map_at_3_diff1": -0.105457, + "nauc_map_at_5_max": 0.381326, + "nauc_map_at_5_std": 0.259769, + "nauc_map_at_5_diff1": -0.111968, + "nauc_map_at_10_max": 0.384656, + "nauc_map_at_10_std": 0.262279, + "nauc_map_at_10_diff1": -0.107969, + "nauc_map_at_20_max": 0.384294, + "nauc_map_at_20_std": 0.263394, + "nauc_map_at_20_diff1": -0.111912, + "nauc_map_at_100_max": 0.38485, + "nauc_map_at_100_std": 0.266301, + "nauc_map_at_100_diff1": -0.11209, + "nauc_map_at_1000_max": 0.384353, + "nauc_map_at_1000_std": 0.265929, + "nauc_map_at_1000_diff1": -0.112038, + "nauc_recall_at_1_max": 0.290107, + "nauc_recall_at_1_std": 0.22427, + "nauc_recall_at_1_diff1": -0.104152, + "nauc_recall_at_3_max": 0.410704, + "nauc_recall_at_3_std": 0.268154, + "nauc_recall_at_3_diff1": -0.114365, + "nauc_recall_at_5_max": 0.483982, + "nauc_recall_at_5_std": 0.304489, + "nauc_recall_at_5_diff1": -0.135305, + "nauc_recall_at_10_max": 0.502674, + "nauc_recall_at_10_std": 0.315021, + "nauc_recall_at_10_diff1": -0.118445, + "nauc_recall_at_20_max": 0.520938, + "nauc_recall_at_20_std": 0.345033, + "nauc_recall_at_20_diff1": -0.176838, + "nauc_recall_at_100_max": 0.630484, + "nauc_recall_at_100_std": 0.53605, + "nauc_recall_at_100_diff1": -0.216631, + "nauc_recall_at_1000_max": 0.727362, + "nauc_recall_at_1000_std": 0.806421, + "nauc_recall_at_1000_diff1": -0.707664, + "nauc_precision_at_1_max": 0.290107, + "nauc_precision_at_1_std": 0.22427, + "nauc_precision_at_1_diff1": -0.104152, + "nauc_precision_at_3_max": 0.410704, + "nauc_precision_at_3_std": 0.268154, + "nauc_precision_at_3_diff1": -0.114365, + "nauc_precision_at_5_max": 0.483982, + "nauc_precision_at_5_std": 0.304489, + "nauc_precision_at_5_diff1": -0.135305, + "nauc_precision_at_10_max": 0.502674, + "nauc_precision_at_10_std": 0.315021, + "nauc_precision_at_10_diff1": -0.118445, + "nauc_precision_at_20_max": 0.520938, + "nauc_precision_at_20_std": 0.345033, + "nauc_precision_at_20_diff1": -0.176838, + "nauc_precision_at_100_max": 0.630484, + "nauc_precision_at_100_std": 0.53605, + "nauc_precision_at_100_diff1": -0.216631, + "nauc_precision_at_1000_max": 0.727362, + "nauc_precision_at_1000_std": 0.806421, + "nauc_precision_at_1000_diff1": -0.707664, + "nauc_mrr_at_1_max": 0.272027, + "nauc_mrr_at_1_std": 0.185942, + "nauc_mrr_at_1_diff1": -0.078152, + "nauc_mrr_at_3_max": 0.359192, + "nauc_mrr_at_3_std": 0.230782, + "nauc_mrr_at_3_diff1": -0.088113, + "nauc_mrr_at_5_max": 0.373482, + "nauc_mrr_at_5_std": 0.239655, + "nauc_mrr_at_5_diff1": -0.098186, + "nauc_mrr_at_10_max": 0.377904, + "nauc_mrr_at_10_std": 0.243566, + "nauc_mrr_at_10_diff1": -0.095706, + "nauc_mrr_at_20_max": 0.377771, + "nauc_mrr_at_20_std": 0.244723, + "nauc_mrr_at_20_diff1": -0.099119, + "nauc_mrr_at_100_max": 0.378291, + "nauc_mrr_at_100_std": 0.247354, + "nauc_mrr_at_100_diff1": -0.099476, + "nauc_mrr_at_1000_max": 0.377794, + "nauc_mrr_at_1000_std": 0.246955, + "nauc_mrr_at_1000_diff1": -0.099389, + "main_score": 0.32416, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 24.458454608917236, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/FreshStackRetrieval.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/FreshStackRetrieval.json new file mode 100644 index 0000000000..724e1c0835 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/FreshStackRetrieval.json @@ -0,0 +1,161 @@ +{ + "dataset_revision": "7a20df1abe4dafc46f93f9a7965bf9c6968bdf04", + "task_name": "FreshStackRetrieval", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.21131, + "ndcg_at_3": 0.18638, + "ndcg_at_5": 0.17885, + "ndcg_at_10": 0.18302, + "ndcg_at_20": 0.20221, + "ndcg_at_100": 0.28186, + "ndcg_at_1000": 0.37724, + "map_at_1": 0.03442, + "map_at_3": 0.06275, + "map_at_5": 0.07733, + "map_at_10": 0.09662, + "map_at_20": 0.11126, + "map_at_100": 0.1312, + "map_at_1000": 0.13966, + "recall_at_1": 0.03442, + "recall_at_3": 0.07813, + "recall_at_5": 0.11068, + "recall_at_10": 0.1715, + "recall_at_20": 0.23926, + "recall_at_100": 0.46565, + "recall_at_1000": 0.8693, + "precision_at_1": 0.21131, + "precision_at_3": 0.17361, + "precision_at_5": 0.15268, + "precision_at_10": 0.12113, + "precision_at_20": 0.08951, + "precision_at_100": 0.03765, + "precision_at_1000": 0.00712, + "mrr_at_1": 0.21131, + "mrr_at_3": 0.279762, + "mrr_at_5": 0.298437, + "mrr_at_10": 0.311818, + "mrr_at_20": 0.31881, + "mrr_at_100": 0.32403, + "mrr_at_1000": 0.324716, + "nauc_ndcg_at_1_max": 0.034681, + "nauc_ndcg_at_1_std": -0.037056, + "nauc_ndcg_at_1_diff1": 0.154677, + "nauc_ndcg_at_3_max": 0.033954, + "nauc_ndcg_at_3_std": -0.021928, + "nauc_ndcg_at_3_diff1": 0.102114, + "nauc_ndcg_at_5_max": 0.040244, + "nauc_ndcg_at_5_std": -0.026622, + "nauc_ndcg_at_5_diff1": 0.087412, + "nauc_ndcg_at_10_max": 0.039201, + "nauc_ndcg_at_10_std": -0.026528, + "nauc_ndcg_at_10_diff1": 0.078269, + "nauc_ndcg_at_20_max": 0.037466, + "nauc_ndcg_at_20_std": -0.028082, + "nauc_ndcg_at_20_diff1": 0.079518, + "nauc_ndcg_at_100_max": 0.071164, + "nauc_ndcg_at_100_std": 0.030437, + "nauc_ndcg_at_100_diff1": 0.084198, + "nauc_ndcg_at_1000_max": 0.102922, + "nauc_ndcg_at_1000_std": 0.085086, + "nauc_ndcg_at_1000_diff1": 0.084445, + "nauc_map_at_1_max": 0.000777, + "nauc_map_at_1_std": -0.150536, + "nauc_map_at_1_diff1": 0.215919, + "nauc_map_at_3_max": 0.016328, + "nauc_map_at_3_std": -0.133099, + "nauc_map_at_3_diff1": 0.180952, + "nauc_map_at_5_max": 0.03219, + "nauc_map_at_5_std": -0.118999, + "nauc_map_at_5_diff1": 0.153531, + "nauc_map_at_10_max": 0.042539, + "nauc_map_at_10_std": -0.079884, + "nauc_map_at_10_diff1": 0.116646, + "nauc_map_at_20_max": 0.05018, + "nauc_map_at_20_std": -0.058117, + "nauc_map_at_20_diff1": 0.097647, + "nauc_map_at_100_max": 0.063593, + "nauc_map_at_100_std": -0.023627, + "nauc_map_at_100_diff1": 0.091377, + "nauc_map_at_1000_max": 0.070135, + "nauc_map_at_1000_std": -0.009867, + "nauc_map_at_1000_diff1": 0.090922, + "nauc_recall_at_1_max": 0.000777, + "nauc_recall_at_1_std": -0.150536, + "nauc_recall_at_1_diff1": 0.215919, + "nauc_recall_at_3_max": 0.009202, + "nauc_recall_at_3_std": -0.12601, + "nauc_recall_at_3_diff1": 0.159411, + "nauc_recall_at_5_max": 0.011627, + "nauc_recall_at_5_std": -0.113416, + "nauc_recall_at_5_diff1": 0.109716, + "nauc_recall_at_10_max": 0.02163, + "nauc_recall_at_10_std": -0.061452, + "nauc_recall_at_10_diff1": 0.063787, + "nauc_recall_at_20_max": 0.023827, + "nauc_recall_at_20_std": -0.029379, + "nauc_recall_at_20_diff1": 0.042699, + "nauc_recall_at_100_max": 0.102737, + "nauc_recall_at_100_std": 0.084391, + "nauc_recall_at_100_diff1": 0.059543, + "nauc_recall_at_1000_max": 0.271869, + "nauc_recall_at_1000_std": 0.354907, + "nauc_recall_at_1000_diff1": 0.054258, + "nauc_precision_at_1_max": 0.034681, + "nauc_precision_at_1_std": -0.037056, + "nauc_precision_at_1_diff1": 0.154677, + "nauc_precision_at_3_max": 0.040328, + "nauc_precision_at_3_std": -0.002626, + "nauc_precision_at_3_diff1": 0.071098, + "nauc_precision_at_5_max": 0.04862, + "nauc_precision_at_5_std": 0.01009, + "nauc_precision_at_5_diff1": 0.033339, + "nauc_precision_at_10_max": 0.06175, + "nauc_precision_at_10_std": 0.072049, + "nauc_precision_at_10_diff1": -0.013053, + "nauc_precision_at_20_max": 0.07734, + "nauc_precision_at_20_std": 0.115941, + "nauc_precision_at_20_diff1": -0.023253, + "nauc_precision_at_100_max": 0.121326, + "nauc_precision_at_100_std": 0.218014, + "nauc_precision_at_100_diff1": -0.017556, + "nauc_precision_at_1000_max": 0.150349, + "nauc_precision_at_1000_std": 0.273161, + "nauc_precision_at_1000_diff1": -0.019303, + "nauc_mrr_at_1_max": 0.034681, + "nauc_mrr_at_1_std": -0.037056, + "nauc_mrr_at_1_diff1": 0.154677, + "nauc_mrr_at_3_max": 0.031003, + "nauc_mrr_at_3_std": -0.021568, + "nauc_mrr_at_3_diff1": 0.120334, + "nauc_mrr_at_5_max": 0.025594, + "nauc_mrr_at_5_std": -0.023101, + "nauc_mrr_at_5_diff1": 0.113493, + "nauc_mrr_at_10_max": 0.021448, + "nauc_mrr_at_10_std": -0.025776, + "nauc_mrr_at_10_diff1": 0.114574, + "nauc_mrr_at_20_max": 0.023537, + "nauc_mrr_at_20_std": -0.022011, + "nauc_mrr_at_20_diff1": 0.115039, + "nauc_mrr_at_100_max": 0.024331, + "nauc_mrr_at_100_std": -0.022094, + "nauc_mrr_at_100_diff1": 0.116758, + "nauc_mrr_at_1000_max": 0.024284, + "nauc_mrr_at_1000_std": -0.022274, + "nauc_mrr_at_1000_diff1": 0.116727, + "main_score": 0.18302, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code", + "javascript-Code", + "go-Code" + ] + } + ] + }, + "evaluation_time": 180.57317686080933, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/HumanEvalRetrieval.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/HumanEvalRetrieval.json new file mode 100644 index 0000000000..35492cddb9 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/HumanEvalRetrieval.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "ed1f48aca747f10bac146795328e2f03326e7625", + "task_name": "HumanEvalRetrieval", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.55696, + "ndcg_at_3": 0.66696, + "ndcg_at_5": 0.69584, + "ndcg_at_10": 0.71823, + "ndcg_at_20": 0.73797, + "ndcg_at_100": 0.74637, + "ndcg_at_1000": 0.74637, + "map_at_1": 0.55696, + "map_at_3": 0.64135, + "map_at_5": 0.65749, + "map_at_10": 0.66665, + "map_at_20": 0.67236, + "map_at_100": 0.67358, + "map_at_1000": 0.67358, + "recall_at_1": 0.55696, + "recall_at_3": 0.74051, + "recall_at_5": 0.81013, + "recall_at_10": 0.87975, + "recall_at_20": 0.9557, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.55696, + "precision_at_3": 0.24684, + "precision_at_5": 0.16203, + "precision_at_10": 0.08797, + "precision_at_20": 0.04778, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.556962, + "mrr_at_3": 0.64135, + "mrr_at_5": 0.657489, + "mrr_at_10": 0.666654, + "mrr_at_20": 0.672361, + "mrr_at_100": 0.673584, + "mrr_at_1000": 0.673584, + "nauc_ndcg_at_1_max": 0.281886, + "nauc_ndcg_at_1_std": -0.114665, + "nauc_ndcg_at_1_diff1": 0.591634, + "nauc_ndcg_at_3_max": 0.3192, + "nauc_ndcg_at_3_std": -0.056025, + "nauc_ndcg_at_3_diff1": 0.535739, + "nauc_ndcg_at_5_max": 0.281202, + "nauc_ndcg_at_5_std": -0.072348, + "nauc_ndcg_at_5_diff1": 0.516659, + "nauc_ndcg_at_10_max": 0.320486, + "nauc_ndcg_at_10_std": -0.036931, + "nauc_ndcg_at_10_diff1": 0.528012, + "nauc_ndcg_at_20_max": 0.331555, + "nauc_ndcg_at_20_std": -0.035264, + "nauc_ndcg_at_20_diff1": 0.537593, + "nauc_ndcg_at_100_max": 0.309353, + "nauc_ndcg_at_100_std": -0.064337, + "nauc_ndcg_at_100_diff1": 0.541147, + "nauc_ndcg_at_1000_max": 0.309353, + "nauc_ndcg_at_1000_std": -0.064337, + "nauc_ndcg_at_1000_diff1": 0.541147, + "nauc_map_at_1_max": 0.281886, + "nauc_map_at_1_std": -0.114665, + "nauc_map_at_1_diff1": 0.591634, + "nauc_map_at_3_max": 0.313729, + "nauc_map_at_3_std": -0.069196, + "nauc_map_at_3_diff1": 0.549875, + "nauc_map_at_5_max": 0.293152, + "nauc_map_at_5_std": -0.079984, + "nauc_map_at_5_diff1": 0.540664, + "nauc_map_at_10_max": 0.305553, + "nauc_map_at_10_std": -0.069644, + "nauc_map_at_10_diff1": 0.544651, + "nauc_map_at_20_max": 0.307879, + "nauc_map_at_20_std": -0.069623, + "nauc_map_at_20_diff1": 0.547423, + "nauc_map_at_100_max": 0.305203, + "nauc_map_at_100_std": -0.073275, + "nauc_map_at_100_diff1": 0.547687, + "nauc_map_at_1000_max": 0.305203, + "nauc_map_at_1000_std": -0.073275, + "nauc_map_at_1000_diff1": 0.547687, + "nauc_recall_at_1_max": 0.281886, + "nauc_recall_at_1_std": -0.114665, + "nauc_recall_at_1_diff1": 0.591634, + "nauc_recall_at_3_max": 0.336226, + "nauc_recall_at_3_std": -0.010594, + "nauc_recall_at_3_diff1": 0.485971, + "nauc_recall_at_5_max": 0.218895, + "nauc_recall_at_5_std": -0.040505, + "nauc_recall_at_5_diff1": 0.407609, + "nauc_recall_at_10_max": 0.448895, + "nauc_recall_at_10_std": 0.215268, + "nauc_recall_at_10_diff1": 0.430254, + "nauc_recall_at_20_max": 0.864348, + "nauc_recall_at_20_std": 0.65122, + "nauc_recall_at_20_diff1": 0.436997, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.281886, + "nauc_precision_at_1_std": -0.114665, + "nauc_precision_at_1_diff1": 0.591634, + "nauc_precision_at_3_max": 0.336226, + "nauc_precision_at_3_std": -0.010594, + "nauc_precision_at_3_diff1": 0.485971, + "nauc_precision_at_5_max": 0.218895, + "nauc_precision_at_5_std": -0.040505, + "nauc_precision_at_5_diff1": 0.407609, + "nauc_precision_at_10_max": 0.448895, + "nauc_precision_at_10_std": 0.215268, + "nauc_precision_at_10_diff1": 0.430254, + "nauc_precision_at_20_max": 0.864348, + "nauc_precision_at_20_std": 0.65122, + "nauc_precision_at_20_diff1": 0.436997, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.281886, + "nauc_mrr_at_1_std": -0.114665, + "nauc_mrr_at_1_diff1": 0.591634, + "nauc_mrr_at_3_max": 0.313729, + "nauc_mrr_at_3_std": -0.069196, + "nauc_mrr_at_3_diff1": 0.549875, + "nauc_mrr_at_5_max": 0.293152, + "nauc_mrr_at_5_std": -0.079984, + "nauc_mrr_at_5_diff1": 0.540664, + "nauc_mrr_at_10_max": 0.305553, + "nauc_mrr_at_10_std": -0.069644, + "nauc_mrr_at_10_diff1": 0.544651, + "nauc_mrr_at_20_max": 0.307879, + "nauc_mrr_at_20_std": -0.069623, + "nauc_mrr_at_20_diff1": 0.547423, + "nauc_mrr_at_100_max": 0.305203, + "nauc_mrr_at_100_std": -0.073275, + "nauc_mrr_at_100_diff1": 0.547687, + "nauc_mrr_at_1000_max": 0.305203, + "nauc_mrr_at_1000_std": -0.073275, + "nauc_mrr_at_1000_diff1": 0.547687, + "main_score": 0.71823, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 0.7495989799499512, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/MBPPRetrieval.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/MBPPRetrieval.json new file mode 100644 index 0000000000..ba575c285e --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/MBPPRetrieval.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "586a1fd6a0c63fdeda3b49c0293559a81c79cdec", + "task_name": "MBPPRetrieval", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.54928, + "ndcg_at_3": 0.67209, + "ndcg_at_5": 0.70373, + "ndcg_at_10": 0.72065, + "ndcg_at_20": 0.73315, + "ndcg_at_100": 0.74246, + "ndcg_at_1000": 0.74495, + "map_at_1": 0.54928, + "map_at_3": 0.64305, + "map_at_5": 0.66081, + "map_at_10": 0.66779, + "map_at_20": 0.67138, + "map_at_100": 0.67279, + "map_at_1000": 0.67289, + "recall_at_1": 0.54928, + "recall_at_3": 0.75565, + "recall_at_5": 0.83162, + "recall_at_10": 0.88398, + "recall_at_20": 0.93224, + "recall_at_100": 0.98049, + "recall_at_1000": 1.0, + "precision_at_1": 0.54928, + "precision_at_3": 0.25188, + "precision_at_5": 0.16632, + "precision_at_10": 0.0884, + "precision_at_20": 0.04661, + "precision_at_100": 0.0098, + "precision_at_1000": 0.001, + "mrr_at_1": 0.549281, + "mrr_at_3": 0.643053, + "mrr_at_5": 0.660815, + "mrr_at_10": 0.667786, + "mrr_at_20": 0.671378, + "mrr_at_100": 0.672794, + "mrr_at_1000": 0.672887, + "nauc_ndcg_at_1_max": 0.298429, + "nauc_ndcg_at_1_std": -0.10966, + "nauc_ndcg_at_1_diff1": 0.60925, + "nauc_ndcg_at_3_max": 0.353325, + "nauc_ndcg_at_3_std": -0.099254, + "nauc_ndcg_at_3_diff1": 0.557442, + "nauc_ndcg_at_5_max": 0.365163, + "nauc_ndcg_at_5_std": -0.096452, + "nauc_ndcg_at_5_diff1": 0.565216, + "nauc_ndcg_at_10_max": 0.365955, + "nauc_ndcg_at_10_std": -0.083522, + "nauc_ndcg_at_10_diff1": 0.560208, + "nauc_ndcg_at_20_max": 0.367045, + "nauc_ndcg_at_20_std": -0.080515, + "nauc_ndcg_at_20_diff1": 0.564092, + "nauc_ndcg_at_100_max": 0.358948, + "nauc_ndcg_at_100_std": -0.08455, + "nauc_ndcg_at_100_diff1": 0.57222, + "nauc_ndcg_at_1000_max": 0.353849, + "nauc_ndcg_at_1000_std": -0.089682, + "nauc_ndcg_at_1000_diff1": 0.571035, + "nauc_map_at_1_max": 0.298429, + "nauc_map_at_1_std": -0.10966, + "nauc_map_at_1_diff1": 0.60925, + "nauc_map_at_3_max": 0.339195, + "nauc_map_at_3_std": -0.101426, + "nauc_map_at_3_diff1": 0.57082, + "nauc_map_at_5_max": 0.344708, + "nauc_map_at_5_std": -0.100164, + "nauc_map_at_5_diff1": 0.575564, + "nauc_map_at_10_max": 0.344486, + "nauc_map_at_10_std": -0.095992, + "nauc_map_at_10_diff1": 0.574074, + "nauc_map_at_20_max": 0.344531, + "nauc_map_at_20_std": -0.095426, + "nauc_map_at_20_diff1": 0.57515, + "nauc_map_at_100_max": 0.343628, + "nauc_map_at_100_std": -0.095747, + "nauc_map_at_100_diff1": 0.575946, + "nauc_map_at_1000_max": 0.343465, + "nauc_map_at_1000_std": -0.095901, + "nauc_map_at_1000_diff1": 0.575895, + "nauc_recall_at_1_max": 0.298429, + "nauc_recall_at_1_std": -0.10966, + "nauc_recall_at_1_diff1": 0.60925, + "nauc_recall_at_3_max": 0.405188, + "nauc_recall_at_3_std": -0.091582, + "nauc_recall_at_3_diff1": 0.508317, + "nauc_recall_at_5_max": 0.465768, + "nauc_recall_at_5_std": -0.07824, + "nauc_recall_at_5_diff1": 0.517312, + "nauc_recall_at_10_max": 0.511221, + "nauc_recall_at_10_std": 0.013077, + "nauc_recall_at_10_diff1": 0.466539, + "nauc_recall_at_20_max": 0.621874, + "nauc_recall_at_20_std": 0.115419, + "nauc_recall_at_20_diff1": 0.451055, + "nauc_recall_at_100_max": 0.773305, + "nauc_recall_at_100_std": 0.336032, + "nauc_recall_at_100_diff1": 0.656538, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.298429, + "nauc_precision_at_1_std": -0.10966, + "nauc_precision_at_1_diff1": 0.60925, + "nauc_precision_at_3_max": 0.405188, + "nauc_precision_at_3_std": -0.091582, + "nauc_precision_at_3_diff1": 0.508317, + "nauc_precision_at_5_max": 0.465768, + "nauc_precision_at_5_std": -0.07824, + "nauc_precision_at_5_diff1": 0.517312, + "nauc_precision_at_10_max": 0.511221, + "nauc_precision_at_10_std": 0.013077, + "nauc_precision_at_10_diff1": 0.466539, + "nauc_precision_at_20_max": 0.621874, + "nauc_precision_at_20_std": 0.115419, + "nauc_precision_at_20_diff1": 0.451055, + "nauc_precision_at_100_max": 0.773305, + "nauc_precision_at_100_std": 0.336032, + "nauc_precision_at_100_diff1": 0.656538, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.298429, + "nauc_mrr_at_1_std": -0.10966, + "nauc_mrr_at_1_diff1": 0.60925, + "nauc_mrr_at_3_max": 0.339195, + "nauc_mrr_at_3_std": -0.101426, + "nauc_mrr_at_3_diff1": 0.57082, + "nauc_mrr_at_5_max": 0.344708, + "nauc_mrr_at_5_std": -0.100164, + "nauc_mrr_at_5_diff1": 0.575564, + "nauc_mrr_at_10_max": 0.344486, + "nauc_mrr_at_10_std": -0.095992, + "nauc_mrr_at_10_diff1": 0.574074, + "nauc_mrr_at_20_max": 0.344531, + "nauc_mrr_at_20_std": -0.095426, + "nauc_mrr_at_20_diff1": 0.57515, + "nauc_mrr_at_100_max": 0.343628, + "nauc_mrr_at_100_std": -0.095747, + "nauc_mrr_at_100_diff1": 0.575946, + "nauc_mrr_at_1000_max": 0.343465, + "nauc_mrr_at_1000_std": -0.095901, + "nauc_mrr_at_1000_diff1": 0.575895, + "main_score": 0.72065, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 4.244866132736206, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/StackOverflowQA.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/StackOverflowQA.json new file mode 100644 index 0000000000..1e018a507f --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/StackOverflowQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "db8f169f3894c14a00251061f957b2063eef2bd5", + "task_name": "StackOverflowQA", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.46289, + "ndcg_at_3": 0.52574, + "ndcg_at_5": 0.54324, + "ndcg_at_10": 0.56534, + "ndcg_at_20": 0.57834, + "ndcg_at_100": 0.59911, + "ndcg_at_1000": 0.61266, + "map_at_1": 0.46289, + "map_at_3": 0.51036, + "map_at_5": 0.52004, + "map_at_10": 0.52927, + "map_at_20": 0.53295, + "map_at_100": 0.53577, + "map_at_1000": 0.53623, + "recall_at_1": 0.46289, + "recall_at_3": 0.57021, + "recall_at_5": 0.61284, + "recall_at_10": 0.68054, + "recall_at_20": 0.73119, + "recall_at_100": 0.84403, + "recall_at_1000": 0.95336, + "precision_at_1": 0.46289, + "precision_at_3": 0.19007, + "precision_at_5": 0.12257, + "precision_at_10": 0.06805, + "precision_at_20": 0.03656, + "precision_at_100": 0.00844, + "precision_at_1000": 0.00095, + "mrr_at_1": 0.462889, + "mrr_at_3": 0.510364, + "mrr_at_5": 0.520043, + "mrr_at_10": 0.529274, + "mrr_at_20": 0.532949, + "mrr_at_100": 0.535767, + "mrr_at_1000": 0.536227, + "nauc_ndcg_at_1_max": 0.523296, + "nauc_ndcg_at_1_std": -0.088776, + "nauc_ndcg_at_1_diff1": 0.755274, + "nauc_ndcg_at_3_max": 0.499148, + "nauc_ndcg_at_3_std": -0.097051, + "nauc_ndcg_at_3_diff1": 0.703913, + "nauc_ndcg_at_5_max": 0.498808, + "nauc_ndcg_at_5_std": -0.082937, + "nauc_ndcg_at_5_diff1": 0.693211, + "nauc_ndcg_at_10_max": 0.496158, + "nauc_ndcg_at_10_std": -0.076948, + "nauc_ndcg_at_10_diff1": 0.683294, + "nauc_ndcg_at_20_max": 0.501587, + "nauc_ndcg_at_20_std": -0.070697, + "nauc_ndcg_at_20_diff1": 0.68489, + "nauc_ndcg_at_100_max": 0.497898, + "nauc_ndcg_at_100_std": -0.07468, + "nauc_ndcg_at_100_diff1": 0.685952, + "nauc_ndcg_at_1000_max": 0.502122, + "nauc_ndcg_at_1000_std": -0.072364, + "nauc_ndcg_at_1000_diff1": 0.692349, + "nauc_map_at_1_max": 0.523296, + "nauc_map_at_1_std": -0.088776, + "nauc_map_at_1_diff1": 0.755274, + "nauc_map_at_3_max": 0.504969, + "nauc_map_at_3_std": -0.094886, + "nauc_map_at_3_diff1": 0.716259, + "nauc_map_at_5_max": 0.504858, + "nauc_map_at_5_std": -0.087257, + "nauc_map_at_5_diff1": 0.710542, + "nauc_map_at_10_max": 0.503931, + "nauc_map_at_10_std": -0.084851, + "nauc_map_at_10_diff1": 0.706731, + "nauc_map_at_20_max": 0.505368, + "nauc_map_at_20_std": -0.083297, + "nauc_map_at_20_diff1": 0.707249, + "nauc_map_at_100_max": 0.505003, + "nauc_map_at_100_std": -0.083761, + "nauc_map_at_100_diff1": 0.707399, + "nauc_map_at_1000_max": 0.505123, + "nauc_map_at_1000_std": -0.083672, + "nauc_map_at_1000_diff1": 0.707581, + "nauc_recall_at_1_max": 0.523296, + "nauc_recall_at_1_std": -0.088776, + "nauc_recall_at_1_diff1": 0.755274, + "nauc_recall_at_3_max": 0.481562, + "nauc_recall_at_3_std": -0.103666, + "nauc_recall_at_3_diff1": 0.666632, + "nauc_recall_at_5_max": 0.479637, + "nauc_recall_at_5_std": -0.066957, + "nauc_recall_at_5_diff1": 0.636787, + "nauc_recall_at_10_max": 0.467082, + "nauc_recall_at_10_std": -0.043622, + "nauc_recall_at_10_diff1": 0.59381, + "nauc_recall_at_20_max": 0.491286, + "nauc_recall_at_20_std": -0.006562, + "nauc_recall_at_20_diff1": 0.590078, + "nauc_recall_at_100_max": 0.443632, + "nauc_recall_at_100_std": -0.014762, + "nauc_recall_at_100_diff1": 0.549858, + "nauc_recall_at_1000_max": 0.536637, + "nauc_recall_at_1000_std": 0.218271, + "nauc_recall_at_1000_diff1": 0.578551, + "nauc_precision_at_1_max": 0.523296, + "nauc_precision_at_1_std": -0.088776, + "nauc_precision_at_1_diff1": 0.755274, + "nauc_precision_at_3_max": 0.481562, + "nauc_precision_at_3_std": -0.103666, + "nauc_precision_at_3_diff1": 0.666632, + "nauc_precision_at_5_max": 0.479637, + "nauc_precision_at_5_std": -0.066957, + "nauc_precision_at_5_diff1": 0.636787, + "nauc_precision_at_10_max": 0.467082, + "nauc_precision_at_10_std": -0.043622, + "nauc_precision_at_10_diff1": 0.59381, + "nauc_precision_at_20_max": 0.491286, + "nauc_precision_at_20_std": -0.006562, + "nauc_precision_at_20_diff1": 0.590078, + "nauc_precision_at_100_max": 0.443632, + "nauc_precision_at_100_std": -0.014762, + "nauc_precision_at_100_diff1": 0.549858, + "nauc_precision_at_1000_max": 0.536637, + "nauc_precision_at_1000_std": 0.218271, + "nauc_precision_at_1000_diff1": 0.578551, + "nauc_mrr_at_1_max": 0.523296, + "nauc_mrr_at_1_std": -0.088776, + "nauc_mrr_at_1_diff1": 0.755274, + "nauc_mrr_at_3_max": 0.504969, + "nauc_mrr_at_3_std": -0.094886, + "nauc_mrr_at_3_diff1": 0.716259, + "nauc_mrr_at_5_max": 0.504858, + "nauc_mrr_at_5_std": -0.087257, + "nauc_mrr_at_5_diff1": 0.710542, + "nauc_mrr_at_10_max": 0.503931, + "nauc_mrr_at_10_std": -0.084851, + "nauc_mrr_at_10_diff1": 0.706731, + "nauc_mrr_at_20_max": 0.505368, + "nauc_mrr_at_20_std": -0.083297, + "nauc_mrr_at_20_diff1": 0.707249, + "nauc_mrr_at_100_max": 0.505003, + "nauc_mrr_at_100_std": -0.083761, + "nauc_mrr_at_100_diff1": 0.707399, + "nauc_mrr_at_1000_max": 0.505123, + "nauc_mrr_at_1000_std": -0.083672, + "nauc_mrr_at_1000_diff1": 0.707581, + "main_score": 0.56534, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 196.23864197731018, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/SyntheticText2SQL.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/SyntheticText2SQL.json new file mode 100644 index 0000000000..9d7ec30558 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/SyntheticText2SQL.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "686b87296c3a0191b5d9415a00526c62db9fce09", + "task_name": "SyntheticText2SQL", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.09127, + "ndcg_at_3": 0.3663, + "ndcg_at_5": 0.39833, + "ndcg_at_10": 0.42789, + "ndcg_at_20": 0.4458, + "ndcg_at_100": 0.46784, + "ndcg_at_1000": 0.47816, + "map_at_1": 0.09127, + "map_at_3": 0.30428, + "map_at_5": 0.32205, + "map_at_10": 0.33434, + "map_at_20": 0.33934, + "map_at_100": 0.34246, + "map_at_1000": 0.34285, + "recall_at_1": 0.09127, + "recall_at_3": 0.5435, + "recall_at_5": 0.62126, + "recall_at_10": 0.71219, + "recall_at_20": 0.78243, + "recall_at_100": 0.90019, + "recall_at_1000": 0.98137, + "precision_at_1": 0.09127, + "precision_at_3": 0.18117, + "precision_at_5": 0.12425, + "precision_at_10": 0.07122, + "precision_at_20": 0.03912, + "precision_at_100": 0.009, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.233123, + "mrr_at_3": 0.384863, + "mrr_at_5": 0.401518, + "mrr_at_10": 0.413021, + "mrr_at_20": 0.417822, + "mrr_at_100": 0.420847, + "mrr_at_1000": 0.421237, + "nauc_ndcg_at_1_max": 0.123256, + "nauc_ndcg_at_1_std": -0.075598, + "nauc_ndcg_at_1_diff1": 0.016799, + "nauc_ndcg_at_3_max": 0.296542, + "nauc_ndcg_at_3_std": -0.086179, + "nauc_ndcg_at_3_diff1": -0.524618, + "nauc_ndcg_at_5_max": 0.289706, + "nauc_ndcg_at_5_std": -0.089671, + "nauc_ndcg_at_5_diff1": -0.491487, + "nauc_ndcg_at_10_max": 0.281436, + "nauc_ndcg_at_10_std": -0.079911, + "nauc_ndcg_at_10_diff1": -0.46806, + "nauc_ndcg_at_20_max": 0.27784, + "nauc_ndcg_at_20_std": -0.075187, + "nauc_ndcg_at_20_diff1": -0.455432, + "nauc_ndcg_at_100_max": 0.277382, + "nauc_ndcg_at_100_std": -0.071498, + "nauc_ndcg_at_100_diff1": -0.442647, + "nauc_ndcg_at_1000_max": 0.2773, + "nauc_ndcg_at_1000_std": -0.072023, + "nauc_ndcg_at_1000_diff1": -0.442349, + "nauc_map_at_1_max": 0.123256, + "nauc_map_at_1_std": -0.075598, + "nauc_map_at_1_diff1": 0.016799, + "nauc_map_at_3_max": 0.274345, + "nauc_map_at_3_std": -0.083597, + "nauc_map_at_3_diff1": -0.460911, + "nauc_map_at_5_max": 0.269634, + "nauc_map_at_5_std": -0.085711, + "nauc_map_at_5_diff1": -0.439365, + "nauc_map_at_10_max": 0.265473, + "nauc_map_at_10_std": -0.081578, + "nauc_map_at_10_diff1": -0.427616, + "nauc_map_at_20_max": 0.264237, + "nauc_map_at_20_std": -0.080234, + "nauc_map_at_20_diff1": -0.423459, + "nauc_map_at_100_max": 0.263815, + "nauc_map_at_100_std": -0.080101, + "nauc_map_at_100_diff1": -0.421304, + "nauc_map_at_1000_max": 0.263849, + "nauc_map_at_1000_std": -0.080043, + "nauc_map_at_1000_diff1": -0.421232, + "nauc_recall_at_1_max": 0.123256, + "nauc_recall_at_1_std": -0.075598, + "nauc_recall_at_1_diff1": 0.016799, + "nauc_recall_at_3_max": 0.343059, + "nauc_recall_at_3_std": -0.092045, + "nauc_recall_at_3_diff1": -0.656047, + "nauc_recall_at_5_max": 0.333771, + "nauc_recall_at_5_std": -0.099519, + "nauc_recall_at_5_diff1": -0.601131, + "nauc_recall_at_10_max": 0.318493, + "nauc_recall_at_10_std": -0.071769, + "nauc_recall_at_10_diff1": -0.555242, + "nauc_recall_at_20_max": 0.311728, + "nauc_recall_at_20_std": -0.048941, + "nauc_recall_at_20_diff1": -0.520873, + "nauc_recall_at_100_max": 0.346662, + "nauc_recall_at_100_std": 0.02805, + "nauc_recall_at_100_diff1": -0.447011, + "nauc_recall_at_1000_max": 0.604021, + "nauc_recall_at_1000_std": 0.363982, + "nauc_recall_at_1000_diff1": -0.462525, + "nauc_precision_at_1_max": 0.123256, + "nauc_precision_at_1_std": -0.075598, + "nauc_precision_at_1_diff1": 0.016799, + "nauc_precision_at_3_max": 0.343059, + "nauc_precision_at_3_std": -0.092045, + "nauc_precision_at_3_diff1": -0.656047, + "nauc_precision_at_5_max": 0.333771, + "nauc_precision_at_5_std": -0.099519, + "nauc_precision_at_5_diff1": -0.601131, + "nauc_precision_at_10_max": 0.318493, + "nauc_precision_at_10_std": -0.071769, + "nauc_precision_at_10_diff1": -0.555242, + "nauc_precision_at_20_max": 0.311728, + "nauc_precision_at_20_std": -0.048941, + "nauc_precision_at_20_diff1": -0.520873, + "nauc_precision_at_100_max": 0.346662, + "nauc_precision_at_100_std": 0.02805, + "nauc_precision_at_100_diff1": -0.447011, + "nauc_precision_at_1000_max": 0.604021, + "nauc_precision_at_1000_std": 0.363982, + "nauc_precision_at_1000_diff1": -0.462525, + "nauc_mrr_at_1_max": 0.213753, + "nauc_mrr_at_1_std": -0.050502, + "nauc_mrr_at_1_diff1": -0.381364, + "nauc_mrr_at_3_max": 0.282553, + "nauc_mrr_at_3_std": -0.070962, + "nauc_mrr_at_3_diff1": -0.530361, + "nauc_mrr_at_5_max": 0.277991, + "nauc_mrr_at_5_std": -0.071874, + "nauc_mrr_at_5_diff1": -0.516401, + "nauc_mrr_at_10_max": 0.275373, + "nauc_mrr_at_10_std": -0.06829, + "nauc_mrr_at_10_diff1": -0.509904, + "nauc_mrr_at_20_max": 0.274423, + "nauc_mrr_at_20_std": -0.067326, + "nauc_mrr_at_20_diff1": -0.507842, + "nauc_mrr_at_100_max": 0.274298, + "nauc_mrr_at_100_std": -0.067031, + "nauc_mrr_at_100_diff1": -0.506871, + "nauc_mrr_at_1000_max": 0.274338, + "nauc_mrr_at_1000_std": -0.066969, + "nauc_mrr_at_1000_diff1": -0.506927, + "main_score": 0.42789, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "sql-Code" + ] + } + ] + }, + "evaluation_time": 307.2980921268463, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/WikiSQLRetrieval.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/WikiSQLRetrieval.json new file mode 100644 index 0000000000..6d083b1e4b --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/WikiSQLRetrieval.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "4e099ab42dffd49d72c1472f451371e53343e3d7", + "task_name": "WikiSQLRetrieval", + "mteb_version": "1.38.57", + "scores": { + "test": [ + { + "ndcg_at_1": 0.81055, + "ndcg_at_3": 0.86311, + "ndcg_at_5": 0.87179, + "ndcg_at_10": 0.87916, + "ndcg_at_20": 0.88446, + "ndcg_at_100": 0.88891, + "ndcg_at_1000": 0.89031, + "map_at_1": 0.81055, + "map_at_3": 0.85067, + "map_at_5": 0.8555, + "map_at_10": 0.85851, + "map_at_20": 0.85996, + "map_at_100": 0.86058, + "map_at_1000": 0.86062, + "recall_at_1": 0.81055, + "recall_at_3": 0.89893, + "recall_at_5": 0.91992, + "recall_at_10": 0.94287, + "recall_at_20": 0.96387, + "recall_at_100": 0.98779, + "recall_at_1000": 0.99902, + "precision_at_1": 0.81055, + "precision_at_3": 0.29964, + "precision_at_5": 0.18398, + "precision_at_10": 0.09429, + "precision_at_20": 0.04819, + "precision_at_100": 0.00988, + "precision_at_1000": 0.001, + "mrr_at_1": 0.80957, + "mrr_at_3": 0.850342, + "mrr_at_5": 0.855078, + "mrr_at_10": 0.858161, + "mrr_at_20": 0.85962, + "mrr_at_100": 0.860236, + "mrr_at_1000": 0.860282, + "nauc_ndcg_at_1_max": 0.625762, + "nauc_ndcg_at_1_std": -0.137364, + "nauc_ndcg_at_1_diff1": 0.892648, + "nauc_ndcg_at_3_max": 0.671948, + "nauc_ndcg_at_3_std": -0.150605, + "nauc_ndcg_at_3_diff1": 0.864835, + "nauc_ndcg_at_5_max": 0.674016, + "nauc_ndcg_at_5_std": -0.141507, + "nauc_ndcg_at_5_diff1": 0.867779, + "nauc_ndcg_at_10_max": 0.67466, + "nauc_ndcg_at_10_std": -0.133075, + "nauc_ndcg_at_10_diff1": 0.867992, + "nauc_ndcg_at_20_max": 0.673847, + "nauc_ndcg_at_20_std": -0.135426, + "nauc_ndcg_at_20_diff1": 0.870344, + "nauc_ndcg_at_100_max": 0.667499, + "nauc_ndcg_at_100_std": -0.136445, + "nauc_ndcg_at_100_diff1": 0.872442, + "nauc_ndcg_at_1000_max": 0.665777, + "nauc_ndcg_at_1000_std": -0.138498, + "nauc_ndcg_at_1000_diff1": 0.872842, + "nauc_map_at_1_max": 0.625762, + "nauc_map_at_1_std": -0.137364, + "nauc_map_at_1_diff1": 0.892648, + "nauc_map_at_3_max": 0.660333, + "nauc_map_at_3_std": -0.147837, + "nauc_map_at_3_diff1": 0.872806, + "nauc_map_at_5_max": 0.661078, + "nauc_map_at_5_std": -0.143153, + "nauc_map_at_5_diff1": 0.874572, + "nauc_map_at_10_max": 0.661095, + "nauc_map_at_10_std": -0.139852, + "nauc_map_at_10_diff1": 0.874827, + "nauc_map_at_20_max": 0.660779, + "nauc_map_at_20_std": -0.140521, + "nauc_map_at_20_diff1": 0.875455, + "nauc_map_at_100_max": 0.659977, + "nauc_map_at_100_std": -0.140753, + "nauc_map_at_100_diff1": 0.875707, + "nauc_map_at_1000_max": 0.659927, + "nauc_map_at_1000_std": -0.140802, + "nauc_map_at_1000_diff1": 0.87572, + "nauc_recall_at_1_max": 0.625762, + "nauc_recall_at_1_std": -0.137364, + "nauc_recall_at_1_diff1": 0.892648, + "nauc_recall_at_3_max": 0.718084, + "nauc_recall_at_3_std": -0.161208, + "nauc_recall_at_3_diff1": 0.832489, + "nauc_recall_at_5_max": 0.736974, + "nauc_recall_at_5_std": -0.130451, + "nauc_recall_at_5_diff1": 0.834881, + "nauc_recall_at_10_max": 0.765252, + "nauc_recall_at_10_std": -0.077338, + "nauc_recall_at_10_diff1": 0.822663, + "nauc_recall_at_20_max": 0.807812, + "nauc_recall_at_20_std": -0.071596, + "nauc_recall_at_20_diff1": 0.824534, + "nauc_recall_at_100_max": 0.77826, + "nauc_recall_at_100_std": 0.019299, + "nauc_recall_at_100_diff1": 0.833593, + "nauc_recall_at_1000_max": 0.561406, + "nauc_recall_at_1000_std": -0.075726, + "nauc_recall_at_1000_diff1": 0.711709, + "nauc_precision_at_1_max": 0.625762, + "nauc_precision_at_1_std": -0.137364, + "nauc_precision_at_1_diff1": 0.892648, + "nauc_precision_at_3_max": 0.718084, + "nauc_precision_at_3_std": -0.161208, + "nauc_precision_at_3_diff1": 0.832489, + "nauc_precision_at_5_max": 0.736974, + "nauc_precision_at_5_std": -0.130451, + "nauc_precision_at_5_diff1": 0.834881, + "nauc_precision_at_10_max": 0.765252, + "nauc_precision_at_10_std": -0.077338, + "nauc_precision_at_10_diff1": 0.822663, + "nauc_precision_at_20_max": 0.807812, + "nauc_precision_at_20_std": -0.071596, + "nauc_precision_at_20_diff1": 0.824534, + "nauc_precision_at_100_max": 0.77826, + "nauc_precision_at_100_std": 0.019299, + "nauc_precision_at_100_diff1": 0.833593, + "nauc_precision_at_1000_max": 0.561406, + "nauc_precision_at_1000_std": -0.075726, + "nauc_precision_at_1000_diff1": 0.711709, + "nauc_mrr_at_1_max": 0.619187, + "nauc_mrr_at_1_std": -0.141936, + "nauc_mrr_at_1_diff1": 0.893872, + "nauc_mrr_at_3_max": 0.657026, + "nauc_mrr_at_3_std": -0.149161, + "nauc_mrr_at_3_diff1": 0.873429, + "nauc_mrr_at_5_max": 0.6577, + "nauc_mrr_at_5_std": -0.145657, + "nauc_mrr_at_5_diff1": 0.875209, + "nauc_mrr_at_10_max": 0.657605, + "nauc_mrr_at_10_std": -0.141478, + "nauc_mrr_at_10_diff1": 0.875486, + "nauc_mrr_at_20_max": 0.657239, + "nauc_mrr_at_20_std": -0.142187, + "nauc_mrr_at_20_diff1": 0.876117, + "nauc_mrr_at_100_max": 0.656423, + "nauc_mrr_at_100_std": -0.142427, + "nauc_mrr_at_100_diff1": 0.87637, + "nauc_mrr_at_1000_max": 0.656372, + "nauc_mrr_at_1000_std": -0.142477, + "nauc_mrr_at_1000_diff1": 0.876384, + "main_score": 0.87916, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "sql-Code" + ] + } + ] + }, + "evaluation_time": 29.275694131851196, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/model_meta.json b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/model_meta.json new file mode 100644 index 0000000000..d3841941e2 --- /dev/null +++ b/results/fyaronskiy__english_code_retriever/be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c/model_meta.json @@ -0,0 +1 @@ +{"name": "fyaronskiy/english_code_retriever", "revision": "be653fab7d27a7348a0c2c3d16b9f92a7f10cb0c", "release_date": "2025-07-10", "languages": ["eng-Latn"], "n_parameters": 149000000, "memory_usage_mb": 568, "max_tokens": 8192, "embed_dim": 768, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": "https://huggingface.co/datasets/code-search-net/code_search_net", "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/fyaronskiy/english_code_retriever", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"CodeSearchNet": ["train"]}, "adapted_from": "answerdotai/ModernBERT-base", "superseded_by": null, "is_cross_encoder": false, "modalities": ["text"], "loader": "sentence_transformers_loader"} \ No newline at end of file