diff --git a/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/AppsRetrieval.json b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/AppsRetrieval.json new file mode 100644 index 0000000000..ed862ce613 --- /dev/null +++ b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/AppsRetrieval.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "f22508f96b7a36c2415181ed8bb76f76e04ae2d5", + "task_name": "AppsRetrieval", + "mteb_version": "1.34.21", + "scores": { + "test": [ + { + "ndcg_at_1": 0.31793, + "ndcg_at_3": 0.38992, + "ndcg_at_5": 0.41029, + "ndcg_at_10": 0.43265, + "ndcg_at_20": 0.45177, + "ndcg_at_100": 0.481, + "ndcg_at_1000": 0.5013, + "map_at_1": 0.31793, + "map_at_3": 0.37238, + "map_at_5": 0.38369, + "map_at_10": 0.39293, + "map_at_20": 0.39821, + "map_at_100": 0.40205, + "map_at_1000": 0.40276, + "recall_at_1": 0.31793, + "recall_at_3": 0.44064, + "recall_at_5": 0.49004, + "recall_at_10": 0.5591, + "recall_at_20": 0.63453, + "recall_at_100": 0.79495, + "recall_at_1000": 0.95777, + "precision_at_1": 0.31793, + "precision_at_3": 0.14688, + "precision_at_5": 0.09801, + "precision_at_10": 0.05591, + "precision_at_20": 0.03173, + "precision_at_100": 0.00795, + "precision_at_1000": 0.00096, + "mrr_at_1": 0.317928, + "mrr_at_3": 0.372377, + "mrr_at_5": 0.383692, + "mrr_at_10": 0.392931, + "mrr_at_20": 0.398211, + "mrr_at_100": 0.402053, + "mrr_at_1000": 0.402763, + "nauc_ndcg_at_1_max": 0.337556, + "nauc_ndcg_at_1_std": 0.172291, + "nauc_ndcg_at_1_diff1": 0.651338, + "nauc_ndcg_at_3_max": 0.328771, + "nauc_ndcg_at_3_std": 0.197452, + "nauc_ndcg_at_3_diff1": 0.58139, + "nauc_ndcg_at_5_max": 0.322644, + "nauc_ndcg_at_5_std": 0.20288, + "nauc_ndcg_at_5_diff1": 0.569157, + "nauc_ndcg_at_10_max": 0.32038, + "nauc_ndcg_at_10_std": 0.213871, + "nauc_ndcg_at_10_diff1": 0.565018, + "nauc_ndcg_at_20_max": 0.318862, + "nauc_ndcg_at_20_std": 0.215271, + "nauc_ndcg_at_20_diff1": 0.562118, + "nauc_ndcg_at_100_max": 0.318928, + "nauc_ndcg_at_100_std": 0.220729, + "nauc_ndcg_at_100_diff1": 0.5596, + "nauc_ndcg_at_1000_max": 0.320221, + "nauc_ndcg_at_1000_std": 0.212593, + "nauc_ndcg_at_1000_diff1": 0.569066, + "nauc_map_at_1_max": 0.337556, + "nauc_map_at_1_std": 0.172291, + "nauc_map_at_1_diff1": 0.651338, + "nauc_map_at_3_max": 0.329915, + "nauc_map_at_3_std": 0.19057, + "nauc_map_at_3_diff1": 0.597253, + "nauc_map_at_5_max": 0.326532, + "nauc_map_at_5_std": 0.193541, + "nauc_map_at_5_diff1": 0.59048, + "nauc_map_at_10_max": 0.325467, + "nauc_map_at_10_std": 0.198159, + "nauc_map_at_10_diff1": 0.588954, + "nauc_map_at_20_max": 0.32498, + "nauc_map_at_20_std": 0.198481, + "nauc_map_at_20_diff1": 0.588176, + "nauc_map_at_100_max": 0.324868, + "nauc_map_at_100_std": 0.199134, + "nauc_map_at_100_diff1": 0.587872, + "nauc_map_at_1000_max": 0.324874, + "nauc_map_at_1000_std": 0.198873, + "nauc_map_at_1000_diff1": 0.588186, + "nauc_recall_at_1_max": 0.337556, + "nauc_recall_at_1_std": 0.172291, + "nauc_recall_at_1_diff1": 0.651338, + "nauc_recall_at_3_max": 0.32589, + "nauc_recall_at_3_std": 0.217474, + "nauc_recall_at_3_diff1": 0.536393, + "nauc_recall_at_5_max": 0.31088, + "nauc_recall_at_5_std": 0.231087, + "nauc_recall_at_5_diff1": 0.50605, + "nauc_recall_at_10_max": 0.303671, + "nauc_recall_at_10_std": 0.267403, + "nauc_recall_at_10_diff1": 0.488432, + "nauc_recall_at_20_max": 0.295883, + "nauc_recall_at_20_std": 0.279345, + "nauc_recall_at_20_diff1": 0.468055, + "nauc_recall_at_100_max": 0.288607, + "nauc_recall_at_100_std": 0.361388, + "nauc_recall_at_100_diff1": 0.398347, + "nauc_recall_at_1000_max": 0.276225, + "nauc_recall_at_1000_std": 0.378295, + "nauc_recall_at_1000_diff1": 0.380408, + "nauc_precision_at_1_max": 0.337556, + "nauc_precision_at_1_std": 0.172291, + "nauc_precision_at_1_diff1": 0.651338, + "nauc_precision_at_3_max": 0.32589, + "nauc_precision_at_3_std": 0.217474, + "nauc_precision_at_3_diff1": 0.536393, + "nauc_precision_at_5_max": 0.31088, + "nauc_precision_at_5_std": 0.231087, + "nauc_precision_at_5_diff1": 0.50605, + "nauc_precision_at_10_max": 0.303671, + "nauc_precision_at_10_std": 0.267403, + "nauc_precision_at_10_diff1": 0.488432, + "nauc_precision_at_20_max": 0.295883, + "nauc_precision_at_20_std": 0.279345, + "nauc_precision_at_20_diff1": 0.468055, + "nauc_precision_at_100_max": 0.288607, + "nauc_precision_at_100_std": 0.361388, + "nauc_precision_at_100_diff1": 0.398347, + "nauc_precision_at_1000_max": 0.276225, + "nauc_precision_at_1000_std": 0.378295, + "nauc_precision_at_1000_diff1": 0.380408, + "nauc_mrr_at_1_max": 0.337556, + "nauc_mrr_at_1_std": 0.172142, + "nauc_mrr_at_1_diff1": 0.651338, + "nauc_mrr_at_3_max": 0.329915, + "nauc_mrr_at_3_std": 0.190497, + "nauc_mrr_at_3_diff1": 0.597253, + "nauc_mrr_at_5_max": 0.326532, + "nauc_mrr_at_5_std": 0.193467, + "nauc_mrr_at_5_diff1": 0.59048, + "nauc_mrr_at_10_max": 0.325467, + "nauc_mrr_at_10_std": 0.198085, + "nauc_mrr_at_10_diff1": 0.588954, + "nauc_mrr_at_20_max": 0.32498, + "nauc_mrr_at_20_std": 0.198406, + "nauc_mrr_at_20_diff1": 0.588176, + "nauc_mrr_at_100_max": 0.324868, + "nauc_mrr_at_100_std": 0.199058, + "nauc_mrr_at_100_diff1": 0.587872, + "nauc_mrr_at_1000_max": 0.324875, + "nauc_mrr_at_1000_std": 0.198798, + "nauc_mrr_at_1000_diff1": 0.588186, + "main_score": 0.43265, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 1138.5480706691742, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/COIRCodeSearchNetRetrieval.json b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/COIRCodeSearchNetRetrieval.json new file mode 100644 index 0000000000..845a398cd1 --- /dev/null +++ b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/COIRCodeSearchNetRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "4adc7bc41202b5c13543c9c886a25f340634dab3", + "task_name": "COIRCodeSearchNetRetrieval", + "mteb_version": "1.34.21", + "scores": { + "test": [ + { + "ndcg_at_1": 0.8906, + "ndcg_at_3": 0.92665, + "ndcg_at_5": 0.93132, + "ndcg_at_10": 0.93543, + "ndcg_at_20": 0.93733, + "ndcg_at_100": 0.93943, + "ndcg_at_1000": 0.94009, + "map_at_1": 0.8906, + "map_at_3": 0.91836, + "map_at_5": 0.92098, + "map_at_10": 0.92269, + "map_at_20": 0.92322, + "map_at_100": 0.92353, + "map_at_1000": 0.92355, + "recall_at_1": 0.8906, + "recall_at_3": 0.9504, + "recall_at_5": 0.96166, + "recall_at_10": 0.97426, + "recall_at_20": 0.9817, + "recall_at_100": 0.99276, + "recall_at_1000": 0.99792, + "precision_at_1": 0.8906, + "precision_at_3": 0.3168, + "precision_at_5": 0.19233, + "precision_at_10": 0.09743, + "precision_at_20": 0.04908, + "precision_at_100": 0.00993, + "precision_at_1000": 0.001, + "mrr_at_1": 0.890602, + "mrr_at_3": 0.918365, + "mrr_at_5": 0.920976, + "mrr_at_10": 0.922692, + "mrr_at_20": 0.92322, + "mrr_at_100": 0.923527, + "mrr_at_1000": 0.923553, + "nauc_ndcg_at_1_max": 0.808779, + "nauc_ndcg_at_1_std": -0.055702, + "nauc_ndcg_at_1_diff1": 0.907191, + "nauc_ndcg_at_3_max": 0.847697, + "nauc_ndcg_at_3_std": -0.054543, + "nauc_ndcg_at_3_diff1": 0.90674, + "nauc_ndcg_at_5_max": 0.845168, + "nauc_ndcg_at_5_std": -0.054652, + "nauc_ndcg_at_5_diff1": 0.907517, + "nauc_ndcg_at_10_max": 0.842944, + "nauc_ndcg_at_10_std": -0.045531, + "nauc_ndcg_at_10_diff1": 0.908864, + "nauc_ndcg_at_20_max": 0.841442, + "nauc_ndcg_at_20_std": -0.043424, + "nauc_ndcg_at_20_diff1": 0.908514, + "nauc_ndcg_at_100_max": 0.838914, + "nauc_ndcg_at_100_std": -0.042292, + "nauc_ndcg_at_100_diff1": 0.907963, + "nauc_ndcg_at_1000_max": 0.837946, + "nauc_ndcg_at_1000_std": -0.044745, + "nauc_ndcg_at_1000_diff1": 0.90779, + "nauc_map_at_1_max": 0.808779, + "nauc_map_at_1_std": -0.055702, + "nauc_map_at_1_diff1": 0.907191, + "nauc_map_at_3_max": 0.836731, + "nauc_map_at_3_std": -0.055872, + "nauc_map_at_3_diff1": 0.906762, + "nauc_map_at_5_max": 0.835122, + "nauc_map_at_5_std": -0.056071, + "nauc_map_at_5_diff1": 0.907119, + "nauc_map_at_10_max": 0.834147, + "nauc_map_at_10_std": -0.052844, + "nauc_map_at_10_diff1": 0.907576, + "nauc_map_at_20_max": 0.833746, + "nauc_map_at_20_std": -0.052403, + "nauc_map_at_20_diff1": 0.907511, + "nauc_map_at_100_max": 0.833435, + "nauc_map_at_100_std": -0.0522, + "nauc_map_at_100_diff1": 0.907443, + "nauc_map_at_1000_max": 0.833402, + "nauc_map_at_1000_std": -0.052259, + "nauc_map_at_1000_diff1": 0.907438, + "nauc_recall_at_1_max": 0.808779, + "nauc_recall_at_1_std": -0.055702, + "nauc_recall_at_1_diff1": 0.907191, + "nauc_recall_at_3_max": 0.898333, + "nauc_recall_at_3_std": -0.047635, + "nauc_recall_at_3_diff1": 0.90672, + "nauc_recall_at_5_max": 0.902476, + "nauc_recall_at_5_std": -0.045205, + "nauc_recall_at_5_diff1": 0.910228, + "nauc_recall_at_10_max": 0.913117, + "nauc_recall_at_10_std": 0.027952, + "nauc_recall_at_10_diff1": 0.921887, + "nauc_recall_at_20_max": 0.921585, + "nauc_recall_at_20_std": 0.085912, + "nauc_recall_at_20_diff1": 0.921773, + "nauc_recall_at_100_max": 0.931143, + "nauc_recall_at_100_std": 0.312715, + "nauc_recall_at_100_diff1": 0.918066, + "nauc_recall_at_1000_max": 0.945875, + "nauc_recall_at_1000_std": 0.612289, + "nauc_recall_at_1000_diff1": 0.902713, + "nauc_precision_at_1_max": 0.808779, + "nauc_precision_at_1_std": -0.055702, + "nauc_precision_at_1_diff1": 0.907191, + "nauc_precision_at_3_max": 0.898333, + "nauc_precision_at_3_std": -0.047635, + "nauc_precision_at_3_diff1": 0.90672, + "nauc_precision_at_5_max": 0.902476, + "nauc_precision_at_5_std": -0.045205, + "nauc_precision_at_5_diff1": 0.910228, + "nauc_precision_at_10_max": 0.913117, + "nauc_precision_at_10_std": 0.027952, + "nauc_precision_at_10_diff1": 0.921887, + "nauc_precision_at_20_max": 0.921585, + "nauc_precision_at_20_std": 0.085912, + "nauc_precision_at_20_diff1": 0.921773, + "nauc_precision_at_100_max": 0.931143, + "nauc_precision_at_100_std": 0.312715, + "nauc_precision_at_100_diff1": 0.918066, + "nauc_precision_at_1000_max": 0.945875, + "nauc_precision_at_1000_std": 0.612289, + "nauc_precision_at_1000_diff1": 0.902713, + "nauc_mrr_at_1_max": 0.808779, + "nauc_mrr_at_1_std": -0.055702, + "nauc_mrr_at_1_diff1": 0.907191, + "nauc_mrr_at_3_max": 0.836731, + "nauc_mrr_at_3_std": -0.055872, + "nauc_mrr_at_3_diff1": 0.906762, + "nauc_mrr_at_5_max": 0.835122, + "nauc_mrr_at_5_std": -0.056071, + "nauc_mrr_at_5_diff1": 0.907119, + "nauc_mrr_at_10_max": 0.834147, + "nauc_mrr_at_10_std": -0.052844, + "nauc_mrr_at_10_diff1": 0.907576, + "nauc_mrr_at_20_max": 0.833746, + "nauc_mrr_at_20_std": -0.052403, + "nauc_mrr_at_20_diff1": 0.907511, + "nauc_mrr_at_100_max": 0.833435, + "nauc_mrr_at_100_std": -0.0522, + "nauc_mrr_at_100_diff1": 0.907443, + "nauc_mrr_at_1000_max": 0.833402, + "nauc_mrr_at_1000_std": -0.052259, + "nauc_mrr_at_1000_diff1": 0.907438, + "main_score": 0.93543, + "hf_subset": "python", + "languages": [ + "python-Code" + ] + }, + { + "ndcg_at_1": 0.66545, + "ndcg_at_3": 0.73708, + "ndcg_at_5": 0.75256, + "ndcg_at_10": 0.76549, + "ndcg_at_20": 0.77279, + "ndcg_at_100": 0.77961, + "ndcg_at_1000": 0.7848, + "map_at_1": 0.66545, + "map_at_3": 0.71969, + "map_at_5": 0.72833, + "map_at_10": 0.73376, + "map_at_20": 0.73576, + "map_at_100": 0.7367, + "map_at_1000": 0.73689, + "recall_at_1": 0.66545, + "recall_at_3": 0.7873, + "recall_at_5": 0.82467, + "recall_at_10": 0.86418, + "recall_at_20": 0.89304, + "recall_at_100": 0.92981, + "recall_at_1000": 0.97113, + "precision_at_1": 0.66545, + "precision_at_3": 0.26243, + "precision_at_5": 0.16493, + "precision_at_10": 0.08642, + "precision_at_20": 0.04465, + "precision_at_100": 0.0093, + "precision_at_1000": 0.00097, + "mrr_at_1": 0.665451, + "mrr_at_3": 0.71969, + "mrr_at_5": 0.728335, + "mrr_at_10": 0.733759, + "mrr_at_20": 0.735763, + "mrr_at_100": 0.7367, + "mrr_at_1000": 0.736886, + "nauc_ndcg_at_1_max": 0.670476, + "nauc_ndcg_at_1_std": -0.002417, + "nauc_ndcg_at_1_diff1": 0.823632, + "nauc_ndcg_at_3_max": 0.694785, + "nauc_ndcg_at_3_std": 0.017195, + "nauc_ndcg_at_3_diff1": 0.782058, + "nauc_ndcg_at_5_max": 0.696655, + "nauc_ndcg_at_5_std": 0.022924, + "nauc_ndcg_at_5_diff1": 0.77801, + "nauc_ndcg_at_10_max": 0.695291, + "nauc_ndcg_at_10_std": 0.027653, + "nauc_ndcg_at_10_diff1": 0.777649, + "nauc_ndcg_at_20_max": 0.693198, + "nauc_ndcg_at_20_std": 0.030513, + "nauc_ndcg_at_20_diff1": 0.778024, + "nauc_ndcg_at_100_max": 0.690965, + "nauc_ndcg_at_100_std": 0.031426, + "nauc_ndcg_at_100_diff1": 0.779384, + "nauc_ndcg_at_1000_max": 0.690992, + "nauc_ndcg_at_1000_std": 0.027787, + "nauc_ndcg_at_1000_diff1": 0.781698, + "nauc_map_at_1_max": 0.670476, + "nauc_map_at_1_std": -0.002417, + "nauc_map_at_1_diff1": 0.823632, + "nauc_map_at_3_max": 0.689569, + "nauc_map_at_3_std": 0.011588, + "nauc_map_at_3_diff1": 0.792941, + "nauc_map_at_5_max": 0.690399, + "nauc_map_at_5_std": 0.014347, + "nauc_map_at_5_diff1": 0.79108, + "nauc_map_at_10_max": 0.689802, + "nauc_map_at_10_std": 0.016077, + "nauc_map_at_10_diff1": 0.791112, + "nauc_map_at_20_max": 0.689211, + "nauc_map_at_20_std": 0.016611, + "nauc_map_at_20_diff1": 0.791314, + "nauc_map_at_100_max": 0.688894, + "nauc_map_at_100_std": 0.016724, + "nauc_map_at_100_diff1": 0.791468, + "nauc_map_at_1000_max": 0.688901, + "nauc_map_at_1000_std": 0.016647, + "nauc_map_at_1000_diff1": 0.79155, + "nauc_recall_at_1_max": 0.670476, + "nauc_recall_at_1_std": -0.002417, + "nauc_recall_at_1_diff1": 0.823632, + "nauc_recall_at_3_max": 0.712588, + "nauc_recall_at_3_std": 0.037252, + "nauc_recall_at_3_diff1": 0.743627, + "nauc_recall_at_5_max": 0.721697, + "nauc_recall_at_5_std": 0.059345, + "nauc_recall_at_5_diff1": 0.724226, + "nauc_recall_at_10_max": 0.721309, + "nauc_recall_at_10_std": 0.090418, + "nauc_recall_at_10_diff1": 0.709345, + "nauc_recall_at_20_max": 0.712593, + "nauc_recall_at_20_std": 0.127566, + "nauc_recall_at_20_diff1": 0.695844, + "nauc_recall_at_100_max": 0.69106, + "nauc_recall_at_100_std": 0.18831, + "nauc_recall_at_100_diff1": 0.676564, + "nauc_recall_at_1000_max": 0.689488, + "nauc_recall_at_1000_std": 0.204263, + "nauc_recall_at_1000_diff1": 0.647992, + "nauc_precision_at_1_max": 0.670476, + "nauc_precision_at_1_std": -0.002417, + "nauc_precision_at_1_diff1": 0.823632, + "nauc_precision_at_3_max": 0.712588, + "nauc_precision_at_3_std": 0.037252, + "nauc_precision_at_3_diff1": 0.743627, + "nauc_precision_at_5_max": 0.721697, + "nauc_precision_at_5_std": 0.059345, + "nauc_precision_at_5_diff1": 0.724226, + "nauc_precision_at_10_max": 0.721309, + "nauc_precision_at_10_std": 0.090418, + "nauc_precision_at_10_diff1": 0.709345, + "nauc_precision_at_20_max": 0.712593, + "nauc_precision_at_20_std": 0.127566, + "nauc_precision_at_20_diff1": 0.695844, + "nauc_precision_at_100_max": 0.69106, + "nauc_precision_at_100_std": 0.18831, + "nauc_precision_at_100_diff1": 0.676564, + "nauc_precision_at_1000_max": 0.689488, + "nauc_precision_at_1000_std": 0.204263, + "nauc_precision_at_1000_diff1": 0.647992, + "nauc_mrr_at_1_max": 0.670476, + "nauc_mrr_at_1_std": -0.002417, + "nauc_mrr_at_1_diff1": 0.823632, + "nauc_mrr_at_3_max": 0.689569, + "nauc_mrr_at_3_std": 0.011588, + "nauc_mrr_at_3_diff1": 0.792941, + "nauc_mrr_at_5_max": 0.690399, + "nauc_mrr_at_5_std": 0.014347, + "nauc_mrr_at_5_diff1": 0.79108, + "nauc_mrr_at_10_max": 0.689802, + "nauc_mrr_at_10_std": 0.016077, + "nauc_mrr_at_10_diff1": 0.791112, + "nauc_mrr_at_20_max": 0.689211, + "nauc_mrr_at_20_std": 0.016611, + "nauc_mrr_at_20_diff1": 0.791314, + "nauc_mrr_at_100_max": 0.688894, + "nauc_mrr_at_100_std": 0.016724, + "nauc_mrr_at_100_diff1": 0.791468, + "nauc_mrr_at_1000_max": 0.688901, + "nauc_mrr_at_1000_std": 0.016647, + "nauc_mrr_at_1000_diff1": 0.79155, + "main_score": 0.76549, + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ] + }, + { + "ndcg_at_1": 0.72236, + "ndcg_at_3": 0.80599, + "ndcg_at_5": 0.81985, + "ndcg_at_10": 0.83093, + "ndcg_at_20": 0.83654, + "ndcg_at_100": 0.84193, + "ndcg_at_1000": 0.84371, + "map_at_1": 0.72236, + "map_at_3": 0.78579, + "map_at_5": 0.79349, + "map_at_10": 0.79811, + "map_at_20": 0.79967, + "map_at_100": 0.80044, + "map_at_1000": 0.80051, + "recall_at_1": 0.72236, + "recall_at_3": 0.86432, + "recall_at_5": 0.89793, + "recall_at_10": 0.93191, + "recall_at_20": 0.95395, + "recall_at_100": 0.98264, + "recall_at_1000": 0.99668, + "precision_at_1": 0.72236, + "precision_at_3": 0.28811, + "precision_at_5": 0.17959, + "precision_at_10": 0.09319, + "precision_at_20": 0.0477, + "precision_at_100": 0.00983, + "precision_at_1000": 0.001, + "mrr_at_1": 0.722359, + "mrr_at_3": 0.785788, + "mrr_at_5": 0.793489, + "mrr_at_10": 0.798111, + "mrr_at_20": 0.799671, + "mrr_at_100": 0.80044, + "mrr_at_1000": 0.800506, + "nauc_ndcg_at_1_max": 0.490147, + "nauc_ndcg_at_1_std": -0.245809, + "nauc_ndcg_at_1_diff1": 0.79224, + "nauc_ndcg_at_3_max": 0.525008, + "nauc_ndcg_at_3_std": -0.273216, + "nauc_ndcg_at_3_diff1": 0.763052, + "nauc_ndcg_at_5_max": 0.519512, + "nauc_ndcg_at_5_std": -0.277969, + "nauc_ndcg_at_5_diff1": 0.761273, + "nauc_ndcg_at_10_max": 0.519017, + "nauc_ndcg_at_10_std": -0.269299, + "nauc_ndcg_at_10_diff1": 0.765428, + "nauc_ndcg_at_20_max": 0.518372, + "nauc_ndcg_at_20_std": -0.266056, + "nauc_ndcg_at_20_diff1": 0.768941, + "nauc_ndcg_at_100_max": 0.516306, + "nauc_ndcg_at_100_std": -0.263055, + "nauc_ndcg_at_100_diff1": 0.769935, + "nauc_ndcg_at_1000_max": 0.515665, + "nauc_ndcg_at_1000_std": -0.263292, + "nauc_ndcg_at_1000_diff1": 0.770369, + "nauc_map_at_1_max": 0.490147, + "nauc_map_at_1_std": -0.245809, + "nauc_map_at_1_diff1": 0.79224, + "nauc_map_at_3_max": 0.514823, + "nauc_map_at_3_std": -0.265655, + "nauc_map_at_3_diff1": 0.771022, + "nauc_map_at_5_max": 0.511538, + "nauc_map_at_5_std": -0.267939, + "nauc_map_at_5_diff1": 0.770414, + "nauc_map_at_10_max": 0.511166, + "nauc_map_at_10_std": -0.264718, + "nauc_map_at_10_diff1": 0.772174, + "nauc_map_at_20_max": 0.511034, + "nauc_map_at_20_std": -0.263894, + "nauc_map_at_20_diff1": 0.773033, + "nauc_map_at_100_max": 0.510753, + "nauc_map_at_100_std": -0.263543, + "nauc_map_at_100_diff1": 0.773124, + "nauc_map_at_1000_max": 0.510741, + "nauc_map_at_1000_std": -0.263546, + "nauc_map_at_1000_diff1": 0.773132, + "nauc_recall_at_1_max": 0.490147, + "nauc_recall_at_1_std": -0.245809, + "nauc_recall_at_1_diff1": 0.79224, + "nauc_recall_at_3_max": 0.568203, + "nauc_recall_at_3_std": -0.305006, + "nauc_recall_at_3_diff1": 0.7296, + "nauc_recall_at_5_max": 0.560595, + "nauc_recall_at_5_std": -0.331214, + "nauc_recall_at_5_diff1": 0.713122, + "nauc_recall_at_10_max": 0.577462, + "nauc_recall_at_10_std": -0.295759, + "nauc_recall_at_10_diff1": 0.717359, + "nauc_recall_at_20_max": 0.594781, + "nauc_recall_at_20_std": -0.268569, + "nauc_recall_at_20_diff1": 0.73908, + "nauc_recall_at_100_max": 0.631419, + "nauc_recall_at_100_std": -0.141908, + "nauc_recall_at_100_diff1": 0.739117, + "nauc_recall_at_1000_max": 0.87741, + "nauc_recall_at_1000_std": 0.276417, + "nauc_recall_at_1000_diff1": 0.771349, + "nauc_precision_at_1_max": 0.490147, + "nauc_precision_at_1_std": -0.245809, + "nauc_precision_at_1_diff1": 0.79224, + "nauc_precision_at_3_max": 0.568203, + "nauc_precision_at_3_std": -0.305006, + "nauc_precision_at_3_diff1": 0.7296, + "nauc_precision_at_5_max": 0.560595, + "nauc_precision_at_5_std": -0.331214, + "nauc_precision_at_5_diff1": 0.713122, + "nauc_precision_at_10_max": 0.577462, + "nauc_precision_at_10_std": -0.295759, + "nauc_precision_at_10_diff1": 0.717359, + "nauc_precision_at_20_max": 0.594781, + "nauc_precision_at_20_std": -0.268569, + "nauc_precision_at_20_diff1": 0.73908, + "nauc_precision_at_100_max": 0.631419, + "nauc_precision_at_100_std": -0.141908, + "nauc_precision_at_100_diff1": 0.739117, + "nauc_precision_at_1000_max": 0.87741, + "nauc_precision_at_1000_std": 0.276417, + "nauc_precision_at_1000_diff1": 0.771349, + "nauc_mrr_at_1_max": 0.490147, + "nauc_mrr_at_1_std": -0.245809, + "nauc_mrr_at_1_diff1": 0.79224, + "nauc_mrr_at_3_max": 0.514823, + "nauc_mrr_at_3_std": -0.265655, + "nauc_mrr_at_3_diff1": 0.771022, + "nauc_mrr_at_5_max": 0.511538, + "nauc_mrr_at_5_std": -0.267939, + "nauc_mrr_at_5_diff1": 0.770414, + "nauc_mrr_at_10_max": 0.511166, + "nauc_mrr_at_10_std": -0.264718, + "nauc_mrr_at_10_diff1": 0.772174, + "nauc_mrr_at_20_max": 0.511034, + "nauc_mrr_at_20_std": -0.263894, + "nauc_mrr_at_20_diff1": 0.773033, + "nauc_mrr_at_100_max": 0.510753, + "nauc_mrr_at_100_std": -0.263543, + "nauc_mrr_at_100_diff1": 0.773124, + "nauc_mrr_at_1000_max": 0.510741, + "nauc_mrr_at_1000_std": -0.263546, + "nauc_mrr_at_1000_diff1": 0.773132, + "main_score": 0.83093, + "hf_subset": "go", + "languages": [ + "go-Code" + ] + }, + { + "ndcg_at_1": 0.68517, + "ndcg_at_3": 0.76174, + "ndcg_at_5": 0.7772, + "ndcg_at_10": 0.79075, + "ndcg_at_20": 0.79855, + "ndcg_at_100": 0.80567, + "ndcg_at_1000": 0.80858, + "map_at_1": 0.68517, + "map_at_3": 0.74346, + "map_at_5": 0.7521, + "map_at_10": 0.75782, + "map_at_20": 0.75996, + "map_at_100": 0.76101, + "map_at_1000": 0.76111, + "recall_at_1": 0.68517, + "recall_at_3": 0.81443, + "recall_at_5": 0.8517, + "recall_at_10": 0.89294, + "recall_at_20": 0.92387, + "recall_at_100": 0.96114, + "recall_at_1000": 0.98493, + "precision_at_1": 0.68517, + "precision_at_3": 0.27148, + "precision_at_5": 0.17034, + "precision_at_10": 0.08929, + "precision_at_20": 0.04619, + "precision_at_100": 0.00961, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.68517, + "mrr_at_3": 0.743458, + "mrr_at_5": 0.752102, + "mrr_at_10": 0.757821, + "mrr_at_20": 0.759955, + "mrr_at_100": 0.761011, + "mrr_at_1000": 0.761106, + "nauc_ndcg_at_1_max": 0.651629, + "nauc_ndcg_at_1_std": -0.095467, + "nauc_ndcg_at_1_diff1": 0.785588, + "nauc_ndcg_at_3_max": 0.670352, + "nauc_ndcg_at_3_std": -0.091795, + "nauc_ndcg_at_3_diff1": 0.742951, + "nauc_ndcg_at_5_max": 0.677507, + "nauc_ndcg_at_5_std": -0.080659, + "nauc_ndcg_at_5_diff1": 0.744517, + "nauc_ndcg_at_10_max": 0.678871, + "nauc_ndcg_at_10_std": -0.055663, + "nauc_ndcg_at_10_diff1": 0.741, + "nauc_ndcg_at_20_max": 0.676776, + "nauc_ndcg_at_20_std": -0.058669, + "nauc_ndcg_at_20_diff1": 0.742612, + "nauc_ndcg_at_100_max": 0.674707, + "nauc_ndcg_at_100_std": -0.062395, + "nauc_ndcg_at_100_diff1": 0.744752, + "nauc_ndcg_at_1000_max": 0.674848, + "nauc_ndcg_at_1000_std": -0.065995, + "nauc_ndcg_at_1000_diff1": 0.748661, + "nauc_map_at_1_max": 0.651629, + "nauc_map_at_1_std": -0.095467, + "nauc_map_at_1_diff1": 0.785588, + "nauc_map_at_3_max": 0.666079, + "nauc_map_at_3_std": -0.093015, + "nauc_map_at_3_diff1": 0.754526, + "nauc_map_at_5_max": 0.669938, + "nauc_map_at_5_std": -0.087197, + "nauc_map_at_5_diff1": 0.755696, + "nauc_map_at_10_max": 0.670322, + "nauc_map_at_10_std": -0.077915, + "nauc_map_at_10_diff1": 0.754696, + "nauc_map_at_20_max": 0.669765, + "nauc_map_at_20_std": -0.078746, + "nauc_map_at_20_diff1": 0.755218, + "nauc_map_at_100_max": 0.669478, + "nauc_map_at_100_std": -0.079323, + "nauc_map_at_100_diff1": 0.755494, + "nauc_map_at_1000_max": 0.669482, + "nauc_map_at_1000_std": -0.079342, + "nauc_map_at_1000_diff1": 0.755594, + "nauc_recall_at_1_max": 0.651629, + "nauc_recall_at_1_std": -0.095467, + "nauc_recall_at_1_diff1": 0.785588, + "nauc_recall_at_3_max": 0.685744, + "nauc_recall_at_3_std": -0.087144, + "nauc_recall_at_3_diff1": 0.70002, + "nauc_recall_at_5_max": 0.711173, + "nauc_recall_at_5_std": -0.049584, + "nauc_recall_at_5_diff1": 0.695693, + "nauc_recall_at_10_max": 0.729297, + "nauc_recall_at_10_std": 0.093301, + "nauc_recall_at_10_diff1": 0.659877, + "nauc_recall_at_20_max": 0.72914, + "nauc_recall_at_20_std": 0.122176, + "nauc_recall_at_20_diff1": 0.642652, + "nauc_recall_at_100_max": 0.730793, + "nauc_recall_at_100_std": 0.213856, + "nauc_recall_at_100_diff1": 0.599584, + "nauc_recall_at_1000_max": 0.83074, + "nauc_recall_at_1000_std": 0.258705, + "nauc_recall_at_1000_diff1": 0.7308, + "nauc_precision_at_1_max": 0.651629, + "nauc_precision_at_1_std": -0.095467, + "nauc_precision_at_1_diff1": 0.785588, + "nauc_precision_at_3_max": 0.685744, + "nauc_precision_at_3_std": -0.087144, + "nauc_precision_at_3_diff1": 0.70002, + "nauc_precision_at_5_max": 0.711173, + "nauc_precision_at_5_std": -0.049584, + "nauc_precision_at_5_diff1": 0.695693, + "nauc_precision_at_10_max": 0.729297, + "nauc_precision_at_10_std": 0.093301, + "nauc_precision_at_10_diff1": 0.659877, + "nauc_precision_at_20_max": 0.72914, + "nauc_precision_at_20_std": 0.122176, + "nauc_precision_at_20_diff1": 0.642652, + "nauc_precision_at_100_max": 0.730793, + "nauc_precision_at_100_std": 0.213856, + "nauc_precision_at_100_diff1": 0.599584, + "nauc_precision_at_1000_max": 0.83074, + "nauc_precision_at_1000_std": 0.258705, + "nauc_precision_at_1000_diff1": 0.7308, + "nauc_mrr_at_1_max": 0.651629, + "nauc_mrr_at_1_std": -0.095467, + "nauc_mrr_at_1_diff1": 0.785588, + "nauc_mrr_at_3_max": 0.666079, + "nauc_mrr_at_3_std": -0.093015, + "nauc_mrr_at_3_diff1": 0.754526, + "nauc_mrr_at_5_max": 0.669938, + "nauc_mrr_at_5_std": -0.087197, + "nauc_mrr_at_5_diff1": 0.755696, + "nauc_mrr_at_10_max": 0.670322, + "nauc_mrr_at_10_std": -0.077915, + "nauc_mrr_at_10_diff1": 0.754696, + "nauc_mrr_at_20_max": 0.669765, + "nauc_mrr_at_20_std": -0.078746, + "nauc_mrr_at_20_diff1": 0.755218, + "nauc_mrr_at_100_max": 0.669478, + "nauc_mrr_at_100_std": -0.079323, + "nauc_mrr_at_100_diff1": 0.755494, + "nauc_mrr_at_1000_max": 0.669482, + "nauc_mrr_at_1000_std": -0.079342, + "nauc_mrr_at_1000_diff1": 0.755594, + "main_score": 0.79075, + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ] + }, + { + "ndcg_at_1": 0.74094, + "ndcg_at_3": 0.81086, + "ndcg_at_5": 0.82187, + "ndcg_at_10": 0.82965, + "ndcg_at_20": 0.83508, + "ndcg_at_100": 0.84097, + "ndcg_at_1000": 0.84346, + "map_at_1": 0.74094, + "map_at_3": 0.79449, + "map_at_5": 0.80062, + "map_at_10": 0.80388, + "map_at_20": 0.80537, + "map_at_100": 0.80622, + "map_at_1000": 0.8063, + "recall_at_1": 0.74094, + "recall_at_3": 0.85787, + "recall_at_5": 0.88453, + "recall_at_10": 0.90835, + "recall_at_20": 0.9298, + "recall_at_100": 0.96111, + "recall_at_1000": 0.9811, + "precision_at_1": 0.74094, + "precision_at_3": 0.28596, + "precision_at_5": 0.17691, + "precision_at_10": 0.09084, + "precision_at_20": 0.04649, + "precision_at_100": 0.00961, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.74094, + "mrr_at_3": 0.794493, + "mrr_at_5": 0.800622, + "mrr_at_10": 0.803881, + "mrr_at_20": 0.805374, + "mrr_at_100": 0.806215, + "mrr_at_1000": 0.806304, + "nauc_ndcg_at_1_max": 0.608677, + "nauc_ndcg_at_1_std": -0.162077, + "nauc_ndcg_at_1_diff1": 0.819165, + "nauc_ndcg_at_3_max": 0.663931, + "nauc_ndcg_at_3_std": -0.127495, + "nauc_ndcg_at_3_diff1": 0.786928, + "nauc_ndcg_at_5_max": 0.665572, + "nauc_ndcg_at_5_std": -0.119295, + "nauc_ndcg_at_5_diff1": 0.787295, + "nauc_ndcg_at_10_max": 0.665975, + "nauc_ndcg_at_10_std": -0.104875, + "nauc_ndcg_at_10_diff1": 0.787229, + "nauc_ndcg_at_20_max": 0.664858, + "nauc_ndcg_at_20_std": -0.102161, + "nauc_ndcg_at_20_diff1": 0.789013, + "nauc_ndcg_at_100_max": 0.661962, + "nauc_ndcg_at_100_std": -0.101444, + "nauc_ndcg_at_100_diff1": 0.790804, + "nauc_ndcg_at_1000_max": 0.659618, + "nauc_ndcg_at_1000_std": -0.106734, + "nauc_ndcg_at_1000_diff1": 0.791968, + "nauc_map_at_1_max": 0.608677, + "nauc_map_at_1_std": -0.162077, + "nauc_map_at_1_diff1": 0.819165, + "nauc_map_at_3_max": 0.648956, + "nauc_map_at_3_std": -0.138049, + "nauc_map_at_3_diff1": 0.795599, + "nauc_map_at_5_max": 0.64928, + "nauc_map_at_5_std": -0.134446, + "nauc_map_at_5_diff1": 0.796046, + "nauc_map_at_10_max": 0.649174, + "nauc_map_at_10_std": -0.129201, + "nauc_map_at_10_diff1": 0.796122, + "nauc_map_at_20_max": 0.648792, + "nauc_map_at_20_std": -0.128794, + "nauc_map_at_20_diff1": 0.796642, + "nauc_map_at_100_max": 0.648402, + "nauc_map_at_100_std": -0.128849, + "nauc_map_at_100_diff1": 0.79692, + "nauc_map_at_1000_max": 0.64833, + "nauc_map_at_1000_std": -0.129003, + "nauc_map_at_1000_diff1": 0.796961, + "nauc_recall_at_1_max": 0.608677, + "nauc_recall_at_1_std": -0.162077, + "nauc_recall_at_1_diff1": 0.819165, + "nauc_recall_at_3_max": 0.722436, + "nauc_recall_at_3_std": -0.085535, + "nauc_recall_at_3_diff1": 0.753095, + "nauc_recall_at_5_max": 0.740571, + "nauc_recall_at_5_std": -0.046701, + "nauc_recall_at_5_diff1": 0.747417, + "nauc_recall_at_10_max": 0.760361, + "nauc_recall_at_10_std": 0.044542, + "nauc_recall_at_10_diff1": 0.738037, + "nauc_recall_at_20_max": 0.779197, + "nauc_recall_at_20_std": 0.11383, + "nauc_recall_at_20_diff1": 0.737844, + "nauc_recall_at_100_max": 0.811471, + "nauc_recall_at_100_std": 0.303968, + "nauc_recall_at_100_diff1": 0.730533, + "nauc_recall_at_1000_max": 0.825259, + "nauc_recall_at_1000_std": 0.402445, + "nauc_recall_at_1000_diff1": 0.73657, + "nauc_precision_at_1_max": 0.608677, + "nauc_precision_at_1_std": -0.162077, + "nauc_precision_at_1_diff1": 0.819165, + "nauc_precision_at_3_max": 0.722436, + "nauc_precision_at_3_std": -0.085535, + "nauc_precision_at_3_diff1": 0.753095, + "nauc_precision_at_5_max": 0.740571, + "nauc_precision_at_5_std": -0.046701, + "nauc_precision_at_5_diff1": 0.747417, + "nauc_precision_at_10_max": 0.760361, + "nauc_precision_at_10_std": 0.044542, + "nauc_precision_at_10_diff1": 0.738037, + "nauc_precision_at_20_max": 0.779197, + "nauc_precision_at_20_std": 0.11383, + "nauc_precision_at_20_diff1": 0.737844, + "nauc_precision_at_100_max": 0.811471, + "nauc_precision_at_100_std": 0.303968, + "nauc_precision_at_100_diff1": 0.730533, + "nauc_precision_at_1000_max": 0.825259, + "nauc_precision_at_1000_std": 0.402445, + "nauc_precision_at_1000_diff1": 0.73657, + "nauc_mrr_at_1_max": 0.608677, + "nauc_mrr_at_1_std": -0.162077, + "nauc_mrr_at_1_diff1": 0.819165, + "nauc_mrr_at_3_max": 0.648956, + "nauc_mrr_at_3_std": -0.138049, + "nauc_mrr_at_3_diff1": 0.795599, + "nauc_mrr_at_5_max": 0.64928, + "nauc_mrr_at_5_std": -0.134446, + "nauc_mrr_at_5_diff1": 0.796046, + "nauc_mrr_at_10_max": 0.649174, + "nauc_mrr_at_10_std": -0.129201, + "nauc_mrr_at_10_diff1": 0.796122, + "nauc_mrr_at_20_max": 0.648792, + "nauc_mrr_at_20_std": -0.128794, + "nauc_mrr_at_20_diff1": 0.796642, + "nauc_mrr_at_100_max": 0.648402, + "nauc_mrr_at_100_std": -0.128849, + "nauc_mrr_at_100_diff1": 0.79692, + "nauc_mrr_at_1000_max": 0.64833, + "nauc_mrr_at_1000_std": -0.129003, + "nauc_mrr_at_1000_diff1": 0.796961, + "main_score": 0.82965, + "hf_subset": "java", + "languages": [ + "java-Code" + ] + }, + { + "ndcg_at_1": 0.7551, + "ndcg_at_3": 0.82463, + "ndcg_at_5": 0.83657, + "ndcg_at_10": 0.84596, + "ndcg_at_20": 0.85141, + "ndcg_at_100": 0.8574, + "ndcg_at_1000": 0.85908, + "map_at_1": 0.7551, + "map_at_3": 0.80831, + "map_at_5": 0.81495, + "map_at_10": 0.81886, + "map_at_20": 0.82038, + "map_at_100": 0.82126, + "map_at_1000": 0.82132, + "recall_at_1": 0.7551, + "recall_at_3": 0.87156, + "recall_at_5": 0.90046, + "recall_at_10": 0.92936, + "recall_at_20": 0.95069, + "recall_at_100": 0.98223, + "recall_at_1000": 0.99536, + "precision_at_1": 0.7551, + "precision_at_3": 0.29052, + "precision_at_5": 0.18009, + "precision_at_10": 0.09294, + "precision_at_20": 0.04753, + "precision_at_100": 0.00982, + "precision_at_1000": 0.001, + "mrr_at_1": 0.755102, + "mrr_at_3": 0.808311, + "mrr_at_5": 0.814954, + "mrr_at_10": 0.818857, + "mrr_at_20": 0.820377, + "mrr_at_100": 0.821257, + "mrr_at_1000": 0.821322, + "nauc_ndcg_at_1_max": 0.519388, + "nauc_ndcg_at_1_std": -0.270332, + "nauc_ndcg_at_1_diff1": 0.827686, + "nauc_ndcg_at_3_max": 0.573023, + "nauc_ndcg_at_3_std": -0.278932, + "nauc_ndcg_at_3_diff1": 0.799886, + "nauc_ndcg_at_5_max": 0.572875, + "nauc_ndcg_at_5_std": -0.270138, + "nauc_ndcg_at_5_diff1": 0.796647, + "nauc_ndcg_at_10_max": 0.573283, + "nauc_ndcg_at_10_std": -0.262499, + "nauc_ndcg_at_10_diff1": 0.79971, + "nauc_ndcg_at_20_max": 0.57078, + "nauc_ndcg_at_20_std": -0.256423, + "nauc_ndcg_at_20_diff1": 0.800667, + "nauc_ndcg_at_100_max": 0.566407, + "nauc_ndcg_at_100_std": -0.257377, + "nauc_ndcg_at_100_diff1": 0.802602, + "nauc_ndcg_at_1000_max": 0.564491, + "nauc_ndcg_at_1000_std": -0.260897, + "nauc_ndcg_at_1000_diff1": 0.803835, + "nauc_map_at_1_max": 0.519388, + "nauc_map_at_1_std": -0.270332, + "nauc_map_at_1_diff1": 0.827686, + "nauc_map_at_3_max": 0.558059, + "nauc_map_at_3_std": -0.277033, + "nauc_map_at_3_diff1": 0.807151, + "nauc_map_at_5_max": 0.557523, + "nauc_map_at_5_std": -0.27277, + "nauc_map_at_5_diff1": 0.805842, + "nauc_map_at_10_max": 0.5573, + "nauc_map_at_10_std": -0.27001, + "nauc_map_at_10_diff1": 0.807143, + "nauc_map_at_20_max": 0.55661, + "nauc_map_at_20_std": -0.268584, + "nauc_map_at_20_diff1": 0.807413, + "nauc_map_at_100_max": 0.556054, + "nauc_map_at_100_std": -0.268734, + "nauc_map_at_100_diff1": 0.807678, + "nauc_map_at_1000_max": 0.555996, + "nauc_map_at_1000_std": -0.268826, + "nauc_map_at_1000_diff1": 0.80772, + "nauc_recall_at_1_max": 0.519388, + "nauc_recall_at_1_std": -0.270332, + "nauc_recall_at_1_diff1": 0.827686, + "nauc_recall_at_3_max": 0.633442, + "nauc_recall_at_3_std": -0.286286, + "nauc_recall_at_3_diff1": 0.770863, + "nauc_recall_at_5_max": 0.647852, + "nauc_recall_at_5_std": -0.254202, + "nauc_recall_at_5_diff1": 0.751077, + "nauc_recall_at_10_max": 0.681202, + "nauc_recall_at_10_std": -0.201141, + "nauc_recall_at_10_diff1": 0.751626, + "nauc_recall_at_20_max": 0.699454, + "nauc_recall_at_20_std": -0.110043, + "nauc_recall_at_20_diff1": 0.741838, + "nauc_recall_at_100_max": 0.7526, + "nauc_recall_at_100_std": 0.109886, + "nauc_recall_at_100_diff1": 0.71229, + "nauc_recall_at_1000_max": 0.84951, + "nauc_recall_at_1000_std": 0.341655, + "nauc_recall_at_1000_diff1": 0.723143, + "nauc_precision_at_1_max": 0.519388, + "nauc_precision_at_1_std": -0.270332, + "nauc_precision_at_1_diff1": 0.827686, + "nauc_precision_at_3_max": 0.633442, + "nauc_precision_at_3_std": -0.286286, + "nauc_precision_at_3_diff1": 0.770863, + "nauc_precision_at_5_max": 0.647852, + "nauc_precision_at_5_std": -0.254202, + "nauc_precision_at_5_diff1": 0.751077, + "nauc_precision_at_10_max": 0.681202, + "nauc_precision_at_10_std": -0.201141, + "nauc_precision_at_10_diff1": 0.751626, + "nauc_precision_at_20_max": 0.699454, + "nauc_precision_at_20_std": -0.110043, + "nauc_precision_at_20_diff1": 0.741838, + "nauc_precision_at_100_max": 0.7526, + "nauc_precision_at_100_std": 0.109886, + "nauc_precision_at_100_diff1": 0.71229, + "nauc_precision_at_1000_max": 0.84951, + "nauc_precision_at_1000_std": 0.341655, + "nauc_precision_at_1000_diff1": 0.723143, + "nauc_mrr_at_1_max": 0.519388, + "nauc_mrr_at_1_std": -0.270332, + "nauc_mrr_at_1_diff1": 0.827686, + "nauc_mrr_at_3_max": 0.558059, + "nauc_mrr_at_3_std": -0.277033, + "nauc_mrr_at_3_diff1": 0.807151, + "nauc_mrr_at_5_max": 0.557523, + "nauc_mrr_at_5_std": -0.27277, + "nauc_mrr_at_5_diff1": 0.805842, + "nauc_mrr_at_10_max": 0.5573, + "nauc_mrr_at_10_std": -0.27001, + "nauc_mrr_at_10_diff1": 0.807143, + "nauc_mrr_at_20_max": 0.55661, + "nauc_mrr_at_20_std": -0.268584, + "nauc_mrr_at_20_diff1": 0.807413, + "nauc_mrr_at_100_max": 0.556054, + "nauc_mrr_at_100_std": -0.268734, + "nauc_mrr_at_100_diff1": 0.807678, + "nauc_mrr_at_1000_max": 0.555996, + "nauc_mrr_at_1000_std": -0.268826, + "nauc_mrr_at_1000_diff1": 0.80772, + "main_score": 0.84596, + "hf_subset": "php", + "languages": [ + "php-Code" + ] + } + ] + }, + "evaluation_time": 12403.135345935822, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeFeedbackMT.json b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeFeedbackMT.json new file mode 100644 index 0000000000..0c79661643 --- /dev/null +++ b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeFeedbackMT.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b0f12fa0c0dd67f59c95a5c33d02aeeb4c398c5f", + "task_name": "CodeFeedbackMT", + "mteb_version": "1.34.21", + "scores": { + "test": [ + { + "ndcg_at_1": 0.63584, + "ndcg_at_3": 0.71613, + "ndcg_at_5": 0.73245, + "ndcg_at_10": 0.74519, + "ndcg_at_20": 0.75282, + "ndcg_at_100": 0.76368, + "ndcg_at_1000": 0.76918, + "map_at_1": 0.63584, + "map_at_3": 0.69686, + "map_at_5": 0.70593, + "map_at_10": 0.71122, + "map_at_20": 0.71332, + "map_at_100": 0.71483, + "map_at_1000": 0.71503, + "recall_at_1": 0.63584, + "recall_at_3": 0.77171, + "recall_at_5": 0.81125, + "recall_at_10": 0.85049, + "recall_at_20": 0.88062, + "recall_at_100": 0.93907, + "recall_at_1000": 0.98275, + "precision_at_1": 0.63584, + "precision_at_3": 0.25724, + "precision_at_5": 0.16225, + "precision_at_10": 0.08505, + "precision_at_20": 0.04403, + "precision_at_100": 0.00939, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.635836, + "mrr_at_3": 0.696857, + "mrr_at_5": 0.705933, + "mrr_at_10": 0.711218, + "mrr_at_20": 0.713316, + "mrr_at_100": 0.714829, + "mrr_at_1000": 0.715029, + "nauc_ndcg_at_1_max": 0.391852, + "nauc_ndcg_at_1_std": -0.317015, + "nauc_ndcg_at_1_diff1": 0.733826, + "nauc_ndcg_at_3_max": 0.440243, + "nauc_ndcg_at_3_std": -0.346498, + "nauc_ndcg_at_3_diff1": 0.6986, + "nauc_ndcg_at_5_max": 0.438207, + "nauc_ndcg_at_5_std": -0.350201, + "nauc_ndcg_at_5_diff1": 0.690297, + "nauc_ndcg_at_10_max": 0.438182, + "nauc_ndcg_at_10_std": -0.347478, + "nauc_ndcg_at_10_diff1": 0.687701, + "nauc_ndcg_at_20_max": 0.437148, + "nauc_ndcg_at_20_std": -0.346833, + "nauc_ndcg_at_20_diff1": 0.686099, + "nauc_ndcg_at_100_max": 0.43436, + "nauc_ndcg_at_100_std": -0.340935, + "nauc_ndcg_at_100_diff1": 0.688176, + "nauc_ndcg_at_1000_max": 0.432823, + "nauc_ndcg_at_1000_std": -0.338437, + "nauc_ndcg_at_1000_diff1": 0.692633, + "nauc_map_at_1_max": 0.391852, + "nauc_map_at_1_std": -0.317015, + "nauc_map_at_1_diff1": 0.733826, + "nauc_map_at_3_max": 0.42777, + "nauc_map_at_3_std": -0.338755, + "nauc_map_at_3_diff1": 0.707937, + "nauc_map_at_5_max": 0.42635, + "nauc_map_at_5_std": -0.340483, + "nauc_map_at_5_diff1": 0.703841, + "nauc_map_at_10_max": 0.426105, + "nauc_map_at_10_std": -0.339382, + "nauc_map_at_10_diff1": 0.703145, + "nauc_map_at_20_max": 0.425743, + "nauc_map_at_20_std": -0.339223, + "nauc_map_at_20_diff1": 0.702882, + "nauc_map_at_100_max": 0.425336, + "nauc_map_at_100_std": -0.33845, + "nauc_map_at_100_diff1": 0.703194, + "nauc_map_at_1000_max": 0.42529, + "nauc_map_at_1000_std": -0.338373, + "nauc_map_at_1000_diff1": 0.703328, + "nauc_recall_at_1_max": 0.391852, + "nauc_recall_at_1_std": -0.317015, + "nauc_recall_at_1_diff1": 0.733826, + "nauc_recall_at_3_max": 0.4842, + "nauc_recall_at_3_std": -0.373862, + "nauc_recall_at_3_diff1": 0.665556, + "nauc_recall_at_5_max": 0.485398, + "nauc_recall_at_5_std": -0.390044, + "nauc_recall_at_5_diff1": 0.634089, + "nauc_recall_at_10_max": 0.496685, + "nauc_recall_at_10_std": -0.385758, + "nauc_recall_at_10_diff1": 0.608237, + "nauc_recall_at_20_max": 0.502637, + "nauc_recall_at_20_std": -0.388807, + "nauc_recall_at_20_diff1": 0.578734, + "nauc_recall_at_100_max": 0.512842, + "nauc_recall_at_100_std": -0.3245, + "nauc_recall_at_100_diff1": 0.518846, + "nauc_recall_at_1000_max": 0.563566, + "nauc_recall_at_1000_std": -0.059406, + "nauc_recall_at_1000_diff1": 0.502013, + "nauc_precision_at_1_max": 0.391852, + "nauc_precision_at_1_std": -0.317015, + "nauc_precision_at_1_diff1": 0.733826, + "nauc_precision_at_3_max": 0.4842, + "nauc_precision_at_3_std": -0.373862, + "nauc_precision_at_3_diff1": 0.665556, + "nauc_precision_at_5_max": 0.485398, + "nauc_precision_at_5_std": -0.390044, + "nauc_precision_at_5_diff1": 0.634089, + "nauc_precision_at_10_max": 0.496685, + "nauc_precision_at_10_std": -0.385758, + "nauc_precision_at_10_diff1": 0.608237, + "nauc_precision_at_20_max": 0.502637, + "nauc_precision_at_20_std": -0.388807, + "nauc_precision_at_20_diff1": 0.578734, + "nauc_precision_at_100_max": 0.512842, + "nauc_precision_at_100_std": -0.3245, + "nauc_precision_at_100_diff1": 0.518846, + "nauc_precision_at_1000_max": 0.563566, + "nauc_precision_at_1000_std": -0.059406, + "nauc_precision_at_1000_diff1": 0.502013, + "nauc_mrr_at_1_max": 0.391852, + "nauc_mrr_at_1_std": -0.317015, + "nauc_mrr_at_1_diff1": 0.733826, + "nauc_mrr_at_3_max": 0.42777, + "nauc_mrr_at_3_std": -0.338755, + "nauc_mrr_at_3_diff1": 0.707937, + "nauc_mrr_at_5_max": 0.42635, + "nauc_mrr_at_5_std": -0.340483, + "nauc_mrr_at_5_diff1": 0.703841, + "nauc_mrr_at_10_max": 0.426105, + "nauc_mrr_at_10_std": -0.339382, + "nauc_mrr_at_10_diff1": 0.703145, + "nauc_mrr_at_20_max": 0.425743, + "nauc_mrr_at_20_std": -0.339223, + "nauc_mrr_at_20_diff1": 0.702882, + "nauc_mrr_at_100_max": 0.425336, + "nauc_mrr_at_100_std": -0.33845, + "nauc_mrr_at_100_diff1": 0.703194, + "nauc_mrr_at_1000_max": 0.42529, + "nauc_mrr_at_1000_std": -0.338373, + "nauc_mrr_at_1000_diff1": 0.703328, + "main_score": 0.74519, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 7483.419162273407, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeFeedbackST.json b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeFeedbackST.json new file mode 100644 index 0000000000..e1825969d3 --- /dev/null +++ b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeFeedbackST.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "d213819e87aab9010628da8b73ab4eb337c89340", + "task_name": "CodeFeedbackST", + "mteb_version": "1.34.21", + "scores": { + "test": [ + { + "ndcg_at_1": 0.70249, + "ndcg_at_3": 0.81182, + "ndcg_at_5": 0.82882, + "ndcg_at_10": 0.83805, + "ndcg_at_20": 0.8423, + "ndcg_at_100": 0.846, + "ndcg_at_1000": 0.84695, + "map_at_1": 0.70249, + "map_at_3": 0.78599, + "map_at_5": 0.7955, + "map_at_10": 0.79937, + "map_at_20": 0.80057, + "map_at_100": 0.80111, + "map_at_1000": 0.80115, + "recall_at_1": 0.70249, + "recall_at_3": 0.88612, + "recall_at_5": 0.92711, + "recall_at_10": 0.95531, + "recall_at_20": 0.97192, + "recall_at_100": 0.99147, + "recall_at_1000": 0.99869, + "precision_at_1": 0.70249, + "precision_at_3": 0.29537, + "precision_at_5": 0.18542, + "precision_at_10": 0.09553, + "precision_at_20": 0.0486, + "precision_at_100": 0.00991, + "precision_at_1000": 0.001, + "mrr_at_1": 0.701782, + "mrr_at_3": 0.785536, + "mrr_at_5": 0.795019, + "mrr_at_10": 0.798985, + "mrr_at_20": 0.800175, + "mrr_at_100": 0.800717, + "mrr_at_1000": 0.800756, + "nauc_ndcg_at_1_max": 0.173715, + "nauc_ndcg_at_1_std": -0.312006, + "nauc_ndcg_at_1_diff1": 0.797766, + "nauc_ndcg_at_3_max": 0.217598, + "nauc_ndcg_at_3_std": -0.392296, + "nauc_ndcg_at_3_diff1": 0.753953, + "nauc_ndcg_at_5_max": 0.214557, + "nauc_ndcg_at_5_std": -0.396921, + "nauc_ndcg_at_5_diff1": 0.755267, + "nauc_ndcg_at_10_max": 0.207734, + "nauc_ndcg_at_10_std": -0.386846, + "nauc_ndcg_at_10_diff1": 0.76038, + "nauc_ndcg_at_20_max": 0.203032, + "nauc_ndcg_at_20_std": -0.379828, + "nauc_ndcg_at_20_diff1": 0.763369, + "nauc_ndcg_at_100_max": 0.202517, + "nauc_ndcg_at_100_std": -0.370016, + "nauc_ndcg_at_100_diff1": 0.765702, + "nauc_ndcg_at_1000_max": 0.20238, + "nauc_ndcg_at_1000_std": -0.369355, + "nauc_ndcg_at_1000_diff1": 0.766422, + "nauc_map_at_1_max": 0.173715, + "nauc_map_at_1_std": -0.312006, + "nauc_map_at_1_diff1": 0.797766, + "nauc_map_at_3_max": 0.204673, + "nauc_map_at_3_std": -0.368177, + "nauc_map_at_3_diff1": 0.76657, + "nauc_map_at_5_max": 0.202736, + "nauc_map_at_5_std": -0.369135, + "nauc_map_at_5_diff1": 0.76773, + "nauc_map_at_10_max": 0.200114, + "nauc_map_at_10_std": -0.365199, + "nauc_map_at_10_diff1": 0.76975, + "nauc_map_at_20_max": 0.199005, + "nauc_map_at_20_std": -0.36351, + "nauc_map_at_20_diff1": 0.770495, + "nauc_map_at_100_max": 0.198827, + "nauc_map_at_100_std": -0.362366, + "nauc_map_at_100_diff1": 0.770783, + "nauc_map_at_1000_max": 0.198843, + "nauc_map_at_1000_std": -0.362311, + "nauc_map_at_1000_diff1": 0.770805, + "nauc_recall_at_1_max": 0.173715, + "nauc_recall_at_1_std": -0.312006, + "nauc_recall_at_1_diff1": 0.797766, + "nauc_recall_at_3_max": 0.280589, + "nauc_recall_at_3_std": -0.5102, + "nauc_recall_at_3_diff1": 0.692687, + "nauc_recall_at_5_max": 0.29733, + "nauc_recall_at_5_std": -0.597721, + "nauc_recall_at_5_diff1": 0.667419, + "nauc_recall_at_10_max": 0.282187, + "nauc_recall_at_10_std": -0.621554, + "nauc_recall_at_10_diff1": 0.66296, + "nauc_recall_at_20_max": 0.23379, + "nauc_recall_at_20_std": -0.620817, + "nauc_recall_at_20_diff1": 0.663651, + "nauc_recall_at_100_max": 0.293632, + "nauc_recall_at_100_std": -0.327369, + "nauc_recall_at_100_diff1": 0.634136, + "nauc_recall_at_1000_max": 0.545299, + "nauc_recall_at_1000_std": 0.210691, + "nauc_recall_at_1000_diff1": 0.504258, + "nauc_precision_at_1_max": 0.173715, + "nauc_precision_at_1_std": -0.312006, + "nauc_precision_at_1_diff1": 0.797766, + "nauc_precision_at_3_max": 0.280589, + "nauc_precision_at_3_std": -0.5102, + "nauc_precision_at_3_diff1": 0.692687, + "nauc_precision_at_5_max": 0.29733, + "nauc_precision_at_5_std": -0.597721, + "nauc_precision_at_5_diff1": 0.667419, + "nauc_precision_at_10_max": 0.282187, + "nauc_precision_at_10_std": -0.621554, + "nauc_precision_at_10_diff1": 0.66296, + "nauc_precision_at_20_max": 0.23379, + "nauc_precision_at_20_std": -0.620817, + "nauc_precision_at_20_diff1": 0.663651, + "nauc_precision_at_100_max": 0.293632, + "nauc_precision_at_100_std": -0.327369, + "nauc_precision_at_100_diff1": 0.634136, + "nauc_precision_at_1000_max": 0.545299, + "nauc_precision_at_1000_std": 0.210691, + "nauc_precision_at_1000_diff1": 0.504258, + "nauc_mrr_at_1_max": 0.168944, + "nauc_mrr_at_1_std": -0.313918, + "nauc_mrr_at_1_diff1": 0.798921, + "nauc_mrr_at_3_max": 0.200676, + "nauc_mrr_at_3_std": -0.370339, + "nauc_mrr_at_3_diff1": 0.767513, + "nauc_mrr_at_5_max": 0.198509, + "nauc_mrr_at_5_std": -0.371025, + "nauc_mrr_at_5_diff1": 0.768575, + "nauc_mrr_at_10_max": 0.195905, + "nauc_mrr_at_10_std": -0.367154, + "nauc_mrr_at_10_diff1": 0.770562, + "nauc_mrr_at_20_max": 0.194775, + "nauc_mrr_at_20_std": -0.365462, + "nauc_mrr_at_20_diff1": 0.771307, + "nauc_mrr_at_100_max": 0.194583, + "nauc_mrr_at_100_std": -0.36433, + "nauc_mrr_at_100_diff1": 0.771596, + "nauc_mrr_at_1000_max": 0.194597, + "nauc_mrr_at_1000_std": -0.364275, + "nauc_mrr_at_1000_diff1": 0.771618, + "main_score": 0.83805, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 25642.695976495743, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeSearchNetCCRetrieval.json b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeSearchNetCCRetrieval.json new file mode 100644 index 0000000000..cea1ce20c4 --- /dev/null +++ b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeSearchNetCCRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "6e1effa2c03723c5fde48ee912b5ee08d4f211e8", + "task_name": "CodeSearchNetCCRetrieval", + "mteb_version": "1.34.21", + "scores": { + "test": [ + { + "ndcg_at_1": 0.80111, + "ndcg_at_3": 0.86995, + "ndcg_at_5": 0.87879, + "ndcg_at_10": 0.88545, + "ndcg_at_20": 0.88841, + "ndcg_at_100": 0.8917, + "ndcg_at_1000": 0.89303, + "map_at_1": 0.80111, + "map_at_3": 0.85392, + "map_at_5": 0.85884, + "map_at_10": 0.86164, + "map_at_20": 0.86246, + "map_at_100": 0.86294, + "map_at_1000": 0.86299, + "recall_at_1": 0.80111, + "recall_at_3": 0.91594, + "recall_at_5": 0.93739, + "recall_at_10": 0.9577, + "recall_at_20": 0.96937, + "recall_at_100": 0.98673, + "recall_at_1000": 0.99705, + "precision_at_1": 0.80111, + "precision_at_3": 0.30531, + "precision_at_5": 0.18748, + "precision_at_10": 0.09577, + "precision_at_20": 0.04847, + "precision_at_100": 0.00987, + "precision_at_1000": 0.001, + "mrr_at_1": 0.801113, + "mrr_at_3": 0.853879, + "mrr_at_5": 0.858796, + "mrr_at_10": 0.861611, + "mrr_at_20": 0.862432, + "mrr_at_100": 0.862909, + "mrr_at_1000": 0.86296, + "nauc_ndcg_at_1_max": 0.520516, + "nauc_ndcg_at_1_std": -0.135888, + "nauc_ndcg_at_1_diff1": 0.817946, + "nauc_ndcg_at_3_max": 0.596581, + "nauc_ndcg_at_3_std": -0.126946, + "nauc_ndcg_at_3_diff1": 0.798929, + "nauc_ndcg_at_5_max": 0.595957, + "nauc_ndcg_at_5_std": -0.117509, + "nauc_ndcg_at_5_diff1": 0.802052, + "nauc_ndcg_at_10_max": 0.593669, + "nauc_ndcg_at_10_std": -0.110621, + "nauc_ndcg_at_10_diff1": 0.805778, + "nauc_ndcg_at_20_max": 0.589006, + "nauc_ndcg_at_20_std": -0.110293, + "nauc_ndcg_at_20_diff1": 0.806456, + "nauc_ndcg_at_100_max": 0.583619, + "nauc_ndcg_at_100_std": -0.111068, + "nauc_ndcg_at_100_diff1": 0.80713, + "nauc_ndcg_at_1000_max": 0.580675, + "nauc_ndcg_at_1000_std": -0.115028, + "nauc_ndcg_at_1000_diff1": 0.806835, + "nauc_map_at_1_max": 0.520516, + "nauc_map_at_1_std": -0.135888, + "nauc_map_at_1_diff1": 0.817946, + "nauc_map_at_3_max": 0.574464, + "nauc_map_at_3_std": -0.130826, + "nauc_map_at_3_diff1": 0.804252, + "nauc_map_at_5_max": 0.573385, + "nauc_map_at_5_std": -0.126272, + "nauc_map_at_5_diff1": 0.805921, + "nauc_map_at_10_max": 0.572152, + "nauc_map_at_10_std": -0.124174, + "nauc_map_at_10_diff1": 0.807298, + "nauc_map_at_20_max": 0.571017, + "nauc_map_at_20_std": -0.124088, + "nauc_map_at_20_diff1": 0.807459, + "nauc_map_at_100_max": 0.570362, + "nauc_map_at_100_std": -0.124174, + "nauc_map_at_100_diff1": 0.807546, + "nauc_map_at_1000_max": 0.570268, + "nauc_map_at_1000_std": -0.124274, + "nauc_map_at_1000_diff1": 0.807538, + "nauc_recall_at_1_max": 0.520516, + "nauc_recall_at_1_std": -0.135888, + "nauc_recall_at_1_diff1": 0.817946, + "nauc_recall_at_3_max": 0.700783, + "nauc_recall_at_3_std": -0.107693, + "nauc_recall_at_3_diff1": 0.77401, + "nauc_recall_at_5_max": 0.733396, + "nauc_recall_at_5_std": -0.060069, + "nauc_recall_at_5_diff1": 0.779764, + "nauc_recall_at_10_max": 0.780509, + "nauc_recall_at_10_std": 0.024011, + "nauc_recall_at_10_diff1": 0.79871, + "nauc_recall_at_20_max": 0.786109, + "nauc_recall_at_20_std": 0.076667, + "nauc_recall_at_20_diff1": 0.805354, + "nauc_recall_at_100_max": 0.817075, + "nauc_recall_at_100_std": 0.279903, + "nauc_recall_at_100_diff1": 0.830167, + "nauc_recall_at_1000_max": 0.842042, + "nauc_recall_at_1000_std": 0.527745, + "nauc_recall_at_1000_diff1": 0.829204, + "nauc_precision_at_1_max": 0.520516, + "nauc_precision_at_1_std": -0.135888, + "nauc_precision_at_1_diff1": 0.817946, + "nauc_precision_at_3_max": 0.700783, + "nauc_precision_at_3_std": -0.107693, + "nauc_precision_at_3_diff1": 0.77401, + "nauc_precision_at_5_max": 0.733396, + "nauc_precision_at_5_std": -0.060069, + "nauc_precision_at_5_diff1": 0.779764, + "nauc_precision_at_10_max": 0.780509, + "nauc_precision_at_10_std": 0.024011, + "nauc_precision_at_10_diff1": 0.79871, + "nauc_precision_at_20_max": 0.786109, + "nauc_precision_at_20_std": 0.076667, + "nauc_precision_at_20_diff1": 0.805354, + "nauc_precision_at_100_max": 0.817075, + "nauc_precision_at_100_std": 0.279903, + "nauc_precision_at_100_diff1": 0.830167, + "nauc_precision_at_1000_max": 0.842042, + "nauc_precision_at_1000_std": 0.527745, + "nauc_precision_at_1000_diff1": 0.829204, + "nauc_mrr_at_1_max": 0.520516, + "nauc_mrr_at_1_std": -0.135888, + "nauc_mrr_at_1_diff1": 0.817946, + "nauc_mrr_at_3_max": 0.574536, + "nauc_mrr_at_3_std": -0.130521, + "nauc_mrr_at_3_diff1": 0.804281, + "nauc_mrr_at_5_max": 0.573391, + "nauc_mrr_at_5_std": -0.12625, + "nauc_mrr_at_5_diff1": 0.805951, + "nauc_mrr_at_10_max": 0.57218, + "nauc_mrr_at_10_std": -0.124033, + "nauc_mrr_at_10_diff1": 0.807309, + "nauc_mrr_at_20_max": 0.571037, + "nauc_mrr_at_20_std": -0.123939, + "nauc_mrr_at_20_diff1": 0.807465, + "nauc_mrr_at_100_max": 0.570391, + "nauc_mrr_at_100_std": -0.124026, + "nauc_mrr_at_100_diff1": 0.807555, + "nauc_mrr_at_1000_max": 0.570297, + "nauc_mrr_at_1000_std": -0.124125, + "nauc_mrr_at_1000_diff1": 0.807548, + "main_score": 0.88545, + "hf_subset": "python", + "languages": [ + "python-Code" + ] + }, + { + "ndcg_at_1": 0.75296, + "ndcg_at_3": 0.83233, + "ndcg_at_5": 0.84334, + "ndcg_at_10": 0.85238, + "ndcg_at_20": 0.85681, + "ndcg_at_100": 0.86142, + "ndcg_at_1000": 0.86347, + "map_at_1": 0.75296, + "map_at_3": 0.81384, + "map_at_5": 0.81994, + "map_at_10": 0.82367, + "map_at_20": 0.82491, + "map_at_100": 0.82561, + "map_at_1000": 0.82568, + "recall_at_1": 0.75296, + "recall_at_3": 0.88545, + "recall_at_5": 0.91218, + "recall_at_10": 0.94014, + "recall_at_20": 0.95746, + "recall_at_100": 0.98146, + "recall_at_1000": 0.99787, + "precision_at_1": 0.75296, + "precision_at_3": 0.29515, + "precision_at_5": 0.18244, + "precision_at_10": 0.09401, + "precision_at_20": 0.04787, + "precision_at_100": 0.00981, + "precision_at_1000": 0.001, + "mrr_at_1": 0.752963, + "mrr_at_3": 0.813633, + "mrr_at_5": 0.819817, + "mrr_at_10": 0.823554, + "mrr_at_20": 0.824794, + "mrr_at_100": 0.825489, + "mrr_at_1000": 0.82556, + "nauc_ndcg_at_1_max": 0.434928, + "nauc_ndcg_at_1_std": -0.108506, + "nauc_ndcg_at_1_diff1": 0.761959, + "nauc_ndcg_at_3_max": 0.523429, + "nauc_ndcg_at_3_std": -0.105611, + "nauc_ndcg_at_3_diff1": 0.744824, + "nauc_ndcg_at_5_max": 0.522404, + "nauc_ndcg_at_5_std": -0.098101, + "nauc_ndcg_at_5_diff1": 0.743871, + "nauc_ndcg_at_10_max": 0.52135, + "nauc_ndcg_at_10_std": -0.089312, + "nauc_ndcg_at_10_diff1": 0.749961, + "nauc_ndcg_at_20_max": 0.517454, + "nauc_ndcg_at_20_std": -0.08671, + "nauc_ndcg_at_20_diff1": 0.750591, + "nauc_ndcg_at_100_max": 0.509881, + "nauc_ndcg_at_100_std": -0.090176, + "nauc_ndcg_at_100_diff1": 0.750488, + "nauc_ndcg_at_1000_max": 0.506302, + "nauc_ndcg_at_1000_std": -0.093194, + "nauc_ndcg_at_1000_diff1": 0.75066, + "nauc_map_at_1_max": 0.434928, + "nauc_map_at_1_std": -0.108506, + "nauc_map_at_1_diff1": 0.761959, + "nauc_map_at_3_max": 0.499075, + "nauc_map_at_3_std": -0.106764, + "nauc_map_at_3_diff1": 0.749052, + "nauc_map_at_5_max": 0.497801, + "nauc_map_at_5_std": -0.102978, + "nauc_map_at_5_diff1": 0.74867, + "nauc_map_at_10_max": 0.497222, + "nauc_map_at_10_std": -0.099797, + "nauc_map_at_10_diff1": 0.751131, + "nauc_map_at_20_max": 0.496167, + "nauc_map_at_20_std": -0.099164, + "nauc_map_at_20_diff1": 0.75126, + "nauc_map_at_100_max": 0.495119, + "nauc_map_at_100_std": -0.09967, + "nauc_map_at_100_diff1": 0.751213, + "nauc_map_at_1000_max": 0.495021, + "nauc_map_at_1000_std": -0.099755, + "nauc_map_at_1000_diff1": 0.75122, + "nauc_recall_at_1_max": 0.434928, + "nauc_recall_at_1_std": -0.108506, + "nauc_recall_at_1_diff1": 0.761959, + "nauc_recall_at_3_max": 0.628408, + "nauc_recall_at_3_std": -0.100393, + "nauc_recall_at_3_diff1": 0.726945, + "nauc_recall_at_5_max": 0.653536, + "nauc_recall_at_5_std": -0.069783, + "nauc_recall_at_5_diff1": 0.71851, + "nauc_recall_at_10_max": 0.702408, + "nauc_recall_at_10_std": 0.002332, + "nauc_recall_at_10_diff1": 0.746546, + "nauc_recall_at_20_max": 0.727629, + "nauc_recall_at_20_std": 0.068168, + "nauc_recall_at_20_diff1": 0.753554, + "nauc_recall_at_100_max": 0.732662, + "nauc_recall_at_100_std": 0.151421, + "nauc_recall_at_100_diff1": 0.75773, + "nauc_recall_at_1000_max": 0.715582, + "nauc_recall_at_1000_std": 0.534591, + "nauc_recall_at_1000_diff1": 0.883292, + "nauc_precision_at_1_max": 0.434928, + "nauc_precision_at_1_std": -0.108506, + "nauc_precision_at_1_diff1": 0.761959, + "nauc_precision_at_3_max": 0.628408, + "nauc_precision_at_3_std": -0.100393, + "nauc_precision_at_3_diff1": 0.726945, + "nauc_precision_at_5_max": 0.653536, + "nauc_precision_at_5_std": -0.069783, + "nauc_precision_at_5_diff1": 0.71851, + "nauc_precision_at_10_max": 0.702408, + "nauc_precision_at_10_std": 0.002332, + "nauc_precision_at_10_diff1": 0.746546, + "nauc_precision_at_20_max": 0.727629, + "nauc_precision_at_20_std": 0.068168, + "nauc_precision_at_20_diff1": 0.753554, + "nauc_precision_at_100_max": 0.732662, + "nauc_precision_at_100_std": 0.151421, + "nauc_precision_at_100_diff1": 0.75773, + "nauc_precision_at_1000_max": 0.715582, + "nauc_precision_at_1000_std": 0.534591, + "nauc_precision_at_1000_diff1": 0.883292, + "nauc_mrr_at_1_max": 0.434045, + "nauc_mrr_at_1_std": -0.111393, + "nauc_mrr_at_1_diff1": 0.761959, + "nauc_mrr_at_3_max": 0.498418, + "nauc_mrr_at_3_std": -0.109177, + "nauc_mrr_at_3_diff1": 0.749464, + "nauc_mrr_at_5_max": 0.497156, + "nauc_mrr_at_5_std": -0.105262, + "nauc_mrr_at_5_diff1": 0.748936, + "nauc_mrr_at_10_max": 0.49654, + "nauc_mrr_at_10_std": -0.102165, + "nauc_mrr_at_10_diff1": 0.751391, + "nauc_mrr_at_20_max": 0.495481, + "nauc_mrr_at_20_std": -0.10155, + "nauc_mrr_at_20_diff1": 0.751522, + "nauc_mrr_at_100_max": 0.494432, + "nauc_mrr_at_100_std": -0.102067, + "nauc_mrr_at_100_diff1": 0.751476, + "nauc_mrr_at_1000_max": 0.494334, + "nauc_mrr_at_1000_std": -0.102153, + "nauc_mrr_at_1000_diff1": 0.751483, + "main_score": 0.85238, + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ] + }, + { + "ndcg_at_1": 0.71817, + "ndcg_at_3": 0.79596, + "ndcg_at_5": 0.80997, + "ndcg_at_10": 0.82071, + "ndcg_at_20": 0.82722, + "ndcg_at_100": 0.83424, + "ndcg_at_1000": 0.83644, + "map_at_1": 0.71817, + "map_at_3": 0.77752, + "map_at_5": 0.78532, + "map_at_10": 0.78983, + "map_at_20": 0.79165, + "map_at_100": 0.79261, + "map_at_1000": 0.7927, + "recall_at_1": 0.71817, + "recall_at_3": 0.84905, + "recall_at_5": 0.88291, + "recall_at_10": 0.91578, + "recall_at_20": 0.94127, + "recall_at_100": 0.97919, + "recall_at_1000": 0.99655, + "precision_at_1": 0.71817, + "precision_at_3": 0.28302, + "precision_at_5": 0.17658, + "precision_at_10": 0.09158, + "precision_at_20": 0.04706, + "precision_at_100": 0.00979, + "precision_at_1000": 0.001, + "mrr_at_1": 0.718419, + "mrr_at_3": 0.777579, + "mrr_at_5": 0.785576, + "mrr_at_10": 0.790002, + "mrr_at_20": 0.791804, + "mrr_at_100": 0.79277, + "mrr_at_1000": 0.79285, + "nauc_ndcg_at_1_max": 0.462899, + "nauc_ndcg_at_1_std": -0.158057, + "nauc_ndcg_at_1_diff1": 0.751517, + "nauc_ndcg_at_3_max": 0.504038, + "nauc_ndcg_at_3_std": -0.151642, + "nauc_ndcg_at_3_diff1": 0.731708, + "nauc_ndcg_at_5_max": 0.506432, + "nauc_ndcg_at_5_std": -0.146743, + "nauc_ndcg_at_5_diff1": 0.730665, + "nauc_ndcg_at_10_max": 0.504955, + "nauc_ndcg_at_10_std": -0.144179, + "nauc_ndcg_at_10_diff1": 0.730631, + "nauc_ndcg_at_20_max": 0.504419, + "nauc_ndcg_at_20_std": -0.143855, + "nauc_ndcg_at_20_diff1": 0.73322, + "nauc_ndcg_at_100_max": 0.501224, + "nauc_ndcg_at_100_std": -0.141449, + "nauc_ndcg_at_100_diff1": 0.734164, + "nauc_ndcg_at_1000_max": 0.498977, + "nauc_ndcg_at_1000_std": -0.145094, + "nauc_ndcg_at_1000_diff1": 0.734925, + "nauc_map_at_1_max": 0.462899, + "nauc_map_at_1_std": -0.158057, + "nauc_map_at_1_diff1": 0.751517, + "nauc_map_at_3_max": 0.492703, + "nauc_map_at_3_std": -0.154086, + "nauc_map_at_3_diff1": 0.736342, + "nauc_map_at_5_max": 0.49368, + "nauc_map_at_5_std": -0.151706, + "nauc_map_at_5_diff1": 0.736068, + "nauc_map_at_10_max": 0.492909, + "nauc_map_at_10_std": -0.150879, + "nauc_map_at_10_diff1": 0.736065, + "nauc_map_at_20_max": 0.492715, + "nauc_map_at_20_std": -0.150871, + "nauc_map_at_20_diff1": 0.736746, + "nauc_map_at_100_max": 0.492293, + "nauc_map_at_100_std": -0.150683, + "nauc_map_at_100_diff1": 0.736823, + "nauc_map_at_1000_max": 0.492233, + "nauc_map_at_1000_std": -0.150771, + "nauc_map_at_1000_diff1": 0.736853, + "nauc_recall_at_1_max": 0.462899, + "nauc_recall_at_1_std": -0.158057, + "nauc_recall_at_1_diff1": 0.751517, + "nauc_recall_at_3_max": 0.548994, + "nauc_recall_at_3_std": -0.141531, + "nauc_recall_at_3_diff1": 0.713834, + "nauc_recall_at_5_max": 0.56805, + "nauc_recall_at_5_std": -0.121124, + "nauc_recall_at_5_diff1": 0.704842, + "nauc_recall_at_10_max": 0.581137, + "nauc_recall_at_10_std": -0.096612, + "nauc_recall_at_10_diff1": 0.696535, + "nauc_recall_at_20_max": 0.607395, + "nauc_recall_at_20_std": -0.072424, + "nauc_recall_at_20_diff1": 0.70839, + "nauc_recall_at_100_max": 0.672298, + "nauc_recall_at_100_std": 0.156977, + "nauc_recall_at_100_diff1": 0.706957, + "nauc_recall_at_1000_max": 0.749187, + "nauc_recall_at_1000_std": 0.369518, + "nauc_recall_at_1000_diff1": 0.813944, + "nauc_precision_at_1_max": 0.462899, + "nauc_precision_at_1_std": -0.158057, + "nauc_precision_at_1_diff1": 0.751517, + "nauc_precision_at_3_max": 0.548994, + "nauc_precision_at_3_std": -0.141531, + "nauc_precision_at_3_diff1": 0.713834, + "nauc_precision_at_5_max": 0.56805, + "nauc_precision_at_5_std": -0.121124, + "nauc_precision_at_5_diff1": 0.704842, + "nauc_precision_at_10_max": 0.581137, + "nauc_precision_at_10_std": -0.096612, + "nauc_precision_at_10_diff1": 0.696535, + "nauc_precision_at_20_max": 0.607395, + "nauc_precision_at_20_std": -0.072424, + "nauc_precision_at_20_diff1": 0.70839, + "nauc_precision_at_100_max": 0.672298, + "nauc_precision_at_100_std": 0.156977, + "nauc_precision_at_100_diff1": 0.706957, + "nauc_precision_at_1000_max": 0.749187, + "nauc_precision_at_1000_std": 0.369518, + "nauc_precision_at_1000_diff1": 0.813944, + "nauc_mrr_at_1_max": 0.462381, + "nauc_mrr_at_1_std": -0.157398, + "nauc_mrr_at_1_diff1": 0.751059, + "nauc_mrr_at_3_max": 0.492346, + "nauc_mrr_at_3_std": -0.153677, + "nauc_mrr_at_3_diff1": 0.73612, + "nauc_mrr_at_5_max": 0.493337, + "nauc_mrr_at_5_std": -0.151074, + "nauc_mrr_at_5_diff1": 0.735721, + "nauc_mrr_at_10_max": 0.492649, + "nauc_mrr_at_10_std": -0.150452, + "nauc_mrr_at_10_diff1": 0.735723, + "nauc_mrr_at_20_max": 0.492511, + "nauc_mrr_at_20_std": -0.150367, + "nauc_mrr_at_20_diff1": 0.736409, + "nauc_mrr_at_100_max": 0.492089, + "nauc_mrr_at_100_std": -0.150169, + "nauc_mrr_at_100_diff1": 0.736485, + "nauc_mrr_at_1000_max": 0.49203, + "nauc_mrr_at_1000_std": -0.150258, + "nauc_mrr_at_1000_diff1": 0.736515, + "main_score": 0.82071, + "hf_subset": "go", + "languages": [ + "go-Code" + ] + }, + { + "ndcg_at_1": 0.74227, + "ndcg_at_3": 0.81129, + "ndcg_at_5": 0.82347, + "ndcg_at_10": 0.83554, + "ndcg_at_20": 0.84074, + "ndcg_at_100": 0.84668, + "ndcg_at_1000": 0.84924, + "map_at_1": 0.74227, + "map_at_3": 0.79527, + "map_at_5": 0.80209, + "map_at_10": 0.80708, + "map_at_20": 0.8085, + "map_at_100": 0.80935, + "map_at_1000": 0.80943, + "recall_at_1": 0.74227, + "recall_at_3": 0.85726, + "recall_at_5": 0.8866, + "recall_at_10": 0.92387, + "recall_at_20": 0.94449, + "recall_at_100": 0.97621, + "recall_at_1000": 0.99762, + "precision_at_1": 0.74227, + "precision_at_3": 0.28575, + "precision_at_5": 0.17732, + "precision_at_10": 0.09239, + "precision_at_20": 0.04722, + "precision_at_100": 0.00976, + "precision_at_1000": 0.001, + "mrr_at_1": 0.743061, + "mrr_at_3": 0.795929, + "mrr_at_5": 0.802511, + "mrr_at_10": 0.807497, + "mrr_at_20": 0.808914, + "mrr_at_100": 0.809769, + "mrr_at_1000": 0.809844, + "nauc_ndcg_at_1_max": 0.561796, + "nauc_ndcg_at_1_std": -0.155922, + "nauc_ndcg_at_1_diff1": 0.791297, + "nauc_ndcg_at_3_max": 0.597089, + "nauc_ndcg_at_3_std": -0.160787, + "nauc_ndcg_at_3_diff1": 0.757333, + "nauc_ndcg_at_5_max": 0.61997, + "nauc_ndcg_at_5_std": -0.154586, + "nauc_ndcg_at_5_diff1": 0.768737, + "nauc_ndcg_at_10_max": 0.612025, + "nauc_ndcg_at_10_std": -0.141986, + "nauc_ndcg_at_10_diff1": 0.763283, + "nauc_ndcg_at_20_max": 0.607561, + "nauc_ndcg_at_20_std": -0.150949, + "nauc_ndcg_at_20_diff1": 0.763621, + "nauc_ndcg_at_100_max": 0.606459, + "nauc_ndcg_at_100_std": -0.142763, + "nauc_ndcg_at_100_diff1": 0.7675, + "nauc_ndcg_at_1000_max": 0.602404, + "nauc_ndcg_at_1000_std": -0.148205, + "nauc_ndcg_at_1000_diff1": 0.767696, + "nauc_map_at_1_max": 0.561796, + "nauc_map_at_1_std": -0.155922, + "nauc_map_at_1_diff1": 0.791297, + "nauc_map_at_3_max": 0.587908, + "nauc_map_at_3_std": -0.160363, + "nauc_map_at_3_diff1": 0.766202, + "nauc_map_at_5_max": 0.598847, + "nauc_map_at_5_std": -0.157317, + "nauc_map_at_5_diff1": 0.772274, + "nauc_map_at_10_max": 0.595578, + "nauc_map_at_10_std": -0.152674, + "nauc_map_at_10_diff1": 0.770348, + "nauc_map_at_20_max": 0.594436, + "nauc_map_at_20_std": -0.154619, + "nauc_map_at_20_diff1": 0.770324, + "nauc_map_at_100_max": 0.594119, + "nauc_map_at_100_std": -0.153821, + "nauc_map_at_100_diff1": 0.770721, + "nauc_map_at_1000_max": 0.59402, + "nauc_map_at_1000_std": -0.153946, + "nauc_map_at_1000_diff1": 0.770728, + "nauc_recall_at_1_max": 0.561796, + "nauc_recall_at_1_std": -0.155922, + "nauc_recall_at_1_diff1": 0.791297, + "nauc_recall_at_3_max": 0.632476, + "nauc_recall_at_3_std": -0.161883, + "nauc_recall_at_3_diff1": 0.72313, + "nauc_recall_at_5_max": 0.723397, + "nauc_recall_at_5_std": -0.139592, + "nauc_recall_at_5_diff1": 0.755393, + "nauc_recall_at_10_max": 0.721129, + "nauc_recall_at_10_std": -0.05761, + "nauc_recall_at_10_diff1": 0.71647, + "nauc_recall_at_20_max": 0.714452, + "nauc_recall_at_20_std": -0.125229, + "nauc_recall_at_20_diff1": 0.706492, + "nauc_recall_at_100_max": 0.834557, + "nauc_recall_at_100_std": 0.1814, + "nauc_recall_at_100_diff1": 0.763242, + "nauc_recall_at_1000_max": 0.907421, + "nauc_recall_at_1000_std": 0.395699, + "nauc_recall_at_1000_diff1": 0.814842, + "nauc_precision_at_1_max": 0.561796, + "nauc_precision_at_1_std": -0.155922, + "nauc_precision_at_1_diff1": 0.791297, + "nauc_precision_at_3_max": 0.632476, + "nauc_precision_at_3_std": -0.161883, + "nauc_precision_at_3_diff1": 0.72313, + "nauc_precision_at_5_max": 0.723397, + "nauc_precision_at_5_std": -0.139592, + "nauc_precision_at_5_diff1": 0.755393, + "nauc_precision_at_10_max": 0.721129, + "nauc_precision_at_10_std": -0.05761, + "nauc_precision_at_10_diff1": 0.71647, + "nauc_precision_at_20_max": 0.714452, + "nauc_precision_at_20_std": -0.125229, + "nauc_precision_at_20_diff1": 0.706492, + "nauc_precision_at_100_max": 0.834557, + "nauc_precision_at_100_std": 0.1814, + "nauc_precision_at_100_diff1": 0.763242, + "nauc_precision_at_1000_max": 0.907421, + "nauc_precision_at_1000_std": 0.395699, + "nauc_precision_at_1000_diff1": 0.814842, + "nauc_mrr_at_1_max": 0.565167, + "nauc_mrr_at_1_std": -0.152306, + "nauc_mrr_at_1_diff1": 0.789806, + "nauc_mrr_at_3_max": 0.589318, + "nauc_mrr_at_3_std": -0.158188, + "nauc_mrr_at_3_diff1": 0.765325, + "nauc_mrr_at_5_max": 0.600772, + "nauc_mrr_at_5_std": -0.155885, + "nauc_mrr_at_5_diff1": 0.771579, + "nauc_mrr_at_10_max": 0.597563, + "nauc_mrr_at_10_std": -0.151247, + "nauc_mrr_at_10_diff1": 0.769643, + "nauc_mrr_at_20_max": 0.596435, + "nauc_mrr_at_20_std": -0.153183, + "nauc_mrr_at_20_diff1": 0.769612, + "nauc_mrr_at_100_max": 0.596127, + "nauc_mrr_at_100_std": -0.152377, + "nauc_mrr_at_100_diff1": 0.770006, + "nauc_mrr_at_1000_max": 0.596029, + "nauc_mrr_at_1000_std": -0.152502, + "nauc_mrr_at_1000_diff1": 0.770013, + "main_score": 0.83554, + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ] + }, + { + "ndcg_at_1": 0.75171, + "ndcg_at_3": 0.82688, + "ndcg_at_5": 0.83776, + "ndcg_at_10": 0.84544, + "ndcg_at_20": 0.84974, + "ndcg_at_100": 0.85395, + "ndcg_at_1000": 0.85616, + "map_at_1": 0.75171, + "map_at_3": 0.8093, + "map_at_5": 0.81537, + "map_at_10": 0.81859, + "map_at_20": 0.81978, + "map_at_100": 0.82038, + "map_at_1000": 0.82045, + "recall_at_1": 0.75171, + "recall_at_3": 0.87741, + "recall_at_5": 0.9037, + "recall_at_10": 0.92716, + "recall_at_20": 0.94404, + "recall_at_100": 0.9665, + "recall_at_1000": 0.98439, + "precision_at_1": 0.75171, + "precision_at_3": 0.29247, + "precision_at_5": 0.18074, + "precision_at_10": 0.09272, + "precision_at_20": 0.0472, + "precision_at_100": 0.00966, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.751894, + "mrr_at_3": 0.80925, + "mrr_at_5": 0.815412, + "mrr_at_10": 0.81864, + "mrr_at_20": 0.819834, + "mrr_at_100": 0.820434, + "mrr_at_1000": 0.820508, + "nauc_ndcg_at_1_max": 0.506821, + "nauc_ndcg_at_1_std": -0.111366, + "nauc_ndcg_at_1_diff1": 0.791669, + "nauc_ndcg_at_3_max": 0.580723, + "nauc_ndcg_at_3_std": -0.079933, + "nauc_ndcg_at_3_diff1": 0.774472, + "nauc_ndcg_at_5_max": 0.582963, + "nauc_ndcg_at_5_std": -0.065503, + "nauc_ndcg_at_5_diff1": 0.77226, + "nauc_ndcg_at_10_max": 0.582863, + "nauc_ndcg_at_10_std": -0.061547, + "nauc_ndcg_at_10_diff1": 0.777487, + "nauc_ndcg_at_20_max": 0.580287, + "nauc_ndcg_at_20_std": -0.058156, + "nauc_ndcg_at_20_diff1": 0.778931, + "nauc_ndcg_at_100_max": 0.57631, + "nauc_ndcg_at_100_std": -0.054867, + "nauc_ndcg_at_100_diff1": 0.780134, + "nauc_ndcg_at_1000_max": 0.574025, + "nauc_ndcg_at_1000_std": -0.059269, + "nauc_ndcg_at_1000_diff1": 0.780971, + "nauc_map_at_1_max": 0.506821, + "nauc_map_at_1_std": -0.111366, + "nauc_map_at_1_diff1": 0.791669, + "nauc_map_at_3_max": 0.560271, + "nauc_map_at_3_std": -0.090478, + "nauc_map_at_3_diff1": 0.779103, + "nauc_map_at_5_max": 0.560815, + "nauc_map_at_5_std": -0.083639, + "nauc_map_at_5_diff1": 0.778093, + "nauc_map_at_10_max": 0.560335, + "nauc_map_at_10_std": -0.082634, + "nauc_map_at_10_diff1": 0.780061, + "nauc_map_at_20_max": 0.559542, + "nauc_map_at_20_std": -0.082022, + "nauc_map_at_20_diff1": 0.780375, + "nauc_map_at_100_max": 0.559006, + "nauc_map_at_100_std": -0.08175, + "nauc_map_at_100_diff1": 0.780509, + "nauc_map_at_1000_max": 0.558938, + "nauc_map_at_1000_std": -0.081874, + "nauc_map_at_1000_diff1": 0.780524, + "nauc_recall_at_1_max": 0.506821, + "nauc_recall_at_1_std": -0.111366, + "nauc_recall_at_1_diff1": 0.791669, + "nauc_recall_at_3_max": 0.665932, + "nauc_recall_at_3_std": -0.034768, + "nauc_recall_at_3_diff1": 0.755266, + "nauc_recall_at_5_max": 0.694508, + "nauc_recall_at_5_std": 0.030903, + "nauc_recall_at_5_diff1": 0.742639, + "nauc_recall_at_10_max": 0.730032, + "nauc_recall_at_10_std": 0.08675, + "nauc_recall_at_10_diff1": 0.764971, + "nauc_recall_at_20_max": 0.749837, + "nauc_recall_at_20_std": 0.165383, + "nauc_recall_at_20_diff1": 0.776249, + "nauc_recall_at_100_max": 0.777873, + "nauc_recall_at_100_std": 0.387331, + "nauc_recall_at_100_diff1": 0.801161, + "nauc_recall_at_1000_max": 0.851616, + "nauc_recall_at_1000_std": 0.588162, + "nauc_recall_at_1000_diff1": 0.88725, + "nauc_precision_at_1_max": 0.506821, + "nauc_precision_at_1_std": -0.111366, + "nauc_precision_at_1_diff1": 0.791669, + "nauc_precision_at_3_max": 0.665932, + "nauc_precision_at_3_std": -0.034768, + "nauc_precision_at_3_diff1": 0.755266, + "nauc_precision_at_5_max": 0.694508, + "nauc_precision_at_5_std": 0.030903, + "nauc_precision_at_5_diff1": 0.742639, + "nauc_precision_at_10_max": 0.730032, + "nauc_precision_at_10_std": 0.08675, + "nauc_precision_at_10_diff1": 0.764971, + "nauc_precision_at_20_max": 0.749837, + "nauc_precision_at_20_std": 0.165383, + "nauc_precision_at_20_diff1": 0.776249, + "nauc_precision_at_100_max": 0.777873, + "nauc_precision_at_100_std": 0.387331, + "nauc_precision_at_100_diff1": 0.801161, + "nauc_precision_at_1000_max": 0.851616, + "nauc_precision_at_1000_std": 0.588162, + "nauc_precision_at_1000_diff1": 0.88725, + "nauc_mrr_at_1_max": 0.507429, + "nauc_mrr_at_1_std": -0.108783, + "nauc_mrr_at_1_diff1": 0.791316, + "nauc_mrr_at_3_max": 0.560908, + "nauc_mrr_at_3_std": -0.088909, + "nauc_mrr_at_3_diff1": 0.778571, + "nauc_mrr_at_5_max": 0.561448, + "nauc_mrr_at_5_std": -0.082126, + "nauc_mrr_at_5_diff1": 0.777701, + "nauc_mrr_at_10_max": 0.560939, + "nauc_mrr_at_10_std": -0.080959, + "nauc_mrr_at_10_diff1": 0.779702, + "nauc_mrr_at_20_max": 0.560148, + "nauc_mrr_at_20_std": -0.080327, + "nauc_mrr_at_20_diff1": 0.780016, + "nauc_mrr_at_100_max": 0.559614, + "nauc_mrr_at_100_std": -0.080047, + "nauc_mrr_at_100_diff1": 0.780147, + "nauc_mrr_at_1000_max": 0.559546, + "nauc_mrr_at_1000_std": -0.080171, + "nauc_mrr_at_1000_diff1": 0.780163, + "main_score": 0.84544, + "hf_subset": "java", + "languages": [ + "java-Code" + ] + }, + { + "ndcg_at_1": 0.64072, + "ndcg_at_3": 0.72679, + "ndcg_at_5": 0.74392, + "ndcg_at_10": 0.75727, + "ndcg_at_20": 0.76647, + "ndcg_at_100": 0.77596, + "ndcg_at_1000": 0.78, + "map_at_1": 0.64072, + "map_at_3": 0.70633, + "map_at_5": 0.71587, + "map_at_10": 0.72142, + "map_at_20": 0.72396, + "map_at_100": 0.72531, + "map_at_1000": 0.72546, + "recall_at_1": 0.64072, + "recall_at_3": 0.78571, + "recall_at_5": 0.82717, + "recall_at_10": 0.8682, + "recall_at_20": 0.90445, + "recall_at_100": 0.95512, + "recall_at_1000": 0.98701, + "precision_at_1": 0.64072, + "precision_at_3": 0.2619, + "precision_at_5": 0.16543, + "precision_at_10": 0.08682, + "precision_at_20": 0.04522, + "precision_at_100": 0.00955, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.641002, + "mrr_at_3": 0.706436, + "mrr_at_5": 0.716009, + "mrr_at_10": 0.721538, + "mrr_at_20": 0.724085, + "mrr_at_100": 0.725431, + "mrr_at_1000": 0.725581, + "nauc_ndcg_at_1_max": 0.417377, + "nauc_ndcg_at_1_std": -0.179937, + "nauc_ndcg_at_1_diff1": 0.736639, + "nauc_ndcg_at_3_max": 0.450591, + "nauc_ndcg_at_3_std": -0.175289, + "nauc_ndcg_at_3_diff1": 0.698634, + "nauc_ndcg_at_5_max": 0.455764, + "nauc_ndcg_at_5_std": -0.166225, + "nauc_ndcg_at_5_diff1": 0.698618, + "nauc_ndcg_at_10_max": 0.454756, + "nauc_ndcg_at_10_std": -0.159002, + "nauc_ndcg_at_10_diff1": 0.697855, + "nauc_ndcg_at_20_max": 0.450228, + "nauc_ndcg_at_20_std": -0.157143, + "nauc_ndcg_at_20_diff1": 0.697445, + "nauc_ndcg_at_100_max": 0.448713, + "nauc_ndcg_at_100_std": -0.156775, + "nauc_ndcg_at_100_diff1": 0.701688, + "nauc_ndcg_at_1000_max": 0.447273, + "nauc_ndcg_at_1000_std": -0.161259, + "nauc_ndcg_at_1000_diff1": 0.70337, + "nauc_map_at_1_max": 0.417377, + "nauc_map_at_1_std": -0.179937, + "nauc_map_at_1_diff1": 0.736639, + "nauc_map_at_3_max": 0.441581, + "nauc_map_at_3_std": -0.177205, + "nauc_map_at_3_diff1": 0.708333, + "nauc_map_at_5_max": 0.44391, + "nauc_map_at_5_std": -0.172685, + "nauc_map_at_5_diff1": 0.708646, + "nauc_map_at_10_max": 0.44319, + "nauc_map_at_10_std": -0.17012, + "nauc_map_at_10_diff1": 0.708573, + "nauc_map_at_20_max": 0.441996, + "nauc_map_at_20_std": -0.169781, + "nauc_map_at_20_diff1": 0.708572, + "nauc_map_at_100_max": 0.441775, + "nauc_map_at_100_std": -0.169708, + "nauc_map_at_100_diff1": 0.7091, + "nauc_map_at_1000_max": 0.441737, + "nauc_map_at_1000_std": -0.169819, + "nauc_map_at_1000_diff1": 0.709149, + "nauc_recall_at_1_max": 0.417377, + "nauc_recall_at_1_std": -0.179937, + "nauc_recall_at_1_diff1": 0.736639, + "nauc_recall_at_3_max": 0.483304, + "nauc_recall_at_3_std": -0.16798, + "nauc_recall_at_3_diff1": 0.663752, + "nauc_recall_at_5_max": 0.5073, + "nauc_recall_at_5_std": -0.136285, + "nauc_recall_at_5_diff1": 0.656388, + "nauc_recall_at_10_max": 0.517627, + "nauc_recall_at_10_std": -0.091529, + "nauc_recall_at_10_diff1": 0.640698, + "nauc_recall_at_20_max": 0.501088, + "nauc_recall_at_20_std": -0.053413, + "nauc_recall_at_20_diff1": 0.618175, + "nauc_recall_at_100_max": 0.523023, + "nauc_recall_at_100_std": 0.063963, + "nauc_recall_at_100_diff1": 0.6281, + "nauc_recall_at_1000_max": 0.529682, + "nauc_recall_at_1000_std": 0.056354, + "nauc_recall_at_1000_diff1": 0.64713, + "nauc_precision_at_1_max": 0.417377, + "nauc_precision_at_1_std": -0.179937, + "nauc_precision_at_1_diff1": 0.736639, + "nauc_precision_at_3_max": 0.483304, + "nauc_precision_at_3_std": -0.16798, + "nauc_precision_at_3_diff1": 0.663752, + "nauc_precision_at_5_max": 0.5073, + "nauc_precision_at_5_std": -0.136285, + "nauc_precision_at_5_diff1": 0.656388, + "nauc_precision_at_10_max": 0.517627, + "nauc_precision_at_10_std": -0.091529, + "nauc_precision_at_10_diff1": 0.640698, + "nauc_precision_at_20_max": 0.501088, + "nauc_precision_at_20_std": -0.053413, + "nauc_precision_at_20_diff1": 0.618175, + "nauc_precision_at_100_max": 0.523023, + "nauc_precision_at_100_std": 0.063963, + "nauc_precision_at_100_diff1": 0.6281, + "nauc_precision_at_1000_max": 0.529682, + "nauc_precision_at_1000_std": 0.056354, + "nauc_precision_at_1000_diff1": 0.64713, + "nauc_mrr_at_1_max": 0.418261, + "nauc_mrr_at_1_std": -0.179639, + "nauc_mrr_at_1_diff1": 0.73606, + "nauc_mrr_at_3_max": 0.442027, + "nauc_mrr_at_3_std": -0.177026, + "nauc_mrr_at_3_diff1": 0.707917, + "nauc_mrr_at_5_max": 0.444374, + "nauc_mrr_at_5_std": -0.172483, + "nauc_mrr_at_5_diff1": 0.708302, + "nauc_mrr_at_10_max": 0.443679, + "nauc_mrr_at_10_std": -0.16998, + "nauc_mrr_at_10_diff1": 0.708182, + "nauc_mrr_at_20_max": 0.442477, + "nauc_mrr_at_20_std": -0.169601, + "nauc_mrr_at_20_diff1": 0.70816, + "nauc_mrr_at_100_max": 0.442259, + "nauc_mrr_at_100_std": -0.169529, + "nauc_mrr_at_100_diff1": 0.708692, + "nauc_mrr_at_1000_max": 0.442221, + "nauc_mrr_at_1000_std": -0.16964, + "nauc_mrr_at_1000_diff1": 0.708741, + "main_score": 0.75727, + "hf_subset": "php", + "languages": [ + "php-Code" + ] + } + ] + }, + "evaluation_time": 18358.357377052307, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeTransOceanContest.json b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeTransOceanContest.json new file mode 100644 index 0000000000..b855fd0135 --- /dev/null +++ b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeTransOceanContest.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "20da4eb20a4b17300c0986ee148c90867a7f2a4d", + "task_name": "CodeTransOceanContest", + "mteb_version": "1.34.21", + "scores": { + "test": [ + { + "ndcg_at_1": 0.80995, + "ndcg_at_3": 0.85326, + "ndcg_at_5": 0.86261, + "ndcg_at_10": 0.86844, + "ndcg_at_20": 0.87873, + "ndcg_at_100": 0.8845, + "ndcg_at_1000": 0.88498, + "map_at_1": 0.80995, + "map_at_3": 0.84314, + "map_at_5": 0.84834, + "map_at_10": 0.85073, + "map_at_20": 0.85355, + "map_at_100": 0.85431, + "map_at_1000": 0.85432, + "recall_at_1": 0.80995, + "recall_at_3": 0.88235, + "recall_at_5": 0.90498, + "recall_at_10": 0.92308, + "recall_at_20": 0.9638, + "recall_at_100": 0.99548, + "recall_at_1000": 1.0, + "precision_at_1": 0.80995, + "precision_at_3": 0.29412, + "precision_at_5": 0.181, + "precision_at_10": 0.09231, + "precision_at_20": 0.04819, + "precision_at_100": 0.00995, + "precision_at_1000": 0.001, + "mrr_at_1": 0.809955, + "mrr_at_3": 0.843137, + "mrr_at_5": 0.848341, + "mrr_at_10": 0.850733, + "mrr_at_20": 0.853552, + "mrr_at_100": 0.854314, + "mrr_at_1000": 0.854321, + "nauc_ndcg_at_1_max": 0.674226, + "nauc_ndcg_at_1_std": -0.041382, + "nauc_ndcg_at_1_diff1": 0.83388, + "nauc_ndcg_at_3_max": 0.711922, + "nauc_ndcg_at_3_std": -0.00893, + "nauc_ndcg_at_3_diff1": 0.80761, + "nauc_ndcg_at_5_max": 0.69643, + "nauc_ndcg_at_5_std": -0.061053, + "nauc_ndcg_at_5_diff1": 0.80612, + "nauc_ndcg_at_10_max": 0.702181, + "nauc_ndcg_at_10_std": -0.046747, + "nauc_ndcg_at_10_diff1": 0.8111, + "nauc_ndcg_at_20_max": 0.712394, + "nauc_ndcg_at_20_std": -0.034536, + "nauc_ndcg_at_20_diff1": 0.81854, + "nauc_ndcg_at_100_max": 0.704258, + "nauc_ndcg_at_100_std": -0.026751, + "nauc_ndcg_at_100_diff1": 0.816645, + "nauc_ndcg_at_1000_max": 0.702947, + "nauc_ndcg_at_1000_std": -0.031302, + "nauc_ndcg_at_1000_diff1": 0.816411, + "nauc_map_at_1_max": 0.674226, + "nauc_map_at_1_std": -0.041382, + "nauc_map_at_1_diff1": 0.83388, + "nauc_map_at_3_max": 0.703267, + "nauc_map_at_3_std": -0.013679, + "nauc_map_at_3_diff1": 0.815856, + "nauc_map_at_5_max": 0.695188, + "nauc_map_at_5_std": -0.040415, + "nauc_map_at_5_diff1": 0.815002, + "nauc_map_at_10_max": 0.696697, + "nauc_map_at_10_std": -0.035904, + "nauc_map_at_10_diff1": 0.816529, + "nauc_map_at_20_max": 0.69941, + "nauc_map_at_20_std": -0.031924, + "nauc_map_at_20_diff1": 0.818155, + "nauc_map_at_100_max": 0.698438, + "nauc_map_at_100_std": -0.031349, + "nauc_map_at_100_diff1": 0.818053, + "nauc_map_at_1000_max": 0.698423, + "nauc_map_at_1000_std": -0.031403, + "nauc_map_at_1000_diff1": 0.818051, + "nauc_recall_at_1_max": 0.674226, + "nauc_recall_at_1_std": -0.041382, + "nauc_recall_at_1_diff1": 0.83388, + "nauc_recall_at_3_max": 0.74311, + "nauc_recall_at_3_std": 0.006373, + "nauc_recall_at_3_diff1": 0.776407, + "nauc_recall_at_5_max": 0.697099, + "nauc_recall_at_5_std": -0.165429, + "nauc_recall_at_5_diff1": 0.767017, + "nauc_recall_at_10_max": 0.732372, + "nauc_recall_at_10_std": -0.106312, + "nauc_recall_at_10_diff1": 0.78768, + "nauc_recall_at_20_max": 0.881619, + "nauc_recall_at_20_std": -0.047144, + "nauc_recall_at_20_diff1": 0.860651, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 1.0, + "nauc_recall_at_100_diff1": 0.869347, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.674226, + "nauc_precision_at_1_std": -0.041382, + "nauc_precision_at_1_diff1": 0.83388, + "nauc_precision_at_3_max": 0.74311, + "nauc_precision_at_3_std": 0.006373, + "nauc_precision_at_3_diff1": 0.776407, + "nauc_precision_at_5_max": 0.697099, + "nauc_precision_at_5_std": -0.165429, + "nauc_precision_at_5_diff1": 0.767017, + "nauc_precision_at_10_max": 0.732372, + "nauc_precision_at_10_std": -0.106312, + "nauc_precision_at_10_diff1": 0.78768, + "nauc_precision_at_20_max": 0.881619, + "nauc_precision_at_20_std": -0.047144, + "nauc_precision_at_20_diff1": 0.860651, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 0.869347, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.674226, + "nauc_mrr_at_1_std": -0.041382, + "nauc_mrr_at_1_diff1": 0.83388, + "nauc_mrr_at_3_max": 0.703267, + "nauc_mrr_at_3_std": -0.013679, + "nauc_mrr_at_3_diff1": 0.815856, + "nauc_mrr_at_5_max": 0.695188, + "nauc_mrr_at_5_std": -0.040415, + "nauc_mrr_at_5_diff1": 0.815002, + "nauc_mrr_at_10_max": 0.696697, + "nauc_mrr_at_10_std": -0.035904, + "nauc_mrr_at_10_diff1": 0.816529, + "nauc_mrr_at_20_max": 0.69941, + "nauc_mrr_at_20_std": -0.031924, + "nauc_mrr_at_20_diff1": 0.818155, + "nauc_mrr_at_100_max": 0.698438, + "nauc_mrr_at_100_std": -0.031349, + "nauc_mrr_at_100_diff1": 0.818053, + "nauc_mrr_at_1000_max": 0.698423, + "nauc_mrr_at_1000_std": -0.031403, + "nauc_mrr_at_1000_diff1": 0.818051, + "main_score": 0.86844, + "hf_subset": "default", + "languages": [ + "python-Code", + "c++-Code" + ] + } + ] + }, + "evaluation_time": 123.28791236877441, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeTransOceanDL.json b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeTransOceanDL.json new file mode 100644 index 0000000000..2608d558e6 --- /dev/null +++ b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CodeTransOceanDL.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "281562cb8a1265ab5c0824bfa6ddcd9b0a15618f", + "task_name": "CodeTransOceanDL", + "mteb_version": "1.34.21", + "scores": { + "test": [ + { + "ndcg_at_1": 0.08333, + "ndcg_at_3": 0.11329, + "ndcg_at_5": 0.18146, + "ndcg_at_10": 0.33922, + "ndcg_at_20": 0.39399, + "ndcg_at_100": 0.39399, + "ndcg_at_1000": 0.39399, + "map_at_1": 0.08333, + "map_at_3": 0.10463, + "map_at_5": 0.14352, + "map_at_10": 0.20632, + "map_at_20": 0.22362, + "map_at_100": 0.22362, + "map_at_1000": 0.22362, + "recall_at_1": 0.08333, + "recall_at_3": 0.13889, + "recall_at_5": 0.3, + "recall_at_10": 0.8, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.08333, + "precision_at_3": 0.0463, + "precision_at_5": 0.06, + "precision_at_10": 0.08, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.033333, + "mrr_at_3": 0.099074, + "mrr_at_5": 0.116852, + "mrr_at_10": 0.184275, + "mrr_at_20": 0.20313, + "mrr_at_100": 0.20313, + "mrr_at_1000": 0.20313, + "nauc_ndcg_at_1_max": -0.388713, + "nauc_ndcg_at_1_std": -0.011054, + "nauc_ndcg_at_1_diff1": 0.466989, + "nauc_ndcg_at_3_max": -0.419465, + "nauc_ndcg_at_3_std": -0.076967, + "nauc_ndcg_at_3_diff1": 0.421776, + "nauc_ndcg_at_5_max": -0.333053, + "nauc_ndcg_at_5_std": -0.063072, + "nauc_ndcg_at_5_diff1": 0.37135, + "nauc_ndcg_at_10_max": -0.244643, + "nauc_ndcg_at_10_std": -0.078986, + "nauc_ndcg_at_10_diff1": 0.376673, + "nauc_ndcg_at_20_max": -0.346984, + "nauc_ndcg_at_20_std": -0.053791, + "nauc_ndcg_at_20_diff1": 0.401626, + "nauc_ndcg_at_100_max": -0.346984, + "nauc_ndcg_at_100_std": -0.053791, + "nauc_ndcg_at_100_diff1": 0.401626, + "nauc_ndcg_at_1000_max": -0.346984, + "nauc_ndcg_at_1000_std": -0.053791, + "nauc_ndcg_at_1000_diff1": 0.401626, + "nauc_map_at_1_max": -0.388713, + "nauc_map_at_1_std": -0.011054, + "nauc_map_at_1_diff1": 0.466989, + "nauc_map_at_3_max": -0.411535, + "nauc_map_at_3_std": -0.062311, + "nauc_map_at_3_diff1": 0.429703, + "nauc_map_at_5_max": -0.352021, + "nauc_map_at_5_std": -0.054113, + "nauc_map_at_5_diff1": 0.394015, + "nauc_map_at_10_max": -0.319611, + "nauc_map_at_10_std": -0.059001, + "nauc_map_at_10_diff1": 0.399066, + "nauc_map_at_20_max": -0.354103, + "nauc_map_at_20_std": -0.050513, + "nauc_map_at_20_diff1": 0.407569, + "nauc_map_at_100_max": -0.354103, + "nauc_map_at_100_std": -0.050513, + "nauc_map_at_100_diff1": 0.407569, + "nauc_map_at_1000_max": -0.354103, + "nauc_map_at_1000_std": -0.050513, + "nauc_map_at_1000_diff1": 0.407569, + "nauc_recall_at_1_max": -0.388713, + "nauc_recall_at_1_std": -0.011054, + "nauc_recall_at_1_diff1": 0.466989, + "nauc_recall_at_3_max": -0.438291, + "nauc_recall_at_3_std": -0.111161, + "nauc_recall_at_3_diff1": 0.403912, + "nauc_recall_at_5_max": -0.297625, + "nauc_recall_at_5_std": -0.078691, + "nauc_recall_at_5_diff1": 0.330113, + "nauc_recall_at_10_max": 0.032717, + "nauc_recall_at_10_std": -0.148157, + "nauc_recall_at_10_diff1": 0.307276, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": -0.388713, + "nauc_precision_at_1_std": -0.011054, + "nauc_precision_at_1_diff1": 0.466989, + "nauc_precision_at_3_max": -0.438291, + "nauc_precision_at_3_std": -0.111161, + "nauc_precision_at_3_diff1": 0.403912, + "nauc_precision_at_5_max": -0.297625, + "nauc_precision_at_5_std": -0.078691, + "nauc_precision_at_5_diff1": 0.330113, + "nauc_precision_at_10_max": 0.032717, + "nauc_precision_at_10_std": -0.148157, + "nauc_precision_at_10_diff1": 0.307276, + "nauc_precision_at_20_max": NaN, + "nauc_precision_at_20_std": NaN, + "nauc_precision_at_20_diff1": NaN, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": -0.467847, + "nauc_mrr_at_1_std": -0.145783, + "nauc_mrr_at_1_diff1": -0.044674, + "nauc_mrr_at_3_max": -0.279346, + "nauc_mrr_at_3_std": -0.084612, + "nauc_mrr_at_3_diff1": 0.023883, + "nauc_mrr_at_5_max": -0.290716, + "nauc_mrr_at_5_std": -0.093974, + "nauc_mrr_at_5_diff1": 0.07528, + "nauc_mrr_at_10_max": -0.287484, + "nauc_mrr_at_10_std": -0.095941, + "nauc_mrr_at_10_diff1": 0.054789, + "nauc_mrr_at_20_max": -0.309812, + "nauc_mrr_at_20_std": -0.090548, + "nauc_mrr_at_20_diff1": 0.041631, + "nauc_mrr_at_100_max": -0.309812, + "nauc_mrr_at_100_std": -0.090548, + "nauc_mrr_at_100_diff1": 0.041631, + "nauc_mrr_at_1000_max": -0.309812, + "nauc_mrr_at_1000_std": -0.090548, + "nauc_mrr_at_1000_diff1": 0.041631, + "main_score": 0.33922, + "hf_subset": "default", + "languages": [ + "python-Code" + ] + } + ] + }, + "evaluation_time": 98.23712396621704, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CosQA.json b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CosQA.json new file mode 100644 index 0000000000..2a1a972ec7 --- /dev/null +++ b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/CosQA.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "bc5efb7e9d437246ce393ed19d772e08e4a79535", + "task_name": "CosQA", + "mteb_version": "1.34.21", + "scores": { + "test": [ + { + "ndcg_at_1": 0.15, + "ndcg_at_3": 0.23747, + "ndcg_at_5": 0.29092, + "ndcg_at_10": 0.34456, + "ndcg_at_20": 0.37219, + "ndcg_at_100": 0.41251, + "ndcg_at_1000": 0.42197, + "map_at_1": 0.15, + "map_at_3": 0.21467, + "map_at_5": 0.24427, + "map_at_10": 0.26638, + "map_at_20": 0.27386, + "map_at_100": 0.27983, + "map_at_1000": 0.28024, + "recall_at_1": 0.15, + "recall_at_3": 0.304, + "recall_at_5": 0.434, + "recall_at_10": 0.6, + "recall_at_20": 0.71, + "recall_at_100": 0.922, + "recall_at_1000": 0.994, + "precision_at_1": 0.15, + "precision_at_3": 0.10133, + "precision_at_5": 0.0868, + "precision_at_10": 0.06, + "precision_at_20": 0.0355, + "precision_at_100": 0.00922, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.13, + "mrr_at_3": 0.195667, + "mrr_at_5": 0.214267, + "mrr_at_10": 0.242075, + "mrr_at_20": 0.250186, + "mrr_at_100": 0.256436, + "mrr_at_1000": 0.256867, + "nauc_ndcg_at_1_max": 0.032593, + "nauc_ndcg_at_1_std": -0.093607, + "nauc_ndcg_at_1_diff1": 0.345272, + "nauc_ndcg_at_3_max": 0.091909, + "nauc_ndcg_at_3_std": -0.092771, + "nauc_ndcg_at_3_diff1": 0.220051, + "nauc_ndcg_at_5_max": 0.135239, + "nauc_ndcg_at_5_std": -0.069886, + "nauc_ndcg_at_5_diff1": 0.17317, + "nauc_ndcg_at_10_max": 0.126719, + "nauc_ndcg_at_10_std": -0.082603, + "nauc_ndcg_at_10_diff1": 0.165479, + "nauc_ndcg_at_20_max": 0.120633, + "nauc_ndcg_at_20_std": -0.088611, + "nauc_ndcg_at_20_diff1": 0.157967, + "nauc_ndcg_at_100_max": 0.11543, + "nauc_ndcg_at_100_std": -0.067756, + "nauc_ndcg_at_100_diff1": 0.174462, + "nauc_ndcg_at_1000_max": 0.110819, + "nauc_ndcg_at_1000_std": -0.076616, + "nauc_ndcg_at_1000_diff1": 0.190883, + "nauc_map_at_1_max": 0.032593, + "nauc_map_at_1_std": -0.093607, + "nauc_map_at_1_diff1": 0.345272, + "nauc_map_at_3_max": 0.076351, + "nauc_map_at_3_std": -0.094725, + "nauc_map_at_3_diff1": 0.245865, + "nauc_map_at_5_max": 0.101814, + "nauc_map_at_5_std": -0.081052, + "nauc_map_at_5_diff1": 0.217221, + "nauc_map_at_10_max": 0.096141, + "nauc_map_at_10_std": -0.087792, + "nauc_map_at_10_diff1": 0.213795, + "nauc_map_at_20_max": 0.093992, + "nauc_map_at_20_std": -0.0898, + "nauc_map_at_20_diff1": 0.212379, + "nauc_map_at_100_max": 0.093437, + "nauc_map_at_100_std": -0.086653, + "nauc_map_at_100_diff1": 0.214955, + "nauc_map_at_1000_max": 0.093315, + "nauc_map_at_1000_std": -0.086884, + "nauc_map_at_1000_diff1": 0.215572, + "nauc_recall_at_1_max": 0.032593, + "nauc_recall_at_1_std": -0.093607, + "nauc_recall_at_1_diff1": 0.345272, + "nauc_recall_at_3_max": 0.129793, + "nauc_recall_at_3_std": -0.087571, + "nauc_recall_at_3_diff1": 0.158979, + "nauc_recall_at_5_max": 0.218749, + "nauc_recall_at_5_std": -0.041047, + "nauc_recall_at_5_diff1": 0.06671, + "nauc_recall_at_10_max": 0.212395, + "nauc_recall_at_10_std": -0.068281, + "nauc_recall_at_10_diff1": 0.03461, + "nauc_recall_at_20_max": 0.207494, + "nauc_recall_at_20_std": -0.088316, + "nauc_recall_at_20_diff1": -0.029509, + "nauc_recall_at_100_max": 0.288432, + "nauc_recall_at_100_std": 0.2298, + "nauc_recall_at_100_diff1": -0.154888, + "nauc_recall_at_1000_max": 0.851385, + "nauc_recall_at_1000_std": 0.907407, + "nauc_recall_at_1000_diff1": 0.851385, + "nauc_precision_at_1_max": 0.032593, + "nauc_precision_at_1_std": -0.093607, + "nauc_precision_at_1_diff1": 0.345272, + "nauc_precision_at_3_max": 0.129793, + "nauc_precision_at_3_std": -0.087571, + "nauc_precision_at_3_diff1": 0.158979, + "nauc_precision_at_5_max": 0.218749, + "nauc_precision_at_5_std": -0.041047, + "nauc_precision_at_5_diff1": 0.06671, + "nauc_precision_at_10_max": 0.212395, + "nauc_precision_at_10_std": -0.068281, + "nauc_precision_at_10_diff1": 0.03461, + "nauc_precision_at_20_max": 0.207494, + "nauc_precision_at_20_std": -0.088316, + "nauc_precision_at_20_diff1": -0.029509, + "nauc_precision_at_100_max": 0.288432, + "nauc_precision_at_100_std": 0.2298, + "nauc_precision_at_100_diff1": -0.154888, + "nauc_precision_at_1000_max": 0.851385, + "nauc_precision_at_1000_std": 0.907407, + "nauc_precision_at_1000_diff1": 0.851385, + "nauc_mrr_at_1_max": 0.035576, + "nauc_mrr_at_1_std": -0.196091, + "nauc_mrr_at_1_diff1": 0.451573, + "nauc_mrr_at_3_max": 0.033188, + "nauc_mrr_at_3_std": -0.192511, + "nauc_mrr_at_3_diff1": 0.319044, + "nauc_mrr_at_5_max": 0.043265, + "nauc_mrr_at_5_std": -0.189926, + "nauc_mrr_at_5_diff1": 0.292968, + "nauc_mrr_at_10_max": 0.061835, + "nauc_mrr_at_10_std": -0.179, + "nauc_mrr_at_10_diff1": 0.27993, + "nauc_mrr_at_20_max": 0.059341, + "nauc_mrr_at_20_std": -0.179854, + "nauc_mrr_at_20_diff1": 0.280705, + "nauc_mrr_at_100_max": 0.059594, + "nauc_mrr_at_100_std": -0.176327, + "nauc_mrr_at_100_diff1": 0.283339, + "nauc_mrr_at_1000_max": 0.059431, + "nauc_mrr_at_1000_std": -0.176681, + "nauc_mrr_at_1000_diff1": 0.284166, + "main_score": 0.34456, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ] + } + ] + }, + "evaluation_time": 320.1822772026062, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/StackOverflowQA.json b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/StackOverflowQA.json new file mode 100644 index 0000000000..61b8017095 --- /dev/null +++ b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/StackOverflowQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "db8f169f3894c14a00251061f957b2063eef2bd5", + "task_name": "StackOverflowQA", + "mteb_version": "1.34.21", + "scores": { + "test": [ + { + "ndcg_at_1": 0.87563, + "ndcg_at_3": 0.90795, + "ndcg_at_5": 0.91468, + "ndcg_at_10": 0.91924, + "ndcg_at_20": 0.92208, + "ndcg_at_100": 0.92542, + "ndcg_at_1000": 0.92674, + "map_at_1": 0.87563, + "map_at_3": 0.9002, + "map_at_5": 0.90389, + "map_at_10": 0.90577, + "map_at_20": 0.90658, + "map_at_100": 0.90707, + "map_at_1000": 0.90712, + "recall_at_1": 0.87563, + "recall_at_3": 0.93029, + "recall_at_5": 0.94684, + "recall_at_10": 0.96088, + "recall_at_20": 0.97192, + "recall_at_100": 0.98947, + "recall_at_1000": 1.0, + "precision_at_1": 0.87563, + "precision_at_3": 0.3101, + "precision_at_5": 0.18937, + "precision_at_10": 0.09609, + "precision_at_20": 0.0486, + "precision_at_100": 0.00989, + "precision_at_1000": 0.001, + "mrr_at_1": 0.875627, + "mrr_at_3": 0.900201, + "mrr_at_5": 0.903887, + "mrr_at_10": 0.905774, + "mrr_at_20": 0.90658, + "mrr_at_100": 0.907072, + "mrr_at_1000": 0.907119, + "nauc_ndcg_at_1_max": 0.851031, + "nauc_ndcg_at_1_std": -0.033877, + "nauc_ndcg_at_1_diff1": 0.932502, + "nauc_ndcg_at_3_max": 0.866219, + "nauc_ndcg_at_3_std": 0.009147, + "nauc_ndcg_at_3_diff1": 0.923026, + "nauc_ndcg_at_5_max": 0.866081, + "nauc_ndcg_at_5_std": 0.008924, + "nauc_ndcg_at_5_diff1": 0.924341, + "nauc_ndcg_at_10_max": 0.865655, + "nauc_ndcg_at_10_std": 0.008813, + "nauc_ndcg_at_10_diff1": 0.925, + "nauc_ndcg_at_20_max": 0.864518, + "nauc_ndcg_at_20_std": 0.005899, + "nauc_ndcg_at_20_diff1": 0.924749, + "nauc_ndcg_at_100_max": 0.86387, + "nauc_ndcg_at_100_std": 0.005175, + "nauc_ndcg_at_100_diff1": 0.924805, + "nauc_ndcg_at_1000_max": 0.86324, + "nauc_ndcg_at_1000_std": 0.001787, + "nauc_ndcg_at_1000_diff1": 0.925592, + "nauc_map_at_1_max": 0.851031, + "nauc_map_at_1_std": -0.033877, + "nauc_map_at_1_diff1": 0.932502, + "nauc_map_at_3_max": 0.862386, + "nauc_map_at_3_std": -0.002577, + "nauc_map_at_3_diff1": 0.925586, + "nauc_map_at_5_max": 0.862146, + "nauc_map_at_5_std": -0.003411, + "nauc_map_at_5_diff1": 0.926299, + "nauc_map_at_10_max": 0.861963, + "nauc_map_at_10_std": -0.003797, + "nauc_map_at_10_diff1": 0.92663, + "nauc_map_at_20_max": 0.861685, + "nauc_map_at_20_std": -0.004309, + "nauc_map_at_20_diff1": 0.926606, + "nauc_map_at_100_max": 0.861607, + "nauc_map_at_100_std": -0.004296, + "nauc_map_at_100_diff1": 0.926591, + "nauc_map_at_1000_max": 0.861584, + "nauc_map_at_1000_std": -0.004397, + "nauc_map_at_1000_diff1": 0.926621, + "nauc_recall_at_1_max": 0.851031, + "nauc_recall_at_1_std": -0.033877, + "nauc_recall_at_1_diff1": 0.932502, + "nauc_recall_at_3_max": 0.8814, + "nauc_recall_at_3_std": 0.056179, + "nauc_recall_at_3_diff1": 0.912771, + "nauc_recall_at_5_max": 0.885764, + "nauc_recall_at_5_std": 0.071927, + "nauc_recall_at_5_diff1": 0.914899, + "nauc_recall_at_10_max": 0.889552, + "nauc_recall_at_10_std": 0.095213, + "nauc_recall_at_10_diff1": 0.91493, + "nauc_recall_at_20_max": 0.886381, + "nauc_recall_at_20_std": 0.091575, + "nauc_recall_at_20_diff1": 0.907766, + "nauc_recall_at_100_max": 0.897853, + "nauc_recall_at_100_std": 0.191826, + "nauc_recall_at_100_diff1": 0.885188, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.851031, + "nauc_precision_at_1_std": -0.033877, + "nauc_precision_at_1_diff1": 0.932502, + "nauc_precision_at_3_max": 0.8814, + "nauc_precision_at_3_std": 0.056179, + "nauc_precision_at_3_diff1": 0.912771, + "nauc_precision_at_5_max": 0.885764, + "nauc_precision_at_5_std": 0.071927, + "nauc_precision_at_5_diff1": 0.914899, + "nauc_precision_at_10_max": 0.889552, + "nauc_precision_at_10_std": 0.095213, + "nauc_precision_at_10_diff1": 0.91493, + "nauc_precision_at_20_max": 0.886381, + "nauc_precision_at_20_std": 0.091575, + "nauc_precision_at_20_diff1": 0.907766, + "nauc_precision_at_100_max": 0.897853, + "nauc_precision_at_100_std": 0.191826, + "nauc_precision_at_100_diff1": 0.885188, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.851031, + "nauc_mrr_at_1_std": -0.033877, + "nauc_mrr_at_1_diff1": 0.932502, + "nauc_mrr_at_3_max": 0.862386, + "nauc_mrr_at_3_std": -0.002577, + "nauc_mrr_at_3_diff1": 0.925586, + "nauc_mrr_at_5_max": 0.862146, + "nauc_mrr_at_5_std": -0.003411, + "nauc_mrr_at_5_diff1": 0.926299, + "nauc_mrr_at_10_max": 0.861963, + "nauc_mrr_at_10_std": -0.003797, + "nauc_mrr_at_10_diff1": 0.92663, + "nauc_mrr_at_20_max": 0.861685, + "nauc_mrr_at_20_std": -0.004309, + "nauc_mrr_at_20_diff1": 0.926606, + "nauc_mrr_at_100_max": 0.861607, + "nauc_mrr_at_100_std": -0.004296, + "nauc_mrr_at_100_diff1": 0.926591, + "nauc_mrr_at_1000_max": 0.861584, + "nauc_mrr_at_1000_std": -0.004397, + "nauc_mrr_at_1000_diff1": 0.926621, + "main_score": 0.91924, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1542.1715083122253, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/SyntheticText2SQL.json b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/SyntheticText2SQL.json new file mode 100644 index 0000000000..fbf7f569f2 --- /dev/null +++ b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/SyntheticText2SQL.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "686b87296c3a0191b5d9415a00526c62db9fce09", + "task_name": "SyntheticText2SQL", + "mteb_version": "1.34.21", + "scores": { + "test": [ + { + "ndcg_at_1": 0.33892, + "ndcg_at_3": 0.66954, + "ndcg_at_5": 0.68853, + "ndcg_at_10": 0.70039, + "ndcg_at_20": 0.70524, + "ndcg_at_100": 0.709, + "ndcg_at_1000": 0.70948, + "map_at_1": 0.33892, + "map_at_3": 0.59699, + "map_at_5": 0.60765, + "map_at_10": 0.61262, + "map_at_20": 0.61396, + "map_at_100": 0.61454, + "map_at_1000": 0.61456, + "recall_at_1": 0.33892, + "recall_at_3": 0.87592, + "recall_at_5": 0.92155, + "recall_at_10": 0.95778, + "recall_at_20": 0.97693, + "recall_at_100": 0.99641, + "recall_at_1000": 1.0, + "precision_at_1": 0.33892, + "precision_at_3": 0.29197, + "precision_at_5": 0.18431, + "precision_at_10": 0.09578, + "precision_at_20": 0.04885, + "precision_at_100": 0.00996, + "precision_at_1000": 0.001, + "mrr_at_1": 0.417194, + "mrr_at_3": 0.639093, + "mrr_at_5": 0.649322, + "mrr_at_10": 0.653985, + "mrr_at_20": 0.655269, + "mrr_at_100": 0.655848, + "mrr_at_1000": 0.655868, + "nauc_ndcg_at_1_max": 0.104324, + "nauc_ndcg_at_1_std": -0.111962, + "nauc_ndcg_at_1_diff1": -0.013647, + "nauc_ndcg_at_3_max": 0.241791, + "nauc_ndcg_at_3_std": -0.200721, + "nauc_ndcg_at_3_diff1": -0.411591, + "nauc_ndcg_at_5_max": 0.234182, + "nauc_ndcg_at_5_std": -0.189092, + "nauc_ndcg_at_5_diff1": -0.358851, + "nauc_ndcg_at_10_max": 0.219902, + "nauc_ndcg_at_10_std": -0.174524, + "nauc_ndcg_at_10_diff1": -0.313697, + "nauc_ndcg_at_20_max": 0.209849, + "nauc_ndcg_at_20_std": -0.171902, + "nauc_ndcg_at_20_diff1": -0.295502, + "nauc_ndcg_at_100_max": 0.201755, + "nauc_ndcg_at_100_std": -0.167308, + "nauc_ndcg_at_100_diff1": -0.279829, + "nauc_ndcg_at_1000_max": 0.200314, + "nauc_ndcg_at_1000_std": -0.167824, + "nauc_ndcg_at_1000_diff1": -0.277636, + "nauc_map_at_1_max": 0.104324, + "nauc_map_at_1_std": -0.111962, + "nauc_map_at_1_diff1": -0.013647, + "nauc_map_at_3_max": 0.202708, + "nauc_map_at_3_std": -0.175979, + "nauc_map_at_3_diff1": -0.303238, + "nauc_map_at_5_max": 0.197674, + "nauc_map_at_5_std": -0.169447, + "nauc_map_at_5_diff1": -0.274138, + "nauc_map_at_10_max": 0.192142, + "nauc_map_at_10_std": -0.164022, + "nauc_map_at_10_diff1": -0.257269, + "nauc_map_at_20_max": 0.189807, + "nauc_map_at_20_std": -0.163342, + "nauc_map_at_20_diff1": -0.253096, + "nauc_map_at_100_max": 0.188862, + "nauc_map_at_100_std": -0.162737, + "nauc_map_at_100_diff1": -0.251153, + "nauc_map_at_1000_max": 0.188816, + "nauc_map_at_1000_std": -0.162754, + "nauc_map_at_1000_diff1": -0.251079, + "nauc_recall_at_1_max": 0.104324, + "nauc_recall_at_1_std": -0.111962, + "nauc_recall_at_1_diff1": -0.013647, + "nauc_recall_at_3_max": 0.463781, + "nauc_recall_at_3_std": -0.340754, + "nauc_recall_at_3_diff1": -1.022154, + "nauc_recall_at_5_max": 0.536559, + "nauc_recall_at_5_std": -0.34784, + "nauc_recall_at_5_diff1": -1.036549, + "nauc_recall_at_10_max": 0.605171, + "nauc_recall_at_10_std": -0.293902, + "nauc_recall_at_10_diff1": -1.015589, + "nauc_recall_at_20_max": 0.617849, + "nauc_recall_at_20_std": -0.316741, + "nauc_recall_at_20_diff1": -1.035405, + "nauc_recall_at_100_max": 0.739502, + "nauc_recall_at_100_std": 0.018983, + "nauc_recall_at_100_diff1": -1.090712, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.104324, + "nauc_precision_at_1_std": -0.111962, + "nauc_precision_at_1_diff1": -0.013647, + "nauc_precision_at_3_max": 0.463781, + "nauc_precision_at_3_std": -0.340754, + "nauc_precision_at_3_diff1": -1.022154, + "nauc_precision_at_5_max": 0.536559, + "nauc_precision_at_5_std": -0.34784, + "nauc_precision_at_5_diff1": -1.036549, + "nauc_precision_at_10_max": 0.605171, + "nauc_precision_at_10_std": -0.293902, + "nauc_precision_at_10_diff1": -1.015589, + "nauc_precision_at_20_max": 0.617849, + "nauc_precision_at_20_std": -0.316741, + "nauc_precision_at_20_diff1": -1.035405, + "nauc_precision_at_100_max": 0.739502, + "nauc_precision_at_100_std": 0.018983, + "nauc_precision_at_100_diff1": -1.090712, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.143524, + "nauc_mrr_at_1_std": -0.113829, + "nauc_mrr_at_1_diff1": -0.218979, + "nauc_mrr_at_3_max": 0.228754, + "nauc_mrr_at_3_std": -0.17241, + "nauc_mrr_at_3_diff1": -0.440914, + "nauc_mrr_at_5_max": 0.225116, + "nauc_mrr_at_5_std": -0.166427, + "nauc_mrr_at_5_diff1": -0.418173, + "nauc_mrr_at_10_max": 0.220253, + "nauc_mrr_at_10_std": -0.161717, + "nauc_mrr_at_10_diff1": -0.405213, + "nauc_mrr_at_20_max": 0.218055, + "nauc_mrr_at_20_std": -0.161018, + "nauc_mrr_at_20_diff1": -0.402144, + "nauc_mrr_at_100_max": 0.217187, + "nauc_mrr_at_100_std": -0.160414, + "nauc_mrr_at_100_diff1": -0.400585, + "nauc_mrr_at_1000_max": 0.217144, + "nauc_mrr_at_1000_std": -0.16043, + "nauc_mrr_at_1000_diff1": -0.400525, + "main_score": 0.70039, + "hf_subset": "default", + "languages": [ + "eng-Latn", + "sql-Code" + ] + } + ] + }, + "evaluation_time": 928.1394202709198, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/model_meta.json b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/model_meta.json new file mode 100644 index 0000000000..565fc67c52 --- /dev/null +++ b/results/Qodo__Qodo-Embed-1-1.5B/84bbef079b32e8823ec226d4e9e92902706b0eb6/model_meta.json @@ -0,0 +1,28 @@ +{ + "name": "Qodo/Qodo-Embed-1-1.5B", + "revision": "84bbef079b32e8823ec226d4e9e92902706b0eb6", + "release_date": "2025-02-19", + "languages": null, + "n_parameters": 1780000000, + "memory_usage_mb": 6776.0, + "max_tokens": 32000.0, + "embed_dim": 1536, + "license": "QodoAI-Open-RAIL-M", + "open_weights": true, + "public_training_code": null, + "public_training_data": null, + "framework": [ + "Sentence Transformers", + "PyTorch" + ], + "reference": "https://huggingface.co/Qodo/Qodo-Embed-1-1.5B", + "similarity_fn_name": "cosine", + "use_instructions": false, + "training_datasets": null, + "adapted_from":"Alibaba-NLP/gte-Qwen2-1.5B-instruct", + "superseded_by": null, + "modalities": [ + "text" + ], + "loader": null +} \ No newline at end of file