diff --git a/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/CmedqaRetrieval.json b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/CmedqaRetrieval.json new file mode 100644 index 0000000000..811261417f --- /dev/null +++ b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/CmedqaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301", + "task_name": "CmedqaRetrieval", + "mteb_version": "1.28.0", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.3821, + "ndcg_at_3": 0.38438, + "ndcg_at_5": 0.40443, + "ndcg_at_10": 0.43306, + "ndcg_at_20": 0.46183, + "ndcg_at_100": 0.50646, + "ndcg_at_1000": 0.5282, + "map_at_1": 0.25153, + "map_at_3": 0.33038, + "map_at_5": 0.35228, + "map_at_10": 0.36924, + "map_at_20": 0.3796, + "map_at_100": 0.38792, + "map_at_1000": 0.38914, + "recall_at_1": 0.25153, + "recall_at_3": 0.38253, + "recall_at_5": 0.44508, + "recall_at_10": 0.53012, + "recall_at_20": 0.62915, + "recall_at_100": 0.83487, + "recall_at_1000": 0.98059, + "precision_at_1": 0.3821, + "precision_at_3": 0.21605, + "precision_at_5": 0.15664, + "precision_at_10": 0.09585, + "precision_at_20": 0.05731, + "precision_at_100": 0.01545, + "precision_at_1000": 0.00183, + "mrr_at_1": 0.382096, + "mrr_at_3": 0.436776, + "mrr_at_5": 0.449929, + "mrr_at_10": 0.460602, + "mrr_at_20": 0.466425, + "mrr_at_100": 0.470609, + "mrr_at_1000": 0.471081, + "nauc_ndcg_at_1_max": 0.472941, + "nauc_ndcg_at_1_std": 0.011506, + "nauc_ndcg_at_1_diff1": 0.65073, + "nauc_ndcg_at_3_max": 0.400306, + "nauc_ndcg_at_3_std": -0.021844, + "nauc_ndcg_at_3_diff1": 0.556355, + "nauc_ndcg_at_5_max": 0.384437, + "nauc_ndcg_at_5_std": -0.028594, + "nauc_ndcg_at_5_diff1": 0.543432, + "nauc_ndcg_at_10_max": 0.379058, + "nauc_ndcg_at_10_std": -0.022464, + "nauc_ndcg_at_10_diff1": 0.532127, + "nauc_ndcg_at_20_max": 0.386325, + "nauc_ndcg_at_20_std": -0.014661, + "nauc_ndcg_at_20_diff1": 0.530613, + "nauc_ndcg_at_100_max": 0.402745, + "nauc_ndcg_at_100_std": 0.012013, + "nauc_ndcg_at_100_diff1": 0.535367, + "nauc_ndcg_at_1000_max": 0.407732, + "nauc_ndcg_at_1000_std": 0.004336, + "nauc_ndcg_at_1000_diff1": 0.544837, + "nauc_map_at_1_max": 0.297667, + "nauc_map_at_1_std": -0.065036, + "nauc_map_at_1_diff1": 0.55916, + "nauc_map_at_3_max": 0.348091, + "nauc_map_at_3_std": -0.059474, + "nauc_map_at_3_diff1": 0.540836, + "nauc_map_at_5_max": 0.359765, + "nauc_map_at_5_std": -0.051715, + "nauc_map_at_5_diff1": 0.538996, + "nauc_map_at_10_max": 0.367189, + "nauc_map_at_10_std": -0.042277, + "nauc_map_at_10_diff1": 0.535705, + "nauc_map_at_20_max": 0.373278, + "nauc_map_at_20_std": -0.037319, + "nauc_map_at_20_diff1": 0.536623, + "nauc_map_at_100_max": 0.378037, + "nauc_map_at_100_std": -0.030759, + "nauc_map_at_100_diff1": 0.538127, + "nauc_map_at_1000_max": 0.378518, + "nauc_map_at_1000_std": -0.030761, + "nauc_map_at_1000_diff1": 0.538489, + "nauc_recall_at_1_max": 0.297667, + "nauc_recall_at_1_std": -0.065036, + "nauc_recall_at_1_diff1": 0.55916, + "nauc_recall_at_3_max": 0.302726, + "nauc_recall_at_3_std": -0.064455, + "nauc_recall_at_3_diff1": 0.47646, + "nauc_recall_at_5_max": 0.295304, + "nauc_recall_at_5_std": -0.053595, + "nauc_recall_at_5_diff1": 0.446824, + "nauc_recall_at_10_max": 0.280634, + "nauc_recall_at_10_std": -0.02122, + "nauc_recall_at_10_diff1": 0.40478, + "nauc_recall_at_20_max": 0.278269, + "nauc_recall_at_20_std": -0.000429, + "nauc_recall_at_20_diff1": 0.372018, + "nauc_recall_at_100_max": 0.30131, + "nauc_recall_at_100_std": 0.193349, + "nauc_recall_at_100_diff1": 0.292745, + "nauc_recall_at_1000_max": 0.33964, + "nauc_recall_at_1000_std": 0.532129, + "nauc_recall_at_1000_diff1": 0.197514, + "nauc_precision_at_1_max": 0.472941, + "nauc_precision_at_1_std": 0.011506, + "nauc_precision_at_1_diff1": 0.65073, + "nauc_precision_at_3_max": 0.454966, + "nauc_precision_at_3_std": 0.05432, + "nauc_precision_at_3_diff1": 0.440588, + "nauc_precision_at_5_max": 0.439257, + "nauc_precision_at_5_std": 0.081048, + "nauc_precision_at_5_diff1": 0.375711, + "nauc_precision_at_10_max": 0.412745, + "nauc_precision_at_10_std": 0.120234, + "nauc_precision_at_10_diff1": 0.299403, + "nauc_precision_at_20_max": 0.398018, + "nauc_precision_at_20_std": 0.152937, + "nauc_precision_at_20_diff1": 0.244872, + "nauc_precision_at_100_max": 0.35012, + "nauc_precision_at_100_std": 0.240025, + "nauc_precision_at_100_diff1": 0.132948, + "nauc_precision_at_1000_max": 0.291727, + "nauc_precision_at_1000_std": 0.206349, + "nauc_precision_at_1000_diff1": 0.0646, + "nauc_mrr_at_1_max": 0.472941, + "nauc_mrr_at_1_std": 0.011506, + "nauc_mrr_at_1_diff1": 0.65073, + "nauc_mrr_at_3_max": 0.456503, + "nauc_mrr_at_3_std": 0.005387, + "nauc_mrr_at_3_diff1": 0.618489, + "nauc_mrr_at_5_max": 0.453089, + "nauc_mrr_at_5_std": 0.007656, + "nauc_mrr_at_5_diff1": 0.61148, + "nauc_mrr_at_10_max": 0.452298, + "nauc_mrr_at_10_std": 0.0115, + "nauc_mrr_at_10_diff1": 0.607758, + "nauc_mrr_at_20_max": 0.452277, + "nauc_mrr_at_20_std": 0.011928, + "nauc_mrr_at_20_diff1": 0.606927, + "nauc_mrr_at_100_max": 0.453238, + "nauc_mrr_at_100_std": 0.013097, + "nauc_mrr_at_100_diff1": 0.607616, + "nauc_mrr_at_1000_max": 0.453298, + "nauc_mrr_at_1000_std": 0.012893, + "nauc_mrr_at_1000_diff1": 0.607901, + "main_score": 0.43306, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 16793.309911966324, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/CovidRetrieval.json b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/CovidRetrieval.json new file mode 100644 index 0000000000..c49bb25955 --- /dev/null +++ b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/CovidRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "1271c7809071a13532e05f25fb53511ffce77117", + "task_name": "CovidRetrieval", + "mteb_version": "1.28.0", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.77977, + "ndcg_at_3": 0.84489, + "ndcg_at_5": 0.86183, + "ndcg_at_10": 0.87159, + "ndcg_at_20": 0.87638, + "ndcg_at_100": 0.87976, + "ndcg_at_1000": 0.88026, + "map_at_1": 0.77713, + "map_at_3": 0.82947, + "map_at_5": 0.83909, + "map_at_10": 0.84332, + "map_at_20": 0.84472, + "map_at_100": 0.8452, + "map_at_1000": 0.84522, + "recall_at_1": 0.77713, + "recall_at_3": 0.88857, + "recall_at_5": 0.92914, + "recall_at_10": 0.95838, + "recall_at_20": 0.97682, + "recall_at_100": 0.99473, + "recall_at_1000": 0.99895, + "precision_at_1": 0.77977, + "precision_at_3": 0.29856, + "precision_at_5": 0.18757, + "precision_at_10": 0.09684, + "precision_at_20": 0.04937, + "precision_at_100": 0.01005, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.779768, + "mrr_at_3": 0.82947, + "mrr_at_5": 0.839322, + "mrr_at_10": 0.843297, + "mrr_at_20": 0.844602, + "mrr_at_100": 0.845085, + "mrr_at_1000": 0.845098, + "nauc_ndcg_at_1_max": 0.106257, + "nauc_ndcg_at_1_std": -0.37587, + "nauc_ndcg_at_1_diff1": 0.876739, + "nauc_ndcg_at_3_max": 0.114731, + "nauc_ndcg_at_3_std": -0.394782, + "nauc_ndcg_at_3_diff1": 0.851518, + "nauc_ndcg_at_5_max": 0.133553, + "nauc_ndcg_at_5_std": -0.380635, + "nauc_ndcg_at_5_diff1": 0.847753, + "nauc_ndcg_at_10_max": 0.135939, + "nauc_ndcg_at_10_std": -0.353862, + "nauc_ndcg_at_10_diff1": 0.852025, + "nauc_ndcg_at_20_max": 0.123043, + "nauc_ndcg_at_20_std": -0.344532, + "nauc_ndcg_at_20_diff1": 0.854869, + "nauc_ndcg_at_100_max": 0.12359, + "nauc_ndcg_at_100_std": -0.362481, + "nauc_ndcg_at_100_diff1": 0.856223, + "nauc_ndcg_at_1000_max": 0.12554, + "nauc_ndcg_at_1000_std": -0.364419, + "nauc_ndcg_at_1000_diff1": 0.856547, + "nauc_map_at_1_max": 0.107918, + "nauc_map_at_1_std": -0.380017, + "nauc_map_at_1_diff1": 0.88115, + "nauc_map_at_3_max": 0.112948, + "nauc_map_at_3_std": -0.387444, + "nauc_map_at_3_diff1": 0.859009, + "nauc_map_at_5_max": 0.123632, + "nauc_map_at_5_std": -0.381401, + "nauc_map_at_5_diff1": 0.857334, + "nauc_map_at_10_max": 0.125679, + "nauc_map_at_10_std": -0.37196, + "nauc_map_at_10_diff1": 0.858887, + "nauc_map_at_20_max": 0.122576, + "nauc_map_at_20_std": -0.369194, + "nauc_map_at_20_diff1": 0.859492, + "nauc_map_at_100_max": 0.123137, + "nauc_map_at_100_std": -0.371156, + "nauc_map_at_100_diff1": 0.859659, + "nauc_map_at_1000_max": 0.123179, + "nauc_map_at_1000_std": -0.371229, + "nauc_map_at_1000_diff1": 0.859666, + "nauc_recall_at_1_max": 0.107918, + "nauc_recall_at_1_std": -0.380017, + "nauc_recall_at_1_diff1": 0.88115, + "nauc_recall_at_3_max": 0.115712, + "nauc_recall_at_3_std": -0.420403, + "nauc_recall_at_3_diff1": 0.822085, + "nauc_recall_at_5_max": 0.193703, + "nauc_recall_at_5_std": -0.364651, + "nauc_recall_at_5_diff1": 0.790229, + "nauc_recall_at_10_max": 0.247021, + "nauc_recall_at_10_std": -0.130473, + "nauc_recall_at_10_diff1": 0.790931, + "nauc_recall_at_20_max": 0.083776, + "nauc_recall_at_20_std": 0.195544, + "nauc_recall_at_20_diff1": 0.801544, + "nauc_recall_at_100_max": -0.164048, + "nauc_recall_at_100_std": -0.04217, + "nauc_recall_at_100_diff1": 0.773525, + "nauc_recall_at_1000_max": 0.357809, + "nauc_recall_at_1000_std": -0.171046, + "nauc_recall_at_1000_diff1": 0.722168, + "nauc_precision_at_1_max": 0.106257, + "nauc_precision_at_1_std": -0.37587, + "nauc_precision_at_1_diff1": 0.876739, + "nauc_precision_at_3_max": 0.123466, + "nauc_precision_at_3_std": -0.352022, + "nauc_precision_at_3_diff1": 0.693658, + "nauc_precision_at_5_max": 0.218156, + "nauc_precision_at_5_std": -0.267725, + "nauc_precision_at_5_diff1": 0.556505, + "nauc_precision_at_10_max": 0.280424, + "nauc_precision_at_10_std": -0.039005, + "nauc_precision_at_10_diff1": 0.400286, + "nauc_precision_at_20_max": 0.178965, + "nauc_precision_at_20_std": 0.200597, + "nauc_precision_at_20_diff1": 0.197173, + "nauc_precision_at_100_max": 0.195903, + "nauc_precision_at_100_std": 0.151328, + "nauc_precision_at_100_diff1": -0.268167, + "nauc_precision_at_1000_max": 0.302629, + "nauc_precision_at_1000_std": 0.186426, + "nauc_precision_at_1000_diff1": -0.49766, + "nauc_mrr_at_1_max": 0.114448, + "nauc_mrr_at_1_std": -0.384039, + "nauc_mrr_at_1_diff1": 0.876739, + "nauc_mrr_at_3_max": 0.120501, + "nauc_mrr_at_3_std": -0.396139, + "nauc_mrr_at_3_diff1": 0.858329, + "nauc_mrr_at_5_max": 0.125492, + "nauc_mrr_at_5_std": -0.386156, + "nauc_mrr_at_5_diff1": 0.856892, + "nauc_mrr_at_10_max": 0.126474, + "nauc_mrr_at_10_std": -0.377216, + "nauc_mrr_at_10_diff1": 0.858803, + "nauc_mrr_at_20_max": 0.123774, + "nauc_mrr_at_20_std": -0.375013, + "nauc_mrr_at_20_diff1": 0.8595, + "nauc_mrr_at_100_max": 0.124338, + "nauc_mrr_at_100_std": -0.376993, + "nauc_mrr_at_100_diff1": 0.859667, + "nauc_mrr_at_1000_max": 0.124381, + "nauc_mrr_at_1000_std": -0.377067, + "nauc_mrr_at_1000_diff1": 0.859674, + "main_score": 0.87159, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 17476.262266874313, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/DuRetrieval.json b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/DuRetrieval.json new file mode 100644 index 0000000000..90dc37d67e --- /dev/null +++ b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/DuRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced", + "task_name": "DuRetrieval", + "mteb_version": "1.28.0", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.9195, + "ndcg_at_3": 0.89564, + "ndcg_at_5": 0.8874, + "ndcg_at_10": 0.90816, + "ndcg_at_20": 0.92049, + "ndcg_at_100": 0.92944, + "ndcg_at_1000": 0.93101, + "map_at_1": 0.27469, + "map_at_3": 0.58833, + "map_at_5": 0.74146, + "map_at_10": 0.8449, + "map_at_20": 0.86583, + "map_at_100": 0.8715, + "map_at_1000": 0.87167, + "recall_at_1": 0.27469, + "recall_at_3": 0.61253, + "recall_at_5": 0.78605, + "recall_at_10": 0.91984, + "recall_at_20": 0.96157, + "recall_at_100": 0.99012, + "recall_at_1000": 0.9985, + "precision_at_1": 0.9195, + "precision_at_3": 0.79933, + "precision_at_5": 0.6752, + "precision_at_10": 0.43125, + "precision_at_20": 0.23365, + "precision_at_100": 0.04873, + "precision_at_1000": 0.00491, + "mrr_at_1": 0.9195, + "mrr_at_3": 0.946833, + "mrr_at_5": 0.948033, + "mrr_at_10": 0.948327, + "mrr_at_20": 0.948645, + "mrr_at_100": 0.948776, + "mrr_at_1000": 0.948791, + "nauc_ndcg_at_1_max": 0.383147, + "nauc_ndcg_at_1_std": 0.254415, + "nauc_ndcg_at_1_diff1": 0.200724, + "nauc_ndcg_at_3_max": 0.284644, + "nauc_ndcg_at_3_std": 0.163111, + "nauc_ndcg_at_3_diff1": 0.04136, + "nauc_ndcg_at_5_max": 0.208656, + "nauc_ndcg_at_5_std": 0.074921, + "nauc_ndcg_at_5_diff1": 0.069999, + "nauc_ndcg_at_10_max": 0.319021, + "nauc_ndcg_at_10_std": 0.098952, + "nauc_ndcg_at_10_diff1": 0.074908, + "nauc_ndcg_at_20_max": 0.351138, + "nauc_ndcg_at_20_std": 0.159928, + "nauc_ndcg_at_20_diff1": 0.058599, + "nauc_ndcg_at_100_max": 0.354467, + "nauc_ndcg_at_100_std": 0.202691, + "nauc_ndcg_at_100_diff1": 0.046742, + "nauc_ndcg_at_1000_max": 0.348608, + "nauc_ndcg_at_1000_std": 0.197889, + "nauc_ndcg_at_1000_diff1": 0.04455, + "nauc_map_at_1_max": -0.145448, + "nauc_map_at_1_std": -0.36809, + "nauc_map_at_1_diff1": 0.48945, + "nauc_map_at_3_max": -0.043561, + "nauc_map_at_3_std": -0.365114, + "nauc_map_at_3_diff1": 0.345076, + "nauc_map_at_5_max": 0.047926, + "nauc_map_at_5_std": -0.29224, + "nauc_map_at_5_diff1": 0.236996, + "nauc_map_at_10_max": 0.233542, + "nauc_map_at_10_std": -0.081883, + "nauc_map_at_10_diff1": 0.103199, + "nauc_map_at_20_max": 0.261761, + "nauc_map_at_20_std": 0.043185, + "nauc_map_at_20_diff1": 0.057029, + "nauc_map_at_100_max": 0.26251, + "nauc_map_at_100_std": 0.073792, + "nauc_map_at_100_diff1": 0.044363, + "nauc_map_at_1000_max": 0.26215, + "nauc_map_at_1000_std": 0.073779, + "nauc_map_at_1000_diff1": 0.043981, + "nauc_recall_at_1_max": -0.145448, + "nauc_recall_at_1_std": -0.36809, + "nauc_recall_at_1_diff1": 0.48945, + "nauc_recall_at_3_max": -0.081063, + "nauc_recall_at_3_std": -0.399847, + "nauc_recall_at_3_diff1": 0.361148, + "nauc_recall_at_5_max": -0.018721, + "nauc_recall_at_5_std": -0.375321, + "nauc_recall_at_5_diff1": 0.274684, + "nauc_recall_at_10_max": 0.274409, + "nauc_recall_at_10_std": -0.174715, + "nauc_recall_at_10_diff1": 0.181816, + "nauc_recall_at_20_max": 0.420335, + "nauc_recall_at_20_std": 0.132844, + "nauc_recall_at_20_diff1": 0.111351, + "nauc_recall_at_100_max": 0.634582, + "nauc_recall_at_100_std": 0.541563, + "nauc_recall_at_100_diff1": 0.087789, + "nauc_recall_at_1000_max": 0.737315, + "nauc_recall_at_1000_std": 0.783703, + "nauc_recall_at_1000_diff1": 0.151756, + "nauc_precision_at_1_max": 0.383147, + "nauc_precision_at_1_std": 0.254415, + "nauc_precision_at_1_diff1": 0.200724, + "nauc_precision_at_3_max": 0.337041, + "nauc_precision_at_3_std": 0.407562, + "nauc_precision_at_3_diff1": -0.415615, + "nauc_precision_at_5_max": 0.266911, + "nauc_precision_at_5_std": 0.440977, + "nauc_precision_at_5_diff1": -0.428939, + "nauc_precision_at_10_max": 0.235844, + "nauc_precision_at_10_std": 0.486534, + "nauc_precision_at_10_diff1": -0.393653, + "nauc_precision_at_20_max": 0.186268, + "nauc_precision_at_20_std": 0.522362, + "nauc_precision_at_20_diff1": -0.381004, + "nauc_precision_at_100_max": 0.153183, + "nauc_precision_at_100_std": 0.524707, + "nauc_precision_at_100_diff1": -0.375629, + "nauc_precision_at_1000_max": 0.145627, + "nauc_precision_at_1000_std": 0.519881, + "nauc_precision_at_1000_diff1": -0.376074, + "nauc_mrr_at_1_max": 0.383147, + "nauc_mrr_at_1_std": 0.254415, + "nauc_mrr_at_1_diff1": 0.200724, + "nauc_mrr_at_3_max": 0.484265, + "nauc_mrr_at_3_std": 0.32605, + "nauc_mrr_at_3_diff1": 0.205285, + "nauc_mrr_at_5_max": 0.476027, + "nauc_mrr_at_5_std": 0.325635, + "nauc_mrr_at_5_diff1": 0.197022, + "nauc_mrr_at_10_max": 0.474584, + "nauc_mrr_at_10_std": 0.325057, + "nauc_mrr_at_10_diff1": 0.198632, + "nauc_mrr_at_20_max": 0.471959, + "nauc_mrr_at_20_std": 0.323991, + "nauc_mrr_at_20_diff1": 0.19741, + "nauc_mrr_at_100_max": 0.471371, + "nauc_mrr_at_100_std": 0.32264, + "nauc_mrr_at_100_diff1": 0.196933, + "nauc_mrr_at_1000_max": 0.471235, + "nauc_mrr_at_1000_std": 0.322476, + "nauc_mrr_at_1000_diff1": 0.196877, + "main_score": 0.90816, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 17510.853296995163, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/EcomRetrieval.json b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/EcomRetrieval.json new file mode 100644 index 0000000000..dec998dde8 --- /dev/null +++ b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/EcomRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9", + "task_name": "EcomRetrieval", + "mteb_version": "1.28.0", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.559, + "ndcg_at_3": 0.66019, + "ndcg_at_5": 0.6848, + "ndcg_at_10": 0.70678, + "ndcg_at_20": 0.72024, + "ndcg_at_100": 0.72933, + "ndcg_at_1000": 0.73204, + "map_at_1": 0.559, + "map_at_3": 0.63567, + "map_at_5": 0.64927, + "map_at_10": 0.65832, + "map_at_20": 0.66205, + "map_at_100": 0.66331, + "map_at_1000": 0.6634, + "recall_at_1": 0.559, + "recall_at_3": 0.731, + "recall_at_5": 0.791, + "recall_at_10": 0.859, + "recall_at_20": 0.912, + "recall_at_100": 0.961, + "recall_at_1000": 0.983, + "precision_at_1": 0.559, + "precision_at_3": 0.24367, + "precision_at_5": 0.1582, + "precision_at_10": 0.0859, + "precision_at_20": 0.0456, + "precision_at_100": 0.00961, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.559, + "mrr_at_3": 0.635667, + "mrr_at_5": 0.649267, + "mrr_at_10": 0.658319, + "mrr_at_20": 0.662046, + "mrr_at_100": 0.663313, + "mrr_at_1000": 0.663405, + "nauc_ndcg_at_1_max": 0.07969, + "nauc_ndcg_at_1_std": -0.138408, + "nauc_ndcg_at_1_diff1": 0.74252, + "nauc_ndcg_at_3_max": 0.095736, + "nauc_ndcg_at_3_std": -0.111447, + "nauc_ndcg_at_3_diff1": 0.691241, + "nauc_ndcg_at_5_max": 0.115841, + "nauc_ndcg_at_5_std": -0.109319, + "nauc_ndcg_at_5_diff1": 0.685011, + "nauc_ndcg_at_10_max": 0.12167, + "nauc_ndcg_at_10_std": -0.101057, + "nauc_ndcg_at_10_diff1": 0.69022, + "nauc_ndcg_at_20_max": 0.133886, + "nauc_ndcg_at_20_std": -0.084852, + "nauc_ndcg_at_20_diff1": 0.695569, + "nauc_ndcg_at_100_max": 0.128341, + "nauc_ndcg_at_100_std": -0.078253, + "nauc_ndcg_at_100_diff1": 0.697176, + "nauc_ndcg_at_1000_max": 0.125105, + "nauc_ndcg_at_1000_std": -0.085495, + "nauc_ndcg_at_1000_diff1": 0.699608, + "nauc_map_at_1_max": 0.07969, + "nauc_map_at_1_std": -0.138408, + "nauc_map_at_1_diff1": 0.74252, + "nauc_map_at_3_max": 0.091602, + "nauc_map_at_3_std": -0.119092, + "nauc_map_at_3_diff1": 0.704804, + "nauc_map_at_5_max": 0.100994, + "nauc_map_at_5_std": -0.118746, + "nauc_map_at_5_diff1": 0.702168, + "nauc_map_at_10_max": 0.103198, + "nauc_map_at_10_std": -0.11574, + "nauc_map_at_10_diff1": 0.704827, + "nauc_map_at_20_max": 0.105658, + "nauc_map_at_20_std": -0.11232, + "nauc_map_at_20_diff1": 0.706199, + "nauc_map_at_100_max": 0.104852, + "nauc_map_at_100_std": -0.111232, + "nauc_map_at_100_diff1": 0.706486, + "nauc_map_at_1000_max": 0.104838, + "nauc_map_at_1000_std": -0.111379, + "nauc_map_at_1000_diff1": 0.706609, + "nauc_recall_at_1_max": 0.07969, + "nauc_recall_at_1_std": -0.138408, + "nauc_recall_at_1_diff1": 0.74252, + "nauc_recall_at_3_max": 0.110164, + "nauc_recall_at_3_std": -0.084394, + "nauc_recall_at_3_diff1": 0.643724, + "nauc_recall_at_5_max": 0.18305, + "nauc_recall_at_5_std": -0.068603, + "nauc_recall_at_5_diff1": 0.612438, + "nauc_recall_at_10_max": 0.238141, + "nauc_recall_at_10_std": -0.009059, + "nauc_recall_at_10_diff1": 0.60688, + "nauc_recall_at_20_max": 0.429479, + "nauc_recall_at_20_std": 0.209999, + "nauc_recall_at_20_diff1": 0.617557, + "nauc_recall_at_100_max": 0.638859, + "nauc_recall_at_100_std": 0.736108, + "nauc_recall_at_100_diff1": 0.564067, + "nauc_recall_at_1000_max": 0.895645, + "nauc_recall_at_1000_std": 0.992311, + "nauc_recall_at_1000_diff1": 0.599138, + "nauc_precision_at_1_max": 0.07969, + "nauc_precision_at_1_std": -0.138408, + "nauc_precision_at_1_diff1": 0.74252, + "nauc_precision_at_3_max": 0.110164, + "nauc_precision_at_3_std": -0.084394, + "nauc_precision_at_3_diff1": 0.643724, + "nauc_precision_at_5_max": 0.18305, + "nauc_precision_at_5_std": -0.068603, + "nauc_precision_at_5_diff1": 0.612438, + "nauc_precision_at_10_max": 0.238141, + "nauc_precision_at_10_std": -0.009059, + "nauc_precision_at_10_diff1": 0.60688, + "nauc_precision_at_20_max": 0.429479, + "nauc_precision_at_20_std": 0.209999, + "nauc_precision_at_20_diff1": 0.617557, + "nauc_precision_at_100_max": 0.638859, + "nauc_precision_at_100_std": 0.736108, + "nauc_precision_at_100_diff1": 0.564067, + "nauc_precision_at_1000_max": 0.895645, + "nauc_precision_at_1000_std": 0.992311, + "nauc_precision_at_1000_diff1": 0.599138, + "nauc_mrr_at_1_max": 0.07969, + "nauc_mrr_at_1_std": -0.138408, + "nauc_mrr_at_1_diff1": 0.74252, + "nauc_mrr_at_3_max": 0.091602, + "nauc_mrr_at_3_std": -0.119092, + "nauc_mrr_at_3_diff1": 0.704804, + "nauc_mrr_at_5_max": 0.100994, + "nauc_mrr_at_5_std": -0.118746, + "nauc_mrr_at_5_diff1": 0.702168, + "nauc_mrr_at_10_max": 0.103198, + "nauc_mrr_at_10_std": -0.11574, + "nauc_mrr_at_10_diff1": 0.704827, + "nauc_mrr_at_20_max": 0.105658, + "nauc_mrr_at_20_std": -0.11232, + "nauc_mrr_at_20_diff1": 0.706199, + "nauc_mrr_at_100_max": 0.104852, + "nauc_mrr_at_100_std": -0.111232, + "nauc_mrr_at_100_diff1": 0.706486, + "nauc_mrr_at_1000_max": 0.104838, + "nauc_mrr_at_1000_std": -0.111379, + "nauc_mrr_at_1000_diff1": 0.706609, + "main_score": 0.70678, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 2566.890925168991, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/MMarcoRetrieval.json b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/MMarcoRetrieval.json new file mode 100644 index 0000000000..1b24083041 --- /dev/null +++ b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/MMarcoRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "539bbde593d947e2a124ba72651aafc09eb33fc2", + "task_name": "MMarcoRetrieval", + "mteb_version": "1.28.0", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.75745, + "ndcg_at_3": 0.82188, + "ndcg_at_5": 0.83869, + "ndcg_at_10": 0.85118, + "ndcg_at_20": 0.85624, + "ndcg_at_100": 0.8605, + "ndcg_at_1000": 0.86177, + "map_at_1": 0.7333, + "map_at_3": 0.80333, + "map_at_5": 0.81349, + "map_at_10": 0.81906, + "map_at_20": 0.82059, + "map_at_100": 0.82127, + "map_at_1000": 0.82132, + "recall_at_1": 0.7333, + "recall_at_3": 0.86823, + "recall_at_5": 0.90814, + "recall_at_10": 0.94509, + "recall_at_20": 0.96422, + "recall_at_100": 0.986, + "recall_at_1000": 0.99599, + "precision_at_1": 0.75745, + "precision_at_3": 0.30626, + "precision_at_5": 0.19281, + "precision_at_10": 0.1005, + "precision_at_20": 0.05133, + "precision_at_100": 0.01051, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.75745, + "mrr_at_3": 0.810148, + "mrr_at_5": 0.818658, + "mrr_at_10": 0.823666, + "mrr_at_20": 0.824972, + "mrr_at_100": 0.825545, + "mrr_at_1000": 0.825588, + "nauc_ndcg_at_1_max": 0.131958, + "nauc_ndcg_at_1_std": -0.272173, + "nauc_ndcg_at_1_diff1": 0.835531, + "nauc_ndcg_at_3_max": 0.171078, + "nauc_ndcg_at_3_std": -0.241926, + "nauc_ndcg_at_3_diff1": 0.804983, + "nauc_ndcg_at_5_max": 0.193079, + "nauc_ndcg_at_5_std": -0.215239, + "nauc_ndcg_at_5_diff1": 0.806484, + "nauc_ndcg_at_10_max": 0.195153, + "nauc_ndcg_at_10_std": -0.199978, + "nauc_ndcg_at_10_diff1": 0.809953, + "nauc_ndcg_at_20_max": 0.195474, + "nauc_ndcg_at_20_std": -0.193015, + "nauc_ndcg_at_20_diff1": 0.812567, + "nauc_ndcg_at_100_max": 0.185694, + "nauc_ndcg_at_100_std": -0.204957, + "nauc_ndcg_at_100_diff1": 0.813873, + "nauc_ndcg_at_1000_max": 0.181522, + "nauc_ndcg_at_1000_std": -0.213527, + "nauc_ndcg_at_1000_diff1": 0.814459, + "nauc_map_at_1_max": 0.091671, + "nauc_map_at_1_std": -0.276298, + "nauc_map_at_1_diff1": 0.825107, + "nauc_map_at_3_max": 0.151633, + "nauc_map_at_3_std": -0.258352, + "nauc_map_at_3_diff1": 0.812211, + "nauc_map_at_5_max": 0.164272, + "nauc_map_at_5_std": -0.243902, + "nauc_map_at_5_diff1": 0.813282, + "nauc_map_at_10_max": 0.164997, + "nauc_map_at_10_std": -0.238602, + "nauc_map_at_10_diff1": 0.814676, + "nauc_map_at_20_max": 0.164924, + "nauc_map_at_20_std": -0.236935, + "nauc_map_at_20_diff1": 0.81543, + "nauc_map_at_100_max": 0.16381, + "nauc_map_at_100_std": -0.23832, + "nauc_map_at_100_diff1": 0.815622, + "nauc_map_at_1000_max": 0.163702, + "nauc_map_at_1000_std": -0.238562, + "nauc_map_at_1000_diff1": 0.815641, + "nauc_recall_at_1_max": 0.091671, + "nauc_recall_at_1_std": -0.276298, + "nauc_recall_at_1_diff1": 0.825107, + "nauc_recall_at_3_max": 0.22362, + "nauc_recall_at_3_std": -0.193576, + "nauc_recall_at_3_diff1": 0.767854, + "nauc_recall_at_5_max": 0.325794, + "nauc_recall_at_5_std": -0.074792, + "nauc_recall_at_5_diff1": 0.757277, + "nauc_recall_at_10_max": 0.426548, + "nauc_recall_at_10_std": 0.125123, + "nauc_recall_at_10_diff1": 0.757877, + "nauc_recall_at_20_max": 0.55737, + "nauc_recall_at_20_std": 0.390218, + "nauc_recall_at_20_diff1": 0.771814, + "nauc_recall_at_100_max": 0.641065, + "nauc_recall_at_100_std": 0.701918, + "nauc_recall_at_100_diff1": 0.781114, + "nauc_recall_at_1000_max": 0.698721, + "nauc_recall_at_1000_std": 0.772501, + "nauc_recall_at_1000_diff1": 0.84894, + "nauc_precision_at_1_max": 0.131958, + "nauc_precision_at_1_std": -0.272173, + "nauc_precision_at_1_diff1": 0.835531, + "nauc_precision_at_3_max": 0.224513, + "nauc_precision_at_3_std": -0.06339, + "nauc_precision_at_3_diff1": 0.365773, + "nauc_precision_at_5_max": 0.270464, + "nauc_precision_at_5_std": 0.052645, + "nauc_precision_at_5_diff1": 0.226914, + "nauc_precision_at_10_max": 0.28171, + "nauc_precision_at_10_std": 0.170803, + "nauc_precision_at_10_diff1": 0.061321, + "nauc_precision_at_20_max": 0.279335, + "nauc_precision_at_20_std": 0.249318, + "nauc_precision_at_20_diff1": -0.044056, + "nauc_precision_at_100_max": 0.218066, + "nauc_precision_at_100_std": 0.24659, + "nauc_precision_at_100_diff1": -0.193589, + "nauc_precision_at_1000_max": 0.179584, + "nauc_precision_at_1000_std": 0.204178, + "nauc_precision_at_1000_diff1": -0.275299, + "nauc_mrr_at_1_max": 0.131958, + "nauc_mrr_at_1_std": -0.272173, + "nauc_mrr_at_1_diff1": 0.835531, + "nauc_mrr_at_3_max": 0.168011, + "nauc_mrr_at_3_std": -0.244238, + "nauc_mrr_at_3_diff1": 0.815653, + "nauc_mrr_at_5_max": 0.176255, + "nauc_mrr_at_5_std": -0.233454, + "nauc_mrr_at_5_diff1": 0.816503, + "nauc_mrr_at_10_max": 0.175126, + "nauc_mrr_at_10_std": -0.229439, + "nauc_mrr_at_10_diff1": 0.818044, + "nauc_mrr_at_20_max": 0.174687, + "nauc_mrr_at_20_std": -0.228673, + "nauc_mrr_at_20_diff1": 0.81851, + "nauc_mrr_at_100_max": 0.173429, + "nauc_mrr_at_100_std": -0.230229, + "nauc_mrr_at_100_diff1": 0.818634, + "nauc_mrr_at_1000_max": 0.173315, + "nauc_mrr_at_1000_std": -0.230451, + "nauc_mrr_at_1000_diff1": 0.818656, + "main_score": 0.85118, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 7827.2281358242035, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/MedicalRetrieval.json b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/MedicalRetrieval.json new file mode 100644 index 0000000000..1e78e21061 --- /dev/null +++ b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/MedicalRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6", + "task_name": "MedicalRetrieval", + "mteb_version": "1.28.0", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.55, + "ndcg_at_3": 0.61334, + "ndcg_at_5": 0.62591, + "ndcg_at_10": 0.63913, + "ndcg_at_20": 0.64748, + "ndcg_at_100": 0.66675, + "ndcg_at_1000": 0.67894, + "map_at_1": 0.55, + "map_at_3": 0.5985, + "map_at_5": 0.6056, + "map_at_10": 0.61103, + "map_at_20": 0.61332, + "map_at_100": 0.61583, + "map_at_1000": 0.61627, + "recall_at_1": 0.55, + "recall_at_3": 0.656, + "recall_at_5": 0.686, + "recall_at_10": 0.727, + "recall_at_20": 0.76, + "recall_at_100": 0.866, + "recall_at_1000": 0.963, + "precision_at_1": 0.55, + "precision_at_3": 0.21867, + "precision_at_5": 0.1372, + "precision_at_10": 0.0727, + "precision_at_20": 0.038, + "precision_at_100": 0.00866, + "precision_at_1000": 0.00096, + "mrr_at_1": 0.55, + "mrr_at_3": 0.5985, + "mrr_at_5": 0.6056, + "mrr_at_10": 0.611027, + "mrr_at_20": 0.613323, + "mrr_at_100": 0.61583, + "mrr_at_1000": 0.616268, + "nauc_ndcg_at_1_max": 0.457614, + "nauc_ndcg_at_1_std": 0.175893, + "nauc_ndcg_at_1_diff1": 0.846716, + "nauc_ndcg_at_3_max": 0.433722, + "nauc_ndcg_at_3_std": 0.216068, + "nauc_ndcg_at_3_diff1": 0.797078, + "nauc_ndcg_at_5_max": 0.436396, + "nauc_ndcg_at_5_std": 0.219309, + "nauc_ndcg_at_5_diff1": 0.790538, + "nauc_ndcg_at_10_max": 0.435044, + "nauc_ndcg_at_10_std": 0.22351, + "nauc_ndcg_at_10_diff1": 0.787137, + "nauc_ndcg_at_20_max": 0.436169, + "nauc_ndcg_at_20_std": 0.221586, + "nauc_ndcg_at_20_diff1": 0.786764, + "nauc_ndcg_at_100_max": 0.431354, + "nauc_ndcg_at_100_std": 0.224145, + "nauc_ndcg_at_100_diff1": 0.7835, + "nauc_ndcg_at_1000_max": 0.436698, + "nauc_ndcg_at_1000_std": 0.2217, + "nauc_ndcg_at_1000_diff1": 0.790599, + "nauc_map_at_1_max": 0.457614, + "nauc_map_at_1_std": 0.175893, + "nauc_map_at_1_diff1": 0.846716, + "nauc_map_at_3_max": 0.440428, + "nauc_map_at_3_std": 0.206219, + "nauc_map_at_3_diff1": 0.808918, + "nauc_map_at_5_max": 0.442069, + "nauc_map_at_5_std": 0.207613, + "nauc_map_at_5_diff1": 0.80545, + "nauc_map_at_10_max": 0.441289, + "nauc_map_at_10_std": 0.209091, + "nauc_map_at_10_diff1": 0.80417, + "nauc_map_at_20_max": 0.441582, + "nauc_map_at_20_std": 0.208327, + "nauc_map_at_20_diff1": 0.804097, + "nauc_map_at_100_max": 0.441052, + "nauc_map_at_100_std": 0.208381, + "nauc_map_at_100_diff1": 0.803634, + "nauc_map_at_1000_max": 0.44124, + "nauc_map_at_1000_std": 0.20846, + "nauc_map_at_1000_diff1": 0.803866, + "nauc_recall_at_1_max": 0.457614, + "nauc_recall_at_1_std": 0.175893, + "nauc_recall_at_1_diff1": 0.846716, + "nauc_recall_at_3_max": 0.412286, + "nauc_recall_at_3_std": 0.246954, + "nauc_recall_at_3_diff1": 0.760107, + "nauc_recall_at_5_max": 0.417596, + "nauc_recall_at_5_std": 0.258669, + "nauc_recall_at_5_diff1": 0.740678, + "nauc_recall_at_10_max": 0.412619, + "nauc_recall_at_10_std": 0.27896, + "nauc_recall_at_10_diff1": 0.723198, + "nauc_recall_at_20_max": 0.416291, + "nauc_recall_at_20_std": 0.277209, + "nauc_recall_at_20_diff1": 0.714933, + "nauc_recall_at_100_max": 0.34897, + "nauc_recall_at_100_std": 0.348631, + "nauc_recall_at_100_diff1": 0.63698, + "nauc_recall_at_1000_max": 0.434905, + "nauc_recall_at_1000_std": 0.443006, + "nauc_recall_at_1000_diff1": 0.667197, + "nauc_precision_at_1_max": 0.457614, + "nauc_precision_at_1_std": 0.175893, + "nauc_precision_at_1_diff1": 0.846716, + "nauc_precision_at_3_max": 0.412286, + "nauc_precision_at_3_std": 0.246954, + "nauc_precision_at_3_diff1": 0.760107, + "nauc_precision_at_5_max": 0.417596, + "nauc_precision_at_5_std": 0.258669, + "nauc_precision_at_5_diff1": 0.740678, + "nauc_precision_at_10_max": 0.412619, + "nauc_precision_at_10_std": 0.27896, + "nauc_precision_at_10_diff1": 0.723198, + "nauc_precision_at_20_max": 0.416291, + "nauc_precision_at_20_std": 0.277209, + "nauc_precision_at_20_diff1": 0.714933, + "nauc_precision_at_100_max": 0.34897, + "nauc_precision_at_100_std": 0.348631, + "nauc_precision_at_100_diff1": 0.63698, + "nauc_precision_at_1000_max": 0.434905, + "nauc_precision_at_1000_std": 0.443006, + "nauc_precision_at_1000_diff1": 0.667197, + "nauc_mrr_at_1_max": 0.457614, + "nauc_mrr_at_1_std": 0.175893, + "nauc_mrr_at_1_diff1": 0.846716, + "nauc_mrr_at_3_max": 0.440428, + "nauc_mrr_at_3_std": 0.206219, + "nauc_mrr_at_3_diff1": 0.808918, + "nauc_mrr_at_5_max": 0.442069, + "nauc_mrr_at_5_std": 0.207613, + "nauc_mrr_at_5_diff1": 0.80545, + "nauc_mrr_at_10_max": 0.441289, + "nauc_mrr_at_10_std": 0.209091, + "nauc_mrr_at_10_diff1": 0.80417, + "nauc_mrr_at_20_max": 0.441582, + "nauc_mrr_at_20_std": 0.208327, + "nauc_mrr_at_20_diff1": 0.804097, + "nauc_mrr_at_100_max": 0.441051, + "nauc_mrr_at_100_std": 0.208382, + "nauc_mrr_at_100_diff1": 0.803634, + "nauc_mrr_at_1000_max": 0.441239, + "nauc_mrr_at_1000_std": 0.208461, + "nauc_mrr_at_1000_diff1": 0.803866, + "main_score": 0.63913, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 7289.948871850967, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/T2Retrieval.json b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/T2Retrieval.json new file mode 100644 index 0000000000..c502a83d8d --- /dev/null +++ b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/T2Retrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "8731a845f1bf500a4f111cf1070785c793d10e64", + "task_name": "T2Retrieval", + "mteb_version": "1.28.0", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.9122, + "ndcg_at_3": 0.88018, + "ndcg_at_5": 0.87023, + "ndcg_at_10": 0.87256, + "ndcg_at_20": 0.88897, + "ndcg_at_100": 0.90386, + "ndcg_at_1000": 0.90811, + "map_at_1": 0.28203, + "map_at_3": 0.5616, + "map_at_5": 0.69287, + "map_at_10": 0.80361, + "map_at_20": 0.83152, + "map_at_100": 0.83894, + "map_at_1000": 0.8394, + "recall_at_1": 0.28203, + "recall_at_3": 0.57821, + "recall_at_5": 0.72616, + "recall_at_10": 0.8656, + "recall_at_20": 0.92145, + "recall_at_100": 0.96827, + "recall_at_1000": 0.98967, + "precision_at_1": 0.9122, + "precision_at_3": 0.76946, + "precision_at_5": 0.64822, + "precision_at_10": 0.43371, + "precision_at_20": 0.23978, + "precision_at_100": 0.0508, + "precision_at_1000": 0.00518, + "mrr_at_1": 0.912195, + "mrr_at_3": 0.932214, + "mrr_at_5": 0.934857, + "mrr_at_10": 0.936156, + "mrr_at_20": 0.936549, + "mrr_at_100": 0.936802, + "mrr_at_1000": 0.936823, + "nauc_ndcg_at_1_max": 0.574777, + "nauc_ndcg_at_1_std": 0.311, + "nauc_ndcg_at_1_diff1": 0.478114, + "nauc_ndcg_at_3_max": 0.478308, + "nauc_ndcg_at_3_std": 0.275389, + "nauc_ndcg_at_3_diff1": 0.135529, + "nauc_ndcg_at_5_max": 0.406728, + "nauc_ndcg_at_5_std": 0.225295, + "nauc_ndcg_at_5_diff1": 0.139263, + "nauc_ndcg_at_10_max": 0.336021, + "nauc_ndcg_at_10_std": 0.159671, + "nauc_ndcg_at_10_diff1": 0.17678, + "nauc_ndcg_at_20_max": 0.390053, + "nauc_ndcg_at_20_std": 0.21218, + "nauc_ndcg_at_20_diff1": 0.175961, + "nauc_ndcg_at_100_max": 0.44208, + "nauc_ndcg_at_100_std": 0.27437, + "nauc_ndcg_at_100_diff1": 0.177034, + "nauc_ndcg_at_1000_max": 0.448086, + "nauc_ndcg_at_1000_std": 0.274606, + "nauc_ndcg_at_1000_diff1": 0.182625, + "nauc_map_at_1_max": -0.248235, + "nauc_map_at_1_std": -0.395167, + "nauc_map_at_1_diff1": 0.553064, + "nauc_map_at_3_max": -0.175097, + "nauc_map_at_3_std": -0.396402, + "nauc_map_at_3_diff1": 0.412552, + "nauc_map_at_5_max": -0.054253, + "nauc_map_at_5_std": -0.306847, + "nauc_map_at_5_diff1": 0.331347, + "nauc_map_at_10_max": 0.20598, + "nauc_map_at_10_std": -0.01943, + "nauc_map_at_10_diff1": 0.190818, + "nauc_map_at_20_max": 0.3055, + "nauc_map_at_20_std": 0.125299, + "nauc_map_at_20_diff1": 0.142657, + "nauc_map_at_100_max": 0.328313, + "nauc_map_at_100_std": 0.15956, + "nauc_map_at_100_diff1": 0.135621, + "nauc_map_at_1000_max": 0.328957, + "nauc_map_at_1000_std": 0.16003, + "nauc_map_at_1000_diff1": 0.135638, + "nauc_recall_at_1_max": -0.248235, + "nauc_recall_at_1_std": -0.395167, + "nauc_recall_at_1_diff1": 0.553064, + "nauc_recall_at_3_max": -0.21037, + "nauc_recall_at_3_std": -0.422523, + "nauc_recall_at_3_diff1": 0.397052, + "nauc_recall_at_5_max": -0.128722, + "nauc_recall_at_5_std": -0.370029, + "nauc_recall_at_5_diff1": 0.323874, + "nauc_recall_at_10_max": 0.112835, + "nauc_recall_at_10_std": -0.102755, + "nauc_recall_at_10_diff1": 0.183331, + "nauc_recall_at_20_max": 0.286028, + "nauc_recall_at_20_std": 0.154075, + "nauc_recall_at_20_diff1": 0.101417, + "nauc_recall_at_100_max": 0.465586, + "nauc_recall_at_100_std": 0.446235, + "nauc_recall_at_100_diff1": 0.050231, + "nauc_recall_at_1000_max": 0.648937, + "nauc_recall_at_1000_std": 0.645098, + "nauc_recall_at_1000_diff1": 0.087287, + "nauc_precision_at_1_max": 0.574777, + "nauc_precision_at_1_std": 0.311, + "nauc_precision_at_1_diff1": 0.478114, + "nauc_precision_at_3_max": 0.516685, + "nauc_precision_at_3_std": 0.481005, + "nauc_precision_at_3_diff1": -0.362615, + "nauc_precision_at_5_max": 0.482878, + "nauc_precision_at_5_std": 0.52525, + "nauc_precision_at_5_diff1": -0.396363, + "nauc_precision_at_10_max": 0.446478, + "nauc_precision_at_10_std": 0.569255, + "nauc_precision_at_10_diff1": -0.382201, + "nauc_precision_at_20_max": 0.43105, + "nauc_precision_at_20_std": 0.601817, + "nauc_precision_at_20_diff1": -0.37841, + "nauc_precision_at_100_max": 0.41861, + "nauc_precision_at_100_std": 0.612084, + "nauc_precision_at_100_diff1": -0.377674, + "nauc_precision_at_1000_max": 0.409026, + "nauc_precision_at_1000_std": 0.602503, + "nauc_precision_at_1000_diff1": -0.37656, + "nauc_mrr_at_1_max": 0.574777, + "nauc_mrr_at_1_std": 0.311, + "nauc_mrr_at_1_diff1": 0.478114, + "nauc_mrr_at_3_max": 0.618122, + "nauc_mrr_at_3_std": 0.353495, + "nauc_mrr_at_3_diff1": 0.479786, + "nauc_mrr_at_5_max": 0.620303, + "nauc_mrr_at_5_std": 0.358677, + "nauc_mrr_at_5_diff1": 0.480773, + "nauc_mrr_at_10_max": 0.619735, + "nauc_mrr_at_10_std": 0.358413, + "nauc_mrr_at_10_diff1": 0.480359, + "nauc_mrr_at_20_max": 0.619171, + "nauc_mrr_at_20_std": 0.358176, + "nauc_mrr_at_20_diff1": 0.480247, + "nauc_mrr_at_100_max": 0.618535, + "nauc_mrr_at_100_std": 0.357606, + "nauc_mrr_at_100_diff1": 0.480105, + "nauc_mrr_at_1000_max": 0.618448, + "nauc_mrr_at_1000_std": 0.357473, + "nauc_mrr_at_1000_diff1": 0.480124, + "main_score": 0.87256, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 30857.8738656044, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/VideoRetrieval.json b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/VideoRetrieval.json new file mode 100644 index 0000000000..eac0008a00 --- /dev/null +++ b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/VideoRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "58c2597a5943a2ba48f4668c3b90d796283c5639", + "task_name": "VideoRetrieval", + "mteb_version": "1.28.0", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.659, + "ndcg_at_3": 0.75395, + "ndcg_at_5": 0.7697, + "ndcg_at_10": 0.78549, + "ndcg_at_20": 0.79489, + "ndcg_at_100": 0.80269, + "ndcg_at_1000": 0.80459, + "map_at_1": 0.659, + "map_at_3": 0.73167, + "map_at_5": 0.74047, + "map_at_10": 0.74713, + "map_at_20": 0.74974, + "map_at_100": 0.75089, + "map_at_1000": 0.75096, + "recall_at_1": 0.659, + "recall_at_3": 0.818, + "recall_at_5": 0.856, + "recall_at_10": 0.904, + "recall_at_20": 0.941, + "recall_at_100": 0.982, + "recall_at_1000": 0.997, + "precision_at_1": 0.659, + "precision_at_3": 0.27267, + "precision_at_5": 0.1712, + "precision_at_10": 0.0904, + "precision_at_20": 0.04705, + "precision_at_100": 0.00982, + "precision_at_1000": 0.001, + "mrr_at_1": 0.658, + "mrr_at_3": 0.731167, + "mrr_at_5": 0.739967, + "mrr_at_10": 0.746633, + "mrr_at_20": 0.749235, + "mrr_at_100": 0.750391, + "mrr_at_1000": 0.750463, + "nauc_ndcg_at_1_max": -0.071029, + "nauc_ndcg_at_1_std": -0.40454, + "nauc_ndcg_at_1_diff1": 0.737435, + "nauc_ndcg_at_3_max": -0.102162, + "nauc_ndcg_at_3_std": -0.5103, + "nauc_ndcg_at_3_diff1": 0.707969, + "nauc_ndcg_at_5_max": -0.081372, + "nauc_ndcg_at_5_std": -0.50728, + "nauc_ndcg_at_5_diff1": 0.704751, + "nauc_ndcg_at_10_max": -0.08305, + "nauc_ndcg_at_10_std": -0.489725, + "nauc_ndcg_at_10_diff1": 0.709306, + "nauc_ndcg_at_20_max": -0.0761, + "nauc_ndcg_at_20_std": -0.480327, + "nauc_ndcg_at_20_diff1": 0.710216, + "nauc_ndcg_at_100_max": -0.076188, + "nauc_ndcg_at_100_std": -0.463162, + "nauc_ndcg_at_100_diff1": 0.711421, + "nauc_ndcg_at_1000_max": -0.078627, + "nauc_ndcg_at_1000_std": -0.467545, + "nauc_ndcg_at_1000_diff1": 0.71352, + "nauc_map_at_1_max": -0.071029, + "nauc_map_at_1_std": -0.40454, + "nauc_map_at_1_diff1": 0.737435, + "nauc_map_at_3_max": -0.096192, + "nauc_map_at_3_std": -0.480776, + "nauc_map_at_3_diff1": 0.71545, + "nauc_map_at_5_max": -0.085417, + "nauc_map_at_5_std": -0.478473, + "nauc_map_at_5_diff1": 0.714239, + "nauc_map_at_10_max": -0.086365, + "nauc_map_at_10_std": -0.471493, + "nauc_map_at_10_diff1": 0.715916, + "nauc_map_at_20_max": -0.08516, + "nauc_map_at_20_std": -0.469588, + "nauc_map_at_20_diff1": 0.716177, + "nauc_map_at_100_max": -0.08499, + "nauc_map_at_100_std": -0.467142, + "nauc_map_at_100_diff1": 0.71638, + "nauc_map_at_1000_max": -0.084997, + "nauc_map_at_1000_std": -0.467198, + "nauc_map_at_1000_diff1": 0.716453, + "nauc_recall_at_1_max": -0.071029, + "nauc_recall_at_1_std": -0.40454, + "nauc_recall_at_1_diff1": 0.737435, + "nauc_recall_at_3_max": -0.123728, + "nauc_recall_at_3_std": -0.624854, + "nauc_recall_at_3_diff1": 0.679381, + "nauc_recall_at_5_max": -0.055908, + "nauc_recall_at_5_std": -0.640017, + "nauc_recall_at_5_diff1": 0.660113, + "nauc_recall_at_10_max": -0.052142, + "nauc_recall_at_10_std": -0.591241, + "nauc_recall_at_10_diff1": 0.670484, + "nauc_recall_at_20_max": 0.060224, + "nauc_recall_at_20_std": -0.529879, + "nauc_recall_at_20_diff1": 0.657686, + "nauc_recall_at_100_max": 0.334085, + "nauc_recall_at_100_std": 0.153258, + "nauc_recall_at_100_diff1": 0.595887, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.956427, + "nauc_recall_at_1000_diff1": 0.912854, + "nauc_precision_at_1_max": -0.071029, + "nauc_precision_at_1_std": -0.40454, + "nauc_precision_at_1_diff1": 0.737435, + "nauc_precision_at_3_max": -0.123728, + "nauc_precision_at_3_std": -0.624854, + "nauc_precision_at_3_diff1": 0.679381, + "nauc_precision_at_5_max": -0.055908, + "nauc_precision_at_5_std": -0.640017, + "nauc_precision_at_5_diff1": 0.660113, + "nauc_precision_at_10_max": -0.052142, + "nauc_precision_at_10_std": -0.591241, + "nauc_precision_at_10_diff1": 0.670484, + "nauc_precision_at_20_max": 0.060224, + "nauc_precision_at_20_std": -0.529879, + "nauc_precision_at_20_diff1": 0.657686, + "nauc_precision_at_100_max": 0.334085, + "nauc_precision_at_100_std": 0.153258, + "nauc_precision_at_100_diff1": 0.595887, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 0.956427, + "nauc_precision_at_1000_diff1": 0.912854, + "nauc_mrr_at_1_max": -0.07705, + "nauc_mrr_at_1_std": -0.404353, + "nauc_mrr_at_1_diff1": 0.739529, + "nauc_mrr_at_3_max": -0.099824, + "nauc_mrr_at_3_std": -0.480572, + "nauc_mrr_at_3_diff1": 0.716749, + "nauc_mrr_at_5_max": -0.089185, + "nauc_mrr_at_5_std": -0.478265, + "nauc_mrr_at_5_diff1": 0.715584, + "nauc_mrr_at_10_max": -0.090233, + "nauc_mrr_at_10_std": -0.471288, + "nauc_mrr_at_10_diff1": 0.717295, + "nauc_mrr_at_20_max": -0.089076, + "nauc_mrr_at_20_std": -0.469382, + "nauc_mrr_at_20_diff1": 0.717573, + "nauc_mrr_at_100_max": -0.088927, + "nauc_mrr_at_100_std": -0.466939, + "nauc_mrr_at_100_diff1": 0.717782, + "nauc_mrr_at_1000_max": -0.088935, + "nauc_mrr_at_1000_std": -0.466994, + "nauc_mrr_at_1000_diff1": 0.717855, + "main_score": 0.78549, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 2466.23748755455, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/model_meta.json b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/model_meta.json new file mode 100644 index 0000000000..b6cf349809 --- /dev/null +++ b/results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/model_meta.json @@ -0,0 +1 @@ +{"name": "infly/inf-retriever-v1", "revision": "d2d074546028c0012b5cc6af78c4fac24896e67f", "release_date": "2024-12-24", "languages": ["eng_Latn", "zho_Hans"], "n_parameters": 7069121024, "memory_usage": null, "max_tokens": 131072.0, "embed_dim": 3584, "license": null, "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/infly/inf-retriever-v1", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": null, "adapted_from": "Alibaba-NLP/gte-Qwen2-7B-instruct", "superseded_by": null, "loader": "sentence_transformers_loader"} \ No newline at end of file