diff --git a/results/openai__text-embedding-3-large/2/CMedQAv2-reranking.json b/results/openai__text-embedding-3-large/2/CMedQAv2-reranking.json new file mode 100644 index 0000000000..c5bebe11b9 --- /dev/null +++ b/results/openai__text-embedding-3-large/2/CMedQAv2-reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "23d186750531a14a0357ca22cd92d712fd512ea0", + "task_name": "CMedQAv2-reranking", + "mteb_version": "1.24.0", + "scores": { + "test": [ + { + "map": 0.751224, + "mrr": 0.788045, + "nAUC_map_max": 0.532528, + "nAUC_map_std": 0.170824, + "nAUC_map_diff1": 0.506192, + "nAUC_mrr_max": 0.619113, + "nAUC_mrr_std": 0.232415, + "nAUC_mrr_diff1": 0.582075, + "main_score": 0.751224, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 1884.4257838726044, + "kg_co2_emissions": 0.00044117606664305007 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-large/2/CmedqaRetrieval.json b/results/openai__text-embedding-3-large/2/CmedqaRetrieval.json new file mode 100644 index 0000000000..0091b900b7 --- /dev/null +++ b/results/openai__text-embedding-3-large/2/CmedqaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301", + "task_name": "CmedqaRetrieval", + "mteb_version": "1.24.0", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.23806, + "ndcg_at_3": 0.24019, + "ndcg_at_5": 0.25216, + "ndcg_at_10": 0.27435, + "ndcg_at_20": 0.29721, + "ndcg_at_100": 0.34303, + "ndcg_at_1000": 0.38144, + "map_at_1": 0.14967, + "map_at_3": 0.20044, + "map_at_5": 0.21298, + "map_at_10": 0.22494, + "map_at_20": 0.2325, + "map_at_100": 0.2402, + "map_at_1000": 0.24198, + "recall_at_1": 0.14967, + "recall_at_3": 0.24011, + "recall_at_5": 0.28091, + "recall_at_10": 0.34567, + "recall_at_20": 0.42386, + "recall_at_100": 0.63685, + "recall_at_1000": 0.90421, + "precision_at_1": 0.23806, + "precision_at_3": 0.1382, + "precision_at_5": 0.09947, + "precision_at_10": 0.06294, + "precision_at_20": 0.039, + "precision_at_100": 0.01199, + "precision_at_1000": 0.0017, + "mrr_at_1": 0.23806, + "mrr_at_3": 0.281279, + "mrr_at_5": 0.291931, + "mrr_at_10": 0.301517, + "mrr_at_20": 0.306952, + "mrr_at_100": 0.311724, + "mrr_at_1000": 0.312605, + "nauc_ndcg_at_1_max": 0.385752, + "nauc_ndcg_at_1_std": -0.040464, + "nauc_ndcg_at_1_diff1": 0.52806, + "nauc_ndcg_at_3_max": 0.34744, + "nauc_ndcg_at_3_std": -0.040927, + "nauc_ndcg_at_3_diff1": 0.446717, + "nauc_ndcg_at_5_max": 0.338609, + "nauc_ndcg_at_5_std": -0.045501, + "nauc_ndcg_at_5_diff1": 0.438516, + "nauc_ndcg_at_10_max": 0.339566, + "nauc_ndcg_at_10_std": -0.043432, + "nauc_ndcg_at_10_diff1": 0.429259, + "nauc_ndcg_at_20_max": 0.343991, + "nauc_ndcg_at_20_std": -0.026292, + "nauc_ndcg_at_20_diff1": 0.421478, + "nauc_ndcg_at_100_max": 0.347669, + "nauc_ndcg_at_100_std": 0.000656, + "nauc_ndcg_at_100_diff1": 0.412529, + "nauc_ndcg_at_1000_max": 0.356152, + "nauc_ndcg_at_1000_std": 0.004637, + "nauc_ndcg_at_1000_diff1": 0.422442, + "nauc_map_at_1_max": 0.301477, + "nauc_map_at_1_std": -0.072747, + "nauc_map_at_1_diff1": 0.493146, + "nauc_map_at_3_max": 0.326267, + "nauc_map_at_3_std": -0.060284, + "nauc_map_at_3_diff1": 0.454711, + "nauc_map_at_5_max": 0.330391, + "nauc_map_at_5_std": -0.058726, + "nauc_map_at_5_diff1": 0.449565, + "nauc_map_at_10_max": 0.336287, + "nauc_map_at_10_std": -0.055241, + "nauc_map_at_10_diff1": 0.443792, + "nauc_map_at_20_max": 0.339646, + "nauc_map_at_20_std": -0.049295, + "nauc_map_at_20_diff1": 0.441548, + "nauc_map_at_100_max": 0.341537, + "nauc_map_at_100_std": -0.043372, + "nauc_map_at_100_diff1": 0.440142, + "nauc_map_at_1000_max": 0.342049, + "nauc_map_at_1000_std": -0.042582, + "nauc_map_at_1000_diff1": 0.440503, + "nauc_recall_at_1_max": 0.301477, + "nauc_recall_at_1_std": -0.072747, + "nauc_recall_at_1_diff1": 0.493146, + "nauc_recall_at_3_max": 0.291041, + "nauc_recall_at_3_std": -0.053511, + "nauc_recall_at_3_diff1": 0.393961, + "nauc_recall_at_5_max": 0.283306, + "nauc_recall_at_5_std": -0.048442, + "nauc_recall_at_5_diff1": 0.364103, + "nauc_recall_at_10_max": 0.284127, + "nauc_recall_at_10_std": -0.03875, + "nauc_recall_at_10_diff1": 0.334214, + "nauc_recall_at_20_max": 0.290039, + "nauc_recall_at_20_std": 0.015398, + "nauc_recall_at_20_diff1": 0.30011, + "nauc_recall_at_100_max": 0.286998, + "nauc_recall_at_100_std": 0.132302, + "nauc_recall_at_100_diff1": 0.233188, + "nauc_recall_at_1000_max": 0.416109, + "nauc_recall_at_1000_std": 0.408188, + "nauc_recall_at_1000_diff1": 0.204341, + "nauc_precision_at_1_max": 0.385752, + "nauc_precision_at_1_std": -0.040464, + "nauc_precision_at_1_diff1": 0.52806, + "nauc_precision_at_3_max": 0.386608, + "nauc_precision_at_3_std": -0.00421, + "nauc_precision_at_3_diff1": 0.386592, + "nauc_precision_at_5_max": 0.370232, + "nauc_precision_at_5_std": 0.001644, + "nauc_precision_at_5_diff1": 0.350039, + "nauc_precision_at_10_max": 0.366455, + "nauc_precision_at_10_std": 0.029112, + "nauc_precision_at_10_diff1": 0.297162, + "nauc_precision_at_20_max": 0.350007, + "nauc_precision_at_20_std": 0.070674, + "nauc_precision_at_20_diff1": 0.254044, + "nauc_precision_at_100_max": 0.27252, + "nauc_precision_at_100_std": 0.160222, + "nauc_precision_at_100_diff1": 0.148347, + "nauc_precision_at_1000_max": 0.198375, + "nauc_precision_at_1000_std": 0.16985, + "nauc_precision_at_1000_diff1": 0.065617, + "nauc_mrr_at_1_max": 0.385752, + "nauc_mrr_at_1_std": -0.040464, + "nauc_mrr_at_1_diff1": 0.52806, + "nauc_mrr_at_3_max": 0.366945, + "nauc_mrr_at_3_std": -0.029993, + "nauc_mrr_at_3_diff1": 0.484221, + "nauc_mrr_at_5_max": 0.365061, + "nauc_mrr_at_5_std": -0.028935, + "nauc_mrr_at_5_diff1": 0.478225, + "nauc_mrr_at_10_max": 0.364478, + "nauc_mrr_at_10_std": -0.027304, + "nauc_mrr_at_10_diff1": 0.473609, + "nauc_mrr_at_20_max": 0.364233, + "nauc_mrr_at_20_std": -0.023302, + "nauc_mrr_at_20_diff1": 0.471052, + "nauc_mrr_at_100_max": 0.364592, + "nauc_mrr_at_100_std": -0.021667, + "nauc_mrr_at_100_diff1": 0.470421, + "nauc_mrr_at_1000_max": 0.364768, + "nauc_mrr_at_1000_std": -0.021747, + "nauc_mrr_at_1000_diff1": 0.470786, + "main_score": 0.27435, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 2936.1834087371826, + "kg_co2_emissions": 0.0005700112053268032 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-large/2/MedicalQARetrieval.json b/results/openai__text-embedding-3-large/2/MedicalQARetrieval.json new file mode 100644 index 0000000000..e5af0cdc99 --- /dev/null +++ b/results/openai__text-embedding-3-large/2/MedicalQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae763399273d8b20506b80cf6f6f9a31a6a2b238", + "task_name": "MedicalQARetrieval", + "mteb_version": "1.24.0", + "scores": { + "test": [ + { + "ndcg_at_1": 0.66406, + "ndcg_at_3": 0.76001, + "ndcg_at_5": 0.78057, + "ndcg_at_10": 0.79657, + "ndcg_at_20": 0.804, + "ndcg_at_100": 0.80889, + "ndcg_at_1000": 0.81129, + "map_at_1": 0.66406, + "map_at_3": 0.73706, + "map_at_5": 0.74861, + "map_at_10": 0.75525, + "map_at_20": 0.75736, + "map_at_100": 0.75807, + "map_at_1000": 0.75816, + "recall_at_1": 0.66406, + "recall_at_3": 0.82617, + "recall_at_5": 0.87549, + "recall_at_10": 0.9248, + "recall_at_20": 0.95361, + "recall_at_100": 0.97949, + "recall_at_1000": 0.99854, + "precision_at_1": 0.66406, + "precision_at_3": 0.27539, + "precision_at_5": 0.1751, + "precision_at_10": 0.09248, + "precision_at_20": 0.04768, + "precision_at_100": 0.00979, + "precision_at_1000": 0.001, + "mrr_at_1": 0.664062, + "mrr_at_3": 0.736735, + "mrr_at_5": 0.748161, + "mrr_at_10": 0.754961, + "mrr_at_20": 0.757079, + "mrr_at_100": 0.757784, + "mrr_at_1000": 0.757873, + "nauc_ndcg_at_1_max": 0.433356, + "nauc_ndcg_at_1_std": -0.091583, + "nauc_ndcg_at_1_diff1": 0.694226, + "nauc_ndcg_at_3_max": 0.420168, + "nauc_ndcg_at_3_std": -0.131054, + "nauc_ndcg_at_3_diff1": 0.646435, + "nauc_ndcg_at_5_max": 0.41975, + "nauc_ndcg_at_5_std": -0.139749, + "nauc_ndcg_at_5_diff1": 0.645507, + "nauc_ndcg_at_10_max": 0.428545, + "nauc_ndcg_at_10_std": -0.150018, + "nauc_ndcg_at_10_diff1": 0.644121, + "nauc_ndcg_at_20_max": 0.427207, + "nauc_ndcg_at_20_std": -0.126902, + "nauc_ndcg_at_20_diff1": 0.646615, + "nauc_ndcg_at_100_max": 0.427837, + "nauc_ndcg_at_100_std": -0.117718, + "nauc_ndcg_at_100_diff1": 0.652459, + "nauc_ndcg_at_1000_max": 0.425608, + "nauc_ndcg_at_1000_std": -0.121263, + "nauc_ndcg_at_1000_diff1": 0.654148, + "nauc_map_at_1_max": 0.433356, + "nauc_map_at_1_std": -0.091583, + "nauc_map_at_1_diff1": 0.694226, + "nauc_map_at_3_max": 0.421986, + "nauc_map_at_3_std": -0.119741, + "nauc_map_at_3_diff1": 0.659279, + "nauc_map_at_5_max": 0.421911, + "nauc_map_at_5_std": -0.123421, + "nauc_map_at_5_diff1": 0.659557, + "nauc_map_at_10_max": 0.424924, + "nauc_map_at_10_std": -0.126781, + "nauc_map_at_10_diff1": 0.659324, + "nauc_map_at_20_max": 0.424638, + "nauc_map_at_20_std": -0.121343, + "nauc_map_at_20_diff1": 0.660062, + "nauc_map_at_100_max": 0.424713, + "nauc_map_at_100_std": -0.119872, + "nauc_map_at_100_diff1": 0.660823, + "nauc_map_at_1000_max": 0.424617, + "nauc_map_at_1000_std": -0.119899, + "nauc_map_at_1000_diff1": 0.660859, + "nauc_recall_at_1_max": 0.433356, + "nauc_recall_at_1_std": -0.091583, + "nauc_recall_at_1_diff1": 0.694226, + "nauc_recall_at_3_max": 0.414072, + "nauc_recall_at_3_std": -0.176215, + "nauc_recall_at_3_diff1": 0.595665, + "nauc_recall_at_5_max": 0.409939, + "nauc_recall_at_5_std": -0.226551, + "nauc_recall_at_5_diff1": 0.573048, + "nauc_recall_at_10_max": 0.470337, + "nauc_recall_at_10_std": -0.351935, + "nauc_recall_at_10_diff1": 0.520716, + "nauc_recall_at_20_max": 0.475081, + "nauc_recall_at_20_std": -0.14001, + "nauc_recall_at_20_diff1": 0.479932, + "nauc_recall_at_100_max": 0.561001, + "nauc_recall_at_100_std": 0.183469, + "nauc_recall_at_100_diff1": 0.513431, + "nauc_recall_at_1000_max": 0.565963, + "nauc_recall_at_1000_std": 0.391581, + "nauc_recall_at_1000_diff1": 0.322598, + "nauc_precision_at_1_max": 0.433356, + "nauc_precision_at_1_std": -0.091583, + "nauc_precision_at_1_diff1": 0.694226, + "nauc_precision_at_3_max": 0.414072, + "nauc_precision_at_3_std": -0.176215, + "nauc_precision_at_3_diff1": 0.595665, + "nauc_precision_at_5_max": 0.409939, + "nauc_precision_at_5_std": -0.226551, + "nauc_precision_at_5_diff1": 0.573048, + "nauc_precision_at_10_max": 0.470337, + "nauc_precision_at_10_std": -0.351935, + "nauc_precision_at_10_diff1": 0.520716, + "nauc_precision_at_20_max": 0.475081, + "nauc_precision_at_20_std": -0.14001, + "nauc_precision_at_20_diff1": 0.479932, + "nauc_precision_at_100_max": 0.561001, + "nauc_precision_at_100_std": 0.183469, + "nauc_precision_at_100_diff1": 0.513431, + "nauc_precision_at_1000_max": 0.565963, + "nauc_precision_at_1000_std": 0.391581, + "nauc_precision_at_1000_diff1": 0.322598, + "nauc_mrr_at_1_max": 0.434137, + "nauc_mrr_at_1_std": -0.08679, + "nauc_mrr_at_1_diff1": 0.694226, + "nauc_mrr_at_3_max": 0.422952, + "nauc_mrr_at_3_std": -0.115251, + "nauc_mrr_at_3_diff1": 0.659957, + "nauc_mrr_at_5_max": 0.423275, + "nauc_mrr_at_5_std": -0.118462, + "nauc_mrr_at_5_diff1": 0.660504, + "nauc_mrr_at_10_max": 0.426052, + "nauc_mrr_at_10_std": -0.122276, + "nauc_mrr_at_10_diff1": 0.659972, + "nauc_mrr_at_20_max": 0.425811, + "nauc_mrr_at_20_std": -0.116797, + "nauc_mrr_at_20_diff1": 0.660715, + "nauc_mrr_at_100_max": 0.425891, + "nauc_mrr_at_100_std": -0.115313, + "nauc_mrr_at_100_diff1": 0.661478, + "nauc_mrr_at_1000_max": 0.425795, + "nauc_mrr_at_1000_std": -0.115338, + "nauc_mrr_at_1000_diff1": 0.661514, + "main_score": 0.79657, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 81.16353392601013, + "kg_co2_emissions": 2.6804755836003245e-05 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-large/2/MedrxivClusteringS2S.v2.json b/results/openai__text-embedding-3-large/2/MedrxivClusteringS2S.v2.json new file mode 100644 index 0000000000..90d61b85eb --- /dev/null +++ b/results/openai__text-embedding-3-large/2/MedrxivClusteringS2S.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "task_name": "MedrxivClusteringS2S.v2", + "mteb_version": "1.24.0", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.340695, + 0.345533, + 0.339033, + 0.349097, + 0.348423, + 0.328494, + 0.347611, + 0.338524, + 0.345661, + 0.351792 + ] + }, + "v_measure": 0.343486, + "v_measure_std": 0.006541, + "main_score": 0.343486, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 43.02136588096619, + "kg_co2_emissions": 9.90941361771497e-06 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-large/2/PublicHealthQA.json b/results/openai__text-embedding-3-large/2/PublicHealthQA.json new file mode 100644 index 0000000000..23b1afa815 --- /dev/null +++ b/results/openai__text-embedding-3-large/2/PublicHealthQA.json @@ -0,0 +1,1187 @@ +{ + "dataset_revision": "main", + "task_name": "PublicHealthQA", + "mteb_version": "1.24.0", + "scores": { + "test": [ + { + "ndcg_at_1": 0.74419, + "ndcg_at_3": 0.85396, + "ndcg_at_5": 0.86796, + "ndcg_at_10": 0.87965, + "ndcg_at_20": 0.87965, + "ndcg_at_100": 0.87965, + "ndcg_at_1000": 0.87965, + "map_at_1": 0.74419, + "map_at_3": 0.82752, + "map_at_5": 0.83508, + "map_at_10": 0.84013, + "map_at_20": 0.84013, + "map_at_100": 0.84013, + "map_at_1000": 0.84013, + "recall_at_1": 0.74419, + "recall_at_3": 0.93023, + "recall_at_5": 0.96512, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.74419, + "precision_at_3": 0.31008, + "precision_at_5": 0.19302, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.744186, + "mrr_at_3": 0.827519, + "mrr_at_5": 0.835078, + "mrr_at_10": 0.84013, + "mrr_at_20": 0.84013, + "mrr_at_100": 0.84013, + "mrr_at_1000": 0.84013, + "nauc_ndcg_at_1_max": 0.524839, + "nauc_ndcg_at_1_std": -0.034106, + "nauc_ndcg_at_1_diff1": 0.857785, + "nauc_ndcg_at_3_max": 0.60516, + "nauc_ndcg_at_3_std": 0.10566, + "nauc_ndcg_at_3_diff1": 0.836955, + "nauc_ndcg_at_5_max": 0.559523, + "nauc_ndcg_at_5_std": 0.007167, + "nauc_ndcg_at_5_diff1": 0.836571, + "nauc_ndcg_at_10_max": 0.568783, + "nauc_ndcg_at_10_std": 0.036991, + "nauc_ndcg_at_10_diff1": 0.843891, + "nauc_ndcg_at_20_max": 0.568783, + "nauc_ndcg_at_20_std": 0.036991, + "nauc_ndcg_at_20_diff1": 0.843891, + "nauc_ndcg_at_100_max": 0.568783, + "nauc_ndcg_at_100_std": 0.036991, + "nauc_ndcg_at_100_diff1": 0.843891, + "nauc_ndcg_at_1000_max": 0.568783, + "nauc_ndcg_at_1000_std": 0.036991, + "nauc_ndcg_at_1000_diff1": 0.843891, + "nauc_map_at_1_max": 0.524839, + "nauc_map_at_1_std": -0.034106, + "nauc_map_at_1_diff1": 0.857785, + "nauc_map_at_3_max": 0.583601, + "nauc_map_at_3_std": 0.06706, + "nauc_map_at_3_diff1": 0.841343, + "nauc_map_at_5_max": 0.56265, + "nauc_map_at_5_std": 0.022155, + "nauc_map_at_5_diff1": 0.842004, + "nauc_map_at_10_max": 0.566109, + "nauc_map_at_10_std": 0.032965, + "nauc_map_at_10_diff1": 0.844714, + "nauc_map_at_20_max": 0.566109, + "nauc_map_at_20_std": 0.032965, + "nauc_map_at_20_diff1": 0.844714, + "nauc_map_at_100_max": 0.566109, + "nauc_map_at_100_std": 0.032965, + "nauc_map_at_100_diff1": 0.844714, + "nauc_map_at_1000_max": 0.566109, + "nauc_map_at_1000_std": 0.032965, + "nauc_map_at_1000_diff1": 0.844714, + "nauc_recall_at_1_max": 0.524839, + "nauc_recall_at_1_std": -0.034106, + "nauc_recall_at_1_diff1": 0.857785, + "nauc_recall_at_3_max": 0.739919, + "nauc_recall_at_3_std": 0.348241, + "nauc_recall_at_3_diff1": 0.81098, + "nauc_recall_at_5_max": 0.479838, + "nauc_recall_at_5_std": -0.259718, + "nauc_recall_at_5_diff1": 0.770847, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.524839, + "nauc_precision_at_1_std": -0.034106, + "nauc_precision_at_1_diff1": 0.857785, + "nauc_precision_at_3_max": 0.739919, + "nauc_precision_at_3_std": 0.348241, + "nauc_precision_at_3_diff1": 0.81098, + "nauc_precision_at_5_max": 0.479838, + "nauc_precision_at_5_std": -0.259718, + "nauc_precision_at_5_diff1": 0.770847, + "nauc_precision_at_10_max": NaN, + "nauc_precision_at_10_std": NaN, + "nauc_precision_at_10_diff1": NaN, + "nauc_precision_at_20_max": NaN, + "nauc_precision_at_20_std": NaN, + "nauc_precision_at_20_diff1": NaN, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.524839, + "nauc_mrr_at_1_std": -0.034106, + "nauc_mrr_at_1_diff1": 0.857785, + "nauc_mrr_at_3_max": 0.583601, + "nauc_mrr_at_3_std": 0.06706, + "nauc_mrr_at_3_diff1": 0.841343, + "nauc_mrr_at_5_max": 0.56265, + "nauc_mrr_at_5_std": 0.022155, + "nauc_mrr_at_5_diff1": 0.842004, + "nauc_mrr_at_10_max": 0.566109, + "nauc_mrr_at_10_std": 0.032965, + "nauc_mrr_at_10_diff1": 0.844714, + "nauc_mrr_at_20_max": 0.566109, + "nauc_mrr_at_20_std": 0.032965, + "nauc_mrr_at_20_diff1": 0.844714, + "nauc_mrr_at_100_max": 0.566109, + "nauc_mrr_at_100_std": 0.032965, + "nauc_mrr_at_100_diff1": 0.844714, + "nauc_mrr_at_1000_max": 0.566109, + "nauc_mrr_at_1000_std": 0.032965, + "nauc_mrr_at_1000_diff1": 0.844714, + "main_score": 0.87965, + "hf_subset": "arabic", + "languages": [ + "ara-Arab" + ] + }, + { + "ndcg_at_1": 0.71166, + "ndcg_at_3": 0.84312, + "ndcg_at_5": 0.86345, + "ndcg_at_10": 0.86933, + "ndcg_at_20": 0.86933, + "ndcg_at_100": 0.8707, + "ndcg_at_1000": 0.8707, + "map_at_1": 0.71166, + "map_at_3": 0.81391, + "map_at_5": 0.82526, + "map_at_10": 0.82764, + "map_at_20": 0.82764, + "map_at_100": 0.82793, + "map_at_1000": 0.82793, + "recall_at_1": 0.71166, + "recall_at_3": 0.92638, + "recall_at_5": 0.97546, + "recall_at_10": 0.99387, + "recall_at_20": 0.99387, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.71166, + "precision_at_3": 0.30879, + "precision_at_5": 0.19509, + "precision_at_10": 0.09939, + "precision_at_20": 0.04969, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.711656, + "mrr_at_3": 0.813906, + "mrr_at_5": 0.825256, + "mrr_at_10": 0.827641, + "mrr_at_20": 0.827641, + "mrr_at_100": 0.827934, + "mrr_at_1000": 0.827934, + "nauc_ndcg_at_1_max": 0.54515, + "nauc_ndcg_at_1_std": 0.105863, + "nauc_ndcg_at_1_diff1": 0.661424, + "nauc_ndcg_at_3_max": 0.572808, + "nauc_ndcg_at_3_std": 0.015899, + "nauc_ndcg_at_3_diff1": 0.69822, + "nauc_ndcg_at_5_max": 0.570509, + "nauc_ndcg_at_5_std": 0.077186, + "nauc_ndcg_at_5_diff1": 0.682953, + "nauc_ndcg_at_10_max": 0.555938, + "nauc_ndcg_at_10_std": 0.074145, + "nauc_ndcg_at_10_diff1": 0.683399, + "nauc_ndcg_at_20_max": 0.555938, + "nauc_ndcg_at_20_std": 0.074145, + "nauc_ndcg_at_20_diff1": 0.683399, + "nauc_ndcg_at_100_max": 0.56112, + "nauc_ndcg_at_100_std": 0.068425, + "nauc_ndcg_at_100_diff1": 0.682938, + "nauc_ndcg_at_1000_max": 0.56112, + "nauc_ndcg_at_1000_std": 0.068425, + "nauc_ndcg_at_1000_diff1": 0.682938, + "nauc_map_at_1_max": 0.54515, + "nauc_map_at_1_std": 0.105863, + "nauc_map_at_1_diff1": 0.661424, + "nauc_map_at_3_max": 0.566147, + "nauc_map_at_3_std": 0.04492, + "nauc_map_at_3_diff1": 0.687693, + "nauc_map_at_5_max": 0.56457, + "nauc_map_at_5_std": 0.073653, + "nauc_map_at_5_diff1": 0.680704, + "nauc_map_at_10_max": 0.559783, + "nauc_map_at_10_std": 0.071447, + "nauc_map_at_10_diff1": 0.681251, + "nauc_map_at_20_max": 0.559783, + "nauc_map_at_20_std": 0.071447, + "nauc_map_at_20_diff1": 0.681251, + "nauc_map_at_100_max": 0.560619, + "nauc_map_at_100_std": 0.070527, + "nauc_map_at_100_diff1": 0.681173, + "nauc_map_at_1000_max": 0.560619, + "nauc_map_at_1000_std": 0.070527, + "nauc_map_at_1000_diff1": 0.681173, + "nauc_recall_at_1_max": 0.54515, + "nauc_recall_at_1_std": 0.105863, + "nauc_recall_at_1_diff1": 0.661424, + "nauc_recall_at_3_max": 0.613332, + "nauc_recall_at_3_std": -0.17102, + "nauc_recall_at_3_diff1": 0.76461, + "nauc_recall_at_5_max": 0.668805, + "nauc_recall_at_5_std": 0.202117, + "nauc_recall_at_5_diff1": 0.700554, + "nauc_recall_at_10_max": 0.121397, + "nauc_recall_at_10_std": 0.553822, + "nauc_recall_at_10_diff1": 0.722037, + "nauc_recall_at_20_max": 0.121397, + "nauc_recall_at_20_std": 0.553822, + "nauc_recall_at_20_diff1": 0.722037, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.54515, + "nauc_precision_at_1_std": 0.105863, + "nauc_precision_at_1_diff1": 0.661424, + "nauc_precision_at_3_max": 0.613332, + "nauc_precision_at_3_std": -0.17102, + "nauc_precision_at_3_diff1": 0.76461, + "nauc_precision_at_5_max": 0.668805, + "nauc_precision_at_5_std": 0.202117, + "nauc_precision_at_5_diff1": 0.700554, + "nauc_precision_at_10_max": 0.121397, + "nauc_precision_at_10_std": 0.553822, + "nauc_precision_at_10_diff1": 0.722037, + "nauc_precision_at_20_max": 0.121397, + "nauc_precision_at_20_std": 0.553822, + "nauc_precision_at_20_diff1": 0.722037, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.54515, + "nauc_mrr_at_1_std": 0.105863, + "nauc_mrr_at_1_diff1": 0.661424, + "nauc_mrr_at_3_max": 0.566147, + "nauc_mrr_at_3_std": 0.04492, + "nauc_mrr_at_3_diff1": 0.687693, + "nauc_mrr_at_5_max": 0.56457, + "nauc_mrr_at_5_std": 0.073653, + "nauc_mrr_at_5_diff1": 0.680704, + "nauc_mrr_at_10_max": 0.559783, + "nauc_mrr_at_10_std": 0.071447, + "nauc_mrr_at_10_diff1": 0.681251, + "nauc_mrr_at_20_max": 0.559783, + "nauc_mrr_at_20_std": 0.071447, + "nauc_mrr_at_20_diff1": 0.681251, + "nauc_mrr_at_100_max": 0.560619, + "nauc_mrr_at_100_std": 0.070527, + "nauc_mrr_at_100_diff1": 0.681173, + "nauc_mrr_at_1000_max": 0.560619, + "nauc_mrr_at_1000_std": 0.070527, + "nauc_mrr_at_1000_diff1": 0.681173, + "main_score": 0.86933, + "hf_subset": "chinese", + "languages": [ + "zho-Hans" + ] + }, + { + "ndcg_at_1": 0.77907, + "ndcg_at_3": 0.87216, + "ndcg_at_5": 0.88892, + "ndcg_at_10": 0.89845, + "ndcg_at_20": 0.89845, + "ndcg_at_100": 0.89845, + "ndcg_at_1000": 0.89845, + "map_at_1": 0.77907, + "map_at_3": 0.85174, + "map_at_5": 0.86105, + "map_at_10": 0.86505, + "map_at_20": 0.86505, + "map_at_100": 0.86505, + "map_at_1000": 0.86505, + "recall_at_1": 0.77907, + "recall_at_3": 0.93023, + "recall_at_5": 0.97093, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.77907, + "precision_at_3": 0.31008, + "precision_at_5": 0.19419, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.77907, + "mrr_at_3": 0.851744, + "mrr_at_5": 0.861047, + "mrr_at_10": 0.865049, + "mrr_at_20": 0.865049, + "mrr_at_100": 0.865049, + "mrr_at_1000": 0.865049, + "nauc_ndcg_at_1_max": 0.458955, + "nauc_ndcg_at_1_std": -0.079293, + "nauc_ndcg_at_1_diff1": 0.76383, + "nauc_ndcg_at_3_max": 0.473559, + "nauc_ndcg_at_3_std": -0.070023, + "nauc_ndcg_at_3_diff1": 0.717024, + "nauc_ndcg_at_5_max": 0.481287, + "nauc_ndcg_at_5_std": -0.050724, + "nauc_ndcg_at_5_diff1": 0.710607, + "nauc_ndcg_at_10_max": 0.454488, + "nauc_ndcg_at_10_std": -0.084964, + "nauc_ndcg_at_10_diff1": 0.728031, + "nauc_ndcg_at_20_max": 0.454488, + "nauc_ndcg_at_20_std": -0.084964, + "nauc_ndcg_at_20_diff1": 0.728031, + "nauc_ndcg_at_100_max": 0.454488, + "nauc_ndcg_at_100_std": -0.084964, + "nauc_ndcg_at_100_diff1": 0.728031, + "nauc_ndcg_at_1000_max": 0.454488, + "nauc_ndcg_at_1000_std": -0.084964, + "nauc_ndcg_at_1000_diff1": 0.728031, + "nauc_map_at_1_max": 0.458955, + "nauc_map_at_1_std": -0.079293, + "nauc_map_at_1_diff1": 0.76383, + "nauc_map_at_3_max": 0.462338, + "nauc_map_at_3_std": -0.078027, + "nauc_map_at_3_diff1": 0.726482, + "nauc_map_at_5_max": 0.462943, + "nauc_map_at_5_std": -0.071781, + "nauc_map_at_5_diff1": 0.723945, + "nauc_map_at_10_max": 0.45378, + "nauc_map_at_10_std": -0.084951, + "nauc_map_at_10_diff1": 0.730474, + "nauc_map_at_20_max": 0.45378, + "nauc_map_at_20_std": -0.084951, + "nauc_map_at_20_diff1": 0.730474, + "nauc_map_at_100_max": 0.45378, + "nauc_map_at_100_std": -0.084951, + "nauc_map_at_100_diff1": 0.730474, + "nauc_map_at_1000_max": 0.45378, + "nauc_map_at_1000_std": -0.084951, + "nauc_map_at_1000_diff1": 0.730474, + "nauc_recall_at_1_max": 0.458955, + "nauc_recall_at_1_std": -0.079293, + "nauc_recall_at_1_diff1": 0.76383, + "nauc_recall_at_3_max": 0.542885, + "nauc_recall_at_3_std": -0.020243, + "nauc_recall_at_3_diff1": 0.668521, + "nauc_recall_at_5_max": 0.744528, + "nauc_recall_at_5_std": 0.249909, + "nauc_recall_at_5_diff1": 0.551435, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.458955, + "nauc_precision_at_1_std": -0.079293, + "nauc_precision_at_1_diff1": 0.76383, + "nauc_precision_at_3_max": 0.542885, + "nauc_precision_at_3_std": -0.020243, + "nauc_precision_at_3_diff1": 0.668521, + "nauc_precision_at_5_max": 0.744528, + "nauc_precision_at_5_std": 0.249909, + "nauc_precision_at_5_diff1": 0.551435, + "nauc_precision_at_10_max": NaN, + "nauc_precision_at_10_std": NaN, + "nauc_precision_at_10_diff1": NaN, + "nauc_precision_at_20_max": NaN, + "nauc_precision_at_20_std": NaN, + "nauc_precision_at_20_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.458955, + "nauc_mrr_at_1_std": -0.079293, + "nauc_mrr_at_1_diff1": 0.76383, + "nauc_mrr_at_3_max": 0.462338, + "nauc_mrr_at_3_std": -0.078027, + "nauc_mrr_at_3_diff1": 0.726482, + "nauc_mrr_at_5_max": 0.462943, + "nauc_mrr_at_5_std": -0.071781, + "nauc_mrr_at_5_diff1": 0.723945, + "nauc_mrr_at_10_max": 0.45378, + "nauc_mrr_at_10_std": -0.084951, + "nauc_mrr_at_10_diff1": 0.730474, + "nauc_mrr_at_20_max": 0.45378, + "nauc_mrr_at_20_std": -0.084951, + "nauc_mrr_at_20_diff1": 0.730474, + "nauc_mrr_at_100_max": 0.45378, + "nauc_mrr_at_100_std": -0.084951, + "nauc_mrr_at_100_diff1": 0.730474, + "nauc_mrr_at_1000_max": 0.45378, + "nauc_mrr_at_1000_std": -0.084951, + "nauc_mrr_at_1000_diff1": 0.730474, + "main_score": 0.89845, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.81176, + "ndcg_at_3": 0.90056, + "ndcg_at_5": 0.91018, + "ndcg_at_10": 0.9141, + "ndcg_at_20": 0.9141, + "ndcg_at_100": 0.9141, + "ndcg_at_1000": 0.9141, + "map_at_1": 0.81176, + "map_at_3": 0.87843, + "map_at_5": 0.88373, + "map_at_10": 0.88541, + "map_at_20": 0.88541, + "map_at_100": 0.88541, + "map_at_1000": 0.88541, + "recall_at_1": 0.81176, + "recall_at_3": 0.96471, + "recall_at_5": 0.98824, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.81176, + "precision_at_3": 0.32157, + "precision_at_5": 0.19765, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.811765, + "mrr_at_3": 0.878431, + "mrr_at_5": 0.883725, + "mrr_at_10": 0.885406, + "mrr_at_20": 0.885406, + "mrr_at_100": 0.885406, + "mrr_at_1000": 0.885406, + "nauc_ndcg_at_1_max": 0.406326, + "nauc_ndcg_at_1_std": -0.203654, + "nauc_ndcg_at_1_diff1": 0.902153, + "nauc_ndcg_at_3_max": 0.341984, + "nauc_ndcg_at_3_std": -0.356836, + "nauc_ndcg_at_3_diff1": 0.863283, + "nauc_ndcg_at_5_max": 0.399611, + "nauc_ndcg_at_5_std": -0.233586, + "nauc_ndcg_at_5_diff1": 0.888132, + "nauc_ndcg_at_10_max": 0.401639, + "nauc_ndcg_at_10_std": -0.23669, + "nauc_ndcg_at_10_diff1": 0.882742, + "nauc_ndcg_at_20_max": 0.401639, + "nauc_ndcg_at_20_std": -0.23669, + "nauc_ndcg_at_20_diff1": 0.882742, + "nauc_ndcg_at_100_max": 0.401639, + "nauc_ndcg_at_100_std": -0.23669, + "nauc_ndcg_at_100_diff1": 0.882742, + "nauc_ndcg_at_1000_max": 0.401639, + "nauc_ndcg_at_1000_std": -0.23669, + "nauc_ndcg_at_1000_diff1": 0.882742, + "nauc_map_at_1_max": 0.406326, + "nauc_map_at_1_std": -0.203654, + "nauc_map_at_1_diff1": 0.902153, + "nauc_map_at_3_max": 0.376044, + "nauc_map_at_3_std": -0.288964, + "nauc_map_at_3_diff1": 0.874747, + "nauc_map_at_5_max": 0.402246, + "nauc_map_at_5_std": -0.232396, + "nauc_map_at_5_diff1": 0.885656, + "nauc_map_at_10_max": 0.402939, + "nauc_map_at_10_std": -0.233375, + "nauc_map_at_10_diff1": 0.883884, + "nauc_map_at_20_max": 0.402939, + "nauc_map_at_20_std": -0.233375, + "nauc_map_at_20_diff1": 0.883884, + "nauc_map_at_100_max": 0.402939, + "nauc_map_at_100_std": -0.233375, + "nauc_map_at_100_diff1": 0.883884, + "nauc_map_at_1000_max": 0.402939, + "nauc_map_at_1000_std": -0.233375, + "nauc_map_at_1000_diff1": 0.883884, + "nauc_recall_at_1_max": 0.406326, + "nauc_recall_at_1_std": -0.203654, + "nauc_recall_at_1_diff1": 0.902153, + "nauc_recall_at_3_max": 0.006384, + "nauc_recall_at_3_std": -1.013183, + "nauc_recall_at_3_diff1": 0.758884, + "nauc_recall_at_5_max": 0.357501, + "nauc_recall_at_5_std": -0.169175, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.406326, + "nauc_precision_at_1_std": -0.203654, + "nauc_precision_at_1_diff1": 0.902153, + "nauc_precision_at_3_max": 0.006384, + "nauc_precision_at_3_std": -1.013183, + "nauc_precision_at_3_diff1": 0.758884, + "nauc_precision_at_5_max": 0.357501, + "nauc_precision_at_5_std": -0.169175, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_10_max": NaN, + "nauc_precision_at_10_std": NaN, + "nauc_precision_at_10_diff1": NaN, + "nauc_precision_at_20_max": NaN, + "nauc_precision_at_20_std": NaN, + "nauc_precision_at_20_diff1": NaN, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.406326, + "nauc_mrr_at_1_std": -0.203654, + "nauc_mrr_at_1_diff1": 0.902153, + "nauc_mrr_at_3_max": 0.376044, + "nauc_mrr_at_3_std": -0.288964, + "nauc_mrr_at_3_diff1": 0.874747, + "nauc_mrr_at_5_max": 0.402246, + "nauc_mrr_at_5_std": -0.232396, + "nauc_mrr_at_5_diff1": 0.885656, + "nauc_mrr_at_10_max": 0.402939, + "nauc_mrr_at_10_std": -0.233375, + "nauc_mrr_at_10_diff1": 0.883884, + "nauc_mrr_at_20_max": 0.402939, + "nauc_mrr_at_20_std": -0.233375, + "nauc_mrr_at_20_diff1": 0.883884, + "nauc_mrr_at_100_max": 0.402939, + "nauc_mrr_at_100_std": -0.233375, + "nauc_mrr_at_100_diff1": 0.883884, + "nauc_mrr_at_1000_max": 0.402939, + "nauc_mrr_at_1000_std": -0.233375, + "nauc_mrr_at_1000_diff1": 0.883884, + "main_score": 0.9141, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.71429, + "ndcg_at_3": 0.8256, + "ndcg_at_5": 0.83119, + "ndcg_at_10": 0.85308, + "ndcg_at_20": 0.85955, + "ndcg_at_100": 0.85955, + "ndcg_at_1000": 0.85955, + "map_at_1": 0.71429, + "map_at_3": 0.80087, + "map_at_5": 0.80411, + "map_at_10": 0.81367, + "map_at_20": 0.81539, + "map_at_100": 0.81539, + "map_at_1000": 0.81539, + "recall_at_1": 0.71429, + "recall_at_3": 0.8961, + "recall_at_5": 0.90909, + "recall_at_10": 0.97403, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.71429, + "precision_at_3": 0.2987, + "precision_at_5": 0.18182, + "precision_at_10": 0.0974, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.714286, + "mrr_at_3": 0.800866, + "mrr_at_5": 0.804113, + "mrr_at_10": 0.813672, + "mrr_at_20": 0.815393, + "mrr_at_100": 0.815393, + "mrr_at_1000": 0.815393, + "nauc_ndcg_at_1_max": 0.11575, + "nauc_ndcg_at_1_std": -0.104658, + "nauc_ndcg_at_1_diff1": 0.677351, + "nauc_ndcg_at_3_max": 0.179531, + "nauc_ndcg_at_3_std": -0.241375, + "nauc_ndcg_at_3_diff1": 0.66349, + "nauc_ndcg_at_5_max": 0.181481, + "nauc_ndcg_at_5_std": -0.229835, + "nauc_ndcg_at_5_diff1": 0.65132, + "nauc_ndcg_at_10_max": 0.115815, + "nauc_ndcg_at_10_std": -0.183971, + "nauc_ndcg_at_10_diff1": 0.652075, + "nauc_ndcg_at_20_max": 0.144278, + "nauc_ndcg_at_20_std": -0.186153, + "nauc_ndcg_at_20_diff1": 0.665764, + "nauc_ndcg_at_100_max": 0.144278, + "nauc_ndcg_at_100_std": -0.186153, + "nauc_ndcg_at_100_diff1": 0.665764, + "nauc_ndcg_at_1000_max": 0.144278, + "nauc_ndcg_at_1000_std": -0.186153, + "nauc_ndcg_at_1000_diff1": 0.665764, + "nauc_map_at_1_max": 0.11575, + "nauc_map_at_1_std": -0.104658, + "nauc_map_at_1_diff1": 0.677351, + "nauc_map_at_3_max": 0.158918, + "nauc_map_at_3_std": -0.20121, + "nauc_map_at_3_diff1": 0.668028, + "nauc_map_at_5_max": 0.159529, + "nauc_map_at_5_std": -0.194611, + "nauc_map_at_5_diff1": 0.66193, + "nauc_map_at_10_max": 0.134938, + "nauc_map_at_10_std": -0.18035, + "nauc_map_at_10_diff1": 0.663702, + "nauc_map_at_20_max": 0.142429, + "nauc_map_at_20_std": -0.179556, + "nauc_map_at_20_diff1": 0.667248, + "nauc_map_at_100_max": 0.142429, + "nauc_map_at_100_std": -0.179556, + "nauc_map_at_100_diff1": 0.667248, + "nauc_map_at_1000_max": 0.142429, + "nauc_map_at_1000_std": -0.179556, + "nauc_map_at_1000_diff1": 0.667248, + "nauc_recall_at_1_max": 0.11575, + "nauc_recall_at_1_std": -0.104658, + "nauc_recall_at_1_diff1": 0.677351, + "nauc_recall_at_3_max": 0.282136, + "nauc_recall_at_3_std": -0.438329, + "nauc_recall_at_3_diff1": 0.640858, + "nauc_recall_at_5_max": 0.304493, + "nauc_recall_at_5_std": -0.420884, + "nauc_recall_at_5_diff1": 0.589552, + "nauc_recall_at_10_max": -0.365271, + "nauc_recall_at_10_std": -0.082183, + "nauc_recall_at_10_diff1": 0.416709, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.11575, + "nauc_precision_at_1_std": -0.104658, + "nauc_precision_at_1_diff1": 0.677351, + "nauc_precision_at_3_max": 0.282136, + "nauc_precision_at_3_std": -0.438329, + "nauc_precision_at_3_diff1": 0.640858, + "nauc_precision_at_5_max": 0.304493, + "nauc_precision_at_5_std": -0.420884, + "nauc_precision_at_5_diff1": 0.589552, + "nauc_precision_at_10_max": -0.365271, + "nauc_precision_at_10_std": -0.082183, + "nauc_precision_at_10_diff1": 0.416709, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.11575, + "nauc_mrr_at_1_std": -0.104658, + "nauc_mrr_at_1_diff1": 0.677351, + "nauc_mrr_at_3_max": 0.158918, + "nauc_mrr_at_3_std": -0.20121, + "nauc_mrr_at_3_diff1": 0.668028, + "nauc_mrr_at_5_max": 0.159529, + "nauc_mrr_at_5_std": -0.194611, + "nauc_mrr_at_5_diff1": 0.66193, + "nauc_mrr_at_10_max": 0.134938, + "nauc_mrr_at_10_std": -0.18035, + "nauc_mrr_at_10_diff1": 0.663702, + "nauc_mrr_at_20_max": 0.142429, + "nauc_mrr_at_20_std": -0.179556, + "nauc_mrr_at_20_diff1": 0.667248, + "nauc_mrr_at_100_max": 0.142429, + "nauc_mrr_at_100_std": -0.179556, + "nauc_mrr_at_100_diff1": 0.667248, + "nauc_mrr_at_1000_max": 0.142429, + "nauc_mrr_at_1000_std": -0.179556, + "nauc_mrr_at_1000_diff1": 0.667248, + "main_score": 0.85308, + "hf_subset": "korean", + "languages": [ + "kor-Hang" + ] + }, + { + "ndcg_at_1": 0.72308, + "ndcg_at_3": 0.85293, + "ndcg_at_5": 0.85293, + "ndcg_at_10": 0.86353, + "ndcg_at_20": 0.86722, + "ndcg_at_100": 0.87063, + "ndcg_at_1000": 0.87063, + "map_at_1": 0.72308, + "map_at_3": 0.82308, + "map_at_5": 0.82308, + "map_at_10": 0.82784, + "map_at_20": 0.82874, + "map_at_100": 0.82944, + "map_at_1000": 0.82944, + "recall_at_1": 0.72308, + "recall_at_3": 0.93846, + "recall_at_5": 0.93846, + "recall_at_10": 0.96923, + "recall_at_20": 0.98462, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.72308, + "precision_at_3": 0.31282, + "precision_at_5": 0.18769, + "precision_at_10": 0.09692, + "precision_at_20": 0.04923, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.723077, + "mrr_at_3": 0.823077, + "mrr_at_5": 0.823077, + "mrr_at_10": 0.827839, + "mrr_at_20": 0.828744, + "mrr_at_100": 0.829443, + "mrr_at_1000": 0.829443, + "nauc_ndcg_at_1_max": 0.369021, + "nauc_ndcg_at_1_std": -0.021819, + "nauc_ndcg_at_1_diff1": 0.75029, + "nauc_ndcg_at_3_max": 0.520017, + "nauc_ndcg_at_3_std": 0.177014, + "nauc_ndcg_at_3_diff1": 0.769612, + "nauc_ndcg_at_5_max": 0.520017, + "nauc_ndcg_at_5_std": 0.177014, + "nauc_ndcg_at_5_diff1": 0.769612, + "nauc_ndcg_at_10_max": 0.490203, + "nauc_ndcg_at_10_std": 0.142547, + "nauc_ndcg_at_10_diff1": 0.755169, + "nauc_ndcg_at_20_max": 0.474465, + "nauc_ndcg_at_20_std": 0.124617, + "nauc_ndcg_at_20_diff1": 0.756151, + "nauc_ndcg_at_100_max": 0.459071, + "nauc_ndcg_at_100_std": 0.098975, + "nauc_ndcg_at_100_diff1": 0.752805, + "nauc_ndcg_at_1000_max": 0.459071, + "nauc_ndcg_at_1000_std": 0.098975, + "nauc_ndcg_at_1000_diff1": 0.752805, + "nauc_map_at_1_max": 0.369021, + "nauc_map_at_1_std": -0.021819, + "nauc_map_at_1_diff1": 0.75029, + "nauc_map_at_3_max": 0.464878, + "nauc_map_at_3_std": 0.105081, + "nauc_map_at_3_diff1": 0.757411, + "nauc_map_at_5_max": 0.464878, + "nauc_map_at_5_std": 0.105081, + "nauc_map_at_5_diff1": 0.757411, + "nauc_map_at_10_max": 0.452296, + "nauc_map_at_10_std": 0.089908, + "nauc_map_at_10_diff1": 0.751767, + "nauc_map_at_20_max": 0.449063, + "nauc_map_at_20_std": 0.086169, + "nauc_map_at_20_diff1": 0.751935, + "nauc_map_at_100_max": 0.446538, + "nauc_map_at_100_std": 0.081982, + "nauc_map_at_100_diff1": 0.751392, + "nauc_map_at_1000_max": 0.446538, + "nauc_map_at_1000_std": 0.081982, + "nauc_map_at_1000_diff1": 0.751392, + "nauc_recall_at_1_max": 0.369021, + "nauc_recall_at_1_std": -0.021819, + "nauc_recall_at_1_diff1": 0.75029, + "nauc_recall_at_3_max": 0.930833, + "nauc_recall_at_3_std": 0.712229, + "nauc_recall_at_3_diff1": 0.866027, + "nauc_recall_at_5_max": 0.930833, + "nauc_recall_at_5_std": 0.712229, + "nauc_recall_at_5_diff1": 0.866027, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": 0.861665, + "nauc_recall_at_10_diff1": 0.796859, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 1.0, + "nauc_recall_at_20_diff1": 0.870389, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.369021, + "nauc_precision_at_1_std": -0.021819, + "nauc_precision_at_1_diff1": 0.75029, + "nauc_precision_at_3_max": 0.930833, + "nauc_precision_at_3_std": 0.712229, + "nauc_precision_at_3_diff1": 0.866027, + "nauc_precision_at_5_max": 0.930833, + "nauc_precision_at_5_std": 0.712229, + "nauc_precision_at_5_diff1": 0.866027, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 0.861665, + "nauc_precision_at_10_diff1": 0.796859, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 0.870389, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.369021, + "nauc_mrr_at_1_std": -0.021819, + "nauc_mrr_at_1_diff1": 0.75029, + "nauc_mrr_at_3_max": 0.464878, + "nauc_mrr_at_3_std": 0.105081, + "nauc_mrr_at_3_diff1": 0.757411, + "nauc_mrr_at_5_max": 0.464878, + "nauc_mrr_at_5_std": 0.105081, + "nauc_mrr_at_5_diff1": 0.757411, + "nauc_mrr_at_10_max": 0.452296, + "nauc_mrr_at_10_std": 0.089908, + "nauc_mrr_at_10_diff1": 0.751767, + "nauc_mrr_at_20_max": 0.449063, + "nauc_mrr_at_20_std": 0.086169, + "nauc_mrr_at_20_diff1": 0.751935, + "nauc_mrr_at_100_max": 0.446538, + "nauc_mrr_at_100_std": 0.081982, + "nauc_mrr_at_100_diff1": 0.751392, + "nauc_mrr_at_1000_max": 0.446538, + "nauc_mrr_at_1000_std": 0.081982, + "nauc_mrr_at_1000_diff1": 0.751392, + "main_score": 0.86353, + "hf_subset": "russian", + "languages": [ + "rus-Cyrl" + ] + }, + { + "ndcg_at_1": 0.75926, + "ndcg_at_3": 0.86802, + "ndcg_at_5": 0.87545, + "ndcg_at_10": 0.88773, + "ndcg_at_20": 0.88773, + "ndcg_at_100": 0.88773, + "ndcg_at_1000": 0.88773, + "map_at_1": 0.75926, + "map_at_3": 0.84156, + "map_at_5": 0.84558, + "map_at_10": 0.85082, + "map_at_20": 0.85082, + "map_at_100": 0.85082, + "map_at_1000": 0.85082, + "recall_at_1": 0.75926, + "recall_at_3": 0.94444, + "recall_at_5": 0.96296, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.75926, + "precision_at_3": 0.31481, + "precision_at_5": 0.19259, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.759259, + "mrr_at_3": 0.841564, + "mrr_at_5": 0.845576, + "mrr_at_10": 0.850823, + "mrr_at_20": 0.850823, + "mrr_at_100": 0.850823, + "mrr_at_1000": 0.850823, + "nauc_ndcg_at_1_max": 0.518325, + "nauc_ndcg_at_1_std": 0.096303, + "nauc_ndcg_at_1_diff1": 0.729535, + "nauc_ndcg_at_3_max": 0.525067, + "nauc_ndcg_at_3_std": 0.025636, + "nauc_ndcg_at_3_diff1": 0.726223, + "nauc_ndcg_at_5_max": 0.500131, + "nauc_ndcg_at_5_std": 0.00741, + "nauc_ndcg_at_5_diff1": 0.718833, + "nauc_ndcg_at_10_max": 0.50837, + "nauc_ndcg_at_10_std": 0.04857, + "nauc_ndcg_at_10_diff1": 0.722401, + "nauc_ndcg_at_20_max": 0.50837, + "nauc_ndcg_at_20_std": 0.04857, + "nauc_ndcg_at_20_diff1": 0.722401, + "nauc_ndcg_at_100_max": 0.50837, + "nauc_ndcg_at_100_std": 0.04857, + "nauc_ndcg_at_100_diff1": 0.722401, + "nauc_ndcg_at_1000_max": 0.50837, + "nauc_ndcg_at_1000_std": 0.04857, + "nauc_ndcg_at_1000_diff1": 0.722401, + "nauc_map_at_1_max": 0.518325, + "nauc_map_at_1_std": 0.096303, + "nauc_map_at_1_diff1": 0.729535, + "nauc_map_at_3_max": 0.517364, + "nauc_map_at_3_std": 0.047468, + "nauc_map_at_3_diff1": 0.72435, + "nauc_map_at_5_max": 0.506103, + "nauc_map_at_5_std": 0.039269, + "nauc_map_at_5_diff1": 0.721397, + "nauc_map_at_10_max": 0.508888, + "nauc_map_at_10_std": 0.053498, + "nauc_map_at_10_diff1": 0.722665, + "nauc_map_at_20_max": 0.508888, + "nauc_map_at_20_std": 0.053498, + "nauc_map_at_20_diff1": 0.722665, + "nauc_map_at_100_max": 0.508888, + "nauc_map_at_100_std": 0.053498, + "nauc_map_at_100_diff1": 0.722665, + "nauc_map_at_1000_max": 0.508888, + "nauc_map_at_1000_std": 0.053498, + "nauc_map_at_1000_diff1": 0.722665, + "nauc_recall_at_1_max": 0.518325, + "nauc_recall_at_1_std": 0.096303, + "nauc_recall_at_1_diff1": 0.729535, + "nauc_recall_at_3_max": 0.589053, + "nauc_recall_at_3_std": -0.135767, + "nauc_recall_at_3_diff1": 0.743696, + "nauc_recall_at_5_max": 0.429771, + "nauc_recall_at_5_std": -0.344427, + "nauc_recall_at_5_diff1": 0.689845, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.518325, + "nauc_precision_at_1_std": 0.096303, + "nauc_precision_at_1_diff1": 0.729535, + "nauc_precision_at_3_max": 0.589053, + "nauc_precision_at_3_std": -0.135767, + "nauc_precision_at_3_diff1": 0.743696, + "nauc_precision_at_5_max": 0.429771, + "nauc_precision_at_5_std": -0.344427, + "nauc_precision_at_5_diff1": 0.689845, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.518325, + "nauc_mrr_at_1_std": 0.096303, + "nauc_mrr_at_1_diff1": 0.729535, + "nauc_mrr_at_3_max": 0.517364, + "nauc_mrr_at_3_std": 0.047468, + "nauc_mrr_at_3_diff1": 0.72435, + "nauc_mrr_at_5_max": 0.506103, + "nauc_mrr_at_5_std": 0.039269, + "nauc_mrr_at_5_diff1": 0.721397, + "nauc_mrr_at_10_max": 0.508888, + "nauc_mrr_at_10_std": 0.053498, + "nauc_mrr_at_10_diff1": 0.722665, + "nauc_mrr_at_20_max": 0.508888, + "nauc_mrr_at_20_std": 0.053498, + "nauc_mrr_at_20_diff1": 0.722665, + "nauc_mrr_at_100_max": 0.508888, + "nauc_mrr_at_100_std": 0.053498, + "nauc_mrr_at_100_diff1": 0.722665, + "nauc_mrr_at_1000_max": 0.508888, + "nauc_mrr_at_1000_std": 0.053498, + "nauc_mrr_at_1000_diff1": 0.722665, + "main_score": 0.88773, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.71429, + "ndcg_at_3": 0.83209, + "ndcg_at_5": 0.83209, + "ndcg_at_10": 0.85338, + "ndcg_at_20": 0.85985, + "ndcg_at_100": 0.85985, + "ndcg_at_1000": 0.85985, + "map_at_1": 0.71429, + "map_at_3": 0.80519, + "map_at_5": 0.80519, + "map_at_10": 0.81414, + "map_at_20": 0.81587, + "map_at_100": 0.81587, + "map_at_1000": 0.81587, + "recall_at_1": 0.71429, + "recall_at_3": 0.90909, + "recall_at_5": 0.90909, + "recall_at_10": 0.97403, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.71429, + "precision_at_3": 0.30303, + "precision_at_5": 0.18182, + "precision_at_10": 0.0974, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.714286, + "mrr_at_3": 0.805195, + "mrr_at_5": 0.805195, + "mrr_at_10": 0.814136, + "mrr_at_20": 0.815868, + "mrr_at_100": 0.815868, + "mrr_at_1000": 0.815868, + "nauc_ndcg_at_1_max": 0.233162, + "nauc_ndcg_at_1_std": -0.264235, + "nauc_ndcg_at_1_diff1": 0.683021, + "nauc_ndcg_at_3_max": 0.35194, + "nauc_ndcg_at_3_std": -0.331735, + "nauc_ndcg_at_3_diff1": 0.711275, + "nauc_ndcg_at_5_max": 0.35194, + "nauc_ndcg_at_5_std": -0.331735, + "nauc_ndcg_at_5_diff1": 0.711275, + "nauc_ndcg_at_10_max": 0.283088, + "nauc_ndcg_at_10_std": -0.3628, + "nauc_ndcg_at_10_diff1": 0.686803, + "nauc_ndcg_at_20_max": 0.298026, + "nauc_ndcg_at_20_std": -0.329306, + "nauc_ndcg_at_20_diff1": 0.699061, + "nauc_ndcg_at_100_max": 0.298026, + "nauc_ndcg_at_100_std": -0.329306, + "nauc_ndcg_at_100_diff1": 0.699061, + "nauc_ndcg_at_1000_max": 0.298026, + "nauc_ndcg_at_1000_std": -0.329306, + "nauc_ndcg_at_1000_diff1": 0.699061, + "nauc_map_at_1_max": 0.233162, + "nauc_map_at_1_std": -0.264235, + "nauc_map_at_1_diff1": 0.683021, + "nauc_map_at_3_max": 0.31629, + "nauc_map_at_3_std": -0.315941, + "nauc_map_at_3_diff1": 0.704515, + "nauc_map_at_5_max": 0.31629, + "nauc_map_at_5_std": -0.315941, + "nauc_map_at_5_diff1": 0.704515, + "nauc_map_at_10_max": 0.290905, + "nauc_map_at_10_std": -0.330043, + "nauc_map_at_10_diff1": 0.695916, + "nauc_map_at_20_max": 0.29379, + "nauc_map_at_20_std": -0.3215, + "nauc_map_at_20_diff1": 0.698689, + "nauc_map_at_100_max": 0.29379, + "nauc_map_at_100_std": -0.3215, + "nauc_map_at_100_diff1": 0.698689, + "nauc_map_at_1000_max": 0.29379, + "nauc_map_at_1000_std": -0.3215, + "nauc_map_at_1000_diff1": 0.698689, + "nauc_recall_at_1_max": 0.233162, + "nauc_recall_at_1_std": -0.264235, + "nauc_recall_at_1_diff1": 0.683021, + "nauc_recall_at_3_max": 0.546832, + "nauc_recall_at_3_std": -0.414026, + "nauc_recall_at_3_diff1": 0.746669, + "nauc_recall_at_5_max": 0.546832, + "nauc_recall_at_5_std": -0.414026, + "nauc_recall_at_5_diff1": 0.746669, + "nauc_recall_at_10_max": -0.020472, + "nauc_recall_at_10_std": -0.948562, + "nauc_recall_at_10_diff1": 0.456381, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.233162, + "nauc_precision_at_1_std": -0.264235, + "nauc_precision_at_1_diff1": 0.683021, + "nauc_precision_at_3_max": 0.546832, + "nauc_precision_at_3_std": -0.414026, + "nauc_precision_at_3_diff1": 0.746669, + "nauc_precision_at_5_max": 0.546832, + "nauc_precision_at_5_std": -0.414026, + "nauc_precision_at_5_diff1": 0.746669, + "nauc_precision_at_10_max": -0.020472, + "nauc_precision_at_10_std": -0.948562, + "nauc_precision_at_10_diff1": 0.456381, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.233162, + "nauc_mrr_at_1_std": -0.264235, + "nauc_mrr_at_1_diff1": 0.683021, + "nauc_mrr_at_3_max": 0.31629, + "nauc_mrr_at_3_std": -0.315941, + "nauc_mrr_at_3_diff1": 0.704515, + "nauc_mrr_at_5_max": 0.31629, + "nauc_mrr_at_5_std": -0.315941, + "nauc_mrr_at_5_diff1": 0.704515, + "nauc_mrr_at_10_max": 0.290905, + "nauc_mrr_at_10_std": -0.330043, + "nauc_mrr_at_10_diff1": 0.695916, + "nauc_mrr_at_20_max": 0.29379, + "nauc_mrr_at_20_std": -0.3215, + "nauc_mrr_at_20_diff1": 0.698689, + "nauc_mrr_at_100_max": 0.29379, + "nauc_mrr_at_100_std": -0.3215, + "nauc_mrr_at_100_diff1": 0.698689, + "nauc_mrr_at_1000_max": 0.29379, + "nauc_mrr_at_1000_std": -0.3215, + "nauc_mrr_at_1000_diff1": 0.698689, + "main_score": 0.85338, + "hf_subset": "vietnamese", + "languages": [ + "vie-Latn" + ] + } + ] + }, + "evaluation_time": 60.81188201904297, + "kg_co2_emissions": 3.584997699480917e-05 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-large/2/SciFact-PL.json b/results/openai__text-embedding-3-large/2/SciFact-PL.json new file mode 100644 index 0000000000..0df810d10b --- /dev/null +++ b/results/openai__text-embedding-3-large/2/SciFact-PL.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "47932a35f045ef8ed01ba82bf9ff67f6e109207e", + "task_name": "SciFact-PL", + "mteb_version": "1.24.0", + "scores": { + "test": [ + { + "ndcg_at_1": 0.59, + "ndcg_at_3": 0.65833, + "ndcg_at_5": 0.68712, + "ndcg_at_10": 0.71036, + "ndcg_at_20": 0.72337, + "ndcg_at_100": 0.73778, + "ndcg_at_1000": 0.74107, + "map_at_1": 0.55661, + "map_at_3": 0.63046, + "map_at_5": 0.64847, + "map_at_10": 0.66132, + "map_at_20": 0.66541, + "map_at_100": 0.66748, + "map_at_1000": 0.66762, + "recall_at_1": 0.55661, + "recall_at_3": 0.70767, + "recall_at_5": 0.78178, + "recall_at_10": 0.846, + "recall_at_20": 0.895, + "recall_at_100": 0.97167, + "recall_at_1000": 0.99667, + "precision_at_1": 0.59, + "precision_at_3": 0.25889, + "precision_at_5": 0.17267, + "precision_at_10": 0.09667, + "precision_at_20": 0.051, + "precision_at_100": 0.011, + "precision_at_1000": 0.00113, + "mrr_at_1": 0.59, + "mrr_at_3": 0.652778, + "mrr_at_5": 0.669278, + "mrr_at_10": 0.675717, + "mrr_at_20": 0.679221, + "mrr_at_100": 0.68111, + "mrr_at_1000": 0.68123, + "nauc_ndcg_at_1_max": 0.523046, + "nauc_ndcg_at_1_std": 0.103947, + "nauc_ndcg_at_1_diff1": 0.703406, + "nauc_ndcg_at_3_max": 0.480365, + "nauc_ndcg_at_3_std": 0.00514, + "nauc_ndcg_at_3_diff1": 0.648658, + "nauc_ndcg_at_5_max": 0.490607, + "nauc_ndcg_at_5_std": 0.013015, + "nauc_ndcg_at_5_diff1": 0.638363, + "nauc_ndcg_at_10_max": 0.515913, + "nauc_ndcg_at_10_std": 0.045596, + "nauc_ndcg_at_10_diff1": 0.657448, + "nauc_ndcg_at_20_max": 0.513068, + "nauc_ndcg_at_20_std": 0.052055, + "nauc_ndcg_at_20_diff1": 0.661647, + "nauc_ndcg_at_100_max": 0.514564, + "nauc_ndcg_at_100_std": 0.06049, + "nauc_ndcg_at_100_diff1": 0.669072, + "nauc_ndcg_at_1000_max": 0.51023, + "nauc_ndcg_at_1000_std": 0.053438, + "nauc_ndcg_at_1000_diff1": 0.666737, + "nauc_map_at_1_max": 0.457377, + "nauc_map_at_1_std": 0.01516, + "nauc_map_at_1_diff1": 0.690591, + "nauc_map_at_3_max": 0.470714, + "nauc_map_at_3_std": -0.007129, + "nauc_map_at_3_diff1": 0.659517, + "nauc_map_at_5_max": 0.481056, + "nauc_map_at_5_std": 0.004464, + "nauc_map_at_5_diff1": 0.655637, + "nauc_map_at_10_max": 0.498482, + "nauc_map_at_10_std": 0.03061, + "nauc_map_at_10_diff1": 0.664263, + "nauc_map_at_20_max": 0.497091, + "nauc_map_at_20_std": 0.032929, + "nauc_map_at_20_diff1": 0.665828, + "nauc_map_at_100_max": 0.496717, + "nauc_map_at_100_std": 0.03351, + "nauc_map_at_100_diff1": 0.666665, + "nauc_map_at_1000_max": 0.496564, + "nauc_map_at_1000_std": 0.033268, + "nauc_map_at_1000_diff1": 0.666587, + "nauc_recall_at_1_max": 0.457377, + "nauc_recall_at_1_std": 0.01516, + "nauc_recall_at_1_diff1": 0.690591, + "nauc_recall_at_3_max": 0.441462, + "nauc_recall_at_3_std": -0.072052, + "nauc_recall_at_3_diff1": 0.587243, + "nauc_recall_at_5_max": 0.465922, + "nauc_recall_at_5_std": -0.039192, + "nauc_recall_at_5_diff1": 0.53672, + "nauc_recall_at_10_max": 0.556015, + "nauc_recall_at_10_std": 0.039424, + "nauc_recall_at_10_diff1": 0.605722, + "nauc_recall_at_20_max": 0.554628, + "nauc_recall_at_20_std": 0.082151, + "nauc_recall_at_20_diff1": 0.616169, + "nauc_recall_at_100_max": 0.79983, + "nauc_recall_at_100_std": 0.540369, + "nauc_recall_at_100_diff1": 0.817131, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.722222, + "nauc_recall_at_1000_diff1": 1.0, + "nauc_precision_at_1_max": 0.523046, + "nauc_precision_at_1_std": 0.103947, + "nauc_precision_at_1_diff1": 0.703406, + "nauc_precision_at_3_max": 0.443221, + "nauc_precision_at_3_std": 0.100271, + "nauc_precision_at_3_diff1": 0.441121, + "nauc_precision_at_5_max": 0.471615, + "nauc_precision_at_5_std": 0.200827, + "nauc_precision_at_5_diff1": 0.340777, + "nauc_precision_at_10_max": 0.422708, + "nauc_precision_at_10_std": 0.346397, + "nauc_precision_at_10_diff1": 0.211181, + "nauc_precision_at_20_max": 0.377026, + "nauc_precision_at_20_std": 0.395904, + "nauc_precision_at_20_diff1": 0.149225, + "nauc_precision_at_100_max": 0.321236, + "nauc_precision_at_100_std": 0.518386, + "nauc_precision_at_100_diff1": 0.010821, + "nauc_precision_at_1000_max": 0.265656, + "nauc_precision_at_1000_std": 0.514404, + "nauc_precision_at_1000_diff1": -0.075589, + "nauc_mrr_at_1_max": 0.523046, + "nauc_mrr_at_1_std": 0.103947, + "nauc_mrr_at_1_diff1": 0.703406, + "nauc_mrr_at_3_max": 0.509639, + "nauc_mrr_at_3_std": 0.058132, + "nauc_mrr_at_3_diff1": 0.670841, + "nauc_mrr_at_5_max": 0.515342, + "nauc_mrr_at_5_std": 0.07014, + "nauc_mrr_at_5_diff1": 0.663434, + "nauc_mrr_at_10_max": 0.517475, + "nauc_mrr_at_10_std": 0.070861, + "nauc_mrr_at_10_diff1": 0.669589, + "nauc_mrr_at_20_max": 0.516653, + "nauc_mrr_at_20_std": 0.072841, + "nauc_mrr_at_20_diff1": 0.67026, + "nauc_mrr_at_100_max": 0.516678, + "nauc_mrr_at_100_std": 0.073743, + "nauc_mrr_at_100_diff1": 0.67157, + "nauc_mrr_at_1000_max": 0.516485, + "nauc_mrr_at_1000_std": 0.073406, + "nauc_mrr_at_1000_diff1": 0.671487, + "main_score": 0.71036, + "hf_subset": "default", + "languages": [ + "pol-Latn" + ] + } + ] + }, + "evaluation_time": 114.53178191184998, + "kg_co2_emissions": 2.8807605272851364e-05 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-large/2/TRECCOVID-PL.json b/results/openai__text-embedding-3-large/2/TRECCOVID-PL.json new file mode 100644 index 0000000000..a74a11b75d --- /dev/null +++ b/results/openai__text-embedding-3-large/2/TRECCOVID-PL.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "81bcb408f33366c2a20ac54adafad1ae7e877fdd", + "task_name": "TRECCOVID-PL", + "mteb_version": "1.24.0", + "scores": { + "test": [ + { + "ndcg_at_1": 0.64, + "ndcg_at_3": 0.64989, + "ndcg_at_5": 0.63358, + "ndcg_at_10": 0.62882, + "ndcg_at_20": 0.61704, + "ndcg_at_100": 0.51998, + "ndcg_at_1000": 0.50326, + "map_at_1": 0.00187, + "map_at_3": 0.00515, + "map_at_5": 0.00788, + "map_at_10": 0.01482, + "map_at_20": 0.02739, + "map_at_100": 0.09677, + "map_at_1000": 0.24703, + "recall_at_1": 0.00187, + "recall_at_3": 0.00575, + "recall_at_5": 0.00912, + "recall_at_10": 0.01801, + "recall_at_20": 0.03462, + "recall_at_100": 0.13451, + "recall_at_1000": 0.48493, + "precision_at_1": 0.68, + "precision_at_3": 0.70667, + "precision_at_5": 0.68, + "precision_at_10": 0.676, + "precision_at_20": 0.661, + "precision_at_100": 0.5364, + "precision_at_1000": 0.21994, + "mrr_at_1": 0.68, + "mrr_at_3": 0.796667, + "mrr_at_5": 0.806667, + "mrr_at_10": 0.806667, + "mrr_at_20": 0.806667, + "mrr_at_100": 0.806667, + "mrr_at_1000": 0.806667, + "nauc_ndcg_at_1_max": 0.060193, + "nauc_ndcg_at_1_std": 0.440254, + "nauc_ndcg_at_1_diff1": -0.300741, + "nauc_ndcg_at_3_max": 0.213895, + "nauc_ndcg_at_3_std": 0.379248, + "nauc_ndcg_at_3_diff1": -0.168525, + "nauc_ndcg_at_5_max": 0.162615, + "nauc_ndcg_at_5_std": 0.390057, + "nauc_ndcg_at_5_diff1": -0.127423, + "nauc_ndcg_at_10_max": 0.284283, + "nauc_ndcg_at_10_std": 0.525797, + "nauc_ndcg_at_10_diff1": -0.070334, + "nauc_ndcg_at_20_max": 0.367596, + "nauc_ndcg_at_20_std": 0.626766, + "nauc_ndcg_at_20_diff1": 0.017312, + "nauc_ndcg_at_100_max": 0.411994, + "nauc_ndcg_at_100_std": 0.740127, + "nauc_ndcg_at_100_diff1": -0.013901, + "nauc_ndcg_at_1000_max": 0.453516, + "nauc_ndcg_at_1000_std": 0.724224, + "nauc_ndcg_at_1000_diff1": 0.181491, + "nauc_map_at_1_max": -0.030862, + "nauc_map_at_1_std": 0.210045, + "nauc_map_at_1_diff1": -0.113243, + "nauc_map_at_3_max": 0.037002, + "nauc_map_at_3_std": 0.198542, + "nauc_map_at_3_diff1": -0.039105, + "nauc_map_at_5_max": 0.069159, + "nauc_map_at_5_std": 0.206814, + "nauc_map_at_5_diff1": 0.014319, + "nauc_map_at_10_max": 0.141707, + "nauc_map_at_10_std": 0.315946, + "nauc_map_at_10_diff1": 0.063391, + "nauc_map_at_20_max": 0.21385, + "nauc_map_at_20_std": 0.400456, + "nauc_map_at_20_diff1": 0.1246, + "nauc_map_at_100_max": 0.36697, + "nauc_map_at_100_std": 0.645554, + "nauc_map_at_100_diff1": 0.072372, + "nauc_map_at_1000_max": 0.487977, + "nauc_map_at_1000_std": 0.752295, + "nauc_map_at_1000_diff1": 0.050703, + "nauc_recall_at_1_max": -0.030862, + "nauc_recall_at_1_std": 0.210045, + "nauc_recall_at_1_diff1": -0.113243, + "nauc_recall_at_3_max": 0.036483, + "nauc_recall_at_3_std": 0.15905, + "nauc_recall_at_3_diff1": 0.03691, + "nauc_recall_at_5_max": 0.035252, + "nauc_recall_at_5_std": 0.186604, + "nauc_recall_at_5_diff1": 0.104478, + "nauc_recall_at_10_max": 0.078329, + "nauc_recall_at_10_std": 0.277899, + "nauc_recall_at_10_diff1": 0.12915, + "nauc_recall_at_20_max": 0.139406, + "nauc_recall_at_20_std": 0.339323, + "nauc_recall_at_20_diff1": 0.201193, + "nauc_recall_at_100_max": 0.275631, + "nauc_recall_at_100_std": 0.554796, + "nauc_recall_at_100_diff1": 0.168378, + "nauc_recall_at_1000_max": 0.435239, + "nauc_recall_at_1000_std": 0.631795, + "nauc_recall_at_1000_diff1": 0.263409, + "nauc_precision_at_1_max": 0.132161, + "nauc_precision_at_1_std": 0.46721, + "nauc_precision_at_1_diff1": -0.448034, + "nauc_precision_at_3_max": 0.266327, + "nauc_precision_at_3_std": 0.257013, + "nauc_precision_at_3_diff1": -0.196836, + "nauc_precision_at_5_max": 0.231555, + "nauc_precision_at_5_std": 0.342644, + "nauc_precision_at_5_diff1": -0.116333, + "nauc_precision_at_10_max": 0.3481, + "nauc_precision_at_10_std": 0.516508, + "nauc_precision_at_10_diff1": -0.060049, + "nauc_precision_at_20_max": 0.420785, + "nauc_precision_at_20_std": 0.625196, + "nauc_precision_at_20_diff1": 0.03809, + "nauc_precision_at_100_max": 0.445653, + "nauc_precision_at_100_std": 0.765137, + "nauc_precision_at_100_diff1": -0.007424, + "nauc_precision_at_1000_max": 0.441851, + "nauc_precision_at_1000_std": 0.447527, + "nauc_precision_at_1000_diff1": -0.060868, + "nauc_mrr_at_1_max": 0.132161, + "nauc_mrr_at_1_std": 0.46721, + "nauc_mrr_at_1_diff1": -0.448034, + "nauc_mrr_at_3_max": 0.182683, + "nauc_mrr_at_3_std": 0.438751, + "nauc_mrr_at_3_diff1": -0.472603, + "nauc_mrr_at_5_max": 0.150948, + "nauc_mrr_at_5_std": 0.444136, + "nauc_mrr_at_5_diff1": -0.449508, + "nauc_mrr_at_10_max": 0.150948, + "nauc_mrr_at_10_std": 0.444136, + "nauc_mrr_at_10_diff1": -0.449508, + "nauc_mrr_at_20_max": 0.150948, + "nauc_mrr_at_20_std": 0.444136, + "nauc_mrr_at_20_diff1": -0.449508, + "nauc_mrr_at_100_max": 0.150948, + "nauc_mrr_at_100_std": 0.444136, + "nauc_mrr_at_100_diff1": -0.449508, + "nauc_mrr_at_1000_max": 0.150948, + "nauc_mrr_at_1000_std": 0.444136, + "nauc_mrr_at_1000_diff1": -0.449508, + "main_score": 0.62882, + "hf_subset": "default", + "languages": [ + "pol-Latn" + ] + } + ] + }, + "evaluation_time": 3320.7924070358276, + "kg_co2_emissions": 0.000863871068961537 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-large/2/model_meta.json b/results/openai__text-embedding-3-large/2/model_meta.json index 0095186b5b..3414eaa4d9 100644 --- a/results/openai__text-embedding-3-large/2/model_meta.json +++ b/results/openai__text-embedding-3-large/2/model_meta.json @@ -1 +1 @@ -{"name": "openai/text-embedding-3-large", "revision": "2", "release_date": "2024-01-25", "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": 8191.0, "embed_dim": 3072, "license": null, "open_weights": false, "public_training_data": null, "public_training_code": null, "framework": ["API"], "reference": "https://openai.com/index/new-embedding-models-and-api-updates/", "similarity_fn_name": null, "use_instructions": false, "zero_shot_benchmarks": null, "adapted_from": null, "superseded_by": null, "loader": "OpenAIWrapper"} \ No newline at end of file +{"name": "openai/text-embedding-3-large", "revision": "2", "release_date": "2024-01-25", "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": 8191.0, "embed_dim": 3072, "license": null, "open_weights": false, "public_training_data": null, "public_training_code": null, "framework": ["API"], "reference": "https://openai.com/index/new-embedding-models-and-api-updates/", "similarity_fn_name": null, "use_instructions": false, "training_datasets": null, "adapted_from": null, "superseded_by": null, "loader": "OpenAIWrapper"} \ No newline at end of file diff --git a/results/openai__text-embedding-3-small/2/CMedQAv2-reranking.json b/results/openai__text-embedding-3-small/2/CMedQAv2-reranking.json new file mode 100644 index 0000000000..48ae52e253 --- /dev/null +++ b/results/openai__text-embedding-3-small/2/CMedQAv2-reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "23d186750531a14a0357ca22cd92d712fd512ea0", + "task_name": "CMedQAv2-reranking", + "mteb_version": "1.24.0", + "scores": { + "test": [ + { + "map": 0.645397, + "mrr": 0.69917, + "nAUC_map_max": 0.531812, + "nAUC_map_std": 0.239093, + "nAUC_map_diff1": 0.495879, + "nAUC_mrr_max": 0.611273, + "nAUC_mrr_std": 0.289637, + "nAUC_mrr_diff1": 0.544373, + "main_score": 0.645397, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 850.8032479286194, + "kg_co2_emissions": 0.00020905935197114105 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-small/2/CmedqaRetrieval.json b/results/openai__text-embedding-3-small/2/CmedqaRetrieval.json new file mode 100644 index 0000000000..b15f8a33fd --- /dev/null +++ b/results/openai__text-embedding-3-small/2/CmedqaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301", + "task_name": "CmedqaRetrieval", + "mteb_version": "1.24.0", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.18805, + "ndcg_at_3": 0.18752, + "ndcg_at_5": 0.19802, + "ndcg_at_10": 0.21839, + "ndcg_at_20": 0.23833, + "ndcg_at_100": 0.27369, + "ndcg_at_1000": 0.31377, + "map_at_1": 0.11733, + "map_at_3": 0.15466, + "map_at_5": 0.16464, + "map_at_10": 0.17509, + "map_at_20": 0.18153, + "map_at_100": 0.18726, + "map_at_1000": 0.18892, + "recall_at_1": 0.11733, + "recall_at_3": 0.18646, + "recall_at_5": 0.22082, + "recall_at_10": 0.28006, + "recall_at_20": 0.34748, + "recall_at_100": 0.51171, + "recall_at_1000": 0.79445, + "precision_at_1": 0.18805, + "precision_at_3": 0.10744, + "precision_at_5": 0.07822, + "precision_at_10": 0.05089, + "precision_at_20": 0.03218, + "precision_at_100": 0.00969, + "precision_at_1000": 0.00149, + "mrr_at_1": 0.188047, + "mrr_at_3": 0.224931, + "mrr_at_5": 0.234921, + "mrr_at_10": 0.243964, + "mrr_at_20": 0.248938, + "mrr_at_100": 0.252821, + "mrr_at_1000": 0.2538, + "nauc_ndcg_at_1_max": 0.349246, + "nauc_ndcg_at_1_std": 0.060809, + "nauc_ndcg_at_1_diff1": 0.509746, + "nauc_ndcg_at_3_max": 0.332691, + "nauc_ndcg_at_3_std": 0.068341, + "nauc_ndcg_at_3_diff1": 0.417519, + "nauc_ndcg_at_5_max": 0.323648, + "nauc_ndcg_at_5_std": 0.068675, + "nauc_ndcg_at_5_diff1": 0.402927, + "nauc_ndcg_at_10_max": 0.319426, + "nauc_ndcg_at_10_std": 0.069243, + "nauc_ndcg_at_10_diff1": 0.388327, + "nauc_ndcg_at_20_max": 0.325922, + "nauc_ndcg_at_20_std": 0.081376, + "nauc_ndcg_at_20_diff1": 0.379362, + "nauc_ndcg_at_100_max": 0.336681, + "nauc_ndcg_at_100_std": 0.108157, + "nauc_ndcg_at_100_diff1": 0.370023, + "nauc_ndcg_at_1000_max": 0.345946, + "nauc_ndcg_at_1000_std": 0.118407, + "nauc_ndcg_at_1000_diff1": 0.374515, + "nauc_map_at_1_max": 0.308333, + "nauc_map_at_1_std": 0.050341, + "nauc_map_at_1_diff1": 0.503041, + "nauc_map_at_3_max": 0.318273, + "nauc_map_at_3_std": 0.05903, + "nauc_map_at_3_diff1": 0.432005, + "nauc_map_at_5_max": 0.318592, + "nauc_map_at_5_std": 0.061216, + "nauc_map_at_5_diff1": 0.422709, + "nauc_map_at_10_max": 0.322025, + "nauc_map_at_10_std": 0.06438, + "nauc_map_at_10_diff1": 0.416621, + "nauc_map_at_20_max": 0.325678, + "nauc_map_at_20_std": 0.069539, + "nauc_map_at_20_diff1": 0.412612, + "nauc_map_at_100_max": 0.328221, + "nauc_map_at_100_std": 0.074764, + "nauc_map_at_100_diff1": 0.410736, + "nauc_map_at_1000_max": 0.328835, + "nauc_map_at_1000_std": 0.075635, + "nauc_map_at_1000_diff1": 0.410924, + "nauc_recall_at_1_max": 0.308333, + "nauc_recall_at_1_std": 0.050341, + "nauc_recall_at_1_diff1": 0.503041, + "nauc_recall_at_3_max": 0.292839, + "nauc_recall_at_3_std": 0.059338, + "nauc_recall_at_3_diff1": 0.351708, + "nauc_recall_at_5_max": 0.285329, + "nauc_recall_at_5_std": 0.066746, + "nauc_recall_at_5_diff1": 0.323656, + "nauc_recall_at_10_max": 0.266605, + "nauc_recall_at_10_std": 0.065957, + "nauc_recall_at_10_diff1": 0.283282, + "nauc_recall_at_20_max": 0.282008, + "nauc_recall_at_20_std": 0.098984, + "nauc_recall_at_20_diff1": 0.258944, + "nauc_recall_at_100_max": 0.311391, + "nauc_recall_at_100_std": 0.197212, + "nauc_recall_at_100_diff1": 0.210185, + "nauc_recall_at_1000_max": 0.408275, + "nauc_recall_at_1000_std": 0.366315, + "nauc_recall_at_1000_diff1": 0.160262, + "nauc_precision_at_1_max": 0.349246, + "nauc_precision_at_1_std": 0.060809, + "nauc_precision_at_1_diff1": 0.509746, + "nauc_precision_at_3_max": 0.358395, + "nauc_precision_at_3_std": 0.079576, + "nauc_precision_at_3_diff1": 0.363112, + "nauc_precision_at_5_max": 0.348285, + "nauc_precision_at_5_std": 0.087899, + "nauc_precision_at_5_diff1": 0.32473, + "nauc_precision_at_10_max": 0.338256, + "nauc_precision_at_10_std": 0.100633, + "nauc_precision_at_10_diff1": 0.273575, + "nauc_precision_at_20_max": 0.334183, + "nauc_precision_at_20_std": 0.12751, + "nauc_precision_at_20_diff1": 0.23315, + "nauc_precision_at_100_max": 0.295217, + "nauc_precision_at_100_std": 0.181607, + "nauc_precision_at_100_diff1": 0.162611, + "nauc_precision_at_1000_max": 0.216973, + "nauc_precision_at_1000_std": 0.159834, + "nauc_precision_at_1000_diff1": 0.090773, + "nauc_mrr_at_1_max": 0.349246, + "nauc_mrr_at_1_std": 0.060809, + "nauc_mrr_at_1_diff1": 0.509746, + "nauc_mrr_at_3_max": 0.336975, + "nauc_mrr_at_3_std": 0.062913, + "nauc_mrr_at_3_diff1": 0.445713, + "nauc_mrr_at_5_max": 0.3356, + "nauc_mrr_at_5_std": 0.066063, + "nauc_mrr_at_5_diff1": 0.437264, + "nauc_mrr_at_10_max": 0.332296, + "nauc_mrr_at_10_std": 0.066542, + "nauc_mrr_at_10_diff1": 0.42965, + "nauc_mrr_at_20_max": 0.332987, + "nauc_mrr_at_20_std": 0.068726, + "nauc_mrr_at_20_diff1": 0.427986, + "nauc_mrr_at_100_max": 0.33344, + "nauc_mrr_at_100_std": 0.070633, + "nauc_mrr_at_100_diff1": 0.42717, + "nauc_mrr_at_1000_max": 0.333711, + "nauc_mrr_at_1000_std": 0.070806, + "nauc_mrr_at_1000_diff1": 0.427461, + "main_score": 0.21839, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 1528.71191072464, + "kg_co2_emissions": 0.00041058124334275565 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-small/2/MedicalQARetrieval.json b/results/openai__text-embedding-3-small/2/MedicalQARetrieval.json new file mode 100644 index 0000000000..c6034b2e50 --- /dev/null +++ b/results/openai__text-embedding-3-small/2/MedicalQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ae763399273d8b20506b80cf6f6f9a31a6a2b238", + "task_name": "MedicalQARetrieval", + "mteb_version": "1.24.0", + "scores": { + "test": [ + { + "ndcg_at_1": 0.50732, + "ndcg_at_3": 0.65075, + "ndcg_at_5": 0.67925, + "ndcg_at_10": 0.69971, + "ndcg_at_20": 0.70998, + "ndcg_at_100": 0.71802, + "ndcg_at_1000": 0.72149, + "map_at_1": 0.50732, + "map_at_3": 0.61515, + "map_at_5": 0.63105, + "map_at_10": 0.63963, + "map_at_20": 0.64254, + "map_at_100": 0.64371, + "map_at_1000": 0.64383, + "recall_at_1": 0.50732, + "recall_at_3": 0.75391, + "recall_at_5": 0.82275, + "recall_at_10": 0.88525, + "recall_at_20": 0.92529, + "recall_at_100": 0.96777, + "recall_at_1000": 0.99561, + "precision_at_1": 0.50732, + "precision_at_3": 0.2513, + "precision_at_5": 0.16455, + "precision_at_10": 0.08853, + "precision_at_20": 0.04626, + "precision_at_100": 0.00968, + "precision_at_1000": 0.001, + "mrr_at_1": 0.507812, + "mrr_at_3": 0.61556, + "mrr_at_5": 0.631209, + "mrr_at_10": 0.639879, + "mrr_at_20": 0.642782, + "mrr_at_100": 0.643957, + "mrr_at_1000": 0.644078, + "nauc_ndcg_at_1_max": 0.37459, + "nauc_ndcg_at_1_std": -0.034314, + "nauc_ndcg_at_1_diff1": 0.507318, + "nauc_ndcg_at_3_max": 0.34695, + "nauc_ndcg_at_3_std": -0.084532, + "nauc_ndcg_at_3_diff1": 0.424556, + "nauc_ndcg_at_5_max": 0.352633, + "nauc_ndcg_at_5_std": -0.066448, + "nauc_ndcg_at_5_diff1": 0.428754, + "nauc_ndcg_at_10_max": 0.370043, + "nauc_ndcg_at_10_std": -0.054082, + "nauc_ndcg_at_10_diff1": 0.428542, + "nauc_ndcg_at_20_max": 0.372771, + "nauc_ndcg_at_20_std": -0.042269, + "nauc_ndcg_at_20_diff1": 0.433259, + "nauc_ndcg_at_100_max": 0.366625, + "nauc_ndcg_at_100_std": -0.042501, + "nauc_ndcg_at_100_diff1": 0.437926, + "nauc_ndcg_at_1000_max": 0.365047, + "nauc_ndcg_at_1000_std": -0.04731, + "nauc_ndcg_at_1000_diff1": 0.441451, + "nauc_map_at_1_max": 0.37459, + "nauc_map_at_1_std": -0.034314, + "nauc_map_at_1_diff1": 0.507318, + "nauc_map_at_3_max": 0.35446, + "nauc_map_at_3_std": -0.0676, + "nauc_map_at_3_diff1": 0.446786, + "nauc_map_at_5_max": 0.357529, + "nauc_map_at_5_std": -0.058183, + "nauc_map_at_5_diff1": 0.449682, + "nauc_map_at_10_max": 0.364001, + "nauc_map_at_10_std": -0.053739, + "nauc_map_at_10_diff1": 0.45019, + "nauc_map_at_20_max": 0.36456, + "nauc_map_at_20_std": -0.051173, + "nauc_map_at_20_diff1": 0.451677, + "nauc_map_at_100_max": 0.363774, + "nauc_map_at_100_std": -0.051112, + "nauc_map_at_100_diff1": 0.452138, + "nauc_map_at_1000_max": 0.363747, + "nauc_map_at_1000_std": -0.051197, + "nauc_map_at_1000_diff1": 0.452231, + "nauc_recall_at_1_max": 0.37459, + "nauc_recall_at_1_std": -0.034314, + "nauc_recall_at_1_diff1": 0.507318, + "nauc_recall_at_3_max": 0.318221, + "nauc_recall_at_3_std": -0.15121, + "nauc_recall_at_3_diff1": 0.339664, + "nauc_recall_at_5_max": 0.330622, + "nauc_recall_at_5_std": -0.103536, + "nauc_recall_at_5_diff1": 0.329226, + "nauc_recall_at_10_max": 0.430262, + "nauc_recall_at_10_std": -0.039337, + "nauc_recall_at_10_diff1": 0.279733, + "nauc_recall_at_20_max": 0.493547, + "nauc_recall_at_20_std": 0.112868, + "nauc_recall_at_20_diff1": 0.255149, + "nauc_recall_at_100_max": 0.437516, + "nauc_recall_at_100_std": 0.292662, + "nauc_recall_at_100_diff1": 0.208995, + "nauc_recall_at_1000_max": 0.21199, + "nauc_recall_at_1000_std": 0.370193, + "nauc_recall_at_1000_diff1": 0.193833, + "nauc_precision_at_1_max": 0.37459, + "nauc_precision_at_1_std": -0.034314, + "nauc_precision_at_1_diff1": 0.507318, + "nauc_precision_at_3_max": 0.318221, + "nauc_precision_at_3_std": -0.15121, + "nauc_precision_at_3_diff1": 0.339664, + "nauc_precision_at_5_max": 0.330622, + "nauc_precision_at_5_std": -0.103536, + "nauc_precision_at_5_diff1": 0.329226, + "nauc_precision_at_10_max": 0.430262, + "nauc_precision_at_10_std": -0.039337, + "nauc_precision_at_10_diff1": 0.279733, + "nauc_precision_at_20_max": 0.493547, + "nauc_precision_at_20_std": 0.112868, + "nauc_precision_at_20_diff1": 0.255149, + "nauc_precision_at_100_max": 0.437516, + "nauc_precision_at_100_std": 0.292662, + "nauc_precision_at_100_diff1": 0.208995, + "nauc_precision_at_1000_max": 0.21199, + "nauc_precision_at_1000_std": 0.370193, + "nauc_precision_at_1000_diff1": 0.193833, + "nauc_mrr_at_1_max": 0.373373, + "nauc_mrr_at_1_std": -0.035552, + "nauc_mrr_at_1_diff1": 0.506169, + "nauc_mrr_at_3_max": 0.353569, + "nauc_mrr_at_3_std": -0.067731, + "nauc_mrr_at_3_diff1": 0.445713, + "nauc_mrr_at_5_max": 0.356914, + "nauc_mrr_at_5_std": -0.058533, + "nauc_mrr_at_5_diff1": 0.449128, + "nauc_mrr_at_10_max": 0.363249, + "nauc_mrr_at_10_std": -0.054277, + "nauc_mrr_at_10_diff1": 0.449439, + "nauc_mrr_at_20_max": 0.363801, + "nauc_mrr_at_20_std": -0.051715, + "nauc_mrr_at_20_diff1": 0.450919, + "nauc_mrr_at_100_max": 0.363011, + "nauc_mrr_at_100_std": -0.051656, + "nauc_mrr_at_100_diff1": 0.451378, + "nauc_mrr_at_1000_max": 0.362984, + "nauc_mrr_at_1000_std": -0.051741, + "nauc_mrr_at_1000_diff1": 0.45147, + "main_score": 0.69971, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 42.043631076812744, + "kg_co2_emissions": 1.4262066368219871e-05 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-small/2/MedrxivClusteringS2S.v2.json b/results/openai__text-embedding-3-small/2/MedrxivClusteringS2S.v2.json new file mode 100644 index 0000000000..e435b0a110 --- /dev/null +++ b/results/openai__text-embedding-3-small/2/MedrxivClusteringS2S.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "task_name": "MedrxivClusteringS2S.v2", + "mteb_version": "1.24.0", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.346442, + 0.356649, + 0.348436, + 0.354428, + 0.34424, + 0.326995, + 0.346925, + 0.345125, + 0.354051, + 0.359267 + ] + }, + "v_measure": 0.348256, + "v_measure_std": 0.00864, + "main_score": 0.348256, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 23.859488010406494, + "kg_co2_emissions": 1.332439004376324e-05 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-small/2/PublicHealthQA.json b/results/openai__text-embedding-3-small/2/PublicHealthQA.json new file mode 100644 index 0000000000..4c8d28132e --- /dev/null +++ b/results/openai__text-embedding-3-small/2/PublicHealthQA.json @@ -0,0 +1,1187 @@ +{ + "dataset_revision": "main", + "task_name": "PublicHealthQA", + "mteb_version": "1.24.0", + "scores": { + "test": [ + { + "ndcg_at_1": 0.54651, + "ndcg_at_3": 0.67649, + "ndcg_at_5": 0.69949, + "ndcg_at_10": 0.714, + "ndcg_at_20": 0.714, + "ndcg_at_100": 0.73752, + "ndcg_at_1000": 0.73752, + "map_at_1": 0.54651, + "map_at_3": 0.64147, + "map_at_5": 0.65368, + "map_at_10": 0.65933, + "map_at_20": 0.65933, + "map_at_100": 0.66337, + "map_at_1000": 0.66337, + "recall_at_1": 0.54651, + "recall_at_3": 0.77907, + "recall_at_5": 0.83721, + "recall_at_10": 0.88372, + "recall_at_20": 0.88372, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.54651, + "precision_at_3": 0.25969, + "precision_at_5": 0.16744, + "precision_at_10": 0.08837, + "precision_at_20": 0.04419, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.546512, + "mrr_at_3": 0.641473, + "mrr_at_5": 0.653682, + "mrr_at_10": 0.659335, + "mrr_at_20": 0.659335, + "mrr_at_100": 0.663375, + "mrr_at_1000": 0.663375, + "nauc_ndcg_at_1_max": 0.116789, + "nauc_ndcg_at_1_std": -0.117823, + "nauc_ndcg_at_1_diff1": 0.491109, + "nauc_ndcg_at_3_max": 0.180642, + "nauc_ndcg_at_3_std": -0.099606, + "nauc_ndcg_at_3_diff1": 0.525277, + "nauc_ndcg_at_5_max": 0.121992, + "nauc_ndcg_at_5_std": -0.149594, + "nauc_ndcg_at_5_diff1": 0.501239, + "nauc_ndcg_at_10_max": 0.158705, + "nauc_ndcg_at_10_std": -0.098723, + "nauc_ndcg_at_10_diff1": 0.490813, + "nauc_ndcg_at_20_max": 0.158705, + "nauc_ndcg_at_20_std": -0.098723, + "nauc_ndcg_at_20_diff1": 0.490813, + "nauc_ndcg_at_100_max": 0.147265, + "nauc_ndcg_at_100_std": -0.111243, + "nauc_ndcg_at_100_diff1": 0.504471, + "nauc_ndcg_at_1000_max": 0.147265, + "nauc_ndcg_at_1000_std": -0.111243, + "nauc_ndcg_at_1000_diff1": 0.504471, + "nauc_map_at_1_max": 0.116789, + "nauc_map_at_1_std": -0.117823, + "nauc_map_at_1_diff1": 0.491109, + "nauc_map_at_3_max": 0.165778, + "nauc_map_at_3_std": -0.099448, + "nauc_map_at_3_diff1": 0.517988, + "nauc_map_at_5_max": 0.137128, + "nauc_map_at_5_std": -0.12379, + "nauc_map_at_5_diff1": 0.505892, + "nauc_map_at_10_max": 0.14931, + "nauc_map_at_10_std": -0.106488, + "nauc_map_at_10_diff1": 0.502567, + "nauc_map_at_20_max": 0.14931, + "nauc_map_at_20_std": -0.106488, + "nauc_map_at_20_diff1": 0.502567, + "nauc_map_at_100_max": 0.146826, + "nauc_map_at_100_std": -0.109314, + "nauc_map_at_100_diff1": 0.505287, + "nauc_map_at_1000_max": 0.146826, + "nauc_map_at_1000_std": -0.109314, + "nauc_map_at_1000_diff1": 0.505287, + "nauc_recall_at_1_max": 0.116789, + "nauc_recall_at_1_std": -0.117823, + "nauc_recall_at_1_diff1": 0.491109, + "nauc_recall_at_3_max": 0.238496, + "nauc_recall_at_3_std": -0.103322, + "nauc_recall_at_3_diff1": 0.553184, + "nauc_recall_at_5_max": 0.025885, + "nauc_recall_at_5_std": -0.297819, + "nauc_recall_at_5_diff1": 0.46956, + "nauc_recall_at_10_max": 0.234294, + "nauc_recall_at_10_std": -0.018709, + "nauc_recall_at_10_diff1": 0.390012, + "nauc_recall_at_20_max": 0.234294, + "nauc_recall_at_20_std": -0.018709, + "nauc_recall_at_20_diff1": 0.390012, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.116789, + "nauc_precision_at_1_std": -0.117823, + "nauc_precision_at_1_diff1": 0.491109, + "nauc_precision_at_3_max": 0.238496, + "nauc_precision_at_3_std": -0.103322, + "nauc_precision_at_3_diff1": 0.553184, + "nauc_precision_at_5_max": 0.025885, + "nauc_precision_at_5_std": -0.297819, + "nauc_precision_at_5_diff1": 0.46956, + "nauc_precision_at_10_max": 0.234294, + "nauc_precision_at_10_std": -0.018709, + "nauc_precision_at_10_diff1": 0.390012, + "nauc_precision_at_20_max": 0.234294, + "nauc_precision_at_20_std": -0.018709, + "nauc_precision_at_20_diff1": 0.390012, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.116789, + "nauc_mrr_at_1_std": -0.117823, + "nauc_mrr_at_1_diff1": 0.491109, + "nauc_mrr_at_3_max": 0.165778, + "nauc_mrr_at_3_std": -0.099448, + "nauc_mrr_at_3_diff1": 0.517988, + "nauc_mrr_at_5_max": 0.137128, + "nauc_mrr_at_5_std": -0.12379, + "nauc_mrr_at_5_diff1": 0.505892, + "nauc_mrr_at_10_max": 0.14931, + "nauc_mrr_at_10_std": -0.106488, + "nauc_mrr_at_10_diff1": 0.502567, + "nauc_mrr_at_20_max": 0.14931, + "nauc_mrr_at_20_std": -0.106488, + "nauc_mrr_at_20_diff1": 0.502567, + "nauc_mrr_at_100_max": 0.146826, + "nauc_mrr_at_100_std": -0.109314, + "nauc_mrr_at_100_diff1": 0.505287, + "nauc_mrr_at_1000_max": 0.146826, + "nauc_mrr_at_1000_std": -0.109314, + "nauc_mrr_at_1000_diff1": 0.505287, + "main_score": 0.714, + "hf_subset": "arabic", + "languages": [ + "ara-Arab" + ] + }, + { + "ndcg_at_1": 0.6135, + "ndcg_at_3": 0.7516, + "ndcg_at_5": 0.77825, + "ndcg_at_10": 0.79456, + "ndcg_at_20": 0.79925, + "ndcg_at_100": 0.80176, + "ndcg_at_1000": 0.80176, + "map_at_1": 0.6135, + "map_at_3": 0.71677, + "map_at_5": 0.73088, + "map_at_10": 0.73785, + "map_at_20": 0.73916, + "map_at_100": 0.73959, + "map_at_1000": 0.73959, + "recall_at_1": 0.6135, + "recall_at_3": 0.85276, + "recall_at_5": 0.92025, + "recall_at_10": 0.96933, + "recall_at_20": 0.98773, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.6135, + "precision_at_3": 0.28425, + "precision_at_5": 0.18405, + "precision_at_10": 0.09693, + "precision_at_20": 0.04939, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.613497, + "mrr_at_3": 0.716769, + "mrr_at_5": 0.730879, + "mrr_at_10": 0.737854, + "mrr_at_20": 0.73916, + "mrr_at_100": 0.739592, + "mrr_at_1000": 0.739592, + "nauc_ndcg_at_1_max": 0.418976, + "nauc_ndcg_at_1_std": 0.017477, + "nauc_ndcg_at_1_diff1": 0.740084, + "nauc_ndcg_at_3_max": 0.453162, + "nauc_ndcg_at_3_std": 0.028144, + "nauc_ndcg_at_3_diff1": 0.729143, + "nauc_ndcg_at_5_max": 0.422294, + "nauc_ndcg_at_5_std": 0.014928, + "nauc_ndcg_at_5_diff1": 0.727634, + "nauc_ndcg_at_10_max": 0.432632, + "nauc_ndcg_at_10_std": 0.026467, + "nauc_ndcg_at_10_diff1": 0.730955, + "nauc_ndcg_at_20_max": 0.429058, + "nauc_ndcg_at_20_std": 0.034076, + "nauc_ndcg_at_20_diff1": 0.723424, + "nauc_ndcg_at_100_max": 0.426951, + "nauc_ndcg_at_100_std": 0.022705, + "nauc_ndcg_at_100_diff1": 0.727478, + "nauc_ndcg_at_1000_max": 0.426951, + "nauc_ndcg_at_1000_std": 0.022705, + "nauc_ndcg_at_1000_diff1": 0.727478, + "nauc_map_at_1_max": 0.418976, + "nauc_map_at_1_std": 0.017477, + "nauc_map_at_1_diff1": 0.740084, + "nauc_map_at_3_max": 0.438217, + "nauc_map_at_3_std": 0.022512, + "nauc_map_at_3_diff1": 0.729015, + "nauc_map_at_5_max": 0.423492, + "nauc_map_at_5_std": 0.01653, + "nauc_map_at_5_diff1": 0.728236, + "nauc_map_at_10_max": 0.426517, + "nauc_map_at_10_std": 0.020143, + "nauc_map_at_10_diff1": 0.729354, + "nauc_map_at_20_max": 0.42583, + "nauc_map_at_20_std": 0.022012, + "nauc_map_at_20_diff1": 0.727719, + "nauc_map_at_100_max": 0.425514, + "nauc_map_at_100_std": 0.020478, + "nauc_map_at_100_diff1": 0.728252, + "nauc_map_at_1000_max": 0.425514, + "nauc_map_at_1000_std": 0.020478, + "nauc_map_at_1000_diff1": 0.728252, + "nauc_recall_at_1_max": 0.418976, + "nauc_recall_at_1_std": 0.017477, + "nauc_recall_at_1_diff1": 0.740084, + "nauc_recall_at_3_max": 0.528778, + "nauc_recall_at_3_std": 0.057119, + "nauc_recall_at_3_diff1": 0.732235, + "nauc_recall_at_5_max": 0.403621, + "nauc_recall_at_5_std": -0.002319, + "nauc_recall_at_5_diff1": 0.726733, + "nauc_recall_at_10_max": 0.566876, + "nauc_recall_at_10_std": 0.177455, + "nauc_recall_at_10_diff1": 0.782393, + "nauc_recall_at_20_max": 0.560699, + "nauc_recall_at_20_std": 0.776911, + "nauc_recall_at_20_diff1": 0.455983, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.418976, + "nauc_precision_at_1_std": 0.017477, + "nauc_precision_at_1_diff1": 0.740084, + "nauc_precision_at_3_max": 0.528778, + "nauc_precision_at_3_std": 0.057119, + "nauc_precision_at_3_diff1": 0.732235, + "nauc_precision_at_5_max": 0.403621, + "nauc_precision_at_5_std": -0.002319, + "nauc_precision_at_5_diff1": 0.726733, + "nauc_precision_at_10_max": 0.566876, + "nauc_precision_at_10_std": 0.177455, + "nauc_precision_at_10_diff1": 0.782393, + "nauc_precision_at_20_max": 0.560699, + "nauc_precision_at_20_std": 0.776911, + "nauc_precision_at_20_diff1": 0.455983, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.418976, + "nauc_mrr_at_1_std": 0.017477, + "nauc_mrr_at_1_diff1": 0.740084, + "nauc_mrr_at_3_max": 0.438217, + "nauc_mrr_at_3_std": 0.022512, + "nauc_mrr_at_3_diff1": 0.729015, + "nauc_mrr_at_5_max": 0.423492, + "nauc_mrr_at_5_std": 0.01653, + "nauc_mrr_at_5_diff1": 0.728236, + "nauc_mrr_at_10_max": 0.426517, + "nauc_mrr_at_10_std": 0.020143, + "nauc_mrr_at_10_diff1": 0.729354, + "nauc_mrr_at_20_max": 0.42583, + "nauc_mrr_at_20_std": 0.022012, + "nauc_mrr_at_20_diff1": 0.727719, + "nauc_mrr_at_100_max": 0.425514, + "nauc_mrr_at_100_std": 0.020478, + "nauc_mrr_at_100_diff1": 0.728252, + "nauc_mrr_at_1000_max": 0.425514, + "nauc_mrr_at_1000_std": 0.020478, + "nauc_mrr_at_1000_diff1": 0.728252, + "main_score": 0.79456, + "hf_subset": "chinese", + "languages": [ + "zho-Hans" + ] + }, + { + "ndcg_at_1": 0.7093, + "ndcg_at_3": 0.83008, + "ndcg_at_5": 0.85635, + "ndcg_at_10": 0.85635, + "ndcg_at_20": 0.8624, + "ndcg_at_100": 0.8624, + "ndcg_at_1000": 0.8624, + "map_at_1": 0.7093, + "map_at_3": 0.80136, + "map_at_5": 0.81589, + "map_at_10": 0.81589, + "map_at_20": 0.81764, + "map_at_100": 0.81764, + "map_at_1000": 0.81764, + "recall_at_1": 0.7093, + "recall_at_3": 0.91279, + "recall_at_5": 0.97674, + "recall_at_10": 0.97674, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.7093, + "precision_at_3": 0.30426, + "precision_at_5": 0.19535, + "precision_at_10": 0.09767, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.709302, + "mrr_at_3": 0.801357, + "mrr_at_5": 0.815891, + "mrr_at_10": 0.815891, + "mrr_at_20": 0.817643, + "mrr_at_100": 0.817643, + "mrr_at_1000": 0.817643, + "nauc_ndcg_at_1_max": 0.544304, + "nauc_ndcg_at_1_std": -0.064521, + "nauc_ndcg_at_1_diff1": 0.733445, + "nauc_ndcg_at_3_max": 0.511654, + "nauc_ndcg_at_3_std": -0.043106, + "nauc_ndcg_at_3_diff1": 0.696504, + "nauc_ndcg_at_5_max": 0.497881, + "nauc_ndcg_at_5_std": -0.063737, + "nauc_ndcg_at_5_diff1": 0.701721, + "nauc_ndcg_at_10_max": 0.497881, + "nauc_ndcg_at_10_std": -0.063737, + "nauc_ndcg_at_10_diff1": 0.701721, + "nauc_ndcg_at_20_max": 0.510498, + "nauc_ndcg_at_20_std": -0.067807, + "nauc_ndcg_at_20_diff1": 0.706334, + "nauc_ndcg_at_100_max": 0.510498, + "nauc_ndcg_at_100_std": -0.067807, + "nauc_ndcg_at_100_diff1": 0.706334, + "nauc_ndcg_at_1000_max": 0.510498, + "nauc_ndcg_at_1000_std": -0.067807, + "nauc_ndcg_at_1000_diff1": 0.706334, + "nauc_map_at_1_max": 0.544304, + "nauc_map_at_1_std": -0.064521, + "nauc_map_at_1_diff1": 0.733445, + "nauc_map_at_3_max": 0.515765, + "nauc_map_at_3_std": -0.05811, + "nauc_map_at_3_diff1": 0.704373, + "nauc_map_at_5_max": 0.50977, + "nauc_map_at_5_std": -0.069225, + "nauc_map_at_5_diff1": 0.707277, + "nauc_map_at_10_max": 0.50977, + "nauc_map_at_10_std": -0.069225, + "nauc_map_at_10_diff1": 0.707277, + "nauc_map_at_20_max": 0.513165, + "nauc_map_at_20_std": -0.069383, + "nauc_map_at_20_diff1": 0.708356, + "nauc_map_at_100_max": 0.513165, + "nauc_map_at_100_std": -0.069383, + "nauc_map_at_100_diff1": 0.708356, + "nauc_map_at_1000_max": 0.513165, + "nauc_map_at_1000_std": -0.069383, + "nauc_map_at_1000_diff1": 0.708356, + "nauc_recall_at_1_max": 0.544304, + "nauc_recall_at_1_std": -0.064521, + "nauc_recall_at_1_diff1": 0.733445, + "nauc_recall_at_3_max": 0.493325, + "nauc_recall_at_3_std": 0.051913, + "nauc_recall_at_3_diff1": 0.654067, + "nauc_recall_at_5_max": 0.268511, + "nauc_recall_at_5_std": 0.059448, + "nauc_recall_at_5_diff1": 0.60938, + "nauc_recall_at_10_max": 0.268511, + "nauc_recall_at_10_std": 0.059448, + "nauc_recall_at_10_diff1": 0.60938, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.544304, + "nauc_precision_at_1_std": -0.064521, + "nauc_precision_at_1_diff1": 0.733445, + "nauc_precision_at_3_max": 0.493325, + "nauc_precision_at_3_std": 0.051913, + "nauc_precision_at_3_diff1": 0.654067, + "nauc_precision_at_5_max": 0.268511, + "nauc_precision_at_5_std": 0.059448, + "nauc_precision_at_5_diff1": 0.60938, + "nauc_precision_at_10_max": 0.268511, + "nauc_precision_at_10_std": 0.059448, + "nauc_precision_at_10_diff1": 0.60938, + "nauc_precision_at_20_max": NaN, + "nauc_precision_at_20_std": NaN, + "nauc_precision_at_20_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.544304, + "nauc_mrr_at_1_std": -0.064521, + "nauc_mrr_at_1_diff1": 0.733445, + "nauc_mrr_at_3_max": 0.515765, + "nauc_mrr_at_3_std": -0.05811, + "nauc_mrr_at_3_diff1": 0.704373, + "nauc_mrr_at_5_max": 0.50977, + "nauc_mrr_at_5_std": -0.069225, + "nauc_mrr_at_5_diff1": 0.707277, + "nauc_mrr_at_10_max": 0.50977, + "nauc_mrr_at_10_std": -0.069225, + "nauc_mrr_at_10_diff1": 0.707277, + "nauc_mrr_at_20_max": 0.513165, + "nauc_mrr_at_20_std": -0.069383, + "nauc_mrr_at_20_diff1": 0.708356, + "nauc_mrr_at_100_max": 0.513165, + "nauc_mrr_at_100_std": -0.069383, + "nauc_mrr_at_100_diff1": 0.708356, + "nauc_mrr_at_1000_max": 0.513165, + "nauc_mrr_at_1000_std": -0.069383, + "nauc_mrr_at_1000_diff1": 0.708356, + "main_score": 0.85635, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.75294, + "ndcg_at_3": 0.8507, + "ndcg_at_5": 0.875, + "ndcg_at_10": 0.88211, + "ndcg_at_20": 0.88211, + "ndcg_at_100": 0.88211, + "ndcg_at_1000": 0.88211, + "map_at_1": 0.75294, + "map_at_3": 0.82745, + "map_at_5": 0.84098, + "map_at_10": 0.84363, + "map_at_20": 0.84363, + "map_at_100": 0.84363, + "map_at_1000": 0.84363, + "recall_at_1": 0.75294, + "recall_at_3": 0.91765, + "recall_at_5": 0.97647, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.75294, + "precision_at_3": 0.30588, + "precision_at_5": 0.19529, + "precision_at_10": 0.1, + "precision_at_20": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.752941, + "mrr_at_3": 0.827451, + "mrr_at_5": 0.84098, + "mrr_at_10": 0.843627, + "mrr_at_20": 0.843627, + "mrr_at_100": 0.843627, + "mrr_at_1000": 0.843627, + "nauc_ndcg_at_1_max": 0.422371, + "nauc_ndcg_at_1_std": -0.098756, + "nauc_ndcg_at_1_diff1": 0.822492, + "nauc_ndcg_at_3_max": 0.463803, + "nauc_ndcg_at_3_std": 0.060654, + "nauc_ndcg_at_3_diff1": 0.821638, + "nauc_ndcg_at_5_max": 0.38499, + "nauc_ndcg_at_5_std": -0.069985, + "nauc_ndcg_at_5_diff1": 0.806185, + "nauc_ndcg_at_10_max": 0.403487, + "nauc_ndcg_at_10_std": -0.049595, + "nauc_ndcg_at_10_diff1": 0.813695, + "nauc_ndcg_at_20_max": 0.403487, + "nauc_ndcg_at_20_std": -0.049595, + "nauc_ndcg_at_20_diff1": 0.813695, + "nauc_ndcg_at_100_max": 0.403487, + "nauc_ndcg_at_100_std": -0.049595, + "nauc_ndcg_at_100_diff1": 0.813695, + "nauc_ndcg_at_1000_max": 0.403487, + "nauc_ndcg_at_1000_std": -0.049595, + "nauc_ndcg_at_1000_diff1": 0.813695, + "nauc_map_at_1_max": 0.422371, + "nauc_map_at_1_std": -0.098756, + "nauc_map_at_1_diff1": 0.822492, + "nauc_map_at_3_max": 0.437168, + "nauc_map_at_3_std": 0.003555, + "nauc_map_at_3_diff1": 0.819725, + "nauc_map_at_5_max": 0.399765, + "nauc_map_at_5_std": -0.058867, + "nauc_map_at_5_diff1": 0.812908, + "nauc_map_at_10_max": 0.404908, + "nauc_map_at_10_std": -0.052678, + "nauc_map_at_10_diff1": 0.814741, + "nauc_map_at_20_max": 0.404908, + "nauc_map_at_20_std": -0.052678, + "nauc_map_at_20_diff1": 0.814741, + "nauc_map_at_100_max": 0.404908, + "nauc_map_at_100_std": -0.052678, + "nauc_map_at_100_diff1": 0.814741, + "nauc_map_at_1000_max": 0.404908, + "nauc_map_at_1000_std": -0.052678, + "nauc_map_at_1000_diff1": 0.814741, + "nauc_recall_at_1_max": 0.422371, + "nauc_recall_at_1_std": -0.098756, + "nauc_recall_at_1_diff1": 0.822492, + "nauc_recall_at_3_max": 0.621757, + "nauc_recall_at_3_std": 0.380463, + "nauc_recall_at_3_diff1": 0.834003, + "nauc_recall_at_5_max": 0.094163, + "nauc_recall_at_5_std": -0.369175, + "nauc_recall_at_5_diff1": 0.678751, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.422371, + "nauc_precision_at_1_std": -0.098756, + "nauc_precision_at_1_diff1": 0.822492, + "nauc_precision_at_3_max": 0.621757, + "nauc_precision_at_3_std": 0.380463, + "nauc_precision_at_3_diff1": 0.834003, + "nauc_precision_at_5_max": 0.094163, + "nauc_precision_at_5_std": -0.369175, + "nauc_precision_at_5_diff1": 0.678751, + "nauc_precision_at_10_max": NaN, + "nauc_precision_at_10_std": NaN, + "nauc_precision_at_10_diff1": NaN, + "nauc_precision_at_20_max": NaN, + "nauc_precision_at_20_std": NaN, + "nauc_precision_at_20_diff1": NaN, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.422371, + "nauc_mrr_at_1_std": -0.098756, + "nauc_mrr_at_1_diff1": 0.822492, + "nauc_mrr_at_3_max": 0.437168, + "nauc_mrr_at_3_std": 0.003555, + "nauc_mrr_at_3_diff1": 0.819725, + "nauc_mrr_at_5_max": 0.399765, + "nauc_mrr_at_5_std": -0.058867, + "nauc_mrr_at_5_diff1": 0.812908, + "nauc_mrr_at_10_max": 0.404908, + "nauc_mrr_at_10_std": -0.052678, + "nauc_mrr_at_10_diff1": 0.814741, + "nauc_mrr_at_20_max": 0.404908, + "nauc_mrr_at_20_std": -0.052678, + "nauc_mrr_at_20_diff1": 0.814741, + "nauc_mrr_at_100_max": 0.404908, + "nauc_mrr_at_100_std": -0.052678, + "nauc_mrr_at_100_diff1": 0.814741, + "nauc_mrr_at_1000_max": 0.404908, + "nauc_mrr_at_1000_std": -0.052678, + "nauc_mrr_at_1000_diff1": 0.814741, + "main_score": 0.88211, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.51948, + "ndcg_at_3": 0.65198, + "ndcg_at_5": 0.67766, + "ndcg_at_10": 0.70646, + "ndcg_at_20": 0.71945, + "ndcg_at_100": 0.73027, + "ndcg_at_1000": 0.73027, + "map_at_1": 0.51948, + "map_at_3": 0.62121, + "map_at_5": 0.63485, + "map_at_10": 0.64639, + "map_at_20": 0.64987, + "map_at_100": 0.65182, + "map_at_1000": 0.65182, + "recall_at_1": 0.51948, + "recall_at_3": 0.74026, + "recall_at_5": 0.80519, + "recall_at_10": 0.8961, + "recall_at_20": 0.94805, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.51948, + "precision_at_3": 0.24675, + "precision_at_5": 0.16104, + "precision_at_10": 0.08961, + "precision_at_20": 0.0474, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.519481, + "mrr_at_3": 0.621212, + "mrr_at_5": 0.634848, + "mrr_at_10": 0.646387, + "mrr_at_20": 0.649869, + "mrr_at_100": 0.651824, + "mrr_at_1000": 0.651824, + "nauc_ndcg_at_1_max": 0.381924, + "nauc_ndcg_at_1_std": 0.046089, + "nauc_ndcg_at_1_diff1": 0.495245, + "nauc_ndcg_at_3_max": 0.351031, + "nauc_ndcg_at_3_std": 0.019001, + "nauc_ndcg_at_3_diff1": 0.422169, + "nauc_ndcg_at_5_max": 0.356591, + "nauc_ndcg_at_5_std": 0.069605, + "nauc_ndcg_at_5_diff1": 0.445273, + "nauc_ndcg_at_10_max": 0.362248, + "nauc_ndcg_at_10_std": 0.09332, + "nauc_ndcg_at_10_diff1": 0.470284, + "nauc_ndcg_at_20_max": 0.365758, + "nauc_ndcg_at_20_std": 0.080483, + "nauc_ndcg_at_20_diff1": 0.446673, + "nauc_ndcg_at_100_max": 0.36162, + "nauc_ndcg_at_100_std": 0.063524, + "nauc_ndcg_at_100_diff1": 0.453869, + "nauc_ndcg_at_1000_max": 0.36162, + "nauc_ndcg_at_1000_std": 0.063524, + "nauc_ndcg_at_1000_diff1": 0.453869, + "nauc_map_at_1_max": 0.381924, + "nauc_map_at_1_std": 0.046089, + "nauc_map_at_1_diff1": 0.495245, + "nauc_map_at_3_max": 0.359666, + "nauc_map_at_3_std": 0.030252, + "nauc_map_at_3_diff1": 0.443656, + "nauc_map_at_5_max": 0.362749, + "nauc_map_at_5_std": 0.055227, + "nauc_map_at_5_diff1": 0.454986, + "nauc_map_at_10_max": 0.363758, + "nauc_map_at_10_std": 0.063436, + "nauc_map_at_10_diff1": 0.461151, + "nauc_map_at_20_max": 0.36459, + "nauc_map_at_20_std": 0.060715, + "nauc_map_at_20_diff1": 0.455743, + "nauc_map_at_100_max": 0.363682, + "nauc_map_at_100_std": 0.057806, + "nauc_map_at_100_diff1": 0.457177, + "nauc_map_at_1000_max": 0.363682, + "nauc_map_at_1000_std": 0.057806, + "nauc_map_at_1000_diff1": 0.457177, + "nauc_recall_at_1_max": 0.381924, + "nauc_recall_at_1_std": 0.046089, + "nauc_recall_at_1_diff1": 0.495245, + "nauc_recall_at_3_max": 0.319368, + "nauc_recall_at_3_std": -0.024215, + "nauc_recall_at_3_diff1": 0.342814, + "nauc_recall_at_5_max": 0.330476, + "nauc_recall_at_5_std": 0.1445, + "nauc_recall_at_5_diff1": 0.408663, + "nauc_recall_at_10_max": 0.367336, + "nauc_recall_at_10_std": 0.359401, + "nauc_recall_at_10_diff1": 0.597125, + "nauc_recall_at_20_max": 0.429251, + "nauc_recall_at_20_std": 0.389559, + "nauc_recall_at_20_diff1": 0.329243, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.381924, + "nauc_precision_at_1_std": 0.046089, + "nauc_precision_at_1_diff1": 0.495245, + "nauc_precision_at_3_max": 0.319368, + "nauc_precision_at_3_std": -0.024215, + "nauc_precision_at_3_diff1": 0.342814, + "nauc_precision_at_5_max": 0.330476, + "nauc_precision_at_5_std": 0.1445, + "nauc_precision_at_5_diff1": 0.408663, + "nauc_precision_at_10_max": 0.367336, + "nauc_precision_at_10_std": 0.359401, + "nauc_precision_at_10_diff1": 0.597125, + "nauc_precision_at_20_max": 0.429251, + "nauc_precision_at_20_std": 0.389559, + "nauc_precision_at_20_diff1": 0.329243, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.381924, + "nauc_mrr_at_1_std": 0.046089, + "nauc_mrr_at_1_diff1": 0.495245, + "nauc_mrr_at_3_max": 0.359666, + "nauc_mrr_at_3_std": 0.030252, + "nauc_mrr_at_3_diff1": 0.443656, + "nauc_mrr_at_5_max": 0.362749, + "nauc_mrr_at_5_std": 0.055227, + "nauc_mrr_at_5_diff1": 0.454986, + "nauc_mrr_at_10_max": 0.363758, + "nauc_mrr_at_10_std": 0.063436, + "nauc_mrr_at_10_diff1": 0.461151, + "nauc_mrr_at_20_max": 0.36459, + "nauc_mrr_at_20_std": 0.060715, + "nauc_mrr_at_20_diff1": 0.455743, + "nauc_mrr_at_100_max": 0.363682, + "nauc_mrr_at_100_std": 0.057806, + "nauc_mrr_at_100_diff1": 0.457177, + "nauc_mrr_at_1000_max": 0.363682, + "nauc_mrr_at_1000_std": 0.057806, + "nauc_mrr_at_1000_diff1": 0.457177, + "main_score": 0.70646, + "hf_subset": "korean", + "languages": [ + "kor-Hang" + ] + }, + { + "ndcg_at_1": 0.63077, + "ndcg_at_3": 0.73919, + "ndcg_at_5": 0.77097, + "ndcg_at_10": 0.79003, + "ndcg_at_20": 0.7938, + "ndcg_at_100": 0.79918, + "ndcg_at_1000": 0.79918, + "map_at_1": 0.63077, + "map_at_3": 0.71282, + "map_at_5": 0.73051, + "map_at_10": 0.73788, + "map_at_20": 0.73884, + "map_at_100": 0.73944, + "map_at_1000": 0.73944, + "recall_at_1": 0.63077, + "recall_at_3": 0.81538, + "recall_at_5": 0.89231, + "recall_at_10": 0.95385, + "recall_at_20": 0.96923, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.63077, + "precision_at_3": 0.27179, + "precision_at_5": 0.17846, + "precision_at_10": 0.09538, + "precision_at_20": 0.04846, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.630769, + "mrr_at_3": 0.712821, + "mrr_at_5": 0.730513, + "mrr_at_10": 0.737882, + "mrr_at_20": 0.738843, + "mrr_at_100": 0.739441, + "mrr_at_1000": 0.739441, + "nauc_ndcg_at_1_max": 0.327114, + "nauc_ndcg_at_1_std": 0.180323, + "nauc_ndcg_at_1_diff1": 0.578423, + "nauc_ndcg_at_3_max": 0.46673, + "nauc_ndcg_at_3_std": 0.268867, + "nauc_ndcg_at_3_diff1": 0.582189, + "nauc_ndcg_at_5_max": 0.454681, + "nauc_ndcg_at_5_std": 0.309734, + "nauc_ndcg_at_5_diff1": 0.563319, + "nauc_ndcg_at_10_max": 0.440586, + "nauc_ndcg_at_10_std": 0.289341, + "nauc_ndcg_at_10_diff1": 0.5737, + "nauc_ndcg_at_20_max": 0.431353, + "nauc_ndcg_at_20_std": 0.280018, + "nauc_ndcg_at_20_diff1": 0.567327, + "nauc_ndcg_at_100_max": 0.415373, + "nauc_ndcg_at_100_std": 0.25924, + "nauc_ndcg_at_100_diff1": 0.572178, + "nauc_ndcg_at_1000_max": 0.415373, + "nauc_ndcg_at_1000_std": 0.25924, + "nauc_ndcg_at_1000_diff1": 0.572178, + "nauc_map_at_1_max": 0.327114, + "nauc_map_at_1_std": 0.180323, + "nauc_map_at_1_diff1": 0.578423, + "nauc_map_at_3_max": 0.418467, + "nauc_map_at_3_std": 0.236823, + "nauc_map_at_3_diff1": 0.580874, + "nauc_map_at_5_max": 0.41037, + "nauc_map_at_5_std": 0.255153, + "nauc_map_at_5_diff1": 0.572498, + "nauc_map_at_10_max": 0.404368, + "nauc_map_at_10_std": 0.247099, + "nauc_map_at_10_diff1": 0.575187, + "nauc_map_at_20_max": 0.402329, + "nauc_map_at_20_std": 0.245014, + "nauc_map_at_20_diff1": 0.573895, + "nauc_map_at_100_max": 0.400874, + "nauc_map_at_100_std": 0.24313, + "nauc_map_at_100_diff1": 0.574342, + "nauc_map_at_1000_max": 0.400874, + "nauc_map_at_1000_std": 0.24313, + "nauc_map_at_1000_diff1": 0.574342, + "nauc_recall_at_1_max": 0.327114, + "nauc_recall_at_1_std": 0.180323, + "nauc_recall_at_1_diff1": 0.578423, + "nauc_recall_at_3_max": 0.666922, + "nauc_recall_at_3_std": 0.402497, + "nauc_recall_at_3_diff1": 0.587649, + "nauc_recall_at_5_max": 0.730513, + "nauc_recall_at_5_std": 0.664915, + "nauc_recall_at_5_diff1": 0.49852, + "nauc_recall_at_10_max": 0.913593, + "nauc_recall_at_10_std": 0.864573, + "nauc_recall_at_10_diff1": 0.567286, + "nauc_recall_at_20_max": 0.935194, + "nauc_recall_at_20_std": 0.935194, + "nauc_recall_at_20_diff1": 0.415735, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.327114, + "nauc_precision_at_1_std": 0.180323, + "nauc_precision_at_1_diff1": 0.578423, + "nauc_precision_at_3_max": 0.666922, + "nauc_precision_at_3_std": 0.402497, + "nauc_precision_at_3_diff1": 0.587649, + "nauc_precision_at_5_max": 0.730513, + "nauc_precision_at_5_std": 0.664915, + "nauc_precision_at_5_diff1": 0.49852, + "nauc_precision_at_10_max": 0.913593, + "nauc_precision_at_10_std": 0.864573, + "nauc_precision_at_10_diff1": 0.567286, + "nauc_precision_at_20_max": 0.935194, + "nauc_precision_at_20_std": 0.935194, + "nauc_precision_at_20_diff1": 0.415735, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.327114, + "nauc_mrr_at_1_std": 0.180323, + "nauc_mrr_at_1_diff1": 0.578423, + "nauc_mrr_at_3_max": 0.418467, + "nauc_mrr_at_3_std": 0.236823, + "nauc_mrr_at_3_diff1": 0.580874, + "nauc_mrr_at_5_max": 0.41037, + "nauc_mrr_at_5_std": 0.255153, + "nauc_mrr_at_5_diff1": 0.572498, + "nauc_mrr_at_10_max": 0.404368, + "nauc_mrr_at_10_std": 0.247099, + "nauc_mrr_at_10_diff1": 0.575187, + "nauc_mrr_at_20_max": 0.402329, + "nauc_mrr_at_20_std": 0.245014, + "nauc_mrr_at_20_diff1": 0.573895, + "nauc_mrr_at_100_max": 0.400874, + "nauc_mrr_at_100_std": 0.24313, + "nauc_mrr_at_100_diff1": 0.574342, + "nauc_mrr_at_1000_max": 0.400874, + "nauc_mrr_at_1000_std": 0.24313, + "nauc_mrr_at_1000_diff1": 0.574342, + "main_score": 0.79003, + "hf_subset": "russian", + "languages": [ + "rus-Cyrl" + ] + }, + { + "ndcg_at_1": 0.6358, + "ndcg_at_3": 0.76014, + "ndcg_at_5": 0.78564, + "ndcg_at_10": 0.80398, + "ndcg_at_20": 0.81066, + "ndcg_at_100": 0.81292, + "ndcg_at_1000": 0.81292, + "map_at_1": 0.6358, + "map_at_3": 0.73045, + "map_at_5": 0.74465, + "map_at_10": 0.75245, + "map_at_20": 0.75451, + "map_at_100": 0.7548, + "map_at_1000": 0.7548, + "recall_at_1": 0.6358, + "recall_at_3": 0.84568, + "recall_at_5": 0.90741, + "recall_at_10": 0.96296, + "recall_at_20": 0.98765, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.6358, + "precision_at_3": 0.28189, + "precision_at_5": 0.18148, + "precision_at_10": 0.0963, + "precision_at_20": 0.04938, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.635802, + "mrr_at_3": 0.730453, + "mrr_at_5": 0.74465, + "mrr_at_10": 0.75245, + "mrr_at_20": 0.754514, + "mrr_at_100": 0.754804, + "mrr_at_1000": 0.754804, + "nauc_ndcg_at_1_max": 0.494215, + "nauc_ndcg_at_1_std": -0.024377, + "nauc_ndcg_at_1_diff1": 0.773857, + "nauc_ndcg_at_3_max": 0.506302, + "nauc_ndcg_at_3_std": -0.061228, + "nauc_ndcg_at_3_diff1": 0.748894, + "nauc_ndcg_at_5_max": 0.495872, + "nauc_ndcg_at_5_std": -0.036777, + "nauc_ndcg_at_5_diff1": 0.731188, + "nauc_ndcg_at_10_max": 0.487984, + "nauc_ndcg_at_10_std": -0.020963, + "nauc_ndcg_at_10_diff1": 0.739909, + "nauc_ndcg_at_20_max": 0.493401, + "nauc_ndcg_at_20_std": -0.026274, + "nauc_ndcg_at_20_diff1": 0.748657, + "nauc_ndcg_at_100_max": 0.490333, + "nauc_ndcg_at_100_std": -0.03677, + "nauc_ndcg_at_100_diff1": 0.75026, + "nauc_ndcg_at_1000_max": 0.490333, + "nauc_ndcg_at_1000_std": -0.03677, + "nauc_ndcg_at_1000_diff1": 0.75026, + "nauc_map_at_1_max": 0.494215, + "nauc_map_at_1_std": -0.024377, + "nauc_map_at_1_diff1": 0.773857, + "nauc_map_at_3_max": 0.497836, + "nauc_map_at_3_std": -0.053457, + "nauc_map_at_3_diff1": 0.754424, + "nauc_map_at_5_max": 0.492426, + "nauc_map_at_5_std": -0.041318, + "nauc_map_at_5_diff1": 0.746471, + "nauc_map_at_10_max": 0.488789, + "nauc_map_at_10_std": -0.036812, + "nauc_map_at_10_diff1": 0.750549, + "nauc_map_at_20_max": 0.490023, + "nauc_map_at_20_std": -0.038333, + "nauc_map_at_20_diff1": 0.752775, + "nauc_map_at_100_max": 0.489733, + "nauc_map_at_100_std": -0.039363, + "nauc_map_at_100_diff1": 0.752946, + "nauc_map_at_1000_max": 0.489733, + "nauc_map_at_1000_std": -0.039363, + "nauc_map_at_1000_diff1": 0.752946, + "nauc_recall_at_1_max": 0.494215, + "nauc_recall_at_1_std": -0.024377, + "nauc_recall_at_1_diff1": 0.773857, + "nauc_recall_at_3_max": 0.547448, + "nauc_recall_at_3_std": -0.093856, + "nauc_recall_at_3_diff1": 0.725424, + "nauc_recall_at_5_max": 0.520525, + "nauc_recall_at_5_std": 0.005355, + "nauc_recall_at_5_diff1": 0.623992, + "nauc_recall_at_10_max": 0.472955, + "nauc_recall_at_10_std": 0.305264, + "nauc_recall_at_10_diff1": 0.56656, + "nauc_recall_at_20_max": 0.711826, + "nauc_recall_at_20_std": 0.711826, + "nauc_recall_at_20_diff1": 0.638523, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.494215, + "nauc_precision_at_1_std": -0.024377, + "nauc_precision_at_1_diff1": 0.773857, + "nauc_precision_at_3_max": 0.547448, + "nauc_precision_at_3_std": -0.093856, + "nauc_precision_at_3_diff1": 0.725424, + "nauc_precision_at_5_max": 0.520525, + "nauc_precision_at_5_std": 0.005355, + "nauc_precision_at_5_diff1": 0.623992, + "nauc_precision_at_10_max": 0.472955, + "nauc_precision_at_10_std": 0.305264, + "nauc_precision_at_10_diff1": 0.56656, + "nauc_precision_at_20_max": 0.711826, + "nauc_precision_at_20_std": 0.711826, + "nauc_precision_at_20_diff1": 0.638523, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.494215, + "nauc_mrr_at_1_std": -0.024377, + "nauc_mrr_at_1_diff1": 0.773857, + "nauc_mrr_at_3_max": 0.497836, + "nauc_mrr_at_3_std": -0.053457, + "nauc_mrr_at_3_diff1": 0.754424, + "nauc_mrr_at_5_max": 0.492426, + "nauc_mrr_at_5_std": -0.041318, + "nauc_mrr_at_5_diff1": 0.746471, + "nauc_mrr_at_10_max": 0.488789, + "nauc_mrr_at_10_std": -0.036812, + "nauc_mrr_at_10_diff1": 0.750549, + "nauc_mrr_at_20_max": 0.490023, + "nauc_mrr_at_20_std": -0.038333, + "nauc_mrr_at_20_diff1": 0.752775, + "nauc_mrr_at_100_max": 0.489733, + "nauc_mrr_at_100_std": -0.039363, + "nauc_mrr_at_100_diff1": 0.752946, + "nauc_mrr_at_1000_max": 0.489733, + "nauc_mrr_at_1000_std": -0.039363, + "nauc_mrr_at_1000_diff1": 0.752946, + "main_score": 0.80398, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.55844, + "ndcg_at_3": 0.65337, + "ndcg_at_5": 0.69138, + "ndcg_at_10": 0.72583, + "ndcg_at_20": 0.74569, + "ndcg_at_100": 0.74817, + "ndcg_at_1000": 0.74817, + "map_at_1": 0.55844, + "map_at_3": 0.63203, + "map_at_5": 0.65346, + "map_at_10": 0.66818, + "map_at_20": 0.67369, + "map_at_100": 0.67404, + "map_at_1000": 0.67404, + "recall_at_1": 0.55844, + "recall_at_3": 0.71429, + "recall_at_5": 0.80519, + "recall_at_10": 0.90909, + "recall_at_20": 0.98701, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.55844, + "precision_at_3": 0.2381, + "precision_at_5": 0.16104, + "precision_at_10": 0.09091, + "precision_at_20": 0.04935, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.558442, + "mrr_at_3": 0.632035, + "mrr_at_5": 0.653463, + "mrr_at_10": 0.668177, + "mrr_at_20": 0.673689, + "mrr_at_100": 0.67404, + "mrr_at_1000": 0.67404, + "nauc_ndcg_at_1_max": 0.38173, + "nauc_ndcg_at_1_std": -0.07483, + "nauc_ndcg_at_1_diff1": 0.649579, + "nauc_ndcg_at_3_max": 0.44396, + "nauc_ndcg_at_3_std": 0.002129, + "nauc_ndcg_at_3_diff1": 0.644225, + "nauc_ndcg_at_5_max": 0.39238, + "nauc_ndcg_at_5_std": -0.088522, + "nauc_ndcg_at_5_diff1": 0.609759, + "nauc_ndcg_at_10_max": 0.350652, + "nauc_ndcg_at_10_std": -0.088393, + "nauc_ndcg_at_10_diff1": 0.611877, + "nauc_ndcg_at_20_max": 0.381462, + "nauc_ndcg_at_20_std": -0.046012, + "nauc_ndcg_at_20_diff1": 0.624056, + "nauc_ndcg_at_100_max": 0.384621, + "nauc_ndcg_at_100_std": -0.053433, + "nauc_ndcg_at_100_diff1": 0.624911, + "nauc_ndcg_at_1000_max": 0.384621, + "nauc_ndcg_at_1000_std": -0.053433, + "nauc_ndcg_at_1000_diff1": 0.624911, + "nauc_map_at_1_max": 0.38173, + "nauc_map_at_1_std": -0.07483, + "nauc_map_at_1_diff1": 0.649579, + "nauc_map_at_3_max": 0.424063, + "nauc_map_at_3_std": -0.019052, + "nauc_map_at_3_diff1": 0.643061, + "nauc_map_at_5_max": 0.395336, + "nauc_map_at_5_std": -0.067395, + "nauc_map_at_5_diff1": 0.62516, + "nauc_map_at_10_max": 0.380807, + "nauc_map_at_10_std": -0.063549, + "nauc_map_at_10_diff1": 0.626284, + "nauc_map_at_20_max": 0.387696, + "nauc_map_at_20_std": -0.052894, + "nauc_map_at_20_diff1": 0.628705, + "nauc_map_at_100_max": 0.388053, + "nauc_map_at_100_std": -0.053723, + "nauc_map_at_100_diff1": 0.628805, + "nauc_map_at_1000_max": 0.388053, + "nauc_map_at_1000_std": -0.053723, + "nauc_map_at_1000_diff1": 0.628805, + "nauc_recall_at_1_max": 0.38173, + "nauc_recall_at_1_std": -0.07483, + "nauc_recall_at_1_diff1": 0.649579, + "nauc_recall_at_3_max": 0.512801, + "nauc_recall_at_3_std": 0.073996, + "nauc_recall_at_3_diff1": 0.649439, + "nauc_recall_at_5_max": 0.372881, + "nauc_recall_at_5_std": -0.198138, + "nauc_recall_at_5_diff1": 0.537445, + "nauc_recall_at_10_max": 0.038633, + "nauc_recall_at_10_std": -0.349197, + "nauc_recall_at_10_diff1": 0.485356, + "nauc_recall_at_20_max": 0.125638, + "nauc_recall_at_20_std": 0.554847, + "nauc_recall_at_20_diff1": 0.554847, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.38173, + "nauc_precision_at_1_std": -0.07483, + "nauc_precision_at_1_diff1": 0.649579, + "nauc_precision_at_3_max": 0.512801, + "nauc_precision_at_3_std": 0.073996, + "nauc_precision_at_3_diff1": 0.649439, + "nauc_precision_at_5_max": 0.372881, + "nauc_precision_at_5_std": -0.198138, + "nauc_precision_at_5_diff1": 0.537445, + "nauc_precision_at_10_max": 0.038633, + "nauc_precision_at_10_std": -0.349197, + "nauc_precision_at_10_diff1": 0.485356, + "nauc_precision_at_20_max": 0.125638, + "nauc_precision_at_20_std": 0.554847, + "nauc_precision_at_20_diff1": 0.554847, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.38173, + "nauc_mrr_at_1_std": -0.07483, + "nauc_mrr_at_1_diff1": 0.649579, + "nauc_mrr_at_3_max": 0.424063, + "nauc_mrr_at_3_std": -0.019052, + "nauc_mrr_at_3_diff1": 0.643061, + "nauc_mrr_at_5_max": 0.395336, + "nauc_mrr_at_5_std": -0.067395, + "nauc_mrr_at_5_diff1": 0.62516, + "nauc_mrr_at_10_max": 0.380807, + "nauc_mrr_at_10_std": -0.063549, + "nauc_mrr_at_10_diff1": 0.626284, + "nauc_mrr_at_20_max": 0.387696, + "nauc_mrr_at_20_std": -0.052894, + "nauc_mrr_at_20_diff1": 0.628705, + "nauc_mrr_at_100_max": 0.388053, + "nauc_mrr_at_100_std": -0.053723, + "nauc_mrr_at_100_diff1": 0.628805, + "nauc_mrr_at_1000_max": 0.388053, + "nauc_mrr_at_1000_std": -0.053723, + "nauc_mrr_at_1000_diff1": 0.628805, + "main_score": 0.72583, + "hf_subset": "vietnamese", + "languages": [ + "vie-Latn" + ] + } + ] + }, + "evaluation_time": 43.67582583427429, + "kg_co2_emissions": 1.5084035193350963e-05 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-small/2/SciFact-PL.json b/results/openai__text-embedding-3-small/2/SciFact-PL.json new file mode 100644 index 0000000000..bfdb21e73b --- /dev/null +++ b/results/openai__text-embedding-3-small/2/SciFact-PL.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "47932a35f045ef8ed01ba82bf9ff67f6e109207e", + "task_name": "SciFact-PL", + "mteb_version": "1.24.0", + "scores": { + "test": [ + { + "ndcg_at_1": 0.47, + "ndcg_at_3": 0.5242, + "ndcg_at_5": 0.56109, + "ndcg_at_10": 0.59367, + "ndcg_at_20": 0.60738, + "ndcg_at_100": 0.62809, + "ndcg_at_1000": 0.63823, + "map_at_1": 0.45, + "map_at_3": 0.50221, + "map_at_5": 0.52564, + "map_at_10": 0.54108, + "map_at_20": 0.54522, + "map_at_100": 0.54829, + "map_at_1000": 0.54866, + "recall_at_1": 0.45, + "recall_at_3": 0.563, + "recall_at_5": 0.65072, + "recall_at_10": 0.74656, + "recall_at_20": 0.79833, + "recall_at_100": 0.90667, + "recall_at_1000": 0.98667, + "precision_at_1": 0.47, + "precision_at_3": 0.20444, + "precision_at_5": 0.146, + "precision_at_10": 0.08433, + "precision_at_20": 0.04517, + "precision_at_100": 0.01033, + "precision_at_1000": 0.00112, + "mrr_at_1": 0.47, + "mrr_at_3": 0.520556, + "mrr_at_5": 0.540389, + "mrr_at_10": 0.552255, + "mrr_at_20": 0.555896, + "mrr_at_100": 0.558431, + "mrr_at_1000": 0.558756, + "nauc_ndcg_at_1_max": 0.519864, + "nauc_ndcg_at_1_std": 0.166766, + "nauc_ndcg_at_1_diff1": 0.63575, + "nauc_ndcg_at_3_max": 0.515946, + "nauc_ndcg_at_3_std": 0.147885, + "nauc_ndcg_at_3_diff1": 0.610587, + "nauc_ndcg_at_5_max": 0.522295, + "nauc_ndcg_at_5_std": 0.149348, + "nauc_ndcg_at_5_diff1": 0.610275, + "nauc_ndcg_at_10_max": 0.534317, + "nauc_ndcg_at_10_std": 0.175474, + "nauc_ndcg_at_10_diff1": 0.603402, + "nauc_ndcg_at_20_max": 0.538613, + "nauc_ndcg_at_20_std": 0.17736, + "nauc_ndcg_at_20_diff1": 0.60605, + "nauc_ndcg_at_100_max": 0.541235, + "nauc_ndcg_at_100_std": 0.192029, + "nauc_ndcg_at_100_diff1": 0.599634, + "nauc_ndcg_at_1000_max": 0.539914, + "nauc_ndcg_at_1000_std": 0.182217, + "nauc_ndcg_at_1000_diff1": 0.607051, + "nauc_map_at_1_max": 0.490637, + "nauc_map_at_1_std": 0.118313, + "nauc_map_at_1_diff1": 0.648358, + "nauc_map_at_3_max": 0.50406, + "nauc_map_at_3_std": 0.130052, + "nauc_map_at_3_diff1": 0.62058, + "nauc_map_at_5_max": 0.517505, + "nauc_map_at_5_std": 0.138507, + "nauc_map_at_5_diff1": 0.622872, + "nauc_map_at_10_max": 0.527892, + "nauc_map_at_10_std": 0.15645, + "nauc_map_at_10_diff1": 0.618523, + "nauc_map_at_20_max": 0.529868, + "nauc_map_at_20_std": 0.158453, + "nauc_map_at_20_diff1": 0.618832, + "nauc_map_at_100_max": 0.530204, + "nauc_map_at_100_std": 0.160512, + "nauc_map_at_100_diff1": 0.61801, + "nauc_map_at_1000_max": 0.530287, + "nauc_map_at_1000_std": 0.160299, + "nauc_map_at_1000_diff1": 0.618247, + "nauc_recall_at_1_max": 0.490637, + "nauc_recall_at_1_std": 0.118313, + "nauc_recall_at_1_diff1": 0.648358, + "nauc_recall_at_3_max": 0.492566, + "nauc_recall_at_3_std": 0.129752, + "nauc_recall_at_3_diff1": 0.585666, + "nauc_recall_at_5_max": 0.498854, + "nauc_recall_at_5_std": 0.131517, + "nauc_recall_at_5_diff1": 0.573394, + "nauc_recall_at_10_max": 0.522746, + "nauc_recall_at_10_std": 0.204022, + "nauc_recall_at_10_diff1": 0.543733, + "nauc_recall_at_20_max": 0.539526, + "nauc_recall_at_20_std": 0.207811, + "nauc_recall_at_20_diff1": 0.555001, + "nauc_recall_at_100_max": 0.580916, + "nauc_recall_at_100_std": 0.448363, + "nauc_recall_at_100_diff1": 0.423019, + "nauc_recall_at_1000_max": 0.563025, + "nauc_recall_at_1000_std": 0.407796, + "nauc_recall_at_1000_diff1": 0.613679, + "nauc_precision_at_1_max": 0.519864, + "nauc_precision_at_1_std": 0.166766, + "nauc_precision_at_1_diff1": 0.63575, + "nauc_precision_at_3_max": 0.531661, + "nauc_precision_at_3_std": 0.243277, + "nauc_precision_at_3_diff1": 0.481406, + "nauc_precision_at_5_max": 0.53673, + "nauc_precision_at_5_std": 0.267857, + "nauc_precision_at_5_diff1": 0.39769, + "nauc_precision_at_10_max": 0.53856, + "nauc_precision_at_10_std": 0.368414, + "nauc_precision_at_10_diff1": 0.265148, + "nauc_precision_at_20_max": 0.517406, + "nauc_precision_at_20_std": 0.386518, + "nauc_precision_at_20_diff1": 0.209375, + "nauc_precision_at_100_max": 0.415546, + "nauc_precision_at_100_std": 0.439108, + "nauc_precision_at_100_diff1": -0.025846, + "nauc_precision_at_1000_max": 0.327064, + "nauc_precision_at_1000_std": 0.405279, + "nauc_precision_at_1000_diff1": -0.167372, + "nauc_mrr_at_1_max": 0.519864, + "nauc_mrr_at_1_std": 0.166766, + "nauc_mrr_at_1_diff1": 0.63575, + "nauc_mrr_at_3_max": 0.53278, + "nauc_mrr_at_3_std": 0.175758, + "nauc_mrr_at_3_diff1": 0.610051, + "nauc_mrr_at_5_max": 0.533425, + "nauc_mrr_at_5_std": 0.177747, + "nauc_mrr_at_5_diff1": 0.607976, + "nauc_mrr_at_10_max": 0.532784, + "nauc_mrr_at_10_std": 0.1815, + "nauc_mrr_at_10_diff1": 0.607233, + "nauc_mrr_at_20_max": 0.533542, + "nauc_mrr_at_20_std": 0.180907, + "nauc_mrr_at_20_diff1": 0.607918, + "nauc_mrr_at_100_max": 0.53373, + "nauc_mrr_at_100_std": 0.182851, + "nauc_mrr_at_100_diff1": 0.607621, + "nauc_mrr_at_1000_max": 0.533847, + "nauc_mrr_at_1000_std": 0.18268, + "nauc_mrr_at_1000_diff1": 0.607832, + "main_score": 0.59367, + "hf_subset": "default", + "languages": [ + "pol-Latn" + ] + } + ] + }, + "evaluation_time": 104.58239269256592, + "kg_co2_emissions": 2.344003000702677e-05 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-small/2/TRECCOVID-PL.json b/results/openai__text-embedding-3-small/2/TRECCOVID-PL.json new file mode 100644 index 0000000000..3108deec13 --- /dev/null +++ b/results/openai__text-embedding-3-small/2/TRECCOVID-PL.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "81bcb408f33366c2a20ac54adafad1ae7e877fdd", + "task_name": "TRECCOVID-PL", + "mteb_version": "1.24.0", + "scores": { + "test": [ + { + "ndcg_at_1": 0.78, + "ndcg_at_3": 0.71989, + "ndcg_at_5": 0.68095, + "ndcg_at_10": 0.65356, + "ndcg_at_20": 0.62129, + "ndcg_at_100": 0.50467, + "ndcg_at_1000": 0.46404, + "map_at_1": 0.00213, + "map_at_3": 0.00573, + "map_at_5": 0.00852, + "map_at_10": 0.01518, + "map_at_20": 0.02614, + "map_at_100": 0.08651, + "map_at_1000": 0.21903, + "recall_at_1": 0.00213, + "recall_at_3": 0.00599, + "recall_at_5": 0.00912, + "recall_at_10": 0.01755, + "recall_at_20": 0.03233, + "recall_at_100": 0.12329, + "recall_at_1000": 0.44138, + "precision_at_1": 0.84, + "precision_at_3": 0.75333, + "precision_at_5": 0.704, + "precision_at_10": 0.686, + "precision_at_20": 0.655, + "precision_at_100": 0.5206, + "precision_at_1000": 0.20908, + "mrr_at_1": 0.84, + "mrr_at_3": 0.866667, + "mrr_at_5": 0.880667, + "mrr_at_10": 0.883524, + "mrr_at_20": 0.884857, + "mrr_at_100": 0.884857, + "mrr_at_1000": 0.884857, + "nauc_ndcg_at_1_max": 0.292749, + "nauc_ndcg_at_1_std": 0.314959, + "nauc_ndcg_at_1_diff1": -0.060841, + "nauc_ndcg_at_3_max": 0.429157, + "nauc_ndcg_at_3_std": 0.334803, + "nauc_ndcg_at_3_diff1": -0.09356, + "nauc_ndcg_at_5_max": 0.528727, + "nauc_ndcg_at_5_std": 0.466716, + "nauc_ndcg_at_5_diff1": -0.1041, + "nauc_ndcg_at_10_max": 0.53889, + "nauc_ndcg_at_10_std": 0.561637, + "nauc_ndcg_at_10_diff1": -0.078808, + "nauc_ndcg_at_20_max": 0.567284, + "nauc_ndcg_at_20_std": 0.650478, + "nauc_ndcg_at_20_diff1": -0.039501, + "nauc_ndcg_at_100_max": 0.590188, + "nauc_ndcg_at_100_std": 0.758455, + "nauc_ndcg_at_100_diff1": -0.025757, + "nauc_ndcg_at_1000_max": 0.708599, + "nauc_ndcg_at_1000_std": 0.847836, + "nauc_ndcg_at_1000_diff1": 0.038486, + "nauc_map_at_1_max": 0.063163, + "nauc_map_at_1_std": 0.005201, + "nauc_map_at_1_diff1": -0.072405, + "nauc_map_at_3_max": 0.165744, + "nauc_map_at_3_std": 0.048194, + "nauc_map_at_3_diff1": -0.078073, + "nauc_map_at_5_max": 0.274421, + "nauc_map_at_5_std": 0.143046, + "nauc_map_at_5_diff1": -0.093521, + "nauc_map_at_10_max": 0.321935, + "nauc_map_at_10_std": 0.268167, + "nauc_map_at_10_diff1": -0.107634, + "nauc_map_at_20_max": 0.436021, + "nauc_map_at_20_std": 0.43069, + "nauc_map_at_20_diff1": -0.072603, + "nauc_map_at_100_max": 0.642207, + "nauc_map_at_100_std": 0.739363, + "nauc_map_at_100_diff1": -0.06869, + "nauc_map_at_1000_max": 0.71575, + "nauc_map_at_1000_std": 0.867395, + "nauc_map_at_1000_diff1": -0.010588, + "nauc_recall_at_1_max": 0.063163, + "nauc_recall_at_1_std": 0.005201, + "nauc_recall_at_1_diff1": -0.072405, + "nauc_recall_at_3_max": 0.149528, + "nauc_recall_at_3_std": 0.006447, + "nauc_recall_at_3_diff1": -0.059121, + "nauc_recall_at_5_max": 0.260854, + "nauc_recall_at_5_std": 0.135323, + "nauc_recall_at_5_diff1": -0.070298, + "nauc_recall_at_10_max": 0.250123, + "nauc_recall_at_10_std": 0.241209, + "nauc_recall_at_10_diff1": -0.059654, + "nauc_recall_at_20_max": 0.337362, + "nauc_recall_at_20_std": 0.365285, + "nauc_recall_at_20_diff1": 0.000495, + "nauc_recall_at_100_max": 0.508152, + "nauc_recall_at_100_std": 0.605928, + "nauc_recall_at_100_diff1": -0.011728, + "nauc_recall_at_1000_max": 0.675373, + "nauc_recall_at_1000_std": 0.800392, + "nauc_recall_at_1000_diff1": 0.063872, + "nauc_precision_at_1_max": 0.408014, + "nauc_precision_at_1_std": 0.361377, + "nauc_precision_at_1_diff1": -0.125675, + "nauc_precision_at_3_max": 0.472243, + "nauc_precision_at_3_std": 0.326243, + "nauc_precision_at_3_diff1": -0.152309, + "nauc_precision_at_5_max": 0.586468, + "nauc_precision_at_5_std": 0.517847, + "nauc_precision_at_5_diff1": -0.140942, + "nauc_precision_at_10_max": 0.547522, + "nauc_precision_at_10_std": 0.61865, + "nauc_precision_at_10_diff1": -0.125959, + "nauc_precision_at_20_max": 0.58915, + "nauc_precision_at_20_std": 0.68534, + "nauc_precision_at_20_diff1": -0.042195, + "nauc_precision_at_100_max": 0.612863, + "nauc_precision_at_100_std": 0.783987, + "nauc_precision_at_100_diff1": -0.030475, + "nauc_precision_at_1000_max": 0.483205, + "nauc_precision_at_1000_std": 0.600237, + "nauc_precision_at_1000_diff1": -0.016783, + "nauc_mrr_at_1_max": 0.408014, + "nauc_mrr_at_1_std": 0.361377, + "nauc_mrr_at_1_diff1": -0.125675, + "nauc_mrr_at_3_max": 0.457987, + "nauc_mrr_at_3_std": 0.275048, + "nauc_mrr_at_3_diff1": -0.084966, + "nauc_mrr_at_5_max": 0.475493, + "nauc_mrr_at_5_std": 0.324651, + "nauc_mrr_at_5_diff1": -0.070673, + "nauc_mrr_at_10_max": 0.462126, + "nauc_mrr_at_10_std": 0.323799, + "nauc_mrr_at_10_diff1": -0.090879, + "nauc_mrr_at_20_max": 0.455653, + "nauc_mrr_at_20_std": 0.315661, + "nauc_mrr_at_20_diff1": -0.09345, + "nauc_mrr_at_100_max": 0.455653, + "nauc_mrr_at_100_std": 0.315661, + "nauc_mrr_at_100_diff1": -0.09345, + "nauc_mrr_at_1000_max": 0.455653, + "nauc_mrr_at_1000_std": 0.315661, + "nauc_mrr_at_1000_diff1": -0.09345, + "main_score": 0.65356, + "hf_subset": "default", + "languages": [ + "pol-Latn" + ] + } + ] + }, + "evaluation_time": 3090.085769891739, + "kg_co2_emissions": 0.0007749121268690063 +} \ No newline at end of file diff --git a/results/openai__text-embedding-3-small/2/model_meta.json b/results/openai__text-embedding-3-small/2/model_meta.json index e1bb929ea3..3dfa93c419 100644 --- a/results/openai__text-embedding-3-small/2/model_meta.json +++ b/results/openai__text-embedding-3-small/2/model_meta.json @@ -1 +1 @@ -{"name": "openai/text-embedding-3-small", "revision": "2", "release_date": "2024-01-25", "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": 8191.0, "embed_dim": 1536, "license": null, "open_weights": false, "public_training_data": null, "public_training_code": null, "framework": ["API"], "reference": "https://openai.com/index/new-embedding-models-and-api-updates/", "similarity_fn_name": "cosine", "use_instructions": false, "zero_shot_benchmarks": null, "adapted_from": null, "superseded_by": null, "loader": "OpenAIWrapper"} \ No newline at end of file +{"name": "openai/text-embedding-3-small", "revision": "2", "release_date": "2024-01-25", "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": 8191.0, "embed_dim": 1536, "license": null, "open_weights": false, "public_training_data": null, "public_training_code": null, "framework": ["API"], "reference": "https://openai.com/index/new-embedding-models-and-api-updates/", "similarity_fn_name": "cosine", "use_instructions": false, "training_datasets": null, "adapted_from": null, "superseded_by": null, "loader": "OpenAIWrapper"} \ No newline at end of file