diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/AFQMC.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/AFQMC.json new file mode 100644 index 0000000000..24d2d33153 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/AFQMC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "b44c3b011063adb25877c13823db83bb193913c4", + "task_name": "AFQMC", + "mteb_version": "1.36.22", + "scores": { + "validation": [ + { + "pearson": 0.567822, + "spearman": 0.608058, + "cosine_pearson": 0.567822, + "cosine_spearman": 0.608058, + "manhattan_pearson": 0.595396, + "manhattan_spearman": 0.607929, + "euclidean_pearson": 0.595506, + "euclidean_spearman": 0.608058, + "main_score": 0.608058, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 3.13663911819458, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/ATEC.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/ATEC.json new file mode 100644 index 0000000000..733c59c189 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/ATEC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865", + "task_name": "ATEC", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "pearson": 0.566971, + "spearman": 0.58711, + "cosine_pearson": 0.566971, + "cosine_spearman": 0.58711, + "manhattan_pearson": 0.632299, + "manhattan_spearman": 0.587079, + "euclidean_pearson": 0.632405, + "euclidean_spearman": 0.58711, + "main_score": 0.58711, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 13.173291206359863, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/BQ.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/BQ.json new file mode 100644 index 0000000000..c268f5d3f2 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/BQ.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55", + "task_name": "BQ", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "pearson": 0.728411, + "spearman": 0.746565, + "cosine_pearson": 0.728411, + "cosine_spearman": 0.746565, + "manhattan_pearson": 0.732957, + "manhattan_spearman": 0.746468, + "euclidean_pearson": 0.733092, + "euclidean_spearman": 0.746565, + "main_score": 0.746565, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 6.398446798324585, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CLSClusteringP2P.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CLSClusteringP2P.json new file mode 100644 index 0000000000..17b1308a97 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CLSClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476", + "task_name": "CLSClusteringP2P", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "v_measure": 0.614736, + "v_measure_std": 0.019422, + "v_measures": [ + 0.59651, + 0.653691, + 0.597714, + 0.632687, + 0.593833, + 0.595094, + 0.603839, + 0.621118, + 0.621222, + 0.631652 + ], + "main_score": 0.614736, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 310.1149594783783, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CLSClusteringS2S.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CLSClusteringS2S.json new file mode 100644 index 0000000000..7754137056 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CLSClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f", + "task_name": "CLSClusteringS2S", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "v_measure": 0.51269, + "v_measure_std": 0.015478, + "v_measures": [ + 0.51566, + 0.5359, + 0.499923, + 0.51602, + 0.50307, + 0.521295, + 0.486235, + 0.503809, + 0.538871, + 0.506119 + ], + "main_score": 0.51269, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 43.29397511482239, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CMedQAv1-reranking.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CMedQAv1-reranking.json new file mode 100644 index 0000000000..dab70e5ee0 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CMedQAv1-reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "8d7f1e942507dac42dc58017c1a001c3717da7df", + "task_name": "CMedQAv1-reranking", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "map": 0.915554, + "mrr": 0.934513, + "nAUC_map_max": 0.445851, + "nAUC_map_std": 0.017932, + "nAUC_map_diff1": 0.591287, + "nAUC_mrr_max": 0.553582, + "nAUC_mrr_std": 0.161746, + "nAUC_mrr_diff1": 0.646831, + "main_score": 0.915554, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 101.35409474372864, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CMedQAv2-reranking.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CMedQAv2-reranking.json new file mode 100644 index 0000000000..8934419942 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CMedQAv2-reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "23d186750531a14a0357ca22cd92d712fd512ea0", + "task_name": "CMedQAv2-reranking", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "map": 0.904873, + "mrr": 0.923269, + "nAUC_map_max": 0.592987, + "nAUC_map_std": -0.049611, + "nAUC_map_diff1": 0.638675, + "nAUC_mrr_max": 0.722323, + "nAUC_mrr_std": 0.069514, + "nAUC_mrr_diff1": 0.718157, + "main_score": 0.904873, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 113.76977372169495, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CmedqaRetrieval.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CmedqaRetrieval.json new file mode 100644 index 0000000000..9a2dbf9947 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CmedqaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301", + "task_name": "CmedqaRetrieval", + "mteb_version": "1.36.22", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.4081, + "ndcg_at_3": 0.42251, + "ndcg_at_5": 0.44383, + "ndcg_at_10": 0.47676, + "ndcg_at_20": 0.50357, + "ndcg_at_100": 0.54568, + "ndcg_at_1000": 0.56223, + "map_at_1": 0.26811, + "map_at_3": 0.3627, + "map_at_5": 0.38719, + "map_at_10": 0.40717, + "map_at_20": 0.41709, + "map_at_100": 0.42532, + "map_at_1000": 0.42629, + "recall_at_1": 0.26811, + "recall_at_3": 0.4267, + "recall_at_5": 0.4941, + "recall_at_10": 0.59185, + "recall_at_20": 0.68332, + "recall_at_100": 0.87605, + "recall_at_1000": 0.98586, + "precision_at_1": 0.4081, + "precision_at_3": 0.24106, + "precision_at_5": 0.17339, + "precision_at_10": 0.10633, + "precision_at_20": 0.06209, + "precision_at_100": 0.0162, + "precision_at_1000": 0.00184, + "mrr_at_1": 0.408102, + "mrr_at_3": 0.47191, + "mrr_at_5": 0.4849, + "mrr_at_10": 0.49661, + "mrr_at_20": 0.501909, + "mrr_at_100": 0.505703, + "mrr_at_1000": 0.506052, + "nauc_ndcg_at_1_max": 0.424881, + "nauc_ndcg_at_1_std": -0.087249, + "nauc_ndcg_at_1_diff1": 0.626349, + "nauc_ndcg_at_3_max": 0.359452, + "nauc_ndcg_at_3_std": -0.11842, + "nauc_ndcg_at_3_diff1": 0.520893, + "nauc_ndcg_at_5_max": 0.350416, + "nauc_ndcg_at_5_std": -0.118633, + "nauc_ndcg_at_5_diff1": 0.510111, + "nauc_ndcg_at_10_max": 0.358575, + "nauc_ndcg_at_10_std": -0.116825, + "nauc_ndcg_at_10_diff1": 0.506618, + "nauc_ndcg_at_20_max": 0.367633, + "nauc_ndcg_at_20_std": -0.106046, + "nauc_ndcg_at_20_diff1": 0.506945, + "nauc_ndcg_at_100_max": 0.380303, + "nauc_ndcg_at_100_std": -0.082649, + "nauc_ndcg_at_100_diff1": 0.513887, + "nauc_ndcg_at_1000_max": 0.381724, + "nauc_ndcg_at_1000_std": -0.088399, + "nauc_ndcg_at_1000_diff1": 0.520586, + "nauc_map_at_1_max": 0.237646, + "nauc_map_at_1_std": -0.140983, + "nauc_map_at_1_diff1": 0.551072, + "nauc_map_at_3_max": 0.302285, + "nauc_map_at_3_std": -0.1442, + "nauc_map_at_3_diff1": 0.515912, + "nauc_map_at_5_max": 0.320856, + "nauc_map_at_5_std": -0.137845, + "nauc_map_at_5_diff1": 0.511565, + "nauc_map_at_10_max": 0.336354, + "nauc_map_at_10_std": -0.132846, + "nauc_map_at_10_diff1": 0.512022, + "nauc_map_at_20_max": 0.343217, + "nauc_map_at_20_std": -0.127319, + "nauc_map_at_20_diff1": 0.512802, + "nauc_map_at_100_max": 0.347333, + "nauc_map_at_100_std": -0.122043, + "nauc_map_at_100_diff1": 0.514152, + "nauc_map_at_1000_max": 0.347654, + "nauc_map_at_1000_std": -0.121923, + "nauc_map_at_1000_diff1": 0.514524, + "nauc_recall_at_1_max": 0.237646, + "nauc_recall_at_1_std": -0.140983, + "nauc_recall_at_1_diff1": 0.551072, + "nauc_recall_at_3_max": 0.261491, + "nauc_recall_at_3_std": -0.146603, + "nauc_recall_at_3_diff1": 0.438165, + "nauc_recall_at_5_max": 0.272705, + "nauc_recall_at_5_std": -0.129521, + "nauc_recall_at_5_diff1": 0.409213, + "nauc_recall_at_10_max": 0.291984, + "nauc_recall_at_10_std": -0.116023, + "nauc_recall_at_10_diff1": 0.379236, + "nauc_recall_at_20_max": 0.303391, + "nauc_recall_at_20_std": -0.079708, + "nauc_recall_at_20_diff1": 0.350854, + "nauc_recall_at_100_max": 0.352362, + "nauc_recall_at_100_std": 0.135053, + "nauc_recall_at_100_diff1": 0.299488, + "nauc_recall_at_1000_max": 0.58974, + "nauc_recall_at_1000_std": 0.596095, + "nauc_recall_at_1000_diff1": 0.144043, + "nauc_precision_at_1_max": 0.424881, + "nauc_precision_at_1_std": -0.087249, + "nauc_precision_at_1_diff1": 0.626349, + "nauc_precision_at_3_max": 0.420386, + "nauc_precision_at_3_std": -0.036846, + "nauc_precision_at_3_diff1": 0.360087, + "nauc_precision_at_5_max": 0.416255, + "nauc_precision_at_5_std": -0.003032, + "nauc_precision_at_5_diff1": 0.293392, + "nauc_precision_at_10_max": 0.411661, + "nauc_precision_at_10_std": 0.034434, + "nauc_precision_at_10_diff1": 0.226256, + "nauc_precision_at_20_max": 0.4003, + "nauc_precision_at_20_std": 0.084466, + "nauc_precision_at_20_diff1": 0.175816, + "nauc_precision_at_100_max": 0.340196, + "nauc_precision_at_100_std": 0.18193, + "nauc_precision_at_100_diff1": 0.078935, + "nauc_precision_at_1000_max": 0.282039, + "nauc_precision_at_1000_std": 0.172393, + "nauc_precision_at_1000_diff1": 0.030861, + "nauc_mrr_at_1_max": 0.424881, + "nauc_mrr_at_1_std": -0.087249, + "nauc_mrr_at_1_diff1": 0.626349, + "nauc_mrr_at_3_max": 0.413903, + "nauc_mrr_at_3_std": -0.087763, + "nauc_mrr_at_3_diff1": 0.583226, + "nauc_mrr_at_5_max": 0.413295, + "nauc_mrr_at_5_std": -0.083373, + "nauc_mrr_at_5_diff1": 0.57792, + "nauc_mrr_at_10_max": 0.416487, + "nauc_mrr_at_10_std": -0.080944, + "nauc_mrr_at_10_diff1": 0.576457, + "nauc_mrr_at_20_max": 0.416401, + "nauc_mrr_at_20_std": -0.080555, + "nauc_mrr_at_20_diff1": 0.576786, + "nauc_mrr_at_100_max": 0.416775, + "nauc_mrr_at_100_std": -0.079303, + "nauc_mrr_at_100_diff1": 0.577511, + "nauc_mrr_at_1000_max": 0.416793, + "nauc_mrr_at_1000_std": -0.079428, + "nauc_mrr_at_1000_diff1": 0.577636, + "main_score": 0.47676, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 397.79305839538574, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/Cmnli.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/Cmnli.json new file mode 100644 index 0000000000..84d2137b77 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/Cmnli.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "41bc36f332156f7adc9e38f53777c959b2ae9766", + "task_name": "Cmnli", + "mteb_version": "1.36.22", + "scores": { + "validation": [ + { + "similarity_accuracy": 0.86362, + "similarity_accuracy_threshold": 0.828911, + "similarity_f1": 0.870104, + "similarity_f1_threshold": 0.818303, + "similarity_precision": 0.838941, + "similarity_recall": 0.903671, + "similarity_ap": 0.926893, + "cosine_accuracy": 0.86362, + "cosine_accuracy_threshold": 0.828911, + "cosine_f1": 0.870104, + "cosine_f1_threshold": 0.818303, + "cosine_precision": 0.838941, + "cosine_recall": 0.903671, + "cosine_ap": 0.926999, + "manhattan_accuracy": 0.862057, + "manhattan_accuracy_threshold": 19.755219, + "manhattan_f1": 0.869849, + "manhattan_f1_threshold": 20.317467, + "manhattan_precision": 0.838871, + "manhattan_recall": 0.903203, + "manhattan_ap": 0.926913, + "euclidean_accuracy": 0.86362, + "euclidean_accuracy_threshold": 0.58496, + "euclidean_f1": 0.870104, + "euclidean_f1_threshold": 0.602822, + "euclidean_precision": 0.838941, + "euclidean_recall": 0.903671, + "euclidean_ap": 0.926999, + "dot_accuracy": 0.86362, + "dot_accuracy_threshold": 0.828911, + "dot_f1": 0.870104, + "dot_f1_threshold": 0.818303, + "dot_precision": 0.838941, + "dot_recall": 0.903671, + "dot_ap": 0.927121, + "max_accuracy": 0.86362, + "max_f1": 0.870104, + "max_precision": 0.838941, + "max_recall": 0.903671, + "max_ap": 0.927121, + "main_score": 0.86362, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 5.97287917137146, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CovidRetrieval.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CovidRetrieval.json new file mode 100644 index 0000000000..d8f239eead --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/CovidRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "1271c7809071a13532e05f25fb53511ffce77117", + "task_name": "CovidRetrieval", + "mteb_version": "1.36.22", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.83983, + "ndcg_at_3": 0.91114, + "ndcg_at_5": 0.91918, + "ndcg_at_10": 0.92318, + "ndcg_at_20": 0.92432, + "ndcg_at_100": 0.92498, + "ndcg_at_1000": 0.92527, + "map_at_1": 0.83588, + "map_at_3": 0.89392, + "map_at_5": 0.89867, + "map_at_10": 0.90048, + "map_at_20": 0.90083, + "map_at_100": 0.90095, + "map_at_1000": 0.90097, + "recall_at_1": 0.83588, + "recall_at_3": 0.95917, + "recall_at_5": 0.97866, + "recall_at_10": 0.99052, + "recall_at_20": 0.99473, + "recall_at_100": 0.99789, + "recall_at_1000": 1.0, + "precision_at_1": 0.83983, + "precision_at_3": 0.3228, + "precision_at_5": 0.19768, + "precision_at_10": 0.10011, + "precision_at_20": 0.05026, + "precision_at_100": 0.01008, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.840885, + "mrr_at_3": 0.895153, + "mrr_at_5": 0.899895, + "mrr_at_10": 0.901541, + "mrr_at_20": 0.901893, + "mrr_at_100": 0.902016, + "mrr_at_1000": 0.902029, + "nauc_ndcg_at_1_max": 0.22114, + "nauc_ndcg_at_1_std": -0.460871, + "nauc_ndcg_at_1_diff1": 0.8642, + "nauc_ndcg_at_3_max": 0.304741, + "nauc_ndcg_at_3_std": -0.493115, + "nauc_ndcg_at_3_diff1": 0.860096, + "nauc_ndcg_at_5_max": 0.291282, + "nauc_ndcg_at_5_std": -0.491963, + "nauc_ndcg_at_5_diff1": 0.862714, + "nauc_ndcg_at_10_max": 0.279524, + "nauc_ndcg_at_10_std": -0.483319, + "nauc_ndcg_at_10_diff1": 0.863365, + "nauc_ndcg_at_20_max": 0.273904, + "nauc_ndcg_at_20_std": -0.472801, + "nauc_ndcg_at_20_diff1": 0.862843, + "nauc_ndcg_at_100_max": 0.270811, + "nauc_ndcg_at_100_std": -0.47355, + "nauc_ndcg_at_100_diff1": 0.862838, + "nauc_ndcg_at_1000_max": 0.268703, + "nauc_ndcg_at_1000_std": -0.478046, + "nauc_ndcg_at_1000_diff1": 0.863085, + "nauc_map_at_1_max": 0.210584, + "nauc_map_at_1_std": -0.47315, + "nauc_map_at_1_diff1": 0.866891, + "nauc_map_at_3_max": 0.269979, + "nauc_map_at_3_std": -0.490485, + "nauc_map_at_3_diff1": 0.862063, + "nauc_map_at_5_max": 0.264356, + "nauc_map_at_5_std": -0.488252, + "nauc_map_at_5_diff1": 0.863277, + "nauc_map_at_10_max": 0.261959, + "nauc_map_at_10_std": -0.484056, + "nauc_map_at_10_diff1": 0.86357, + "nauc_map_at_20_max": 0.260566, + "nauc_map_at_20_std": -0.481548, + "nauc_map_at_20_diff1": 0.863456, + "nauc_map_at_100_max": 0.260185, + "nauc_map_at_100_std": -0.481526, + "nauc_map_at_100_diff1": 0.863457, + "nauc_map_at_1000_max": 0.260107, + "nauc_map_at_1000_std": -0.481672, + "nauc_map_at_1000_diff1": 0.863463, + "nauc_recall_at_1_max": 0.210584, + "nauc_recall_at_1_std": -0.47315, + "nauc_recall_at_1_diff1": 0.866891, + "nauc_recall_at_3_max": 0.498641, + "nauc_recall_at_3_std": -0.550998, + "nauc_recall_at_3_diff1": 0.845714, + "nauc_recall_at_5_max": 0.599192, + "nauc_recall_at_5_std": -0.563376, + "nauc_recall_at_5_diff1": 0.856145, + "nauc_recall_at_10_max": 0.67285, + "nauc_recall_at_10_std": -0.474024, + "nauc_recall_at_10_diff1": 0.863811, + "nauc_recall_at_20_max": 0.700654, + "nauc_recall_at_20_std": 0.067081, + "nauc_recall_at_20_diff1": 0.836574, + "nauc_recall_at_100_max": 0.795717, + "nauc_recall_at_100_std": 0.678904, + "nauc_recall_at_100_diff1": 0.795717, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.22114, + "nauc_precision_at_1_std": -0.460871, + "nauc_precision_at_1_diff1": 0.8642, + "nauc_precision_at_3_max": 0.428545, + "nauc_precision_at_3_std": -0.247343, + "nauc_precision_at_3_diff1": 0.487441, + "nauc_precision_at_5_max": 0.420658, + "nauc_precision_at_5_std": -0.065094, + "nauc_precision_at_5_diff1": 0.262771, + "nauc_precision_at_10_max": 0.387362, + "nauc_precision_at_10_std": 0.232203, + "nauc_precision_at_10_diff1": -0.027996, + "nauc_precision_at_20_max": 0.342549, + "nauc_precision_at_20_std": 0.479315, + "nauc_precision_at_20_diff1": -0.193473, + "nauc_precision_at_100_max": 0.296925, + "nauc_precision_at_100_std": 0.606267, + "nauc_precision_at_100_diff1": -0.357137, + "nauc_precision_at_1000_max": 0.239528, + "nauc_precision_at_1000_std": 0.597908, + "nauc_precision_at_1000_diff1": -0.4898, + "nauc_mrr_at_1_max": 0.21002, + "nauc_mrr_at_1_std": -0.462469, + "nauc_mrr_at_1_diff1": 0.862468, + "nauc_mrr_at_3_max": 0.271889, + "nauc_mrr_at_3_std": -0.478562, + "nauc_mrr_at_3_diff1": 0.860258, + "nauc_mrr_at_5_max": 0.264254, + "nauc_mrr_at_5_std": -0.480841, + "nauc_mrr_at_5_diff1": 0.861397, + "nauc_mrr_at_10_max": 0.258744, + "nauc_mrr_at_10_std": -0.478757, + "nauc_mrr_at_10_diff1": 0.861616, + "nauc_mrr_at_20_max": 0.257338, + "nauc_mrr_at_20_std": -0.476154, + "nauc_mrr_at_20_diff1": 0.86147, + "nauc_mrr_at_100_max": 0.25695, + "nauc_mrr_at_100_std": -0.476124, + "nauc_mrr_at_100_diff1": 0.861468, + "nauc_mrr_at_1000_max": 0.25687, + "nauc_mrr_at_1000_std": -0.476272, + "nauc_mrr_at_1000_diff1": 0.861474, + "main_score": 0.92318, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 408.97021079063416, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/DuRetrieval.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/DuRetrieval.json new file mode 100644 index 0000000000..1a5be826c9 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/DuRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced", + "task_name": "DuRetrieval", + "mteb_version": "1.36.22", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.9225, + "ndcg_at_3": 0.88386, + "ndcg_at_5": 0.87006, + "ndcg_at_10": 0.89047, + "ndcg_at_20": 0.90448, + "ndcg_at_100": 0.91567, + "ndcg_at_1000": 0.91796, + "map_at_1": 0.27076, + "map_at_3": 0.5744, + "map_at_5": 0.72072, + "map_at_10": 0.82146, + "map_at_20": 0.84288, + "map_at_100": 0.84924, + "map_at_1000": 0.84949, + "recall_at_1": 0.27076, + "recall_at_3": 0.59449, + "recall_at_5": 0.76041, + "recall_at_10": 0.89704, + "recall_at_20": 0.94371, + "recall_at_100": 0.98031, + "recall_at_1000": 0.99213, + "precision_at_1": 0.9225, + "precision_at_3": 0.78933, + "precision_at_5": 0.6627, + "precision_at_10": 0.42305, + "precision_at_20": 0.22983, + "precision_at_100": 0.04836, + "precision_at_1000": 0.00489, + "mrr_at_1": 0.9225, + "mrr_at_3": 0.9455, + "mrr_at_5": 0.946675, + "mrr_at_10": 0.947488, + "mrr_at_20": 0.947824, + "mrr_at_100": 0.948039, + "mrr_at_1000": 0.948046, + "nauc_ndcg_at_1_max": 0.606367, + "nauc_ndcg_at_1_std": 0.308979, + "nauc_ndcg_at_1_diff1": 0.257863, + "nauc_ndcg_at_3_max": 0.400202, + "nauc_ndcg_at_3_std": 0.205161, + "nauc_ndcg_at_3_diff1": -0.062742, + "nauc_ndcg_at_5_max": 0.33899, + "nauc_ndcg_at_5_std": 0.128383, + "nauc_ndcg_at_5_diff1": -0.015741, + "nauc_ndcg_at_10_max": 0.397636, + "nauc_ndcg_at_10_std": 0.172856, + "nauc_ndcg_at_10_diff1": -0.004968, + "nauc_ndcg_at_20_max": 0.4502, + "nauc_ndcg_at_20_std": 0.227083, + "nauc_ndcg_at_20_diff1": -0.008181, + "nauc_ndcg_at_100_max": 0.465724, + "nauc_ndcg_at_100_std": 0.265192, + "nauc_ndcg_at_100_diff1": -0.024712, + "nauc_ndcg_at_1000_max": 0.463709, + "nauc_ndcg_at_1000_std": 0.257983, + "nauc_ndcg_at_1000_diff1": -0.020202, + "nauc_map_at_1_max": -0.152742, + "nauc_map_at_1_std": -0.285455, + "nauc_map_at_1_diff1": 0.457565, + "nauc_map_at_3_max": -0.021941, + "nauc_map_at_3_std": -0.228937, + "nauc_map_at_3_diff1": 0.274618, + "nauc_map_at_5_max": 0.123837, + "nauc_map_at_5_std": -0.125777, + "nauc_map_at_5_diff1": 0.150703, + "nauc_map_at_10_max": 0.309136, + "nauc_map_at_10_std": 0.050528, + "nauc_map_at_10_diff1": 0.027406, + "nauc_map_at_20_max": 0.35209, + "nauc_map_at_20_std": 0.132131, + "nauc_map_at_20_diff1": -0.014751, + "nauc_map_at_100_max": 0.356178, + "nauc_map_at_100_std": 0.155432, + "nauc_map_at_100_diff1": -0.028119, + "nauc_map_at_1000_max": 0.355944, + "nauc_map_at_1000_std": 0.155267, + "nauc_map_at_1000_diff1": -0.028134, + "nauc_recall_at_1_max": -0.152742, + "nauc_recall_at_1_std": -0.285455, + "nauc_recall_at_1_diff1": 0.457565, + "nauc_recall_at_3_max": -0.066632, + "nauc_recall_at_3_std": -0.261689, + "nauc_recall_at_3_diff1": 0.267062, + "nauc_recall_at_5_max": 0.058626, + "nauc_recall_at_5_std": -0.188425, + "nauc_recall_at_5_diff1": 0.157709, + "nauc_recall_at_10_max": 0.298047, + "nauc_recall_at_10_std": 0.009233, + "nauc_recall_at_10_diff1": 0.04129, + "nauc_recall_at_20_max": 0.45602, + "nauc_recall_at_20_std": 0.249989, + "nauc_recall_at_20_diff1": -0.039818, + "nauc_recall_at_100_max": 0.65258, + "nauc_recall_at_100_std": 0.595206, + "nauc_recall_at_100_diff1": -0.194245, + "nauc_recall_at_1000_max": 0.826147, + "nauc_recall_at_1000_std": 0.64053, + "nauc_recall_at_1000_diff1": -0.171055, + "nauc_precision_at_1_max": 0.606367, + "nauc_precision_at_1_std": 0.308979, + "nauc_precision_at_1_diff1": 0.257863, + "nauc_precision_at_3_max": 0.436655, + "nauc_precision_at_3_std": 0.371233, + "nauc_precision_at_3_diff1": -0.393187, + "nauc_precision_at_5_max": 0.363328, + "nauc_precision_at_5_std": 0.37366, + "nauc_precision_at_5_diff1": -0.394869, + "nauc_precision_at_10_max": 0.292828, + "nauc_precision_at_10_std": 0.383134, + "nauc_precision_at_10_diff1": -0.360308, + "nauc_precision_at_20_max": 0.24504, + "nauc_precision_at_20_std": 0.397647, + "nauc_precision_at_20_diff1": -0.345849, + "nauc_precision_at_100_max": 0.202713, + "nauc_precision_at_100_std": 0.390704, + "nauc_precision_at_100_diff1": -0.335931, + "nauc_precision_at_1000_max": 0.19026, + "nauc_precision_at_1000_std": 0.379893, + "nauc_precision_at_1000_diff1": -0.331566, + "nauc_mrr_at_1_max": 0.606367, + "nauc_mrr_at_1_std": 0.308979, + "nauc_mrr_at_1_diff1": 0.257863, + "nauc_mrr_at_3_max": 0.674103, + "nauc_mrr_at_3_std": 0.356094, + "nauc_mrr_at_3_diff1": 0.233323, + "nauc_mrr_at_5_max": 0.669084, + "nauc_mrr_at_5_std": 0.356794, + "nauc_mrr_at_5_diff1": 0.229398, + "nauc_mrr_at_10_max": 0.666856, + "nauc_mrr_at_10_std": 0.353545, + "nauc_mrr_at_10_diff1": 0.230364, + "nauc_mrr_at_20_max": 0.666135, + "nauc_mrr_at_20_std": 0.353761, + "nauc_mrr_at_20_diff1": 0.22882, + "nauc_mrr_at_100_max": 0.665731, + "nauc_mrr_at_100_std": 0.353225, + "nauc_mrr_at_100_diff1": 0.22933, + "nauc_mrr_at_1000_max": 0.665692, + "nauc_mrr_at_1000_std": 0.353153, + "nauc_mrr_at_1000_diff1": 0.22932, + "main_score": 0.89047, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 411.4216184616089, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/EcomRetrieval.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/EcomRetrieval.json new file mode 100644 index 0000000000..14fddfe5e2 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/EcomRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9", + "task_name": "EcomRetrieval", + "mteb_version": "1.36.22", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.55, + "ndcg_at_3": 0.65145, + "ndcg_at_5": 0.68295, + "ndcg_at_10": 0.71083, + "ndcg_at_20": 0.72438, + "ndcg_at_100": 0.73341, + "ndcg_at_1000": 0.73494, + "map_at_1": 0.55, + "map_at_3": 0.627, + "map_at_5": 0.6446, + "map_at_10": 0.65615, + "map_at_20": 0.65995, + "map_at_100": 0.66139, + "map_at_1000": 0.66145, + "recall_at_1": 0.55, + "recall_at_3": 0.722, + "recall_at_5": 0.798, + "recall_at_10": 0.884, + "recall_at_20": 0.937, + "recall_at_100": 0.983, + "recall_at_1000": 0.995, + "precision_at_1": 0.55, + "precision_at_3": 0.24067, + "precision_at_5": 0.1596, + "precision_at_10": 0.0884, + "precision_at_20": 0.04685, + "precision_at_100": 0.00983, + "precision_at_1000": 0.001, + "mrr_at_1": 0.55, + "mrr_at_3": 0.627, + "mrr_at_5": 0.6446, + "mrr_at_10": 0.656153, + "mrr_at_20": 0.659947, + "mrr_at_100": 0.661392, + "mrr_at_1000": 0.661449, + "nauc_ndcg_at_1_max": 0.256672, + "nauc_ndcg_at_1_std": -0.161882, + "nauc_ndcg_at_1_diff1": 0.714078, + "nauc_ndcg_at_3_max": 0.284471, + "nauc_ndcg_at_3_std": -0.170597, + "nauc_ndcg_at_3_diff1": 0.692413, + "nauc_ndcg_at_5_max": 0.276609, + "nauc_ndcg_at_5_std": -0.163807, + "nauc_ndcg_at_5_diff1": 0.680358, + "nauc_ndcg_at_10_max": 0.281257, + "nauc_ndcg_at_10_std": -0.160254, + "nauc_ndcg_at_10_diff1": 0.683565, + "nauc_ndcg_at_20_max": 0.276349, + "nauc_ndcg_at_20_std": -0.16108, + "nauc_ndcg_at_20_diff1": 0.691642, + "nauc_ndcg_at_100_max": 0.283733, + "nauc_ndcg_at_100_std": -0.147579, + "nauc_ndcg_at_100_diff1": 0.692348, + "nauc_ndcg_at_1000_max": 0.280352, + "nauc_ndcg_at_1000_std": -0.153566, + "nauc_ndcg_at_1000_diff1": 0.692534, + "nauc_map_at_1_max": 0.256672, + "nauc_map_at_1_std": -0.161882, + "nauc_map_at_1_diff1": 0.714078, + "nauc_map_at_3_max": 0.276791, + "nauc_map_at_3_std": -0.168895, + "nauc_map_at_3_diff1": 0.697682, + "nauc_map_at_5_max": 0.27263, + "nauc_map_at_5_std": -0.165473, + "nauc_map_at_5_diff1": 0.691619, + "nauc_map_at_10_max": 0.274395, + "nauc_map_at_10_std": -0.163218, + "nauc_map_at_10_diff1": 0.692969, + "nauc_map_at_20_max": 0.272958, + "nauc_map_at_20_std": -0.163326, + "nauc_map_at_20_diff1": 0.694885, + "nauc_map_at_100_max": 0.273927, + "nauc_map_at_100_std": -0.161622, + "nauc_map_at_100_diff1": 0.695066, + "nauc_map_at_1000_max": 0.273854, + "nauc_map_at_1000_std": -0.161775, + "nauc_map_at_1000_diff1": 0.695082, + "nauc_recall_at_1_max": 0.256672, + "nauc_recall_at_1_std": -0.161882, + "nauc_recall_at_1_diff1": 0.714078, + "nauc_recall_at_3_max": 0.311248, + "nauc_recall_at_3_std": -0.176181, + "nauc_recall_at_3_diff1": 0.674391, + "nauc_recall_at_5_max": 0.291734, + "nauc_recall_at_5_std": -0.154533, + "nauc_recall_at_5_diff1": 0.630059, + "nauc_recall_at_10_max": 0.327736, + "nauc_recall_at_10_std": -0.139324, + "nauc_recall_at_10_diff1": 0.620086, + "nauc_recall_at_20_max": 0.304981, + "nauc_recall_at_20_std": -0.136958, + "nauc_recall_at_20_diff1": 0.682243, + "nauc_recall_at_100_max": 0.84267, + "nauc_recall_at_100_std": 0.78857, + "nauc_recall_at_100_diff1": 0.685039, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_1000_diff1": 0.700747, + "nauc_precision_at_1_max": 0.256672, + "nauc_precision_at_1_std": -0.161882, + "nauc_precision_at_1_diff1": 0.714078, + "nauc_precision_at_3_max": 0.311248, + "nauc_precision_at_3_std": -0.176181, + "nauc_precision_at_3_diff1": 0.674391, + "nauc_precision_at_5_max": 0.291734, + "nauc_precision_at_5_std": -0.154533, + "nauc_precision_at_5_diff1": 0.630059, + "nauc_precision_at_10_max": 0.327736, + "nauc_precision_at_10_std": -0.139324, + "nauc_precision_at_10_diff1": 0.620086, + "nauc_precision_at_20_max": 0.304981, + "nauc_precision_at_20_std": -0.136958, + "nauc_precision_at_20_diff1": 0.682243, + "nauc_precision_at_100_max": 0.84267, + "nauc_precision_at_100_std": 0.78857, + "nauc_precision_at_100_diff1": 0.685039, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 0.700747, + "nauc_mrr_at_1_max": 0.256672, + "nauc_mrr_at_1_std": -0.161882, + "nauc_mrr_at_1_diff1": 0.714078, + "nauc_mrr_at_3_max": 0.276791, + "nauc_mrr_at_3_std": -0.168895, + "nauc_mrr_at_3_diff1": 0.697682, + "nauc_mrr_at_5_max": 0.27263, + "nauc_mrr_at_5_std": -0.165473, + "nauc_mrr_at_5_diff1": 0.691619, + "nauc_mrr_at_10_max": 0.274395, + "nauc_mrr_at_10_std": -0.163218, + "nauc_mrr_at_10_diff1": 0.692969, + "nauc_mrr_at_20_max": 0.272958, + "nauc_mrr_at_20_std": -0.163326, + "nauc_mrr_at_20_diff1": 0.694885, + "nauc_mrr_at_100_max": 0.273927, + "nauc_mrr_at_100_std": -0.161622, + "nauc_mrr_at_100_diff1": 0.695066, + "nauc_mrr_at_1000_max": 0.273854, + "nauc_mrr_at_1000_std": -0.161775, + "nauc_mrr_at_1000_diff1": 0.695082, + "main_score": 0.71083, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 56.67814564704895, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/IFlyTek.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/IFlyTek.json new file mode 100644 index 0000000000..a79d0df4be --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/IFlyTek.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "421605374b29664c5fc098418fe20ada9bd55f8a", + "task_name": "IFlyTek", + "mteb_version": "1.36.22", + "scores": { + "validation": [ + { + "accuracy": 0.525356, + "f1": 0.407883, + "f1_weighted": 0.513697, + "scores_per_experiment": [ + { + "accuracy": 0.532128, + "f1": 0.411696, + "f1_weighted": 0.519817 + }, + { + "accuracy": 0.5202, + "f1": 0.406018, + "f1_weighted": 0.505917 + }, + { + "accuracy": 0.528665, + "f1": 0.408549, + "f1_weighted": 0.519468 + }, + { + "accuracy": 0.522124, + "f1": 0.404547, + "f1_weighted": 0.511884 + }, + { + "accuracy": 0.523663, + "f1": 0.408603, + "f1_weighted": 0.511399 + } + ], + "main_score": 0.525356, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 92.15740203857422, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/JDReview.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/JDReview.json new file mode 100644 index 0000000000..3b094a0b33 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/JDReview.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b", + "task_name": "JDReview", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "accuracy": 0.902627, + "f1": 0.858237, + "f1_weighted": 0.90809, + "ap": 0.636284, + "ap_weighted": 0.636284, + "scores_per_experiment": [ + { + "accuracy": 0.904315, + "f1": 0.859946, + "f1_weighted": 0.909492, + "ap": 0.639237, + "ap_weighted": 0.639237 + }, + { + "accuracy": 0.904315, + "f1": 0.859071, + "f1_weighted": 0.909259, + "ap": 0.63672, + "ap_weighted": 0.63672 + }, + { + "accuracy": 0.906191, + "f1": 0.863114, + "f1_weighted": 0.911378, + "ap": 0.646429, + "ap_weighted": 0.646429 + }, + { + "accuracy": 0.904315, + "f1": 0.859946, + "f1_weighted": 0.909492, + "ap": 0.639237, + "ap_weighted": 0.639237 + }, + { + "accuracy": 0.900563, + "f1": 0.855345, + "f1_weighted": 0.906177, + "ap": 0.630057, + "ap_weighted": 0.630057 + }, + { + "accuracy": 0.896811, + "f1": 0.853417, + "f1_weighted": 0.903544, + "ap": 0.62919, + "ap_weighted": 0.62919 + }, + { + "accuracy": 0.906191, + "f1": 0.8614, + "f1_weighted": 0.910922, + "ap": 0.641452, + "ap_weighted": 0.641452 + }, + { + "accuracy": 0.904315, + "f1": 0.859071, + "f1_weighted": 0.909259, + "ap": 0.63672, + "ap_weighted": 0.63672 + }, + { + "accuracy": 0.902439, + "f1": 0.857639, + "f1_weighted": 0.907833, + "ap": 0.634613, + "ap_weighted": 0.634613 + }, + { + "accuracy": 0.896811, + "f1": 0.853417, + "f1_weighted": 0.903544, + "ap": 0.62919, + "ap_weighted": 0.62919 + } + ], + "main_score": 0.902627, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 9.53889274597168, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/LCQMC.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/LCQMC.json new file mode 100644 index 0000000000..a05d68ad34 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/LCQMC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "17f9b096f80380fce5ed12a9be8be7784b337daf", + "task_name": "LCQMC", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "pearson": 0.736204, + "spearman": 0.795699, + "cosine_pearson": 0.736204, + "cosine_spearman": 0.795699, + "manhattan_pearson": 0.795438, + "manhattan_spearman": 0.795608, + "euclidean_pearson": 0.7955, + "euclidean_spearman": 0.795699, + "main_score": 0.795699, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 6.692282438278198, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/MMarcoReranking.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/MMarcoReranking.json new file mode 100644 index 0000000000..73737653ba --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/MMarcoReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6", + "task_name": "MMarcoReranking", + "mteb_version": "1.36.22", + "scores": { + "dev": [ + { + "map": 0.40919, + "mrr": 0.406036, + "nAUC_map_max": -0.012102, + "nAUC_map_std": -0.299295, + "nAUC_map_diff1": 0.382706, + "nAUC_mrr_max": -0.020512, + "nAUC_mrr_std": -0.31824, + "nAUC_mrr_diff1": 0.374338, + "main_score": 0.40919, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 179.59954500198364, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/MMarcoRetrieval.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/MMarcoRetrieval.json new file mode 100644 index 0000000000..2d4eec5701 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/MMarcoRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "539bbde593d947e2a124ba72651aafc09eb33fc2", + "task_name": "MMarcoRetrieval", + "mteb_version": "1.36.22", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.71003, + "ndcg_at_3": 0.78784, + "ndcg_at_5": 0.80974, + "ndcg_at_10": 0.82535, + "ndcg_at_20": 0.83198, + "ndcg_at_100": 0.83557, + "ndcg_at_1000": 0.83681, + "map_at_1": 0.68703, + "map_at_3": 0.76562, + "map_at_5": 0.77868, + "map_at_10": 0.7856, + "map_at_20": 0.78758, + "map_at_100": 0.78813, + "map_at_1000": 0.78818, + "recall_at_1": 0.68703, + "recall_at_3": 0.8436, + "recall_at_5": 0.89571, + "recall_at_10": 0.94224, + "recall_at_20": 0.96756, + "recall_at_100": 0.98625, + "recall_at_1000": 0.99594, + "precision_at_1": 0.71003, + "precision_at_3": 0.29713, + "precision_at_5": 0.19006, + "precision_at_10": 0.10014, + "precision_at_20": 0.05148, + "precision_at_100": 0.01051, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.710029, + "mrr_at_3": 0.773806, + "mrr_at_5": 0.784787, + "mrr_at_10": 0.790937, + "mrr_at_20": 0.79265, + "mrr_at_100": 0.793107, + "mrr_at_1000": 0.79315, + "nauc_ndcg_at_1_max": 0.364571, + "nauc_ndcg_at_1_std": -0.229028, + "nauc_ndcg_at_1_diff1": 0.796433, + "nauc_ndcg_at_3_max": 0.382284, + "nauc_ndcg_at_3_std": -0.217087, + "nauc_ndcg_at_3_diff1": 0.757692, + "nauc_ndcg_at_5_max": 0.392134, + "nauc_ndcg_at_5_std": -0.203111, + "nauc_ndcg_at_5_diff1": 0.753753, + "nauc_ndcg_at_10_max": 0.399342, + "nauc_ndcg_at_10_std": -0.193102, + "nauc_ndcg_at_10_diff1": 0.76, + "nauc_ndcg_at_20_max": 0.398019, + "nauc_ndcg_at_20_std": -0.191588, + "nauc_ndcg_at_20_diff1": 0.763716, + "nauc_ndcg_at_100_max": 0.393456, + "nauc_ndcg_at_100_std": -0.192089, + "nauc_ndcg_at_100_diff1": 0.765006, + "nauc_ndcg_at_1000_max": 0.390549, + "nauc_ndcg_at_1000_std": -0.197613, + "nauc_ndcg_at_1000_diff1": 0.765823, + "nauc_map_at_1_max": 0.323225, + "nauc_map_at_1_std": -0.237114, + "nauc_map_at_1_diff1": 0.787131, + "nauc_map_at_3_max": 0.371608, + "nauc_map_at_3_std": -0.226667, + "nauc_map_at_3_diff1": 0.76556, + "nauc_map_at_5_max": 0.37792, + "nauc_map_at_5_std": -0.218555, + "nauc_map_at_5_diff1": 0.764574, + "nauc_map_at_10_max": 0.380912, + "nauc_map_at_10_std": -0.215009, + "nauc_map_at_10_diff1": 0.76728, + "nauc_map_at_20_max": 0.380759, + "nauc_map_at_20_std": -0.214414, + "nauc_map_at_20_diff1": 0.76839, + "nauc_map_at_100_max": 0.380229, + "nauc_map_at_100_std": -0.214194, + "nauc_map_at_100_diff1": 0.768525, + "nauc_map_at_1000_max": 0.380136, + "nauc_map_at_1000_std": -0.214356, + "nauc_map_at_1000_diff1": 0.768555, + "nauc_recall_at_1_max": 0.323225, + "nauc_recall_at_1_std": -0.237114, + "nauc_recall_at_1_diff1": 0.787131, + "nauc_recall_at_3_max": 0.402646, + "nauc_recall_at_3_std": -0.194919, + "nauc_recall_at_3_diff1": 0.713661, + "nauc_recall_at_5_max": 0.45198, + "nauc_recall_at_5_std": -0.131888, + "nauc_recall_at_5_diff1": 0.680359, + "nauc_recall_at_10_max": 0.549291, + "nauc_recall_at_10_std": 0.001971, + "nauc_recall_at_10_diff1": 0.673372, + "nauc_recall_at_20_max": 0.636797, + "nauc_recall_at_20_std": 0.165771, + "nauc_recall_at_20_diff1": 0.674057, + "nauc_recall_at_100_max": 0.701258, + "nauc_recall_at_100_std": 0.562871, + "nauc_recall_at_100_diff1": 0.636972, + "nauc_recall_at_1000_max": 0.6312, + "nauc_recall_at_1000_std": 0.789207, + "nauc_recall_at_1000_diff1": 0.555965, + "nauc_precision_at_1_max": 0.364571, + "nauc_precision_at_1_std": -0.229028, + "nauc_precision_at_1_diff1": 0.796433, + "nauc_precision_at_3_max": 0.357021, + "nauc_precision_at_3_std": -0.078635, + "nauc_precision_at_3_diff1": 0.415648, + "nauc_precision_at_5_max": 0.333819, + "nauc_precision_at_5_std": 0.013899, + "nauc_precision_at_5_diff1": 0.256196, + "nauc_precision_at_10_max": 0.304213, + "nauc_precision_at_10_std": 0.12356, + "nauc_precision_at_10_diff1": 0.08557, + "nauc_precision_at_20_max": 0.25657, + "nauc_precision_at_20_std": 0.191891, + "nauc_precision_at_20_diff1": -0.041476, + "nauc_precision_at_100_max": 0.182841, + "nauc_precision_at_100_std": 0.235049, + "nauc_precision_at_100_diff1": -0.162731, + "nauc_precision_at_1000_max": 0.12692, + "nauc_precision_at_1000_std": 0.207907, + "nauc_precision_at_1000_diff1": -0.232069, + "nauc_mrr_at_1_max": 0.364571, + "nauc_mrr_at_1_std": -0.229028, + "nauc_mrr_at_1_diff1": 0.796433, + "nauc_mrr_at_3_max": 0.383775, + "nauc_mrr_at_3_std": -0.215226, + "nauc_mrr_at_3_diff1": 0.772572, + "nauc_mrr_at_5_max": 0.386608, + "nauc_mrr_at_5_std": -0.210003, + "nauc_mrr_at_5_diff1": 0.770669, + "nauc_mrr_at_10_max": 0.388368, + "nauc_mrr_at_10_std": -0.20751, + "nauc_mrr_at_10_diff1": 0.773113, + "nauc_mrr_at_20_max": 0.387787, + "nauc_mrr_at_20_std": -0.20762, + "nauc_mrr_at_20_diff1": 0.773951, + "nauc_mrr_at_100_max": 0.387259, + "nauc_mrr_at_100_std": -0.207595, + "nauc_mrr_at_100_diff1": 0.774094, + "nauc_mrr_at_1000_max": 0.387178, + "nauc_mrr_at_1000_std": -0.207748, + "nauc_mrr_at_1000_diff1": 0.774122, + "main_score": 0.82535, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 201.38595843315125, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/MedicalRetrieval.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/MedicalRetrieval.json new file mode 100644 index 0000000000..6f1c4f2266 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/MedicalRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6", + "task_name": "MedicalRetrieval", + "mteb_version": "1.36.22", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.571, + "ndcg_at_3": 0.64455, + "ndcg_at_5": 0.66335, + "ndcg_at_10": 0.68272, + "ndcg_at_20": 0.69846, + "ndcg_at_100": 0.71523, + "ndcg_at_1000": 0.72144, + "map_at_1": 0.571, + "map_at_3": 0.62733, + "map_at_5": 0.63768, + "map_at_10": 0.64567, + "map_at_20": 0.65002, + "map_at_100": 0.65235, + "map_at_1000": 0.65262, + "recall_at_1": 0.571, + "recall_at_3": 0.694, + "recall_at_5": 0.74, + "recall_at_10": 0.8, + "recall_at_20": 0.862, + "recall_at_100": 0.952, + "recall_at_1000": 0.999, + "precision_at_1": 0.571, + "precision_at_3": 0.23133, + "precision_at_5": 0.148, + "precision_at_10": 0.08, + "precision_at_20": 0.0431, + "precision_at_100": 0.00952, + "precision_at_1000": 0.001, + "mrr_at_1": 0.571, + "mrr_at_3": 0.6275, + "mrr_at_5": 0.63765, + "mrr_at_10": 0.6458, + "mrr_at_20": 0.650156, + "mrr_at_100": 0.652488, + "mrr_at_1000": 0.652755, + "nauc_ndcg_at_1_max": 0.519357, + "nauc_ndcg_at_1_std": -0.075981, + "nauc_ndcg_at_1_diff1": 0.82035, + "nauc_ndcg_at_3_max": 0.493751, + "nauc_ndcg_at_3_std": -0.084452, + "nauc_ndcg_at_3_diff1": 0.773339, + "nauc_ndcg_at_5_max": 0.48036, + "nauc_ndcg_at_5_std": -0.078411, + "nauc_ndcg_at_5_diff1": 0.770515, + "nauc_ndcg_at_10_max": 0.475251, + "nauc_ndcg_at_10_std": -0.071966, + "nauc_ndcg_at_10_diff1": 0.762659, + "nauc_ndcg_at_20_max": 0.474819, + "nauc_ndcg_at_20_std": -0.063318, + "nauc_ndcg_at_20_diff1": 0.765007, + "nauc_ndcg_at_100_max": 0.484451, + "nauc_ndcg_at_100_std": -0.063645, + "nauc_ndcg_at_100_diff1": 0.77125, + "nauc_ndcg_at_1000_max": 0.486867, + "nauc_ndcg_at_1000_std": -0.069311, + "nauc_ndcg_at_1000_diff1": 0.774345, + "nauc_map_at_1_max": 0.519357, + "nauc_map_at_1_std": -0.075981, + "nauc_map_at_1_diff1": 0.82035, + "nauc_map_at_3_max": 0.501405, + "nauc_map_at_3_std": -0.082879, + "nauc_map_at_3_diff1": 0.78519, + "nauc_map_at_5_max": 0.494362, + "nauc_map_at_5_std": -0.079839, + "nauc_map_at_5_diff1": 0.783727, + "nauc_map_at_10_max": 0.492767, + "nauc_map_at_10_std": -0.077866, + "nauc_map_at_10_diff1": 0.780862, + "nauc_map_at_20_max": 0.492938, + "nauc_map_at_20_std": -0.075445, + "nauc_map_at_20_diff1": 0.78158, + "nauc_map_at_100_max": 0.493911, + "nauc_map_at_100_std": -0.075594, + "nauc_map_at_100_diff1": 0.782285, + "nauc_map_at_1000_max": 0.494013, + "nauc_map_at_1000_std": -0.075646, + "nauc_map_at_1000_diff1": 0.782391, + "nauc_recall_at_1_max": 0.519357, + "nauc_recall_at_1_std": -0.075981, + "nauc_recall_at_1_diff1": 0.82035, + "nauc_recall_at_3_max": 0.468379, + "nauc_recall_at_3_std": -0.089273, + "nauc_recall_at_3_diff1": 0.734961, + "nauc_recall_at_5_max": 0.426994, + "nauc_recall_at_5_std": -0.071512, + "nauc_recall_at_5_diff1": 0.722832, + "nauc_recall_at_10_max": 0.392113, + "nauc_recall_at_10_std": -0.038529, + "nauc_recall_at_10_diff1": 0.679101, + "nauc_recall_at_20_max": 0.354595, + "nauc_recall_at_20_std": 0.03211, + "nauc_recall_at_20_diff1": 0.66405, + "nauc_recall_at_100_max": 0.414741, + "nauc_recall_at_100_std": 0.195485, + "nauc_recall_at_100_diff1": 0.657816, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 0.869281, + "nauc_recall_at_1000_diff1": 0.722222, + "nauc_precision_at_1_max": 0.519357, + "nauc_precision_at_1_std": -0.075981, + "nauc_precision_at_1_diff1": 0.82035, + "nauc_precision_at_3_max": 0.468379, + "nauc_precision_at_3_std": -0.089273, + "nauc_precision_at_3_diff1": 0.734961, + "nauc_precision_at_5_max": 0.426994, + "nauc_precision_at_5_std": -0.071512, + "nauc_precision_at_5_diff1": 0.722832, + "nauc_precision_at_10_max": 0.392113, + "nauc_precision_at_10_std": -0.038529, + "nauc_precision_at_10_diff1": 0.679101, + "nauc_precision_at_20_max": 0.354595, + "nauc_precision_at_20_std": 0.03211, + "nauc_precision_at_20_diff1": 0.66405, + "nauc_precision_at_100_max": 0.414741, + "nauc_precision_at_100_std": 0.195485, + "nauc_precision_at_100_diff1": 0.657816, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 0.869281, + "nauc_precision_at_1000_diff1": 0.722222, + "nauc_mrr_at_1_max": 0.521168, + "nauc_mrr_at_1_std": -0.075981, + "nauc_mrr_at_1_diff1": 0.82035, + "nauc_mrr_at_3_max": 0.50287, + "nauc_mrr_at_3_std": -0.082809, + "nauc_mrr_at_3_diff1": 0.784946, + "nauc_mrr_at_5_max": 0.495997, + "nauc_mrr_at_5_std": -0.079343, + "nauc_mrr_at_5_diff1": 0.783686, + "nauc_mrr_at_10_max": 0.494319, + "nauc_mrr_at_10_std": -0.077719, + "nauc_mrr_at_10_diff1": 0.780641, + "nauc_mrr_at_20_max": 0.494511, + "nauc_mrr_at_20_std": -0.075296, + "nauc_mrr_at_20_diff1": 0.781357, + "nauc_mrr_at_100_max": 0.495494, + "nauc_mrr_at_100_std": -0.075449, + "nauc_mrr_at_100_diff1": 0.782064, + "nauc_mrr_at_1000_max": 0.495598, + "nauc_mrr_at_1000_std": -0.0755, + "nauc_mrr_at_1000_diff1": 0.78217, + "main_score": 0.68272, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 187.7778468132019, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/MultilingualSentiment.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/MultilingualSentiment.json new file mode 100644 index 0000000000..d29a3a27a1 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/MultilingualSentiment.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a", + "task_name": "MultilingualSentiment", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "accuracy": 0.7983, + "f1": 0.795314, + "f1_weighted": 0.795314, + "scores_per_experiment": [ + { + "accuracy": 0.796333, + "f1": 0.795087, + "f1_weighted": 0.795087 + }, + { + "accuracy": 0.79, + "f1": 0.78257, + "f1_weighted": 0.78257 + }, + { + "accuracy": 0.799667, + "f1": 0.794942, + "f1_weighted": 0.794942 + }, + { + "accuracy": 0.797333, + "f1": 0.79529, + "f1_weighted": 0.79529 + }, + { + "accuracy": 0.8, + "f1": 0.797775, + "f1_weighted": 0.797775 + }, + { + "accuracy": 0.792333, + "f1": 0.789185, + "f1_weighted": 0.789185 + }, + { + "accuracy": 0.8, + "f1": 0.797497, + "f1_weighted": 0.797497 + }, + { + "accuracy": 0.802333, + "f1": 0.798941, + "f1_weighted": 0.798941 + }, + { + "accuracy": 0.796667, + "f1": 0.794641, + "f1_weighted": 0.794641 + }, + { + "accuracy": 0.808333, + "f1": 0.807208, + "f1_weighted": 0.807208 + } + ], + "main_score": 0.7983, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ], + "validation": [ + { + "accuracy": 0.787233, + "f1": 0.784088, + "f1_weighted": 0.784088, + "scores_per_experiment": [ + { + "accuracy": 0.786667, + "f1": 0.785799, + "f1_weighted": 0.785799 + }, + { + "accuracy": 0.780667, + "f1": 0.772369, + "f1_weighted": 0.772369 + }, + { + "accuracy": 0.789, + "f1": 0.783989, + "f1_weighted": 0.783989 + }, + { + "accuracy": 0.787333, + "f1": 0.785161, + "f1_weighted": 0.785161 + }, + { + "accuracy": 0.788333, + "f1": 0.785657, + "f1_weighted": 0.785657 + }, + { + "accuracy": 0.78, + "f1": 0.777196, + "f1_weighted": 0.777196 + }, + { + "accuracy": 0.79, + "f1": 0.78749, + "f1_weighted": 0.78749 + }, + { + "accuracy": 0.793, + "f1": 0.789566, + "f1_weighted": 0.789566 + }, + { + "accuracy": 0.786333, + "f1": 0.784128, + "f1_weighted": 0.784128 + }, + { + "accuracy": 0.791, + "f1": 0.78953, + "f1_weighted": 0.78953 + } + ], + "main_score": 0.787233, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 30.730741262435913, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/Ocnli.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/Ocnli.json new file mode 100644 index 0000000000..312b7b0b6d --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/Ocnli.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "66e76a618a34d6d565d5538088562851e6daa7ec", + "task_name": "Ocnli", + "mteb_version": "1.36.22", + "scores": { + "validation": [ + { + "similarity_accuracy": 0.855983, + "similarity_accuracy_threshold": 0.8069, + "similarity_f1": 0.865927, + "similarity_f1_threshold": 0.806503, + "similarity_precision": 0.828351, + "similarity_recall": 0.907075, + "similarity_ap": 0.911253, + "cosine_accuracy": 0.855983, + "cosine_accuracy_threshold": 0.8069, + "cosine_f1": 0.865927, + "cosine_f1_threshold": 0.806503, + "cosine_precision": 0.828351, + "cosine_recall": 0.907075, + "cosine_ap": 0.911252, + "manhattan_accuracy": 0.8549, + "manhattan_accuracy_threshold": 20.654835, + "manhattan_f1": 0.865938, + "manhattan_f1_threshold": 21.230207, + "manhattan_precision": 0.817245, + "manhattan_recall": 0.920803, + "manhattan_ap": 0.910689, + "euclidean_accuracy": 0.855983, + "euclidean_accuracy_threshold": 0.62145, + "euclidean_f1": 0.865927, + "euclidean_f1_threshold": 0.622089, + "euclidean_precision": 0.828351, + "euclidean_recall": 0.907075, + "euclidean_ap": 0.911252, + "dot_accuracy": 0.855983, + "dot_accuracy_threshold": 0.8069, + "dot_f1": 0.865927, + "dot_f1_threshold": 0.806503, + "dot_precision": 0.828351, + "dot_recall": 0.907075, + "dot_ap": 0.911252, + "max_accuracy": 0.855983, + "max_f1": 0.865938, + "max_precision": 0.828351, + "max_recall": 0.920803, + "max_ap": 0.911253, + "main_score": 0.855983, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 1.3805902004241943, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/OnlineShopping.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/OnlineShopping.json new file mode 100644 index 0000000000..8250e45aed --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/OnlineShopping.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e610f2ebd179a8fda30ae534c3878750a96db120", + "task_name": "OnlineShopping", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "accuracy": 0.9501, + "f1": 0.950009, + "f1_weighted": 0.95011, + "ap": 0.934296, + "ap_weighted": 0.934296, + "scores_per_experiment": [ + { + "accuracy": 0.953, + "f1": 0.952896, + "f1_weighted": 0.953002, + "ap": 0.936236, + "ap_weighted": 0.936236 + }, + { + "accuracy": 0.95, + "f1": 0.94992, + "f1_weighted": 0.950016, + "ap": 0.935211, + "ap_weighted": 0.935211 + }, + { + "accuracy": 0.951, + "f1": 0.950892, + "f1_weighted": 0.951002, + "ap": 0.93359, + "ap_weighted": 0.93359 + }, + { + "accuracy": 0.95, + "f1": 0.949885, + "f1_weighted": 0.95, + "ap": 0.931856, + "ap_weighted": 0.931856 + }, + { + "accuracy": 0.951, + "f1": 0.950901, + "f1_weighted": 0.951007, + "ap": 0.934426, + "ap_weighted": 0.934426 + }, + { + "accuracy": 0.945, + "f1": 0.944954, + "f1_weighted": 0.94503, + "ap": 0.933263, + "ap_weighted": 0.933263 + }, + { + "accuracy": 0.951, + "f1": 0.950909, + "f1_weighted": 0.951011, + "ap": 0.935268, + "ap_weighted": 0.935268 + }, + { + "accuracy": 0.946, + "f1": 0.94593, + "f1_weighted": 0.946023, + "ap": 0.93159, + "ap_weighted": 0.93159 + }, + { + "accuracy": 0.951, + "f1": 0.950892, + "f1_weighted": 0.951002, + "ap": 0.93359, + "ap_weighted": 0.93359 + }, + { + "accuracy": 0.953, + "f1": 0.952913, + "f1_weighted": 0.95301, + "ap": 0.937927, + "ap_weighted": 0.937927 + } + ], + "main_score": 0.9501, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 10.539811849594116, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/PAWSX.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/PAWSX.json new file mode 100644 index 0000000000..53a5f076e3 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/PAWSX.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1", + "task_name": "PAWSX", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "pearson": 0.402692, + "spearman": 0.47026, + "cosine_pearson": 0.402692, + "cosine_spearman": 0.470185, + "manhattan_pearson": 0.46214, + "manhattan_spearman": 0.470334, + "euclidean_pearson": 0.461835, + "euclidean_spearman": 0.470186, + "main_score": 0.470185, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 3.0655324459075928, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/QBQTC.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/QBQTC.json new file mode 100644 index 0000000000..37ba3091cc --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/QBQTC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7", + "task_name": "QBQTC", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "pearson": 0.43306, + "spearman": 0.452136, + "cosine_pearson": 0.43306, + "cosine_spearman": 0.452147, + "manhattan_pearson": 0.407161, + "manhattan_spearman": 0.452341, + "euclidean_pearson": 0.407345, + "euclidean_spearman": 0.452158, + "main_score": 0.452147, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 4.067687034606934, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/STSB.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/STSB.json new file mode 100644 index 0000000000..8f3b58f732 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/STSB.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0", + "task_name": "STSB", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "pearson": 0.795757, + "spearman": 0.814339, + "cosine_pearson": 0.795757, + "cosine_spearman": 0.814333, + "manhattan_pearson": 0.811804, + "manhattan_spearman": 0.814181, + "euclidean_pearson": 0.812075, + "euclidean_spearman": 0.814333, + "main_score": 0.814333, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 1.309311866760254, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/T2Reranking.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/T2Reranking.json new file mode 100644 index 0000000000..24b1d143ed --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/T2Reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "76631901a18387f85eaa53e5450019b87ad58ef9", + "task_name": "T2Reranking", + "mteb_version": "1.36.22", + "scores": { + "dev": [ + { + "map": 0.684737, + "mrr": 0.794401, + "nAUC_map_max": 0.406312, + "nAUC_map_std": -0.047851, + "nAUC_map_diff1": -0.133316, + "nAUC_mrr_max": 0.336414, + "nAUC_mrr_std": -0.061487, + "nAUC_mrr_diff1": -0.139707, + "main_score": 0.684737, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 532.4816951751709, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/T2Retrieval.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/T2Retrieval.json new file mode 100644 index 0000000000..d35fd10faf --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/T2Retrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "8731a845f1bf500a4f111cf1070785c793d10e64", + "task_name": "T2Retrieval", + "mteb_version": "1.36.22", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.88966, + "ndcg_at_3": 0.85316, + "ndcg_at_5": 0.84069, + "ndcg_at_10": 0.84296, + "ndcg_at_20": 0.86145, + "ndcg_at_100": 0.87973, + "ndcg_at_1000": 0.88575, + "map_at_1": 0.27041, + "map_at_3": 0.53484, + "map_at_5": 0.65888, + "map_at_10": 0.765, + "map_at_20": 0.7932, + "map_at_100": 0.80193, + "map_at_1000": 0.80257, + "recall_at_1": 0.27041, + "recall_at_3": 0.5545, + "recall_at_5": 0.6977, + "recall_at_10": 0.83772, + "recall_at_20": 0.89904, + "recall_at_100": 0.95659, + "recall_at_1000": 0.98658, + "precision_at_1": 0.88966, + "precision_at_3": 0.74664, + "precision_at_5": 0.62729, + "precision_at_10": 0.42032, + "precision_at_20": 0.23384, + "precision_at_100": 0.05021, + "precision_at_1000": 0.00516, + "mrr_at_1": 0.889663, + "mrr_at_3": 0.915001, + "mrr_at_5": 0.918063, + "mrr_at_10": 0.919711, + "mrr_at_20": 0.920359, + "mrr_at_100": 0.92065, + "mrr_at_1000": 0.920678, + "nauc_ndcg_at_1_max": 0.644584, + "nauc_ndcg_at_1_std": 0.250922, + "nauc_ndcg_at_1_diff1": 0.454055, + "nauc_ndcg_at_3_max": 0.544517, + "nauc_ndcg_at_3_std": 0.221015, + "nauc_ndcg_at_3_diff1": 0.143287, + "nauc_ndcg_at_5_max": 0.467448, + "nauc_ndcg_at_5_std": 0.177236, + "nauc_ndcg_at_5_diff1": 0.145076, + "nauc_ndcg_at_10_max": 0.373639, + "nauc_ndcg_at_10_std": 0.129002, + "nauc_ndcg_at_10_diff1": 0.187465, + "nauc_ndcg_at_20_max": 0.422509, + "nauc_ndcg_at_20_std": 0.180408, + "nauc_ndcg_at_20_diff1": 0.187429, + "nauc_ndcg_at_100_max": 0.483249, + "nauc_ndcg_at_100_std": 0.236967, + "nauc_ndcg_at_100_diff1": 0.183893, + "nauc_ndcg_at_1000_max": 0.49459, + "nauc_ndcg_at_1000_std": 0.234218, + "nauc_ndcg_at_1000_diff1": 0.18851, + "nauc_map_at_1_max": -0.288345, + "nauc_map_at_1_std": -0.297778, + "nauc_map_at_1_diff1": 0.503666, + "nauc_map_at_3_max": -0.19414, + "nauc_map_at_3_std": -0.280129, + "nauc_map_at_3_diff1": 0.376607, + "nauc_map_at_5_max": -0.048381, + "nauc_map_at_5_std": -0.200334, + "nauc_map_at_5_diff1": 0.307093, + "nauc_map_at_10_max": 0.229911, + "nauc_map_at_10_std": 0.004674, + "nauc_map_at_10_diff1": 0.196475, + "nauc_map_at_20_max": 0.335502, + "nauc_map_at_20_std": 0.10248, + "nauc_map_at_20_diff1": 0.15821, + "nauc_map_at_100_max": 0.363463, + "nauc_map_at_100_std": 0.131414, + "nauc_map_at_100_diff1": 0.150916, + "nauc_map_at_1000_max": 0.364622, + "nauc_map_at_1000_std": 0.131566, + "nauc_map_at_1000_diff1": 0.15091, + "nauc_recall_at_1_max": -0.288345, + "nauc_recall_at_1_std": -0.297778, + "nauc_recall_at_1_diff1": 0.503666, + "nauc_recall_at_3_max": -0.243883, + "nauc_recall_at_3_std": -0.305378, + "nauc_recall_at_3_diff1": 0.357697, + "nauc_recall_at_5_max": -0.145869, + "nauc_recall_at_5_std": -0.252552, + "nauc_recall_at_5_diff1": 0.291284, + "nauc_recall_at_10_max": 0.102802, + "nauc_recall_at_10_std": -0.045993, + "nauc_recall_at_10_diff1": 0.179432, + "nauc_recall_at_20_max": 0.259696, + "nauc_recall_at_20_std": 0.150777, + "nauc_recall_at_20_diff1": 0.117769, + "nauc_recall_at_100_max": 0.405347, + "nauc_recall_at_100_std": 0.424529, + "nauc_recall_at_100_diff1": 0.0551, + "nauc_recall_at_1000_max": 0.533139, + "nauc_recall_at_1000_std": 0.622781, + "nauc_recall_at_1000_diff1": 0.025967, + "nauc_precision_at_1_max": 0.644584, + "nauc_precision_at_1_std": 0.250922, + "nauc_precision_at_1_diff1": 0.454055, + "nauc_precision_at_3_max": 0.614326, + "nauc_precision_at_3_std": 0.37097, + "nauc_precision_at_3_diff1": -0.264151, + "nauc_precision_at_5_max": 0.586718, + "nauc_precision_at_5_std": 0.402579, + "nauc_precision_at_5_diff1": -0.314651, + "nauc_precision_at_10_max": 0.558234, + "nauc_precision_at_10_std": 0.439313, + "nauc_precision_at_10_diff1": -0.322221, + "nauc_precision_at_20_max": 0.551797, + "nauc_precision_at_20_std": 0.471787, + "nauc_precision_at_20_diff1": -0.329316, + "nauc_precision_at_100_max": 0.540718, + "nauc_precision_at_100_std": 0.485557, + "nauc_precision_at_100_diff1": -0.334663, + "nauc_precision_at_1000_max": 0.530231, + "nauc_precision_at_1000_std": 0.471836, + "nauc_precision_at_1000_diff1": -0.335506, + "nauc_mrr_at_1_max": 0.644584, + "nauc_mrr_at_1_std": 0.250922, + "nauc_mrr_at_1_diff1": 0.454055, + "nauc_mrr_at_3_max": 0.684759, + "nauc_mrr_at_3_std": 0.291145, + "nauc_mrr_at_3_diff1": 0.453657, + "nauc_mrr_at_5_max": 0.685432, + "nauc_mrr_at_5_std": 0.293503, + "nauc_mrr_at_5_diff1": 0.452113, + "nauc_mrr_at_10_max": 0.68478, + "nauc_mrr_at_10_std": 0.295505, + "nauc_mrr_at_10_diff1": 0.452087, + "nauc_mrr_at_20_max": 0.684012, + "nauc_mrr_at_20_std": 0.295899, + "nauc_mrr_at_20_diff1": 0.452423, + "nauc_mrr_at_100_max": 0.683395, + "nauc_mrr_at_100_std": 0.295126, + "nauc_mrr_at_100_diff1": 0.452215, + "nauc_mrr_at_1000_max": 0.683334, + "nauc_mrr_at_1000_std": 0.29498, + "nauc_mrr_at_1000_diff1": 0.452233, + "main_score": 0.84296, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 746.6982464790344, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/TNews.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/TNews.json new file mode 100644 index 0000000000..e11cef4bad --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/TNews.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "317f262bf1e6126357bbe89e875451e4b0938fe4", + "task_name": "TNews", + "mteb_version": "1.36.22", + "scores": { + "validation": [ + { + "accuracy": 0.55017, + "f1": 0.530463, + "f1_weighted": 0.547708, + "scores_per_experiment": [ + { + "accuracy": 0.5533, + "f1": 0.534337, + "f1_weighted": 0.552578 + }, + { + "accuracy": 0.549, + "f1": 0.528974, + "f1_weighted": 0.545319 + }, + { + "accuracy": 0.5514, + "f1": 0.532257, + "f1_weighted": 0.549528 + }, + { + "accuracy": 0.5477, + "f1": 0.530483, + "f1_weighted": 0.545435 + }, + { + "accuracy": 0.5503, + "f1": 0.531516, + "f1_weighted": 0.550739 + }, + { + "accuracy": 0.5563, + "f1": 0.536944, + "f1_weighted": 0.553634 + }, + { + "accuracy": 0.552, + "f1": 0.530962, + "f1_weighted": 0.548865 + }, + { + "accuracy": 0.5471, + "f1": 0.522916, + "f1_weighted": 0.539316 + }, + { + "accuracy": 0.5429, + "f1": 0.522535, + "f1_weighted": 0.538701 + }, + { + "accuracy": 0.5517, + "f1": 0.533702, + "f1_weighted": 0.552968 + } + ], + "main_score": 0.55017, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 25.658607721328735, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/ThuNewsClusteringP2P.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/ThuNewsClusteringP2P.json new file mode 100644 index 0000000000..22424afdd6 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/ThuNewsClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "5798586b105c0434e4f0fe5e767abe619442cf93", + "task_name": "ThuNewsClusteringP2P", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "v_measure": 0.796852, + "v_measure_std": 0.021688, + "v_measures": [ + 0.821764, + 0.783583, + 0.793552, + 0.812299, + 0.7941, + 0.770864, + 0.778892, + 0.765019, + 0.832595, + 0.815852 + ], + "main_score": 0.796852, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 172.49744844436646, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/ThuNewsClusteringS2S.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/ThuNewsClusteringS2S.json new file mode 100644 index 0000000000..3e010e3cf6 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/ThuNewsClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d", + "task_name": "ThuNewsClusteringS2S", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "v_measure": 0.735795, + "v_measure_std": 0.023915, + "v_measures": [ + 0.733366, + 0.741138, + 0.769973, + 0.731378, + 0.749275, + 0.735144, + 0.736472, + 0.719811, + 0.763139, + 0.678251 + ], + "main_score": 0.735795, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 44.09478139877319, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/VideoRetrieval.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/VideoRetrieval.json new file mode 100644 index 0000000000..a107c081f2 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/VideoRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "58c2597a5943a2ba48f4668c3b90d796283c5639", + "task_name": "VideoRetrieval", + "mteb_version": "1.36.22", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.651, + "ndcg_at_3": 0.76715, + "ndcg_at_5": 0.79005, + "ndcg_at_10": 0.80537, + "ndcg_at_20": 0.81274, + "ndcg_at_100": 0.81672, + "ndcg_at_1000": 0.81672, + "map_at_1": 0.651, + "map_at_3": 0.73883, + "map_at_5": 0.75168, + "map_at_10": 0.75808, + "map_at_20": 0.76011, + "map_at_100": 0.76078, + "map_at_1000": 0.76078, + "recall_at_1": 0.651, + "recall_at_3": 0.849, + "recall_at_5": 0.904, + "recall_at_10": 0.951, + "recall_at_20": 0.98, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.651, + "precision_at_3": 0.283, + "precision_at_5": 0.1808, + "precision_at_10": 0.0951, + "precision_at_20": 0.049, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "mrr_at_1": 0.651, + "mrr_at_3": 0.738833, + "mrr_at_5": 0.751683, + "mrr_at_10": 0.758077, + "mrr_at_20": 0.760111, + "mrr_at_100": 0.760777, + "mrr_at_1000": 0.760777, + "nauc_ndcg_at_1_max": 0.111385, + "nauc_ndcg_at_1_std": -0.440385, + "nauc_ndcg_at_1_diff1": 0.666495, + "nauc_ndcg_at_3_max": 0.08357, + "nauc_ndcg_at_3_std": -0.553195, + "nauc_ndcg_at_3_diff1": 0.648222, + "nauc_ndcg_at_5_max": 0.073946, + "nauc_ndcg_at_5_std": -0.559338, + "nauc_ndcg_at_5_diff1": 0.645316, + "nauc_ndcg_at_10_max": 0.069244, + "nauc_ndcg_at_10_std": -0.558312, + "nauc_ndcg_at_10_diff1": 0.648685, + "nauc_ndcg_at_20_max": 0.075823, + "nauc_ndcg_at_20_std": -0.541269, + "nauc_ndcg_at_20_diff1": 0.649182, + "nauc_ndcg_at_100_max": 0.082242, + "nauc_ndcg_at_100_std": -0.526173, + "nauc_ndcg_at_100_diff1": 0.652121, + "nauc_ndcg_at_1000_max": 0.082242, + "nauc_ndcg_at_1000_std": -0.526173, + "nauc_ndcg_at_1000_diff1": 0.652121, + "nauc_map_at_1_max": 0.111385, + "nauc_map_at_1_std": -0.440385, + "nauc_map_at_1_diff1": 0.666495, + "nauc_map_at_3_max": 0.090268, + "nauc_map_at_3_std": -0.519661, + "nauc_map_at_3_diff1": 0.652961, + "nauc_map_at_5_max": 0.085951, + "nauc_map_at_5_std": -0.520856, + "nauc_map_at_5_diff1": 0.651929, + "nauc_map_at_10_max": 0.084666, + "nauc_map_at_10_std": -0.519602, + "nauc_map_at_10_diff1": 0.653572, + "nauc_map_at_20_max": 0.086166, + "nauc_map_at_20_std": -0.51573, + "nauc_map_at_20_diff1": 0.653829, + "nauc_map_at_100_max": 0.086884, + "nauc_map_at_100_std": -0.513818, + "nauc_map_at_100_diff1": 0.654108, + "nauc_map_at_1000_max": 0.086884, + "nauc_map_at_1000_std": -0.513818, + "nauc_map_at_1000_diff1": 0.654108, + "nauc_recall_at_1_max": 0.111385, + "nauc_recall_at_1_std": -0.440385, + "nauc_recall_at_1_diff1": 0.666495, + "nauc_recall_at_3_max": 0.054683, + "nauc_recall_at_3_std": -0.703036, + "nauc_recall_at_3_diff1": 0.627503, + "nauc_recall_at_5_max": -0.004002, + "nauc_recall_at_5_std": -0.806786, + "nauc_recall_at_5_diff1": 0.602572, + "nauc_recall_at_10_max": -0.126717, + "nauc_recall_at_10_std": -1.02703, + "nauc_recall_at_10_diff1": 0.59097, + "nauc_recall_at_20_max": -0.196289, + "nauc_recall_at_20_std": -1.149463, + "nauc_recall_at_20_diff1": 0.515149, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.111385, + "nauc_precision_at_1_std": -0.440385, + "nauc_precision_at_1_diff1": 0.666495, + "nauc_precision_at_3_max": 0.054683, + "nauc_precision_at_3_std": -0.703036, + "nauc_precision_at_3_diff1": 0.627503, + "nauc_precision_at_5_max": -0.004002, + "nauc_precision_at_5_std": -0.806786, + "nauc_precision_at_5_diff1": 0.602572, + "nauc_precision_at_10_max": -0.126717, + "nauc_precision_at_10_std": -1.02703, + "nauc_precision_at_10_diff1": 0.59097, + "nauc_precision_at_20_max": -0.196289, + "nauc_precision_at_20_std": -1.149463, + "nauc_precision_at_20_diff1": 0.515149, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_mrr_at_1_max": 0.111385, + "nauc_mrr_at_1_std": -0.440385, + "nauc_mrr_at_1_diff1": 0.666495, + "nauc_mrr_at_3_max": 0.090268, + "nauc_mrr_at_3_std": -0.519661, + "nauc_mrr_at_3_diff1": 0.652961, + "nauc_mrr_at_5_max": 0.085951, + "nauc_mrr_at_5_std": -0.520856, + "nauc_mrr_at_5_diff1": 0.651929, + "nauc_mrr_at_10_max": 0.084666, + "nauc_mrr_at_10_std": -0.519602, + "nauc_mrr_at_10_diff1": 0.653572, + "nauc_mrr_at_20_max": 0.086166, + "nauc_mrr_at_20_std": -0.51573, + "nauc_mrr_at_20_diff1": 0.653829, + "nauc_mrr_at_100_max": 0.086884, + "nauc_mrr_at_100_std": -0.513818, + "nauc_mrr_at_100_diff1": 0.654108, + "nauc_mrr_at_1000_max": 0.086884, + "nauc_mrr_at_1000_std": -0.513818, + "nauc_mrr_at_1000_diff1": 0.654108, + "main_score": 0.80537, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 56.61172938346863, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/Waimai.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/Waimai.json new file mode 100644 index 0000000000..8383774907 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/Waimai.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "339287def212450dcaa9df8c22bf93e9980c7023", + "task_name": "Waimai", + "mteb_version": "1.36.22", + "scores": { + "test": [ + { + "accuracy": 0.8974, + "f1": 0.883753, + "f1_weighted": 0.897688, + "ap": 0.760058, + "ap_weighted": 0.760058, + "scores_per_experiment": [ + { + "accuracy": 0.901, + "f1": 0.887625, + "f1_weighted": 0.901194, + "ap": 0.767597, + "ap_weighted": 0.767597 + }, + { + "accuracy": 0.896, + "f1": 0.8811, + "f1_weighted": 0.895832, + "ap": 0.757964, + "ap_weighted": 0.757964 + }, + { + "accuracy": 0.899, + "f1": 0.88606, + "f1_weighted": 0.899499, + "ap": 0.762807, + "ap_weighted": 0.762807 + }, + { + "accuracy": 0.899, + "f1": 0.886232, + "f1_weighted": 0.899572, + "ap": 0.762674, + "ap_weighted": 0.762674 + }, + { + "accuracy": 0.89, + "f1": 0.877275, + "f1_weighted": 0.891106, + "ap": 0.744038, + "ap_weighted": 0.744038 + }, + { + "accuracy": 0.901, + "f1": 0.887089, + "f1_weighted": 0.90096, + "ap": 0.768186, + "ap_weighted": 0.768186 + }, + { + "accuracy": 0.899, + "f1": 0.884992, + "f1_weighted": 0.89904, + "ap": 0.76378, + "ap_weighted": 0.76378 + }, + { + "accuracy": 0.897, + "f1": 0.8829, + "f1_weighted": 0.897122, + "ap": 0.759428, + "ap_weighted": 0.759428 + }, + { + "accuracy": 0.897, + "f1": 0.883626, + "f1_weighted": 0.897434, + "ap": 0.75883, + "ap_weighted": 0.75883 + }, + { + "accuracy": 0.895, + "f1": 0.880626, + "f1_weighted": 0.895124, + "ap": 0.755278, + "ap_weighted": 0.755278 + } + ], + "main_score": 0.8974, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 8.132053852081299, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/model_meta.json b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/model_meta.json new file mode 100644 index 0000000000..ee806881c5 --- /dev/null +++ b/results/richinfoai__ritrieve_zh_v1/f8d5a707656c55705027678e311f9202c8ced12c/model_meta.json @@ -0,0 +1 @@ +{"name": "richinfoai/ritrieve_zh_v1", "revision": "f8d5a707656c55705027678e311f9202c8ced12c", "release_date": "2025-03-25", "languages": ["zho_Hans"], "n_parameters": 326000000, "memory_usage_mb": 1242, "max_tokens": 512, "embed_dim": 1792, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/richinfoai/ritrieve_zh_v1", "similarity_fn_name": "cosine", "use_instructions": false, "training_datasets": {"BQ": [], "LCQMC": [], "PAWSX": [], "STS-B": [], "DuRetrieval": ["train"], "AFQMC": [], "Cmnli": [], "Ocnli": [], "HotpotQA": ["train"], "HotpotQA-NL": ["train"], "FEVER": ["train"], "FEVER-NL": ["train"], "MSMARCO": ["train"], "mMARCO-NL": ["train"], "NQ": ["train"], "NQ-NL": ["train"], "ArguAna": ["train"], "ArguAna-NL": ["train"], "FiQA2018": ["train"], "FiQA2018-NL": ["train"], "SciDocsReranking": ["train"], "StackOverflowDupQuestions": ["train"], "AmazonReviewsClassification": ["train"], "AmazonCounterfactualClassification": ["train"], "Banking77Classification": ["train"], "EmotionClassification": ["train"], "TweetSentimentExtractionClassification": ["train"], "MTOPIntentClassification": ["train"], "ImdbClassification": ["train"], "ToxicConversationsClassification": ["train"], "ArxivClusteringS2S": ["train"], "ArxivClusteringP2P": ["train"], "BiorxivClusteringS2S": ["train"], "BiorxivClusteringP2P": ["train"], "MedrxivClusteringS2S": ["train"], "MedrxivClusteringP2P": ["train"], "BiorxivClusteringS2S.v2": ["train"], "BiorxivClusteringP2P.v2": ["train"], "MedrxivClusteringS2S.v2": ["train"], "MedrxivClusteringP2P.v2": ["train"], "RedditClusteringP2P": ["train"], "RedditClustering": ["train"], "RedditClustering.v2": ["train"], "TwentyNewsgroupsClustering": ["train"], "TwentyNewsgroupsClustering.v2": ["train"], "STS22": ["train"], "STS22.v2": ["train"], "STSBenchmark": ["train"], "MIRACLRetrieval": ["train"], "MIRACLRetrievalHardNegatives": ["train"], "MIRACLReranking": ["train"], "LeCaRDv2": ["train"], "CMedQAv1-reranking": ["train"], "CMedQAv2-reranking": ["train"], "MrTidyRetrieval": ["train"], "T2Reranking": ["train"], "MSMARCOHardNegatives": ["train"], "NanoMSMARCORetrieval": ["train"], "MSMARCO-PL": ["train"], "NQHardNegatives": ["train"], "NanoNQRetrieval": ["train"], "NQ-PL": ["train"], "HotpotQA-PL": ["train"], "HotpotQAHardNegatives": ["train"], "T2Retrieval": ["train"], "MMarcoReranking": ["train"], "CodeSearchNet": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": false, "modalities": ["text"], "loader": null} \ No newline at end of file