From 0b66571786162ffb147d313b2d24acd48a82fd78 Mon Sep 17 00:00:00 2001 From: springxchen Date: Tue, 30 Sep 2025 23:54:33 +0800 Subject: [PATCH] update results for youtu embedding model --- .../AFQMC.json | 26 +++ .../ATEC.json | 26 +++ .../BQ.json | 26 +++ .../CLSClusteringP2P.json | 32 ++++ .../CLSClusteringS2S.json | 32 ++++ .../CMedQAv1-reranking.json | 26 +++ .../CMedQAv2-reranking.json | 26 +++ .../CmedqaRetrieval.json | 158 ++++++++++++++++++ .../Cmnli.json | 58 +++++++ .../CovidRetrieval.json | 158 ++++++++++++++++++ .../DuRetrieval.json | 158 ++++++++++++++++++ .../EcomRetrieval.json | 158 ++++++++++++++++++ .../IFlyTek.json | 48 ++++++ .../JDReview.json | 95 +++++++++++ .../LCQMC.json | 26 +++ .../MMarcoReranking.json | 26 +++ .../MMarcoRetrieval.json | 158 ++++++++++++++++++ .../MedicalRetrieval.json | 158 ++++++++++++++++++ .../MultilingualSentiment.json | 137 +++++++++++++++ .../Ocnli.json | 58 +++++++ .../OnlineShopping.json | 95 +++++++++++ .../PAWSX.json | 26 +++ .../QBQTC.json | 26 +++ .../STSB.json | 26 +++ .../T2Reranking.json | 26 +++ .../T2Retrieval.json | 158 ++++++++++++++++++ .../TNews.json | 73 ++++++++ .../ThuNewsClusteringP2P.json | 32 ++++ .../ThuNewsClusteringS2S.json | 32 ++++ .../VideoRetrieval.json | 158 ++++++++++++++++++ .../Waimai.json | 95 +++++++++++ .../model_meta.json | 1 + 32 files changed, 2338 insertions(+) create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/AFQMC.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/ATEC.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/BQ.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CLSClusteringP2P.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CLSClusteringS2S.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CMedQAv1-reranking.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CMedQAv2-reranking.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CmedqaRetrieval.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/Cmnli.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CovidRetrieval.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/DuRetrieval.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/EcomRetrieval.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/IFlyTek.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/JDReview.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/LCQMC.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MMarcoReranking.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MMarcoRetrieval.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MedicalRetrieval.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MultilingualSentiment.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/Ocnli.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/OnlineShopping.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/PAWSX.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/QBQTC.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/STSB.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/T2Reranking.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/T2Retrieval.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/TNews.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/ThuNewsClusteringP2P.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/ThuNewsClusteringS2S.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/VideoRetrieval.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/Waimai.json create mode 100644 results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/model_meta.json diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/AFQMC.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/AFQMC.json new file mode 100644 index 0000000000..3099ff1d4f --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/AFQMC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "b44c3b011063adb25877c13823db83bb193913c4", + "task_name": "AFQMC", + "mteb_version": "1.38.30", + "scores": { + "validation": [ + { + "pearson": 0.795688, + "spearman": 0.721944, + "cosine_pearson": 0.795688, + "cosine_spearman": 0.721944, + "manhattan_pearson": 0.797082, + "manhattan_spearman": 0.722174, + "euclidean_pearson": 0.797302, + "euclidean_spearman": 0.721944, + "main_score": 0.721944, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 34.21637845039368, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/ATEC.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/ATEC.json new file mode 100644 index 0000000000..af706ed09c --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/ATEC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865", + "task_name": "ATEC", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "pearson": 0.75786, + "spearman": 0.61704, + "cosine_pearson": 0.75786, + "cosine_spearman": 0.61704, + "manhattan_pearson": 0.778669, + "manhattan_spearman": 0.617521, + "euclidean_pearson": 0.779169, + "euclidean_spearman": 0.61704, + "main_score": 0.61704, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 152.8086531162262, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/BQ.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/BQ.json new file mode 100644 index 0000000000..865c253e12 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/BQ.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55", + "task_name": "BQ", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "pearson": 0.739231, + "spearman": 0.722694, + "cosine_pearson": 0.739231, + "cosine_spearman": 0.722694, + "manhattan_pearson": 0.728824, + "manhattan_spearman": 0.723689, + "euclidean_pearson": 0.728151, + "euclidean_spearman": 0.722694, + "main_score": 0.722694, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 68.2534830570221, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CLSClusteringP2P.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CLSClusteringP2P.json new file mode 100644 index 0000000000..27a0a9569b --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CLSClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476", + "task_name": "CLSClusteringP2P", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "v_measure": 0.815311, + "v_measure_std": 0.027662, + "v_measures": [ + 0.804815, + 0.872865, + 0.814021, + 0.829766, + 0.84843, + 0.791818, + 0.808065, + 0.777552, + 0.81997, + 0.785813 + ], + "main_score": 0.815311, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 2388.292834997177, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CLSClusteringS2S.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CLSClusteringS2S.json new file mode 100644 index 0000000000..0ed01e58d4 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CLSClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f", + "task_name": "CLSClusteringS2S", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "v_measure": 0.762722, + "v_measure_std": 0.026018, + "v_measures": [ + 0.735073, + 0.801598, + 0.776533, + 0.791576, + 0.767064, + 0.761668, + 0.753993, + 0.770871, + 0.76363, + 0.705216 + ], + "main_score": 0.762722, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 448.83500051498413, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CMedQAv1-reranking.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CMedQAv1-reranking.json new file mode 100644 index 0000000000..152cd0c3d4 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CMedQAv1-reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "8d7f1e942507dac42dc58017c1a001c3717da7df", + "task_name": "CMedQAv1-reranking", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "map": 0.910863, + "mrr": 0.9271, + "nAUC_map_max": 0.764415, + "nAUC_map_std": 0.516272, + "nAUC_map_diff1": 0.672109, + "nAUC_mrr_max": 0.871521, + "nAUC_mrr_std": 0.652652, + "nAUC_mrr_diff1": 0.75522, + "main_score": 0.910863, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 705.2105937004089, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CMedQAv2-reranking.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CMedQAv2-reranking.json new file mode 100644 index 0000000000..1b7fc4e11d --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CMedQAv2-reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "23d186750531a14a0357ca22cd92d712fd512ea0", + "task_name": "CMedQAv2-reranking", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "map": 0.92563, + "mrr": 0.941517, + "nAUC_map_max": 0.72775, + "nAUC_map_std": 0.418286, + "nAUC_map_diff1": 0.647313, + "nAUC_mrr_max": 0.871088, + "nAUC_mrr_std": 0.599795, + "nAUC_mrr_diff1": 0.755411, + "main_score": 0.92563, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 786.7573840618134, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CmedqaRetrieval.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CmedqaRetrieval.json new file mode 100644 index 0000000000..acff10f1f6 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CmedqaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301", + "task_name": "CmedqaRetrieval", + "mteb_version": "1.38.30", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.46412, + "ndcg_at_3": 0.47453, + "ndcg_at_5": 0.49669, + "ndcg_at_10": 0.52721, + "ndcg_at_20": 0.5523, + "ndcg_at_100": 0.58809, + "ndcg_at_1000": 0.60154, + "map_at_1": 0.31954, + "map_at_3": 0.41696, + "map_at_5": 0.44141, + "map_at_10": 0.46042, + "map_at_20": 0.46961, + "map_at_100": 0.47687, + "map_at_1000": 0.4777, + "recall_at_1": 0.31954, + "recall_at_3": 0.47859, + "recall_at_5": 0.54581, + "recall_at_10": 0.63526, + "recall_at_20": 0.71995, + "recall_at_100": 0.87937, + "recall_at_1000": 0.96705, + "precision_at_1": 0.46412, + "precision_at_3": 0.25806, + "precision_at_5": 0.1839, + "precision_at_10": 0.11055, + "precision_at_20": 0.06378, + "precision_at_100": 0.01616, + "precision_at_1000": 0.0018, + "mrr_at_1": 0.464116, + "mrr_at_3": 0.524756, + "mrr_at_5": 0.53826, + "mrr_at_10": 0.548501, + "mrr_at_20": 0.553803, + "mrr_at_100": 0.556773, + "mrr_at_1000": 0.557046, + "nauc_ndcg_at_1_max": 0.653535, + "nauc_ndcg_at_1_std": 0.095219, + "nauc_ndcg_at_1_diff1": 0.637725, + "nauc_ndcg_at_3_max": 0.590207, + "nauc_ndcg_at_3_std": 0.07966, + "nauc_ndcg_at_3_diff1": 0.533371, + "nauc_ndcg_at_5_max": 0.58684, + "nauc_ndcg_at_5_std": 0.074627, + "nauc_ndcg_at_5_diff1": 0.528752, + "nauc_ndcg_at_10_max": 0.591587, + "nauc_ndcg_at_10_std": 0.085935, + "nauc_ndcg_at_10_diff1": 0.521729, + "nauc_ndcg_at_20_max": 0.601685, + "nauc_ndcg_at_20_std": 0.100926, + "nauc_ndcg_at_20_diff1": 0.525555, + "nauc_ndcg_at_100_max": 0.610371, + "nauc_ndcg_at_100_std": 0.120618, + "nauc_ndcg_at_100_diff1": 0.532211, + "nauc_ndcg_at_1000_max": 0.609798, + "nauc_ndcg_at_1000_std": 0.116226, + "nauc_ndcg_at_1000_diff1": 0.537063, + "nauc_map_at_1_max": 0.445556, + "nauc_map_at_1_std": -0.007292, + "nauc_map_at_1_diff1": 0.541773, + "nauc_map_at_3_max": 0.529008, + "nauc_map_at_3_std": 0.03096, + "nauc_map_at_3_diff1": 0.52064, + "nauc_map_at_5_max": 0.551295, + "nauc_map_at_5_std": 0.043768, + "nauc_map_at_5_diff1": 0.522508, + "nauc_map_at_10_max": 0.565997, + "nauc_map_at_10_std": 0.059126, + "nauc_map_at_10_diff1": 0.522434, + "nauc_map_at_20_max": 0.572552, + "nauc_map_at_20_std": 0.067557, + "nauc_map_at_20_diff1": 0.524159, + "nauc_map_at_100_max": 0.575852, + "nauc_map_at_100_std": 0.073261, + "nauc_map_at_100_diff1": 0.525612, + "nauc_map_at_1000_max": 0.575928, + "nauc_map_at_1000_std": 0.073394, + "nauc_map_at_1000_diff1": 0.525819, + "nauc_recall_at_1_max": 0.445556, + "nauc_recall_at_1_std": -0.007292, + "nauc_recall_at_1_diff1": 0.541773, + "nauc_recall_at_3_max": 0.486915, + "nauc_recall_at_3_std": 0.029079, + "nauc_recall_at_3_diff1": 0.453125, + "nauc_recall_at_5_max": 0.505628, + "nauc_recall_at_5_std": 0.046104, + "nauc_recall_at_5_diff1": 0.439616, + "nauc_recall_at_10_max": 0.517318, + "nauc_recall_at_10_std": 0.090842, + "nauc_recall_at_10_diff1": 0.40147, + "nauc_recall_at_20_max": 0.533289, + "nauc_recall_at_20_std": 0.136432, + "nauc_recall_at_20_diff1": 0.391746, + "nauc_recall_at_100_max": 0.561139, + "nauc_recall_at_100_std": 0.317106, + "nauc_recall_at_100_diff1": 0.358697, + "nauc_recall_at_1000_max": 0.554289, + "nauc_recall_at_1000_std": 0.481672, + "nauc_recall_at_1000_diff1": 0.346357, + "nauc_precision_at_1_max": 0.653535, + "nauc_precision_at_1_std": 0.095219, + "nauc_precision_at_1_diff1": 0.637725, + "nauc_precision_at_3_max": 0.579913, + "nauc_precision_at_3_std": 0.169911, + "nauc_precision_at_3_diff1": 0.362983, + "nauc_precision_at_5_max": 0.545295, + "nauc_precision_at_5_std": 0.197983, + "nauc_precision_at_5_diff1": 0.29292, + "nauc_precision_at_10_max": 0.496932, + "nauc_precision_at_10_std": 0.243733, + "nauc_precision_at_10_diff1": 0.217069, + "nauc_precision_at_20_max": 0.450093, + "nauc_precision_at_20_std": 0.285951, + "nauc_precision_at_20_diff1": 0.162043, + "nauc_precision_at_100_max": 0.313899, + "nauc_precision_at_100_std": 0.321259, + "nauc_precision_at_100_diff1": 0.056807, + "nauc_precision_at_1000_max": 0.222553, + "nauc_precision_at_1000_std": 0.284269, + "nauc_precision_at_1000_diff1": 0.005578, + "nauc_mrr_at_1_max": 0.653535, + "nauc_mrr_at_1_std": 0.095219, + "nauc_mrr_at_1_diff1": 0.637725, + "nauc_mrr_at_3_max": 0.651084, + "nauc_mrr_at_3_std": 0.108722, + "nauc_mrr_at_3_diff1": 0.600569, + "nauc_mrr_at_5_max": 0.652678, + "nauc_mrr_at_5_std": 0.112394, + "nauc_mrr_at_5_diff1": 0.599185, + "nauc_mrr_at_10_max": 0.653072, + "nauc_mrr_at_10_std": 0.116566, + "nauc_mrr_at_10_diff1": 0.596148, + "nauc_mrr_at_20_max": 0.654001, + "nauc_mrr_at_20_std": 0.117851, + "nauc_mrr_at_20_diff1": 0.597414, + "nauc_mrr_at_100_max": 0.65385, + "nauc_mrr_at_100_std": 0.117867, + "nauc_mrr_at_100_diff1": 0.597877, + "nauc_mrr_at_1000_max": 0.653845, + "nauc_mrr_at_1000_std": 0.117711, + "nauc_mrr_at_1000_diff1": 0.598039, + "main_score": 0.52721, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 3092.5504965782166, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/Cmnli.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/Cmnli.json new file mode 100644 index 0000000000..5dabbb2262 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/Cmnli.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "41bc36f332156f7adc9e38f53777c959b2ae9766", + "task_name": "Cmnli", + "mteb_version": "1.38.30", + "scores": { + "validation": [ + { + "similarity_accuracy": 0.876007, + "similarity_accuracy_threshold": 0.604487, + "similarity_f1": 0.882525, + "similarity_f1_threshold": 0.53933, + "similarity_precision": 0.849146, + "similarity_recall": 0.918635, + "similarity_ap": 0.941986, + "cosine_accuracy": 0.876007, + "cosine_accuracy_threshold": 0.604487, + "cosine_f1": 0.882525, + "cosine_f1_threshold": 0.53933, + "cosine_precision": 0.849146, + "cosine_recall": 0.918635, + "cosine_ap": 0.941986, + "manhattan_accuracy": 0.87733, + "manhattan_accuracy_threshold": 31.825895, + "manhattan_f1": 0.882879, + "manhattan_f1_threshold": 34.169777, + "manhattan_precision": 0.845643, + "manhattan_recall": 0.923545, + "manhattan_ap": 0.942202, + "euclidean_accuracy": 0.876007, + "euclidean_accuracy_threshold": 0.889396, + "euclidean_f1": 0.882525, + "euclidean_f1_threshold": 0.959864, + "euclidean_precision": 0.849146, + "euclidean_recall": 0.918635, + "euclidean_ap": 0.941986, + "dot_accuracy": 0.876007, + "dot_accuracy_threshold": 0.604487, + "dot_f1": 0.882525, + "dot_f1_threshold": 0.53933, + "dot_precision": 0.849146, + "dot_recall": 0.918635, + "dot_ap": 0.941976, + "max_accuracy": 0.87733, + "max_f1": 0.882879, + "max_precision": 0.849146, + "max_recall": 0.923545, + "max_ap": 0.942202, + "main_score": 0.87733, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 43.130348205566406, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CovidRetrieval.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CovidRetrieval.json new file mode 100644 index 0000000000..9bdab48ef9 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/CovidRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "1271c7809071a13532e05f25fb53511ffce77117", + "task_name": "CovidRetrieval", + "mteb_version": "1.38.30", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.82824, + "ndcg_at_3": 0.90393, + "ndcg_at_5": 0.91403, + "ndcg_at_10": 0.91935, + "ndcg_at_20": 0.92008, + "ndcg_at_100": 0.92093, + "ndcg_at_1000": 0.92093, + "map_at_1": 0.82613, + "map_at_3": 0.88624, + "map_at_5": 0.89207, + "map_at_10": 0.89455, + "map_at_20": 0.8948, + "map_at_100": 0.89494, + "map_at_1000": 0.89494, + "recall_at_1": 0.82613, + "recall_at_3": 0.95364, + "recall_at_5": 0.97761, + "recall_at_10": 0.99315, + "recall_at_20": 0.99579, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "precision_at_1": 0.82824, + "precision_at_3": 0.31999, + "precision_at_5": 0.19726, + "precision_at_10": 0.10032, + "precision_at_20": 0.05032, + "precision_at_100": 0.01011, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.825079, + "mrr_at_3": 0.884264, + "mrr_at_5": 0.890218, + "mrr_at_10": 0.892385, + "mrr_at_20": 0.892576, + "mrr_at_100": 0.892721, + "mrr_at_1000": 0.892721, + "nauc_ndcg_at_1_max": 0.514795, + "nauc_ndcg_at_1_std": -0.217655, + "nauc_ndcg_at_1_diff1": 0.843137, + "nauc_ndcg_at_3_max": 0.565157, + "nauc_ndcg_at_3_std": -0.270726, + "nauc_ndcg_at_3_diff1": 0.840528, + "nauc_ndcg_at_5_max": 0.545027, + "nauc_ndcg_at_5_std": -0.254537, + "nauc_ndcg_at_5_diff1": 0.841605, + "nauc_ndcg_at_10_max": 0.539625, + "nauc_ndcg_at_10_std": -0.256394, + "nauc_ndcg_at_10_diff1": 0.838764, + "nauc_ndcg_at_20_max": 0.540624, + "nauc_ndcg_at_20_std": -0.246625, + "nauc_ndcg_at_20_diff1": 0.837968, + "nauc_ndcg_at_100_max": 0.537906, + "nauc_ndcg_at_100_std": -0.242955, + "nauc_ndcg_at_100_diff1": 0.839286, + "nauc_ndcg_at_1000_max": 0.537906, + "nauc_ndcg_at_1000_std": -0.242955, + "nauc_ndcg_at_1000_diff1": 0.839286, + "nauc_map_at_1_max": 0.514894, + "nauc_map_at_1_std": -0.221568, + "nauc_map_at_1_diff1": 0.844243, + "nauc_map_at_3_max": 0.548382, + "nauc_map_at_3_std": -0.252594, + "nauc_map_at_3_diff1": 0.839973, + "nauc_map_at_5_max": 0.537931, + "nauc_map_at_5_std": -0.244735, + "nauc_map_at_5_diff1": 0.840581, + "nauc_map_at_10_max": 0.535353, + "nauc_map_at_10_std": -0.244905, + "nauc_map_at_10_diff1": 0.839621, + "nauc_map_at_20_max": 0.535602, + "nauc_map_at_20_std": -0.242356, + "nauc_map_at_20_diff1": 0.839424, + "nauc_map_at_100_max": 0.535276, + "nauc_map_at_100_std": -0.241683, + "nauc_map_at_100_diff1": 0.839619, + "nauc_map_at_1000_max": 0.535276, + "nauc_map_at_1000_std": -0.241683, + "nauc_map_at_1000_diff1": 0.839619, + "nauc_recall_at_1_max": 0.514894, + "nauc_recall_at_1_std": -0.221568, + "nauc_recall_at_1_diff1": 0.844243, + "nauc_recall_at_3_max": 0.681659, + "nauc_recall_at_3_std": -0.401184, + "nauc_recall_at_3_diff1": 0.843875, + "nauc_recall_at_5_max": 0.625455, + "nauc_recall_at_5_std": -0.402978, + "nauc_recall_at_5_diff1": 0.857853, + "nauc_recall_at_10_max": 0.65496, + "nauc_recall_at_10_std": -0.804507, + "nauc_recall_at_10_diff1": 0.80242, + "nauc_recall_at_20_max": 0.791626, + "nauc_recall_at_20_std": -0.514868, + "nauc_recall_at_20_diff1": 0.727958, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.514795, + "nauc_precision_at_1_std": -0.217655, + "nauc_precision_at_1_diff1": 0.843137, + "nauc_precision_at_3_max": 0.516331, + "nauc_precision_at_3_std": -0.281484, + "nauc_precision_at_3_diff1": 0.625582, + "nauc_precision_at_5_max": 0.273532, + "nauc_precision_at_5_std": -0.112143, + "nauc_precision_at_5_diff1": 0.364882, + "nauc_precision_at_10_max": -0.049258, + "nauc_precision_at_10_std": 0.019881, + "nauc_precision_at_10_diff1": -0.104104, + "nauc_precision_at_20_max": -0.109246, + "nauc_precision_at_20_std": 0.209029, + "nauc_precision_at_20_diff1": -0.233466, + "nauc_precision_at_100_max": -0.316579, + "nauc_precision_at_100_std": 0.375632, + "nauc_precision_at_100_diff1": -0.454735, + "nauc_precision_at_1000_max": -0.316579, + "nauc_precision_at_1000_std": 0.375632, + "nauc_precision_at_1000_diff1": -0.454735, + "nauc_mrr_at_1_max": 0.521273, + "nauc_mrr_at_1_std": -0.213606, + "nauc_mrr_at_1_diff1": 0.84825, + "nauc_mrr_at_3_max": 0.557508, + "nauc_mrr_at_3_std": -0.226914, + "nauc_mrr_at_3_diff1": 0.844812, + "nauc_mrr_at_5_max": 0.546541, + "nauc_mrr_at_5_std": -0.227737, + "nauc_mrr_at_5_diff1": 0.844607, + "nauc_mrr_at_10_max": 0.544634, + "nauc_mrr_at_10_std": -0.226431, + "nauc_mrr_at_10_diff1": 0.843973, + "nauc_mrr_at_20_max": 0.5448, + "nauc_mrr_at_20_std": -0.224705, + "nauc_mrr_at_20_diff1": 0.843803, + "nauc_mrr_at_100_max": 0.544493, + "nauc_mrr_at_100_std": -0.22402, + "nauc_mrr_at_100_diff1": 0.844001, + "nauc_mrr_at_1000_max": 0.544493, + "nauc_mrr_at_1000_std": -0.22402, + "nauc_mrr_at_1000_diff1": 0.844001, + "main_score": 0.91935, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 3244.46253657341, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/DuRetrieval.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/DuRetrieval.json new file mode 100644 index 0000000000..92e0e3b97f --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/DuRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced", + "task_name": "DuRetrieval", + "mteb_version": "1.38.30", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.9305, + "ndcg_at_3": 0.9058, + "ndcg_at_5": 0.89746, + "ndcg_at_10": 0.91982, + "ndcg_at_20": 0.93105, + "ndcg_at_100": 0.93859, + "ndcg_at_1000": 0.93986, + "map_at_1": 0.28473, + "map_at_3": 0.6004, + "map_at_5": 0.75255, + "map_at_10": 0.85701, + "map_at_20": 0.87735, + "map_at_100": 0.88254, + "map_at_1000": 0.88272, + "recall_at_1": 0.28473, + "recall_at_3": 0.62176, + "recall_at_5": 0.79521, + "recall_at_10": 0.93212, + "recall_at_20": 0.96985, + "recall_at_100": 0.99295, + "recall_at_1000": 0.99841, + "precision_at_1": 0.9305, + "precision_at_3": 0.80317, + "precision_at_5": 0.6777, + "precision_at_10": 0.43355, + "precision_at_20": 0.2341, + "precision_at_100": 0.04869, + "precision_at_1000": 0.00491, + "mrr_at_1": 0.9305, + "mrr_at_3": 0.957417, + "mrr_at_5": 0.958692, + "mrr_at_10": 0.959023, + "mrr_at_20": 0.959111, + "mrr_at_100": 0.959192, + "mrr_at_1000": 0.959193, + "nauc_ndcg_at_1_max": 0.715135, + "nauc_ndcg_at_1_std": 0.293778, + "nauc_ndcg_at_1_diff1": 0.354953, + "nauc_ndcg_at_3_max": 0.448026, + "nauc_ndcg_at_3_std": 0.151106, + "nauc_ndcg_at_3_diff1": 0.052048, + "nauc_ndcg_at_5_max": 0.378423, + "nauc_ndcg_at_5_std": 0.06193, + "nauc_ndcg_at_5_diff1": 0.080204, + "nauc_ndcg_at_10_max": 0.436679, + "nauc_ndcg_at_10_std": 0.080301, + "nauc_ndcg_at_10_diff1": 0.089775, + "nauc_ndcg_at_20_max": 0.480904, + "nauc_ndcg_at_20_std": 0.134252, + "nauc_ndcg_at_20_diff1": 0.0922, + "nauc_ndcg_at_100_max": 0.488119, + "nauc_ndcg_at_100_std": 0.178221, + "nauc_ndcg_at_100_diff1": 0.085573, + "nauc_ndcg_at_1000_max": 0.489607, + "nauc_ndcg_at_1000_std": 0.178442, + "nauc_ndcg_at_1000_diff1": 0.084777, + "nauc_map_at_1_max": -0.146223, + "nauc_map_at_1_std": -0.388422, + "nauc_map_at_1_diff1": 0.459095, + "nauc_map_at_3_max": -0.010263, + "nauc_map_at_3_std": -0.395255, + "nauc_map_at_3_diff1": 0.336964, + "nauc_map_at_5_max": 0.143442, + "nauc_map_at_5_std": -0.310843, + "nauc_map_at_5_diff1": 0.242226, + "nauc_map_at_10_max": 0.353018, + "nauc_map_at_10_std": -0.092946, + "nauc_map_at_10_diff1": 0.108254, + "nauc_map_at_20_max": 0.401099, + "nauc_map_at_20_std": 0.031319, + "nauc_map_at_20_diff1": 0.073523, + "nauc_map_at_100_max": 0.399766, + "nauc_map_at_100_std": 0.064404, + "nauc_map_at_100_diff1": 0.062905, + "nauc_map_at_1000_max": 0.399685, + "nauc_map_at_1000_std": 0.064557, + "nauc_map_at_1000_diff1": 0.062649, + "nauc_recall_at_1_max": -0.146223, + "nauc_recall_at_1_std": -0.388422, + "nauc_recall_at_1_diff1": 0.459095, + "nauc_recall_at_3_max": -0.06793, + "nauc_recall_at_3_std": -0.436528, + "nauc_recall_at_3_diff1": 0.331513, + "nauc_recall_at_5_max": 0.041635, + "nauc_recall_at_5_std": -0.412001, + "nauc_recall_at_5_diff1": 0.254138, + "nauc_recall_at_10_max": 0.263389, + "nauc_recall_at_10_std": -0.264089, + "nauc_recall_at_10_diff1": 0.109941, + "nauc_recall_at_20_max": 0.439563, + "nauc_recall_at_20_std": 0.018535, + "nauc_recall_at_20_diff1": 0.063472, + "nauc_recall_at_100_max": 0.445912, + "nauc_recall_at_100_std": 0.30565, + "nauc_recall_at_100_diff1": -0.058739, + "nauc_recall_at_1000_max": 0.527409, + "nauc_recall_at_1000_std": 0.539072, + "nauc_recall_at_1000_diff1": -0.501995, + "nauc_precision_at_1_max": 0.715135, + "nauc_precision_at_1_std": 0.293778, + "nauc_precision_at_1_diff1": 0.354953, + "nauc_precision_at_3_max": 0.43997, + "nauc_precision_at_3_std": 0.432257, + "nauc_precision_at_3_diff1": -0.395878, + "nauc_precision_at_5_max": 0.356186, + "nauc_precision_at_5_std": 0.479579, + "nauc_precision_at_5_diff1": -0.406197, + "nauc_precision_at_10_max": 0.263741, + "nauc_precision_at_10_std": 0.52057, + "nauc_precision_at_10_diff1": -0.38177, + "nauc_precision_at_20_max": 0.203683, + "nauc_precision_at_20_std": 0.545241, + "nauc_precision_at_20_diff1": -0.358977, + "nauc_precision_at_100_max": 0.157052, + "nauc_precision_at_100_std": 0.543523, + "nauc_precision_at_100_diff1": -0.346107, + "nauc_precision_at_1000_max": 0.149207, + "nauc_precision_at_1000_std": 0.534781, + "nauc_precision_at_1000_diff1": -0.34238, + "nauc_mrr_at_1_max": 0.715135, + "nauc_mrr_at_1_std": 0.293778, + "nauc_mrr_at_1_diff1": 0.354953, + "nauc_mrr_at_3_max": 0.751701, + "nauc_mrr_at_3_std": 0.337363, + "nauc_mrr_at_3_diff1": 0.358266, + "nauc_mrr_at_5_max": 0.746746, + "nauc_mrr_at_5_std": 0.335716, + "nauc_mrr_at_5_diff1": 0.355231, + "nauc_mrr_at_10_max": 0.745212, + "nauc_mrr_at_10_std": 0.33428, + "nauc_mrr_at_10_diff1": 0.353798, + "nauc_mrr_at_20_max": 0.744669, + "nauc_mrr_at_20_std": 0.333756, + "nauc_mrr_at_20_diff1": 0.353384, + "nauc_mrr_at_100_max": 0.744213, + "nauc_mrr_at_100_std": 0.33262, + "nauc_mrr_at_100_diff1": 0.353464, + "nauc_mrr_at_1000_max": 0.744209, + "nauc_mrr_at_1000_std": 0.332617, + "nauc_mrr_at_1000_diff1": 0.353468, + "main_score": 0.91982, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 3244.9586415290833, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/EcomRetrieval.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/EcomRetrieval.json new file mode 100644 index 0000000000..df467d3719 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/EcomRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9", + "task_name": "EcomRetrieval", + "mteb_version": "1.38.30", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.59, + "ndcg_at_3": 0.70294, + "ndcg_at_5": 0.72519, + "ndcg_at_10": 0.74465, + "ndcg_at_20": 0.75409, + "ndcg_at_100": 0.76195, + "ndcg_at_1000": 0.76391, + "map_at_1": 0.59, + "map_at_3": 0.67617, + "map_at_5": 0.68852, + "map_at_10": 0.69659, + "map_at_20": 0.69922, + "map_at_100": 0.70039, + "map_at_1000": 0.70046, + "recall_at_1": 0.59, + "recall_at_3": 0.78, + "recall_at_5": 0.834, + "recall_at_10": 0.894, + "recall_at_20": 0.931, + "recall_at_100": 0.972, + "recall_at_1000": 0.988, + "precision_at_1": 0.59, + "precision_at_3": 0.26, + "precision_at_5": 0.1668, + "precision_at_10": 0.0894, + "precision_at_20": 0.04655, + "precision_at_100": 0.00972, + "precision_at_1000": 0.00099, + "mrr_at_1": 0.59, + "mrr_at_3": 0.676167, + "mrr_at_5": 0.688517, + "mrr_at_10": 0.69659, + "mrr_at_20": 0.699217, + "mrr_at_100": 0.700392, + "mrr_at_1000": 0.700455, + "nauc_ndcg_at_1_max": 0.256356, + "nauc_ndcg_at_1_std": -0.150058, + "nauc_ndcg_at_1_diff1": 0.586344, + "nauc_ndcg_at_3_max": 0.322765, + "nauc_ndcg_at_3_std": -0.105096, + "nauc_ndcg_at_3_diff1": 0.552669, + "nauc_ndcg_at_5_max": 0.32601, + "nauc_ndcg_at_5_std": -0.101272, + "nauc_ndcg_at_5_diff1": 0.552504, + "nauc_ndcg_at_10_max": 0.330839, + "nauc_ndcg_at_10_std": -0.092648, + "nauc_ndcg_at_10_diff1": 0.555576, + "nauc_ndcg_at_20_max": 0.334446, + "nauc_ndcg_at_20_std": -0.08795, + "nauc_ndcg_at_20_diff1": 0.565688, + "nauc_ndcg_at_100_max": 0.32789, + "nauc_ndcg_at_100_std": -0.081473, + "nauc_ndcg_at_100_diff1": 0.562121, + "nauc_ndcg_at_1000_max": 0.324975, + "nauc_ndcg_at_1000_std": -0.088869, + "nauc_ndcg_at_1000_diff1": 0.562927, + "nauc_map_at_1_max": 0.256356, + "nauc_map_at_1_std": -0.150058, + "nauc_map_at_1_diff1": 0.586344, + "nauc_map_at_3_max": 0.304562, + "nauc_map_at_3_std": -0.118309, + "nauc_map_at_3_diff1": 0.561733, + "nauc_map_at_5_max": 0.3055, + "nauc_map_at_5_std": -0.116919, + "nauc_map_at_5_diff1": 0.562165, + "nauc_map_at_10_max": 0.306739, + "nauc_map_at_10_std": -0.113998, + "nauc_map_at_10_diff1": 0.563431, + "nauc_map_at_20_max": 0.307357, + "nauc_map_at_20_std": -0.112965, + "nauc_map_at_20_diff1": 0.565766, + "nauc_map_at_100_max": 0.306691, + "nauc_map_at_100_std": -0.112047, + "nauc_map_at_100_diff1": 0.565456, + "nauc_map_at_1000_max": 0.306641, + "nauc_map_at_1000_std": -0.112216, + "nauc_map_at_1000_diff1": 0.565482, + "nauc_recall_at_1_max": 0.256356, + "nauc_recall_at_1_std": -0.150058, + "nauc_recall_at_1_diff1": 0.586344, + "nauc_recall_at_3_max": 0.391675, + "nauc_recall_at_3_std": -0.054565, + "nauc_recall_at_3_diff1": 0.518452, + "nauc_recall_at_5_max": 0.422383, + "nauc_recall_at_5_std": -0.026708, + "nauc_recall_at_5_diff1": 0.507215, + "nauc_recall_at_10_max": 0.498962, + "nauc_recall_at_10_std": 0.060753, + "nauc_recall_at_10_diff1": 0.505196, + "nauc_recall_at_20_max": 0.628331, + "nauc_recall_at_20_std": 0.191498, + "nauc_recall_at_20_diff1": 0.599108, + "nauc_recall_at_100_max": 0.788382, + "nauc_recall_at_100_std": 0.807856, + "nauc_recall_at_100_diff1": 0.504719, + "nauc_recall_at_1000_max": 0.978214, + "nauc_recall_at_1000_std": 0.978214, + "nauc_recall_at_1000_diff1": 0.522137, + "nauc_precision_at_1_max": 0.256356, + "nauc_precision_at_1_std": -0.150058, + "nauc_precision_at_1_diff1": 0.586344, + "nauc_precision_at_3_max": 0.391675, + "nauc_precision_at_3_std": -0.054565, + "nauc_precision_at_3_diff1": 0.518452, + "nauc_precision_at_5_max": 0.422383, + "nauc_precision_at_5_std": -0.026708, + "nauc_precision_at_5_diff1": 0.507215, + "nauc_precision_at_10_max": 0.498962, + "nauc_precision_at_10_std": 0.060753, + "nauc_precision_at_10_diff1": 0.505196, + "nauc_precision_at_20_max": 0.628331, + "nauc_precision_at_20_std": 0.191498, + "nauc_precision_at_20_diff1": 0.599108, + "nauc_precision_at_100_max": 0.788382, + "nauc_precision_at_100_std": 0.807856, + "nauc_precision_at_100_diff1": 0.504719, + "nauc_precision_at_1000_max": 0.978214, + "nauc_precision_at_1000_std": 0.978214, + "nauc_precision_at_1000_diff1": 0.522137, + "nauc_mrr_at_1_max": 0.256356, + "nauc_mrr_at_1_std": -0.150058, + "nauc_mrr_at_1_diff1": 0.586344, + "nauc_mrr_at_3_max": 0.304562, + "nauc_mrr_at_3_std": -0.118309, + "nauc_mrr_at_3_diff1": 0.561733, + "nauc_mrr_at_5_max": 0.3055, + "nauc_mrr_at_5_std": -0.116919, + "nauc_mrr_at_5_diff1": 0.562165, + "nauc_mrr_at_10_max": 0.306739, + "nauc_mrr_at_10_std": -0.113998, + "nauc_mrr_at_10_diff1": 0.563431, + "nauc_mrr_at_20_max": 0.307357, + "nauc_mrr_at_20_std": -0.112965, + "nauc_mrr_at_20_diff1": 0.565766, + "nauc_mrr_at_100_max": 0.306691, + "nauc_mrr_at_100_std": -0.112047, + "nauc_mrr_at_100_diff1": 0.565456, + "nauc_mrr_at_1000_max": 0.306641, + "nauc_mrr_at_1000_std": -0.112216, + "nauc_mrr_at_1000_diff1": 0.565482, + "main_score": 0.74465, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 426.16359782218933, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/IFlyTek.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/IFlyTek.json new file mode 100644 index 0000000000..5e01a6e96d --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/IFlyTek.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "421605374b29664c5fc098418fe20ada9bd55f8a", + "task_name": "IFlyTek", + "mteb_version": "1.38.30", + "scores": { + "validation": [ + { + "accuracy": 0.597307, + "f1": 0.435288, + "f1_weighted": 0.600789, + "scores_per_experiment": [ + { + "accuracy": 0.596768, + "f1": 0.436236, + "f1_weighted": 0.600195 + }, + { + "accuracy": 0.600231, + "f1": 0.434711, + "f1_weighted": 0.60312 + }, + { + "accuracy": 0.594844, + "f1": 0.432866, + "f1_weighted": 0.599328 + }, + { + "accuracy": 0.597922, + "f1": 0.435296, + "f1_weighted": 0.601947 + }, + { + "accuracy": 0.596768, + "f1": 0.437332, + "f1_weighted": 0.599354 + } + ], + "main_score": 0.597307, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 661.3727290630341, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/JDReview.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/JDReview.json new file mode 100644 index 0000000000..57ac00e858 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/JDReview.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b", + "task_name": "JDReview", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "accuracy": 0.892308, + "f1": 0.840847, + "f1_weighted": 0.897714, + "ap": 0.598344, + "ap_weighted": 0.598344, + "scores_per_experiment": [ + { + "accuracy": 0.896811, + "f1": 0.84406, + "f1_weighted": 0.901065, + "ap": 0.602923, + "ap_weighted": 0.602923 + }, + { + "accuracy": 0.88743, + "f1": 0.835737, + "f1_weighted": 0.893653, + "ap": 0.588844, + "ap_weighted": 0.588844 + }, + { + "accuracy": 0.881801, + "f1": 0.830111, + "f1_weighted": 0.88901, + "ap": 0.579368, + "ap_weighted": 0.579368 + }, + { + "accuracy": 0.900563, + "f1": 0.846667, + "f1_weighted": 0.903803, + "ap": 0.607481, + "ap_weighted": 0.607481 + }, + { + "accuracy": 0.893058, + "f1": 0.842492, + "f1_weighted": 0.898584, + "ap": 0.601715, + "ap_weighted": 0.601715 + }, + { + "accuracy": 0.891182, + "f1": 0.841213, + "f1_weighted": 0.897198, + "ap": 0.600062, + "ap_weighted": 0.600062 + }, + { + "accuracy": 0.898687, + "f1": 0.846388, + "f1_weighted": 0.902723, + "ap": 0.607596, + "ap_weighted": 0.607596 + }, + { + "accuracy": 0.896811, + "f1": 0.847057, + "f1_weighted": 0.901884, + "ap": 0.610626, + "ap_weighted": 0.610626 + }, + { + "accuracy": 0.891182, + "f1": 0.840228, + "f1_weighted": 0.896938, + "ap": 0.59736, + "ap_weighted": 0.59736 + }, + { + "accuracy": 0.885553, + "f1": 0.834517, + "f1_weighted": 0.892278, + "ap": 0.587466, + "ap_weighted": 0.587466 + } + ], + "main_score": 0.892308, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 44.95045733451843, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/LCQMC.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/LCQMC.json new file mode 100644 index 0000000000..61c071a8ec --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/LCQMC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "17f9b096f80380fce5ed12a9be8be7784b337daf", + "task_name": "LCQMC", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "pearson": 0.773869, + "spearman": 0.774802, + "cosine_pearson": 0.773869, + "cosine_spearman": 0.774802, + "manhattan_pearson": 0.792402, + "manhattan_spearman": 0.775017, + "euclidean_pearson": 0.792159, + "euclidean_spearman": 0.774802, + "main_score": 0.774802, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 88.77106142044067, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MMarcoReranking.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MMarcoReranking.json new file mode 100644 index 0000000000..989bc2394e --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MMarcoReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6", + "task_name": "MMarcoReranking", + "mteb_version": "1.38.30", + "scores": { + "dev": [ + { + "map": 0.435822, + "mrr": 0.426385, + "nAUC_map_max": 0.036779, + "nAUC_map_std": -0.171559, + "nAUC_map_diff1": 0.580751, + "nAUC_mrr_max": 0.042894, + "nAUC_mrr_std": -0.185152, + "nAUC_mrr_diff1": 0.589402, + "main_score": 0.435822, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 1230.5246150493622, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MMarcoRetrieval.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MMarcoRetrieval.json new file mode 100644 index 0000000000..9f70554e66 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MMarcoRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "539bbde593d947e2a124ba72651aafc09eb33fc2", + "task_name": "MMarcoRetrieval", + "mteb_version": "1.38.30", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.7894, + "ndcg_at_3": 0.86075, + "ndcg_at_5": 0.87606, + "ndcg_at_10": 0.88447, + "ndcg_at_20": 0.88728, + "ndcg_at_100": 0.88923, + "ndcg_at_1000": 0.89, + "map_at_1": 0.76507, + "map_at_3": 0.84118, + "map_at_5": 0.85054, + "map_at_10": 0.85449, + "map_at_20": 0.85534, + "map_at_100": 0.85568, + "map_at_1000": 0.85571, + "recall_at_1": 0.76507, + "recall_at_3": 0.91053, + "recall_at_5": 0.94641, + "recall_at_10": 0.97101, + "recall_at_20": 0.98179, + "recall_at_100": 0.99143, + "recall_at_1000": 0.99742, + "precision_at_1": 0.7894, + "precision_at_3": 0.32077, + "precision_at_5": 0.20095, + "precision_at_10": 0.10332, + "precision_at_20": 0.05226, + "precision_at_100": 0.01056, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.789398, + "mrr_at_3": 0.847326, + "mrr_at_5": 0.854955, + "mrr_at_10": 0.858084, + "mrr_at_20": 0.858779, + "mrr_at_100": 0.859058, + "mrr_at_1000": 0.859085, + "nauc_ndcg_at_1_max": 0.411289, + "nauc_ndcg_at_1_std": -0.183514, + "nauc_ndcg_at_1_diff1": 0.802384, + "nauc_ndcg_at_3_max": 0.437992, + "nauc_ndcg_at_3_std": -0.169286, + "nauc_ndcg_at_3_diff1": 0.772186, + "nauc_ndcg_at_5_max": 0.461308, + "nauc_ndcg_at_5_std": -0.156313, + "nauc_ndcg_at_5_diff1": 0.78242, + "nauc_ndcg_at_10_max": 0.462058, + "nauc_ndcg_at_10_std": -0.144016, + "nauc_ndcg_at_10_diff1": 0.784979, + "nauc_ndcg_at_20_max": 0.456108, + "nauc_ndcg_at_20_std": -0.143916, + "nauc_ndcg_at_20_diff1": 0.785401, + "nauc_ndcg_at_100_max": 0.44945, + "nauc_ndcg_at_100_std": -0.147959, + "nauc_ndcg_at_100_diff1": 0.785637, + "nauc_ndcg_at_1000_max": 0.44653, + "nauc_ndcg_at_1000_std": -0.15234, + "nauc_ndcg_at_1000_diff1": 0.785745, + "nauc_map_at_1_max": 0.367639, + "nauc_map_at_1_std": -0.198272, + "nauc_map_at_1_diff1": 0.793496, + "nauc_map_at_3_max": 0.424623, + "nauc_map_at_3_std": -0.179784, + "nauc_map_at_3_diff1": 0.778409, + "nauc_map_at_5_max": 0.437018, + "nauc_map_at_5_std": -0.172075, + "nauc_map_at_5_diff1": 0.78436, + "nauc_map_at_10_max": 0.437074, + "nauc_map_at_10_std": -0.16711, + "nauc_map_at_10_diff1": 0.785287, + "nauc_map_at_20_max": 0.435711, + "nauc_map_at_20_std": -0.166652, + "nauc_map_at_20_diff1": 0.785389, + "nauc_map_at_100_max": 0.434858, + "nauc_map_at_100_std": -0.167077, + "nauc_map_at_100_diff1": 0.785451, + "nauc_map_at_1000_max": 0.434774, + "nauc_map_at_1000_std": -0.167186, + "nauc_map_at_1000_diff1": 0.785448, + "nauc_recall_at_1_max": 0.367639, + "nauc_recall_at_1_std": -0.198272, + "nauc_recall_at_1_diff1": 0.793496, + "nauc_recall_at_3_max": 0.48029, + "nauc_recall_at_3_std": -0.139647, + "nauc_recall_at_3_diff1": 0.725068, + "nauc_recall_at_5_max": 0.630105, + "nauc_recall_at_5_std": -0.062796, + "nauc_recall_at_5_diff1": 0.752611, + "nauc_recall_at_10_max": 0.783084, + "nauc_recall_at_10_std": 0.138681, + "nauc_recall_at_10_diff1": 0.767552, + "nauc_recall_at_20_max": 0.837073, + "nauc_recall_at_20_std": 0.287374, + "nauc_recall_at_20_diff1": 0.771581, + "nauc_recall_at_100_max": 0.859559, + "nauc_recall_at_100_std": 0.509536, + "nauc_recall_at_100_diff1": 0.772283, + "nauc_recall_at_1000_max": 0.897707, + "nauc_recall_at_1000_std": 0.606909, + "nauc_recall_at_1000_diff1": 0.801912, + "nauc_precision_at_1_max": 0.411289, + "nauc_precision_at_1_std": -0.183514, + "nauc_precision_at_1_diff1": 0.802384, + "nauc_precision_at_3_max": 0.28867, + "nauc_precision_at_3_std": 0.009514, + "nauc_precision_at_3_diff1": 0.232804, + "nauc_precision_at_5_max": 0.254256, + "nauc_precision_at_5_std": 0.102605, + "nauc_precision_at_5_diff1": 0.064021, + "nauc_precision_at_10_max": 0.183219, + "nauc_precision_at_10_std": 0.182115, + "nauc_precision_at_10_diff1": -0.085262, + "nauc_precision_at_20_max": 0.128557, + "nauc_precision_at_20_std": 0.208339, + "nauc_precision_at_20_diff1": -0.161518, + "nauc_precision_at_100_max": 0.061048, + "nauc_precision_at_100_std": 0.21197, + "nauc_precision_at_100_diff1": -0.238321, + "nauc_precision_at_1000_max": 0.017132, + "nauc_precision_at_1000_std": 0.195498, + "nauc_precision_at_1000_diff1": -0.290051, + "nauc_mrr_at_1_max": 0.411289, + "nauc_mrr_at_1_std": -0.183514, + "nauc_mrr_at_1_diff1": 0.802384, + "nauc_mrr_at_3_max": 0.434084, + "nauc_mrr_at_3_std": -0.166752, + "nauc_mrr_at_3_diff1": 0.784202, + "nauc_mrr_at_5_max": 0.442098, + "nauc_mrr_at_5_std": -0.1627, + "nauc_mrr_at_5_diff1": 0.788469, + "nauc_mrr_at_10_max": 0.441588, + "nauc_mrr_at_10_std": -0.159684, + "nauc_mrr_at_10_diff1": 0.789377, + "nauc_mrr_at_20_max": 0.440269, + "nauc_mrr_at_20_std": -0.159986, + "nauc_mrr_at_20_diff1": 0.789542, + "nauc_mrr_at_100_max": 0.439458, + "nauc_mrr_at_100_std": -0.160538, + "nauc_mrr_at_100_diff1": 0.789588, + "nauc_mrr_at_1000_max": 0.439375, + "nauc_mrr_at_1000_std": -0.16065, + "nauc_mrr_at_1000_diff1": 0.789583, + "main_score": 0.88447, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 1305.2145018577576, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MedicalRetrieval.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MedicalRetrieval.json new file mode 100644 index 0000000000..3eb35dfa7b --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MedicalRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6", + "task_name": "MedicalRetrieval", + "mteb_version": "1.38.30", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.622, + "ndcg_at_3": 0.70535, + "ndcg_at_5": 0.72257, + "ndcg_at_10": 0.73787, + "ndcg_at_20": 0.74465, + "ndcg_at_100": 0.75329, + "ndcg_at_1000": 0.75907, + "map_at_1": 0.622, + "map_at_3": 0.68567, + "map_at_5": 0.69517, + "map_at_10": 0.70154, + "map_at_20": 0.70338, + "map_at_100": 0.70462, + "map_at_1000": 0.7048, + "recall_at_1": 0.622, + "recall_at_3": 0.762, + "recall_at_5": 0.804, + "recall_at_10": 0.851, + "recall_at_20": 0.878, + "recall_at_100": 0.924, + "recall_at_1000": 0.971, + "precision_at_1": 0.622, + "precision_at_3": 0.254, + "precision_at_5": 0.1608, + "precision_at_10": 0.0851, + "precision_at_20": 0.0439, + "precision_at_100": 0.00924, + "precision_at_1000": 0.00097, + "mrr_at_1": 0.627, + "mrr_at_3": 0.688167, + "mrr_at_5": 0.697667, + "mrr_at_10": 0.70402, + "mrr_at_20": 0.705855, + "mrr_at_100": 0.707092, + "mrr_at_1000": 0.707278, + "nauc_ndcg_at_1_max": 0.667885, + "nauc_ndcg_at_1_std": 0.00503, + "nauc_ndcg_at_1_diff1": 0.803201, + "nauc_ndcg_at_3_max": 0.677553, + "nauc_ndcg_at_3_std": 0.034841, + "nauc_ndcg_at_3_diff1": 0.759333, + "nauc_ndcg_at_5_max": 0.676031, + "nauc_ndcg_at_5_std": 0.059445, + "nauc_ndcg_at_5_diff1": 0.749024, + "nauc_ndcg_at_10_max": 0.678177, + "nauc_ndcg_at_10_std": 0.059612, + "nauc_ndcg_at_10_diff1": 0.746359, + "nauc_ndcg_at_20_max": 0.677531, + "nauc_ndcg_at_20_std": 0.063687, + "nauc_ndcg_at_20_diff1": 0.748021, + "nauc_ndcg_at_100_max": 0.675205, + "nauc_ndcg_at_100_std": 0.061587, + "nauc_ndcg_at_100_diff1": 0.749237, + "nauc_ndcg_at_1000_max": 0.676844, + "nauc_ndcg_at_1000_std": 0.059736, + "nauc_ndcg_at_1000_diff1": 0.755154, + "nauc_map_at_1_max": 0.667885, + "nauc_map_at_1_std": 0.00503, + "nauc_map_at_1_diff1": 0.803201, + "nauc_map_at_3_max": 0.675945, + "nauc_map_at_3_std": 0.028237, + "nauc_map_at_3_diff1": 0.771058, + "nauc_map_at_5_max": 0.675267, + "nauc_map_at_5_std": 0.041025, + "nauc_map_at_5_diff1": 0.766017, + "nauc_map_at_10_max": 0.676061, + "nauc_map_at_10_std": 0.040342, + "nauc_map_at_10_diff1": 0.765403, + "nauc_map_at_20_max": 0.675923, + "nauc_map_at_20_std": 0.041359, + "nauc_map_at_20_diff1": 0.765946, + "nauc_map_at_100_max": 0.675678, + "nauc_map_at_100_std": 0.041155, + "nauc_map_at_100_diff1": 0.766178, + "nauc_map_at_1000_max": 0.675689, + "nauc_map_at_1000_std": 0.041095, + "nauc_map_at_1000_diff1": 0.766327, + "nauc_recall_at_1_max": 0.667885, + "nauc_recall_at_1_std": 0.00503, + "nauc_recall_at_1_diff1": 0.803201, + "nauc_recall_at_3_max": 0.682699, + "nauc_recall_at_3_std": 0.057375, + "nauc_recall_at_3_diff1": 0.718053, + "nauc_recall_at_5_max": 0.677912, + "nauc_recall_at_5_std": 0.136953, + "nauc_recall_at_5_diff1": 0.67892, + "nauc_recall_at_10_max": 0.688654, + "nauc_recall_at_10_std": 0.162995, + "nauc_recall_at_10_diff1": 0.64582, + "nauc_recall_at_20_max": 0.685532, + "nauc_recall_at_20_std": 0.210422, + "nauc_recall_at_20_diff1": 0.637708, + "nauc_recall_at_100_max": 0.653466, + "nauc_recall_at_100_std": 0.25492, + "nauc_recall_at_100_diff1": 0.592922, + "nauc_recall_at_1000_max": 0.726134, + "nauc_recall_at_1000_std": 0.457114, + "nauc_recall_at_1000_diff1": 0.689832, + "nauc_precision_at_1_max": 0.667885, + "nauc_precision_at_1_std": 0.00503, + "nauc_precision_at_1_diff1": 0.803201, + "nauc_precision_at_3_max": 0.682699, + "nauc_precision_at_3_std": 0.057375, + "nauc_precision_at_3_diff1": 0.718053, + "nauc_precision_at_5_max": 0.677912, + "nauc_precision_at_5_std": 0.136953, + "nauc_precision_at_5_diff1": 0.67892, + "nauc_precision_at_10_max": 0.688654, + "nauc_precision_at_10_std": 0.162995, + "nauc_precision_at_10_diff1": 0.64582, + "nauc_precision_at_20_max": 0.685532, + "nauc_precision_at_20_std": 0.210422, + "nauc_precision_at_20_diff1": 0.637708, + "nauc_precision_at_100_max": 0.653466, + "nauc_precision_at_100_std": 0.25492, + "nauc_precision_at_100_diff1": 0.592922, + "nauc_precision_at_1000_max": 0.726134, + "nauc_precision_at_1000_std": 0.457114, + "nauc_precision_at_1000_diff1": 0.689832, + "nauc_mrr_at_1_max": 0.677331, + "nauc_mrr_at_1_std": 0.009262, + "nauc_mrr_at_1_diff1": 0.79394, + "nauc_mrr_at_3_max": 0.681398, + "nauc_mrr_at_3_std": 0.030783, + "nauc_mrr_at_3_diff1": 0.765584, + "nauc_mrr_at_5_max": 0.680865, + "nauc_mrr_at_5_std": 0.043711, + "nauc_mrr_at_5_diff1": 0.760366, + "nauc_mrr_at_10_max": 0.6818, + "nauc_mrr_at_10_std": 0.043064, + "nauc_mrr_at_10_diff1": 0.759654, + "nauc_mrr_at_20_max": 0.681699, + "nauc_mrr_at_20_std": 0.044104, + "nauc_mrr_at_20_diff1": 0.760164, + "nauc_mrr_at_100_max": 0.681479, + "nauc_mrr_at_100_std": 0.043912, + "nauc_mrr_at_100_diff1": 0.76037, + "nauc_mrr_at_1000_max": 0.681495, + "nauc_mrr_at_1000_std": 0.043854, + "nauc_mrr_at_1000_diff1": 0.760516, + "main_score": 0.73787, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 1301.1799216270447, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MultilingualSentiment.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MultilingualSentiment.json new file mode 100644 index 0000000000..629963ff6c --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/MultilingualSentiment.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a", + "task_name": "MultilingualSentiment", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "accuracy": 0.8027, + "f1": 0.804433, + "f1_weighted": 0.804433, + "scores_per_experiment": [ + { + "accuracy": 0.800667, + "f1": 0.803083, + "f1_weighted": 0.803083 + }, + { + "accuracy": 0.805, + "f1": 0.80629, + "f1_weighted": 0.80629 + }, + { + "accuracy": 0.802667, + "f1": 0.803378, + "f1_weighted": 0.803378 + }, + { + "accuracy": 0.804333, + "f1": 0.80656, + "f1_weighted": 0.80656 + }, + { + "accuracy": 0.803, + "f1": 0.804537, + "f1_weighted": 0.804537 + }, + { + "accuracy": 0.799, + "f1": 0.801207, + "f1_weighted": 0.801207 + }, + { + "accuracy": 0.803667, + "f1": 0.804727, + "f1_weighted": 0.804727 + }, + { + "accuracy": 0.806333, + "f1": 0.807444, + "f1_weighted": 0.807444 + }, + { + "accuracy": 0.805, + "f1": 0.807194, + "f1_weighted": 0.807194 + }, + { + "accuracy": 0.797333, + "f1": 0.799906, + "f1_weighted": 0.799906 + } + ], + "main_score": 0.8027, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ], + "validation": [ + { + "accuracy": 0.7944, + "f1": 0.795913, + "f1_weighted": 0.795913, + "scores_per_experiment": [ + { + "accuracy": 0.792, + "f1": 0.794435, + "f1_weighted": 0.794435 + }, + { + "accuracy": 0.796667, + "f1": 0.797656, + "f1_weighted": 0.797656 + }, + { + "accuracy": 0.796667, + "f1": 0.796848, + "f1_weighted": 0.796848 + }, + { + "accuracy": 0.795, + "f1": 0.797219, + "f1_weighted": 0.797219 + }, + { + "accuracy": 0.795333, + "f1": 0.796503, + "f1_weighted": 0.796503 + }, + { + "accuracy": 0.788, + "f1": 0.790517, + "f1_weighted": 0.790517 + }, + { + "accuracy": 0.796667, + "f1": 0.797308, + "f1_weighted": 0.797308 + }, + { + "accuracy": 0.796, + "f1": 0.796823, + "f1_weighted": 0.796823 + }, + { + "accuracy": 0.796, + "f1": 0.798053, + "f1_weighted": 0.798053 + }, + { + "accuracy": 0.791667, + "f1": 0.793768, + "f1_weighted": 0.793768 + } + ], + "main_score": 0.7944, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 148.10591888427734, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/Ocnli.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/Ocnli.json new file mode 100644 index 0000000000..dd5c1cc159 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/Ocnli.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "66e76a618a34d6d565d5538088562851e6daa7ec", + "task_name": "Ocnli", + "mteb_version": "1.38.30", + "scores": { + "validation": [ + { + "similarity_accuracy": 0.845154, + "similarity_accuracy_threshold": 0.533708, + "similarity_f1": 0.858669, + "similarity_f1_threshold": 0.501919, + "similarity_precision": 0.794964, + "similarity_recall": 0.933474, + "similarity_ap": 0.882153, + "cosine_accuracy": 0.845154, + "cosine_accuracy_threshold": 0.533708, + "cosine_f1": 0.858669, + "cosine_f1_threshold": 0.501919, + "cosine_precision": 0.794964, + "cosine_recall": 0.933474, + "cosine_ap": 0.882153, + "manhattan_accuracy": 0.845154, + "manhattan_accuracy_threshold": 31.36273, + "manhattan_f1": 0.860194, + "manhattan_f1_threshold": 35.205849, + "manhattan_precision": 0.796047, + "manhattan_recall": 0.935586, + "manhattan_ap": 0.882357, + "euclidean_accuracy": 0.845154, + "euclidean_accuracy_threshold": 0.965704, + "euclidean_f1": 0.858669, + "euclidean_f1_threshold": 0.998079, + "euclidean_precision": 0.794964, + "euclidean_recall": 0.933474, + "euclidean_ap": 0.882153, + "dot_accuracy": 0.845154, + "dot_accuracy_threshold": 0.533708, + "dot_f1": 0.858669, + "dot_f1_threshold": 0.501919, + "dot_precision": 0.794964, + "dot_recall": 0.933474, + "dot_ap": 0.882153, + "max_accuracy": 0.845154, + "max_f1": 0.860194, + "max_precision": 0.796047, + "max_recall": 0.935586, + "max_ap": 0.882357, + "main_score": 0.845154, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 10.494816303253174, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/OnlineShopping.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/OnlineShopping.json new file mode 100644 index 0000000000..c027506e70 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/OnlineShopping.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e610f2ebd179a8fda30ae534c3878750a96db120", + "task_name": "OnlineShopping", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "accuracy": 0.9413, + "f1": 0.941165, + "f1_weighted": 0.941295, + "ap": 0.921435, + "ap_weighted": 0.921435, + "scores_per_experiment": [ + { + "accuracy": 0.939, + "f1": 0.938741, + "f1_weighted": 0.938932, + "ap": 0.910953, + "ap_weighted": 0.910953 + }, + { + "accuracy": 0.943, + "f1": 0.942904, + "f1_weighted": 0.943016, + "ap": 0.925528, + "ap_weighted": 0.925528 + }, + { + "accuracy": 0.943, + "f1": 0.942801, + "f1_weighted": 0.942963, + "ap": 0.918345, + "ap_weighted": 0.918345 + }, + { + "accuracy": 0.942, + "f1": 0.941855, + "f1_weighted": 0.941994, + "ap": 0.920578, + "ap_weighted": 0.920578 + }, + { + "accuracy": 0.944, + "f1": 0.94391, + "f1_weighted": 0.944018, + "ap": 0.92726, + "ap_weighted": 0.92726 + }, + { + "accuracy": 0.936, + "f1": 0.935974, + "f1_weighted": 0.936036, + "ap": 0.925109, + "ap_weighted": 0.925109 + }, + { + "accuracy": 0.942, + "f1": 0.941866, + "f1_weighted": 0.942, + "ap": 0.921376, + "ap_weighted": 0.921376 + }, + { + "accuracy": 0.941, + "f1": 0.940943, + "f1_weighted": 0.941031, + "ap": 0.927065, + "ap_weighted": 0.927065 + }, + { + "accuracy": 0.939, + "f1": 0.938757, + "f1_weighted": 0.938942, + "ap": 0.9117, + "ap_weighted": 0.9117 + }, + { + "accuracy": 0.944, + "f1": 0.943901, + "f1_weighted": 0.944014, + "ap": 0.926432, + "ap_weighted": 0.926432 + } + ], + "main_score": 0.9413, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 43.98006081581116, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/PAWSX.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/PAWSX.json new file mode 100644 index 0000000000..7c333aa13a --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/PAWSX.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1", + "task_name": "PAWSX", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "pearson": 0.580633, + "spearman": 0.593153, + "cosine_pearson": 0.580633, + "cosine_spearman": 0.593153, + "manhattan_pearson": 0.588125, + "manhattan_spearman": 0.593585, + "euclidean_pearson": 0.587544, + "euclidean_spearman": 0.593153, + "main_score": 0.593153, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 26.81486201286316, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/QBQTC.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/QBQTC.json new file mode 100644 index 0000000000..f1b3fa22d5 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/QBQTC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7", + "task_name": "QBQTC", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "pearson": 0.573109, + "spearman": 0.555974, + "cosine_pearson": 0.573109, + "cosine_spearman": 0.555974, + "manhattan_pearson": 0.558334, + "manhattan_spearman": 0.557276, + "euclidean_pearson": 0.556782, + "euclidean_spearman": 0.555974, + "main_score": 0.555974, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 39.686301469802856, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/STSB.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/STSB.json new file mode 100644 index 0000000000..0105f3195c --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/STSB.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0", + "task_name": "STSB", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "pearson": 0.827743, + "spearman": 0.83176, + "cosine_pearson": 0.827743, + "cosine_spearman": 0.83176, + "manhattan_pearson": 0.822236, + "manhattan_spearman": 0.831955, + "euclidean_pearson": 0.822029, + "euclidean_spearman": 0.83176, + "main_score": 0.83176, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 11.698592185974121, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/T2Reranking.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/T2Reranking.json new file mode 100644 index 0000000000..de47ac1d40 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/T2Reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "76631901a18387f85eaa53e5450019b87ad58ef9", + "task_name": "T2Reranking", + "mteb_version": "1.38.30", + "scores": { + "dev": [ + { + "map": 0.731538, + "mrr": 0.838983, + "nAUC_map_max": 0.200795, + "nAUC_map_std": -0.016705, + "nAUC_map_diff1": -0.028193, + "nAUC_mrr_max": 0.201976, + "nAUC_mrr_std": -0.037462, + "nAUC_mrr_diff1": -0.018514, + "main_score": 0.731538, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 5090.760587692261, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/T2Retrieval.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/T2Retrieval.json new file mode 100644 index 0000000000..c1992f91ee --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/T2Retrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "8731a845f1bf500a4f111cf1070785c793d10e64", + "task_name": "T2Retrieval", + "mteb_version": "1.38.30", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.91838, + "ndcg_at_3": 0.88329, + "ndcg_at_5": 0.87193, + "ndcg_at_10": 0.87502, + "ndcg_at_20": 0.89305, + "ndcg_at_100": 0.90845, + "ndcg_at_1000": 0.9125, + "map_at_1": 0.28971, + "map_at_3": 0.56599, + "map_at_5": 0.69324, + "map_at_10": 0.80074, + "map_at_20": 0.82955, + "map_at_100": 0.83771, + "map_at_1000": 0.83819, + "recall_at_1": 0.28971, + "recall_at_3": 0.58463, + "recall_at_5": 0.73048, + "recall_at_10": 0.8701, + "recall_at_20": 0.92843, + "recall_at_100": 0.97403, + "recall_at_1000": 0.99291, + "precision_at_1": 0.91838, + "precision_at_3": 0.766, + "precision_at_5": 0.64171, + "precision_at_10": 0.42849, + "precision_at_20": 0.23816, + "precision_at_100": 0.05076, + "precision_at_1000": 0.00518, + "mrr_at_1": 0.918376, + "mrr_at_3": 0.940112, + "mrr_at_5": 0.942672, + "mrr_at_10": 0.943922, + "mrr_at_20": 0.944243, + "mrr_at_100": 0.944397, + "mrr_at_1000": 0.944409, + "nauc_ndcg_at_1_max": 0.732948, + "nauc_ndcg_at_1_std": 0.300768, + "nauc_ndcg_at_1_diff1": 0.504808, + "nauc_ndcg_at_3_max": 0.60424, + "nauc_ndcg_at_3_std": 0.248756, + "nauc_ndcg_at_3_diff1": 0.142672, + "nauc_ndcg_at_5_max": 0.53013, + "nauc_ndcg_at_5_std": 0.180736, + "nauc_ndcg_at_5_diff1": 0.145802, + "nauc_ndcg_at_10_max": 0.457654, + "nauc_ndcg_at_10_std": 0.085969, + "nauc_ndcg_at_10_diff1": 0.19012, + "nauc_ndcg_at_20_max": 0.505346, + "nauc_ndcg_at_20_std": 0.151823, + "nauc_ndcg_at_20_diff1": 0.189602, + "nauc_ndcg_at_100_max": 0.553849, + "nauc_ndcg_at_100_std": 0.225011, + "nauc_ndcg_at_100_diff1": 0.189553, + "nauc_ndcg_at_1000_max": 0.561879, + "nauc_ndcg_at_1000_std": 0.225711, + "nauc_ndcg_at_1000_diff1": 0.193673, + "nauc_map_at_1_max": -0.243049, + "nauc_map_at_1_std": -0.404866, + "nauc_map_at_1_diff1": 0.475519, + "nauc_map_at_3_max": -0.1177, + "nauc_map_at_3_std": -0.42014, + "nauc_map_at_3_diff1": 0.366748, + "nauc_map_at_5_max": 0.041021, + "nauc_map_at_5_std": -0.339572, + "nauc_map_at_5_diff1": 0.303778, + "nauc_map_at_10_max": 0.327552, + "nauc_map_at_10_std": -0.073687, + "nauc_map_at_10_diff1": 0.19634, + "nauc_map_at_20_max": 0.431931, + "nauc_map_at_20_std": 0.066423, + "nauc_map_at_20_diff1": 0.156186, + "nauc_map_at_100_max": 0.454526, + "nauc_map_at_100_std": 0.103732, + "nauc_map_at_100_diff1": 0.149183, + "nauc_map_at_1000_max": 0.455203, + "nauc_map_at_1000_std": 0.104236, + "nauc_map_at_1000_diff1": 0.149203, + "nauc_recall_at_1_max": -0.243049, + "nauc_recall_at_1_std": -0.404866, + "nauc_recall_at_1_diff1": 0.475519, + "nauc_recall_at_3_max": -0.169277, + "nauc_recall_at_3_std": -0.449474, + "nauc_recall_at_3_diff1": 0.345721, + "nauc_recall_at_5_max": -0.064629, + "nauc_recall_at_5_std": -0.414597, + "nauc_recall_at_5_diff1": 0.290031, + "nauc_recall_at_10_max": 0.180777, + "nauc_recall_at_10_std": -0.196128, + "nauc_recall_at_10_diff1": 0.189178, + "nauc_recall_at_20_max": 0.334937, + "nauc_recall_at_20_std": 0.05408, + "nauc_recall_at_20_diff1": 0.117631, + "nauc_recall_at_100_max": 0.434158, + "nauc_recall_at_100_std": 0.3651, + "nauc_recall_at_100_diff1": 0.071337, + "nauc_recall_at_1000_max": 0.478888, + "nauc_recall_at_1000_std": 0.547469, + "nauc_recall_at_1000_diff1": 0.032053, + "nauc_precision_at_1_max": 0.732948, + "nauc_precision_at_1_std": 0.300768, + "nauc_precision_at_1_diff1": 0.504808, + "nauc_precision_at_3_max": 0.608106, + "nauc_precision_at_3_std": 0.475713, + "nauc_precision_at_3_diff1": -0.3087, + "nauc_precision_at_5_max": 0.546883, + "nauc_precision_at_5_std": 0.52253, + "nauc_precision_at_5_diff1": -0.34055, + "nauc_precision_at_10_max": 0.482756, + "nauc_precision_at_10_std": 0.575402, + "nauc_precision_at_10_diff1": -0.333675, + "nauc_precision_at_20_max": 0.449017, + "nauc_precision_at_20_std": 0.609476, + "nauc_precision_at_20_diff1": -0.333311, + "nauc_precision_at_100_max": 0.414209, + "nauc_precision_at_100_std": 0.616558, + "nauc_precision_at_100_diff1": -0.330671, + "nauc_precision_at_1000_max": 0.399411, + "nauc_precision_at_1000_std": 0.60507, + "nauc_precision_at_1000_diff1": -0.329206, + "nauc_mrr_at_1_max": 0.732948, + "nauc_mrr_at_1_std": 0.300768, + "nauc_mrr_at_1_diff1": 0.504808, + "nauc_mrr_at_3_max": 0.765159, + "nauc_mrr_at_3_std": 0.339935, + "nauc_mrr_at_3_diff1": 0.500004, + "nauc_mrr_at_5_max": 0.76551, + "nauc_mrr_at_5_std": 0.345117, + "nauc_mrr_at_5_diff1": 0.501251, + "nauc_mrr_at_10_max": 0.763811, + "nauc_mrr_at_10_std": 0.342546, + "nauc_mrr_at_10_diff1": 0.501538, + "nauc_mrr_at_20_max": 0.763103, + "nauc_mrr_at_20_std": 0.34196, + "nauc_mrr_at_20_diff1": 0.501525, + "nauc_mrr_at_100_max": 0.762665, + "nauc_mrr_at_100_std": 0.341314, + "nauc_mrr_at_100_diff1": 0.501559, + "nauc_mrr_at_1000_max": 0.762631, + "nauc_mrr_at_1000_std": 0.341217, + "nauc_mrr_at_1000_diff1": 0.501572, + "main_score": 0.87502, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 6885.843509912491, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/TNews.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/TNews.json new file mode 100644 index 0000000000..eeb27ff24b --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/TNews.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "317f262bf1e6126357bbe89e875451e4b0938fe4", + "task_name": "TNews", + "mteb_version": "1.38.30", + "scores": { + "validation": [ + { + "accuracy": 0.60052, + "f1": 0.586534, + "f1_weighted": 0.600753, + "scores_per_experiment": [ + { + "accuracy": 0.5996, + "f1": 0.584632, + "f1_weighted": 0.599676 + }, + { + "accuracy": 0.6003, + "f1": 0.586883, + "f1_weighted": 0.600252 + }, + { + "accuracy": 0.6015, + "f1": 0.58828, + "f1_weighted": 0.601894 + }, + { + "accuracy": 0.6018, + "f1": 0.589, + "f1_weighted": 0.602318 + }, + { + "accuracy": 0.5991, + "f1": 0.583824, + "f1_weighted": 0.598868 + }, + { + "accuracy": 0.6024, + "f1": 0.589866, + "f1_weighted": 0.602379 + }, + { + "accuracy": 0.6, + "f1": 0.585523, + "f1_weighted": 0.60061 + }, + { + "accuracy": 0.6011, + "f1": 0.588753, + "f1_weighted": 0.601241 + }, + { + "accuracy": 0.5984, + "f1": 0.582971, + "f1_weighted": 0.599067 + }, + { + "accuracy": 0.601, + "f1": 0.585611, + "f1_weighted": 0.601224 + } + ], + "main_score": 0.60052, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 83.39222979545593, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/ThuNewsClusteringP2P.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/ThuNewsClusteringP2P.json new file mode 100644 index 0000000000..08ce97cf2c --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/ThuNewsClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "5798586b105c0434e4f0fe5e767abe619442cf93", + "task_name": "ThuNewsClusteringP2P", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "v_measure": 0.897257, + "v_measure_std": 0.026429, + "v_measures": [ + 0.906127, + 0.881226, + 0.892458, + 0.918832, + 0.900083, + 0.863941, + 0.936267, + 0.844781, + 0.904992, + 0.923859 + ], + "main_score": 0.897257, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 1408.1993434429169, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/ThuNewsClusteringS2S.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/ThuNewsClusteringS2S.json new file mode 100644 index 0000000000..da229783fd --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/ThuNewsClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d", + "task_name": "ThuNewsClusteringS2S", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "v_measure": 0.895518, + "v_measure_std": 0.022667, + "v_measures": [ + 0.93066, + 0.922582, + 0.872378, + 0.861724, + 0.871606, + 0.922358, + 0.899763, + 0.900471, + 0.888647, + 0.88499 + ], + "main_score": 0.895518, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 408.02015805244446, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/VideoRetrieval.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/VideoRetrieval.json new file mode 100644 index 0000000000..08299dbc61 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/VideoRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "58c2597a5943a2ba48f4668c3b90d796283c5639", + "task_name": "VideoRetrieval", + "mteb_version": "1.38.30", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.667, + "ndcg_at_3": 0.77615, + "ndcg_at_5": 0.79428, + "ndcg_at_10": 0.80851, + "ndcg_at_20": 0.81796, + "ndcg_at_100": 0.82005, + "ndcg_at_1000": 0.82121, + "map_at_1": 0.667, + "map_at_3": 0.75017, + "map_at_5": 0.76047, + "map_at_10": 0.76634, + "map_at_20": 0.76898, + "map_at_100": 0.76929, + "map_at_1000": 0.76934, + "recall_at_1": 0.667, + "recall_at_3": 0.851, + "recall_at_5": 0.894, + "recall_at_10": 0.938, + "recall_at_20": 0.975, + "recall_at_100": 0.986, + "recall_at_1000": 0.995, + "precision_at_1": 0.667, + "precision_at_3": 0.28367, + "precision_at_5": 0.1788, + "precision_at_10": 0.0938, + "precision_at_20": 0.04875, + "precision_at_100": 0.00986, + "precision_at_1000": 0.001, + "mrr_at_1": 0.667, + "mrr_at_3": 0.750167, + "mrr_at_5": 0.760467, + "mrr_at_10": 0.766344, + "mrr_at_20": 0.76898, + "mrr_at_100": 0.76929, + "mrr_at_1000": 0.769339, + "nauc_ndcg_at_1_max": 0.274015, + "nauc_ndcg_at_1_std": -0.196296, + "nauc_ndcg_at_1_diff1": 0.689377, + "nauc_ndcg_at_3_max": 0.354737, + "nauc_ndcg_at_3_std": -0.193372, + "nauc_ndcg_at_3_diff1": 0.662637, + "nauc_ndcg_at_5_max": 0.350182, + "nauc_ndcg_at_5_std": -0.183016, + "nauc_ndcg_at_5_diff1": 0.654717, + "nauc_ndcg_at_10_max": 0.349193, + "nauc_ndcg_at_10_std": -0.178763, + "nauc_ndcg_at_10_diff1": 0.666684, + "nauc_ndcg_at_20_max": 0.348345, + "nauc_ndcg_at_20_std": -0.168612, + "nauc_ndcg_at_20_diff1": 0.668221, + "nauc_ndcg_at_100_max": 0.343199, + "nauc_ndcg_at_100_std": -0.17131, + "nauc_ndcg_at_100_diff1": 0.668422, + "nauc_ndcg_at_1000_max": 0.339018, + "nauc_ndcg_at_1000_std": -0.174817, + "nauc_ndcg_at_1000_diff1": 0.668009, + "nauc_map_at_1_max": 0.274015, + "nauc_map_at_1_std": -0.196296, + "nauc_map_at_1_diff1": 0.689377, + "nauc_map_at_3_max": 0.330334, + "nauc_map_at_3_std": -0.195971, + "nauc_map_at_3_diff1": 0.668435, + "nauc_map_at_5_max": 0.326563, + "nauc_map_at_5_std": -0.191518, + "nauc_map_at_5_diff1": 0.664795, + "nauc_map_at_10_max": 0.325999, + "nauc_map_at_10_std": -0.189593, + "nauc_map_at_10_diff1": 0.669323, + "nauc_map_at_20_max": 0.325659, + "nauc_map_at_20_std": -0.187062, + "nauc_map_at_20_diff1": 0.669764, + "nauc_map_at_100_max": 0.325065, + "nauc_map_at_100_std": -0.187297, + "nauc_map_at_100_diff1": 0.669845, + "nauc_map_at_1000_max": 0.324927, + "nauc_map_at_1000_std": -0.187377, + "nauc_map_at_1000_diff1": 0.669815, + "nauc_recall_at_1_max": 0.274015, + "nauc_recall_at_1_std": -0.196296, + "nauc_recall_at_1_diff1": 0.689377, + "nauc_recall_at_3_max": 0.461963, + "nauc_recall_at_3_std": -0.180852, + "nauc_recall_at_3_diff1": 0.638624, + "nauc_recall_at_5_max": 0.485078, + "nauc_recall_at_5_std": -0.128956, + "nauc_recall_at_5_diff1": 0.597079, + "nauc_recall_at_10_max": 0.562792, + "nauc_recall_at_10_std": -0.06789, + "nauc_recall_at_10_diff1": 0.65492, + "nauc_recall_at_20_max": 0.848665, + "nauc_recall_at_20_std": 0.324314, + "nauc_recall_at_20_diff1": 0.671466, + "nauc_recall_at_100_max": 0.929639, + "nauc_recall_at_100_std": 0.530379, + "nauc_recall_at_100_diff1": 0.675737, + "nauc_recall_at_1000_max": 0.973856, + "nauc_recall_at_1000_std": 0.862745, + "nauc_recall_at_1000_diff1": 0.63212, + "nauc_precision_at_1_max": 0.274015, + "nauc_precision_at_1_std": -0.196296, + "nauc_precision_at_1_diff1": 0.689377, + "nauc_precision_at_3_max": 0.461963, + "nauc_precision_at_3_std": -0.180852, + "nauc_precision_at_3_diff1": 0.638624, + "nauc_precision_at_5_max": 0.485078, + "nauc_precision_at_5_std": -0.128956, + "nauc_precision_at_5_diff1": 0.597079, + "nauc_precision_at_10_max": 0.562792, + "nauc_precision_at_10_std": -0.06789, + "nauc_precision_at_10_diff1": 0.65492, + "nauc_precision_at_20_max": 0.848665, + "nauc_precision_at_20_std": 0.324314, + "nauc_precision_at_20_diff1": 0.671466, + "nauc_precision_at_100_max": 0.929639, + "nauc_precision_at_100_std": 0.530379, + "nauc_precision_at_100_diff1": 0.675737, + "nauc_precision_at_1000_max": 0.973856, + "nauc_precision_at_1000_std": 0.862745, + "nauc_precision_at_1000_diff1": 0.63212, + "nauc_mrr_at_1_max": 0.274015, + "nauc_mrr_at_1_std": -0.196296, + "nauc_mrr_at_1_diff1": 0.689377, + "nauc_mrr_at_3_max": 0.330334, + "nauc_mrr_at_3_std": -0.195971, + "nauc_mrr_at_3_diff1": 0.668435, + "nauc_mrr_at_5_max": 0.326563, + "nauc_mrr_at_5_std": -0.191518, + "nauc_mrr_at_5_diff1": 0.664795, + "nauc_mrr_at_10_max": 0.325999, + "nauc_mrr_at_10_std": -0.189593, + "nauc_mrr_at_10_diff1": 0.669323, + "nauc_mrr_at_20_max": 0.325659, + "nauc_mrr_at_20_std": -0.187062, + "nauc_mrr_at_20_diff1": 0.669764, + "nauc_mrr_at_100_max": 0.325065, + "nauc_mrr_at_100_std": -0.187297, + "nauc_mrr_at_100_diff1": 0.669845, + "nauc_mrr_at_1000_max": 0.324927, + "nauc_mrr_at_1000_std": -0.187377, + "nauc_mrr_at_1000_diff1": 0.669815, + "main_score": 0.80851, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 406.888432264328, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/Waimai.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/Waimai.json new file mode 100644 index 0000000000..fed005dd12 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/Waimai.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "339287def212450dcaa9df8c22bf93e9980c7023", + "task_name": "Waimai", + "mteb_version": "1.38.30", + "scores": { + "test": [ + { + "accuracy": 0.8933, + "f1": 0.877596, + "f1_weighted": 0.892927, + "ap": 0.752943, + "ap_weighted": 0.752943, + "scores_per_experiment": [ + { + "accuracy": 0.896, + "f1": 0.8811, + "f1_weighted": 0.895832, + "ap": 0.757964, + "ap_weighted": 0.757964 + }, + { + "accuracy": 0.893, + "f1": 0.875273, + "f1_weighted": 0.89173, + "ap": 0.75429, + "ap_weighted": 0.75429 + }, + { + "accuracy": 0.897, + "f1": 0.882715, + "f1_weighted": 0.897041, + "ap": 0.759598, + "ap_weighted": 0.759598 + }, + { + "accuracy": 0.896, + "f1": 0.8811, + "f1_weighted": 0.895832, + "ap": 0.757964, + "ap_weighted": 0.757964 + }, + { + "accuracy": 0.886, + "f1": 0.872257, + "f1_weighted": 0.886922, + "ap": 0.736233, + "ap_weighted": 0.736233 + }, + { + "accuracy": 0.892, + "f1": 0.873996, + "f1_weighted": 0.890666, + "ap": 0.752268, + "ap_weighted": 0.752268 + }, + { + "accuracy": 0.896, + "f1": 0.88167, + "f1_weighted": 0.896082, + "ap": 0.757431, + "ap_weighted": 0.757431 + }, + { + "accuracy": 0.895, + "f1": 0.880437, + "f1_weighted": 0.895042, + "ap": 0.755436, + "ap_weighted": 0.755436 + }, + { + "accuracy": 0.891, + "f1": 0.874246, + "f1_weighted": 0.890311, + "ap": 0.748541, + "ap_weighted": 0.748541 + }, + { + "accuracy": 0.891, + "f1": 0.873164, + "f1_weighted": 0.889811, + "ap": 0.749708, + "ap_weighted": 0.749708 + } + ], + "main_score": 0.8933, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 21.86222243309021, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/model_meta.json b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/model_meta.json new file mode 100644 index 0000000000..936b81af65 --- /dev/null +++ b/results/Youtu-RAG__Youtu-Embedding-V1/32e04afc24817c187a8422e7bdbb493b19796d47/model_meta.json @@ -0,0 +1 @@ +{"name": "Youtu-RAG/Youtu-Embedding-V1", "revision": "32e04afc24817c187a8422e7bdbb493b19796d47", "release_date": "2025-09-28", "languages": ["zho_Hans"], "n_parameters": 2672957440, "memory_usage_mb": null, "max_tokens": 8192, "embed_dim": 2048, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["PyTorch"], "reference": "https://huggingface.co/tencent/Youtu-Embedding", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"T2Retrieval": ["train"], "DuRetrieval": ["train"], "T2Reranking": ["train"], "MMarcoReranking": ["train"], "CmedqaRetrieval": ["train"], "CMedQAv1-reranking": ["train"], "CMedQAv2-reranking": ["train"], "BQ": ["train"], "LCQMC": ["train"], "PAWSX": ["train"], "STS-B": ["train"], "AFQMC": ["train"], "Cmnli": ["train"], "Ocnli": ["train"]}, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["text"], "loader": null}