From c805b1888d803dfab403a8cd938ff409f605d851 Mon Sep 17 00:00:00 2001 From: zhangpeitian Date: Fri, 25 Apr 2025 01:47:04 +0800 Subject: [PATCH 1/2] update Seed-Embedding results --- .../1/AFQMC.json | 26 +++ .../1/ATEC.json | 26 +++ .../1/AmazonCounterfactualClassification.json | 95 +++++++++++ .../1/AmazonReviewsClassification.json | 73 ++++++++ .../1/ArXivHierarchicalClusteringP2P.json | 46 +++++ .../1/ArXivHierarchicalClusteringS2S.json | 46 +++++ .../1/ArguAna.json | 158 ++++++++++++++++++ .../1/AskUbuntuDupQuestions.json | 26 +++ .../1/BIOSSES.json | 26 +++ .../ByteDance-Seed__Seed-Embedding/1/BQ.json | 26 +++ .../1/Banking77Classification.json | 73 ++++++++ .../1/BiorxivClusteringP2P.v2.json | 34 ++++ .../1/CLSClusteringP2P.json | 32 ++++ .../1/CLSClusteringS2S.json | 32 ++++ .../1/CMedQAv1-reranking.json | 26 +++ .../1/CMedQAv2-reranking.json | 26 +++ .../1/CQADupstackGamingRetrieval.json | 158 ++++++++++++++++++ .../1/CQADupstackUnixRetrieval.json | 158 ++++++++++++++++++ .../1/ClimateFEVERHardNegatives.json | 158 ++++++++++++++++++ .../1/CmedqaRetrieval.json | 158 ++++++++++++++++++ .../1/Cmnli.json | 58 +++++++ .../1/CovidRetrieval.json | 158 ++++++++++++++++++ .../1/DuRetrieval.json | 158 ++++++++++++++++++ .../1/EcomRetrieval.json | 158 ++++++++++++++++++ .../1/FEVERHardNegatives.json | 158 ++++++++++++++++++ .../1/FiQA2018.json | 158 ++++++++++++++++++ .../1/HotpotQAHardNegatives.json | 158 ++++++++++++++++++ .../1/IFlyTek.json | 48 ++++++ .../1/ImdbClassification.json | 95 +++++++++++ .../1/JDReview.json | 95 +++++++++++ .../1/LCQMC.json | 26 +++ .../1/MMarcoReranking.json | 26 +++ .../1/MMarcoRetrieval.json | 158 ++++++++++++++++++ .../1/MTOPDomainClassification.json | 73 ++++++++ .../1/MassiveIntentClassification.json | 135 +++++++++++++++ .../1/MassiveScenarioClassification.json | 135 +++++++++++++++ .../1/MedicalRetrieval.json | 158 ++++++++++++++++++ .../1/MedrxivClusteringP2P.v2.json | 34 ++++ .../1/MedrxivClusteringS2S.v2.json | 34 ++++ .../1/MindSmallReranking.json | 26 +++ .../1/MultilingualSentiment.json | 73 ++++++++ .../1/Ocnli.json | 58 +++++++ .../1/OnlineShopping.json | 95 +++++++++++ .../1/PAWSX.json | 26 +++ .../1/QBQTC.json | 26 +++ .../1/SCIDOCS.json | 158 ++++++++++++++++++ .../1/SICK-R.json | 26 +++ .../1/STS12.json | 26 +++ .../1/STS13.json | 26 +++ .../1/STS14.json | 26 +++ .../1/STS15.json | 26 +++ .../1/STS17.json | 26 +++ .../1/STS22.v2.json | 41 +++++ .../1/STSB.json | 26 +++ .../1/STSBenchmark.json | 26 +++ .../1/SprintDuplicateQuestions.json | 58 +++++++ .../1/StackExchangeClustering.v2.json | 34 ++++ .../1/StackExchangeClusteringP2P.v2.json | 34 ++++ .../1/SummEvalSummarization.v2.json | 24 +++ .../1/T2Reranking.json | 26 +++ .../1/T2Retrieval.json | 158 ++++++++++++++++++ .../1/TNews.json | 73 ++++++++ .../1/TRECCOVID.json | 158 ++++++++++++++++++ .../1/ThuNewsClusteringP2P.json | 32 ++++ .../1/ThuNewsClusteringS2S.json | 32 ++++ .../1/Touche2020Retrieval.v3.json | 158 ++++++++++++++++++ .../1/ToxicConversationsClassification.json | 95 +++++++++++ ...weetSentimentExtractionClassification.json | 73 ++++++++ .../1/TwentyNewsgroupsClustering.v2.json | 34 ++++ .../1/TwitterSemEval2015.json | 58 +++++++ .../1/TwitterURLCorpus.json | 58 +++++++ .../1/VideoRetrieval.json | 158 ++++++++++++++++++ .../1/Waimai.json | 95 +++++++++++ 73 files changed, 5495 insertions(+) create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/AFQMC.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/ATEC.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/AmazonCounterfactualClassification.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/AmazonReviewsClassification.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/ArXivHierarchicalClusteringP2P.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/ArXivHierarchicalClusteringS2S.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/ArguAna.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/AskUbuntuDupQuestions.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/BIOSSES.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/BQ.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/Banking77Classification.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/BiorxivClusteringP2P.v2.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/CLSClusteringP2P.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/CLSClusteringS2S.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/CMedQAv1-reranking.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/CMedQAv2-reranking.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/CQADupstackGamingRetrieval.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/CQADupstackUnixRetrieval.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/ClimateFEVERHardNegatives.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/CmedqaRetrieval.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/Cmnli.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/CovidRetrieval.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/DuRetrieval.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/EcomRetrieval.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/FEVERHardNegatives.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/FiQA2018.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/HotpotQAHardNegatives.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/IFlyTek.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/ImdbClassification.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/JDReview.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/LCQMC.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/MMarcoReranking.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/MMarcoRetrieval.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/MTOPDomainClassification.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/MassiveIntentClassification.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/MassiveScenarioClassification.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/MedicalRetrieval.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/MedrxivClusteringP2P.v2.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/MedrxivClusteringS2S.v2.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/MindSmallReranking.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/MultilingualSentiment.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/Ocnli.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/OnlineShopping.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/PAWSX.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/QBQTC.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/SCIDOCS.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/SICK-R.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/STS12.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/STS13.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/STS14.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/STS15.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/STS17.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/STS22.v2.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/STSB.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/STSBenchmark.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/SprintDuplicateQuestions.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/StackExchangeClustering.v2.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/StackExchangeClusteringP2P.v2.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/SummEvalSummarization.v2.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/T2Reranking.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/T2Retrieval.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/TNews.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/TRECCOVID.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/ThuNewsClusteringP2P.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/ThuNewsClusteringS2S.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/Touche2020Retrieval.v3.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/ToxicConversationsClassification.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/TweetSentimentExtractionClassification.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/TwentyNewsgroupsClustering.v2.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/TwitterSemEval2015.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/TwitterURLCorpus.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/VideoRetrieval.json create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/Waimai.json diff --git a/results/ByteDance-Seed__Seed-Embedding/1/AFQMC.json b/results/ByteDance-Seed__Seed-Embedding/1/AFQMC.json new file mode 100644 index 0000000000..154240846f --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/AFQMC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "b44c3b011063adb25877c13823db83bb193913c4", + "task_name": "AFQMC", + "mteb_version": "1.35.1", + "scores": { + "validation": [ + { + "pearson": 0.522249, + "spearman": 0.563941, + "cosine_pearson": 0.522249, + "cosine_spearman": 0.563941, + "manhattan_pearson": 0.538638, + "manhattan_spearman": 0.561369, + "euclidean_pearson": 0.541385, + "euclidean_spearman": 0.563949, + "main_score": 0.563941, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 7.32570481300354, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/ATEC.json b/results/ByteDance-Seed__Seed-Embedding/1/ATEC.json new file mode 100644 index 0000000000..1e663de990 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/ATEC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865", + "task_name": "ATEC", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "pearson": 0.50333, + "spearman": 0.541535, + "cosine_pearson": 0.50333, + "cosine_spearman": 0.541535, + "manhattan_pearson": 0.549648, + "manhattan_spearman": 0.540144, + "euclidean_pearson": 0.551242, + "euclidean_spearman": 0.541514, + "main_score": 0.541535, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 32.43285655975342, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/AmazonCounterfactualClassification.json b/results/ByteDance-Seed__Seed-Embedding/1/AmazonCounterfactualClassification.json new file mode 100644 index 0000000000..12855e2f70 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/AmazonCounterfactualClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e8379541af4e31359cca9fbcf4b00f2671dba205", + "task_name": "AmazonCounterfactualClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.928955, + "f1": 0.896386, + "f1_weighted": 0.931758, + "ap": 0.723022, + "ap_weighted": 0.723022, + "scores_per_experiment": [ + { + "accuracy": 0.928358, + "f1": 0.895423, + "f1_weighted": 0.931161, + "ap": 0.720459, + "ap_weighted": 0.720459 + }, + { + "accuracy": 0.926866, + "f1": 0.893504, + "f1_weighted": 0.929801, + "ap": 0.716139, + "ap_weighted": 0.716139 + }, + { + "accuracy": 0.938806, + "f1": 0.909103, + "f1_weighted": 0.940745, + "ap": 0.752252, + "ap_weighted": 0.752252 + }, + { + "accuracy": 0.931343, + "f1": 0.899286, + "f1_weighted": 0.933887, + "ap": 0.72926, + "ap_weighted": 0.72926 + }, + { + "accuracy": 0.925373, + "f1": 0.891594, + "f1_weighted": 0.928444, + "ap": 0.711871, + "ap_weighted": 0.711871 + }, + { + "accuracy": 0.931343, + "f1": 0.899286, + "f1_weighted": 0.933887, + "ap": 0.72926, + "ap_weighted": 0.72926 + }, + { + "accuracy": 0.926866, + "f1": 0.893504, + "f1_weighted": 0.929801, + "ap": 0.716139, + "ap_weighted": 0.716139 + }, + { + "accuracy": 0.937313, + "f1": 0.907121, + "f1_weighted": 0.939368, + "ap": 0.747536, + "ap_weighted": 0.747536 + }, + { + "accuracy": 0.929851, + "f1": 0.897851, + "f1_weighted": 0.932667, + "ap": 0.726143, + "ap_weighted": 0.726143 + }, + { + "accuracy": 0.913433, + "f1": 0.877182, + "f1_weighted": 0.917815, + "ap": 0.681157, + "ap_weighted": 0.681157 + } + ], + "main_score": 0.928955, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 18.539973974227905, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/AmazonReviewsClassification.json b/results/ByteDance-Seed__Seed-Embedding/1/AmazonReviewsClassification.json new file mode 100644 index 0000000000..87f16c2174 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/AmazonReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "1399c76144fd37290681b995c656ef9b2e06e26d", + "task_name": "AmazonReviewsClassification", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "accuracy": 0.56052, + "f1": 0.535356, + "f1_weighted": 0.535356, + "scores_per_experiment": [ + { + "accuracy": 0.5678, + "f1": 0.548516, + "f1_weighted": 0.548516 + }, + { + "accuracy": 0.5268, + "f1": 0.480994, + "f1_weighted": 0.480994 + }, + { + "accuracy": 0.5544, + "f1": 0.515861, + "f1_weighted": 0.515861 + }, + { + "accuracy": 0.58, + "f1": 0.556757, + "f1_weighted": 0.556757 + }, + { + "accuracy": 0.5484, + "f1": 0.522065, + "f1_weighted": 0.522065 + }, + { + "accuracy": 0.5558, + "f1": 0.523363, + "f1_weighted": 0.523363 + }, + { + "accuracy": 0.5662, + "f1": 0.556193, + "f1_weighted": 0.556193 + }, + { + "accuracy": 0.5574, + "f1": 0.539821, + "f1_weighted": 0.539821 + }, + { + "accuracy": 0.5772, + "f1": 0.562252, + "f1_weighted": 0.562252 + }, + { + "accuracy": 0.5712, + "f1": 0.547738, + "f1_weighted": 0.547738 + } + ], + "main_score": 0.56052, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 24.05517029762268, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/ArXivHierarchicalClusteringP2P.json b/results/ByteDance-Seed__Seed-Embedding/1/ArXivHierarchicalClusteringP2P.json new file mode 100644 index 0000000000..f79743a18e --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/ArXivHierarchicalClusteringP2P.json @@ -0,0 +1,46 @@ +{ + "dataset_revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8", + "task_name": "ArXivHierarchicalClusteringP2P", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.73266, + 0.695896, + 0.654285, + 0.70933, + 0.727689, + 0.727024, + 0.712068, + 0.700725, + 0.720269, + 0.689926 + ], + "Level 1": [ + 0.604426, + 0.58303, + 0.614237, + 0.633213, + 0.602311, + 0.632342, + 0.605077, + 0.588392, + 0.580594, + 0.611746 + ] + }, + "v_measure": 0.656262, + "v_measure_std": 0.054508, + "main_score": 0.656262, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 12.032912731170654, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/ArXivHierarchicalClusteringS2S.json b/results/ByteDance-Seed__Seed-Embedding/1/ArXivHierarchicalClusteringS2S.json new file mode 100644 index 0000000000..369c417809 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/ArXivHierarchicalClusteringS2S.json @@ -0,0 +1,46 @@ +{ + "dataset_revision": "b73bd54100e5abfa6e3a23dcafb46fe4d2438dc3", + "task_name": "ArXivHierarchicalClusteringS2S", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.658798, + 0.642727, + 0.654446, + 0.681278, + 0.720408, + 0.660879, + 0.638221, + 0.667651, + 0.679952, + 0.637898 + ], + "Level 1": [ + 0.611611, + 0.626843, + 0.611328, + 0.596726, + 0.621872, + 0.626087, + 0.596075, + 0.607581, + 0.606083, + 0.618189 + ] + }, + "v_measure": 0.638233, + "v_measure_std": 0.031846, + "main_score": 0.638233, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.928880453109741, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/ArguAna.json b/results/ByteDance-Seed__Seed-Embedding/1/ArguAna.json new file mode 100644 index 0000000000..b75161397e --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/ArguAna.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a", + "task_name": "ArguAna", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.57895, + "ndcg_at_3": 0.72455, + "ndcg_at_5": 0.75687, + "ndcg_at_10": 0.77888, + "ndcg_at_20": 0.78374, + "ndcg_at_100": 0.78472, + "ndcg_at_1000": 0.78472, + "map_at_1": 0.57895, + "map_at_3": 0.68883, + "map_at_5": 0.70665, + "map_at_10": 0.71609, + "map_at_20": 0.71753, + "map_at_100": 0.71768, + "map_at_1000": 0.71768, + "recall_at_1": 0.57895, + "recall_at_3": 0.82788, + "recall_at_5": 0.90683, + "recall_at_10": 0.97297, + "recall_at_20": 0.99147, + "recall_at_100": 0.99644, + "recall_at_1000": 0.99644, + "precision_at_1": 0.57895, + "precision_at_3": 0.27596, + "precision_at_5": 0.18137, + "precision_at_10": 0.0973, + "precision_at_20": 0.04957, + "precision_at_100": 0.00996, + "precision_at_1000": 0.001, + "mrr_at_1": 0.578947, + "mrr_at_3": 0.688834, + "mrr_at_5": 0.70665, + "mrr_at_10": 0.716094, + "mrr_at_20": 0.717531, + "mrr_at_100": 0.717684, + "mrr_at_1000": 0.717684, + "nauc_ndcg_at_1_max": -0.072778, + "nauc_ndcg_at_1_std": -0.207922, + "nauc_ndcg_at_1_diff1": 0.29081, + "nauc_ndcg_at_3_max": -0.044336, + "nauc_ndcg_at_3_std": -0.18088, + "nauc_ndcg_at_3_diff1": 0.243035, + "nauc_ndcg_at_5_max": -0.043502, + "nauc_ndcg_at_5_std": -0.181422, + "nauc_ndcg_at_5_diff1": 0.249759, + "nauc_ndcg_at_10_max": -0.058321, + "nauc_ndcg_at_10_std": -0.188047, + "nauc_ndcg_at_10_diff1": 0.2512, + "nauc_ndcg_at_20_max": -0.053174, + "nauc_ndcg_at_20_std": -0.179819, + "nauc_ndcg_at_20_diff1": 0.257802, + "nauc_ndcg_at_100_max": -0.054687, + "nauc_ndcg_at_100_std": -0.184222, + "nauc_ndcg_at_100_diff1": 0.257092, + "nauc_ndcg_at_1000_max": -0.054687, + "nauc_ndcg_at_1000_std": -0.184222, + "nauc_ndcg_at_1000_diff1": 0.257092, + "nauc_map_at_1_max": -0.072778, + "nauc_map_at_1_std": -0.207922, + "nauc_map_at_1_diff1": 0.29081, + "nauc_map_at_3_max": -0.053762, + "nauc_map_at_3_std": -0.188522, + "nauc_map_at_3_diff1": 0.253774, + "nauc_map_at_5_max": -0.054309, + "nauc_map_at_5_std": -0.189309, + "nauc_map_at_5_diff1": 0.257599, + "nauc_map_at_10_max": -0.059744, + "nauc_map_at_10_std": -0.191914, + "nauc_map_at_10_diff1": 0.258341, + "nauc_map_at_20_max": -0.058466, + "nauc_map_at_20_std": -0.189903, + "nauc_map_at_20_diff1": 0.259883, + "nauc_map_at_100_max": -0.058612, + "nauc_map_at_100_std": -0.190417, + "nauc_map_at_100_diff1": 0.259814, + "nauc_map_at_1000_max": -0.058612, + "nauc_map_at_1000_std": -0.190417, + "nauc_map_at_1000_diff1": 0.259814, + "nauc_recall_at_1_max": -0.072778, + "nauc_recall_at_1_std": -0.207922, + "nauc_recall_at_1_diff1": 0.29081, + "nauc_recall_at_3_max": -0.001481, + "nauc_recall_at_3_std": -0.146922, + "nauc_recall_at_3_diff1": 0.197245, + "nauc_recall_at_5_max": 0.038268, + "nauc_recall_at_5_std": -0.12412, + "nauc_recall_at_5_diff1": 0.198933, + "nauc_recall_at_10_max": -0.058538, + "nauc_recall_at_10_std": -0.114516, + "nauc_recall_at_10_diff1": 0.106702, + "nauc_recall_at_20_max": 0.318531, + "nauc_recall_at_20_std": 0.654365, + "nauc_recall_at_20_diff1": 0.307642, + "nauc_recall_at_100_max": 0.426109, + "nauc_recall_at_100_std": 0.704948, + "nauc_recall_at_100_diff1": 0.186616, + "nauc_recall_at_1000_max": 0.426109, + "nauc_recall_at_1000_std": 0.704948, + "nauc_recall_at_1000_diff1": 0.186616, + "nauc_precision_at_1_max": -0.072778, + "nauc_precision_at_1_std": -0.207922, + "nauc_precision_at_1_diff1": 0.29081, + "nauc_precision_at_3_max": -0.001481, + "nauc_precision_at_3_std": -0.146922, + "nauc_precision_at_3_diff1": 0.197245, + "nauc_precision_at_5_max": 0.038268, + "nauc_precision_at_5_std": -0.12412, + "nauc_precision_at_5_diff1": 0.198933, + "nauc_precision_at_10_max": -0.058538, + "nauc_precision_at_10_std": -0.114516, + "nauc_precision_at_10_diff1": 0.106702, + "nauc_precision_at_20_max": 0.318531, + "nauc_precision_at_20_std": 0.654365, + "nauc_precision_at_20_diff1": 0.307642, + "nauc_precision_at_100_max": 0.426109, + "nauc_precision_at_100_std": 0.704948, + "nauc_precision_at_100_diff1": 0.186616, + "nauc_precision_at_1000_max": 0.426109, + "nauc_precision_at_1000_std": 0.704948, + "nauc_precision_at_1000_diff1": 0.186616, + "nauc_mrr_at_1_max": -0.072778, + "nauc_mrr_at_1_std": -0.207922, + "nauc_mrr_at_1_diff1": 0.29081, + "nauc_mrr_at_3_max": -0.053762, + "nauc_mrr_at_3_std": -0.188522, + "nauc_mrr_at_3_diff1": 0.253774, + "nauc_mrr_at_5_max": -0.054309, + "nauc_mrr_at_5_std": -0.189309, + "nauc_mrr_at_5_diff1": 0.257599, + "nauc_mrr_at_10_max": -0.059744, + "nauc_mrr_at_10_std": -0.191914, + "nauc_mrr_at_10_diff1": 0.258341, + "nauc_mrr_at_20_max": -0.058466, + "nauc_mrr_at_20_std": -0.189903, + "nauc_mrr_at_20_diff1": 0.259883, + "nauc_mrr_at_100_max": -0.058612, + "nauc_mrr_at_100_std": -0.190417, + "nauc_mrr_at_100_diff1": 0.259814, + "nauc_mrr_at_1000_max": -0.058612, + "nauc_mrr_at_1000_std": -0.190417, + "nauc_mrr_at_1000_diff1": 0.259814, + "main_score": 0.77888, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1420.340444803238, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/AskUbuntuDupQuestions.json b/results/ByteDance-Seed__Seed-Embedding/1/AskUbuntuDupQuestions.json new file mode 100644 index 0000000000..82391bc9ea --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/AskUbuntuDupQuestions.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2000358ca161889fa9c082cb41daa8dcfb161a54", + "task_name": "AskUbuntuDupQuestions", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "map": 0.688668, + "mrr": 0.824797, + "nAUC_map_max": 0.282728, + "nAUC_map_std": 0.175799, + "nAUC_map_diff1": 0.041633, + "nAUC_mrr_max": 0.422197, + "nAUC_mrr_std": 0.288083, + "nAUC_mrr_diff1": 0.0448, + "main_score": 0.688668, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 17.977734804153442, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/BIOSSES.json b/results/ByteDance-Seed__Seed-Embedding/1/BIOSSES.json new file mode 100644 index 0000000000..45c7ad7e4d --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/BIOSSES.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a", + "task_name": "BIOSSES", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.882105, + "spearman": 0.872261, + "cosine_pearson": 0.882105, + "cosine_spearman": 0.872261, + "manhattan_pearson": 0.87393, + "manhattan_spearman": 0.872405, + "euclidean_pearson": 0.874648, + "euclidean_spearman": 0.87146, + "main_score": 0.872261, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.0750174522399902, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/BQ.json b/results/ByteDance-Seed__Seed-Embedding/1/BQ.json new file mode 100644 index 0000000000..ed3ef929de --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/BQ.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55", + "task_name": "BQ", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "pearson": 0.660873, + "spearman": 0.682252, + "cosine_pearson": 0.660873, + "cosine_spearman": 0.682252, + "manhattan_pearson": 0.654615, + "manhattan_spearman": 0.678259, + "euclidean_pearson": 0.658715, + "euclidean_spearman": 0.68218, + "main_score": 0.682252, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 56.92221784591675, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/Banking77Classification.json b/results/ByteDance-Seed__Seed-Embedding/1/Banking77Classification.json new file mode 100644 index 0000000000..946bfda5f7 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/Banking77Classification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "0fd18e25b25c072e09e0d92ab615fda904d66300", + "task_name": "Banking77Classification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.912468, + "f1": 0.910866, + "f1_weighted": 0.910866, + "scores_per_experiment": [ + { + "accuracy": 0.913636, + "f1": 0.912143, + "f1_weighted": 0.912143 + }, + { + "accuracy": 0.909416, + "f1": 0.907919, + "f1_weighted": 0.907919 + }, + { + "accuracy": 0.916234, + "f1": 0.91511, + "f1_weighted": 0.91511 + }, + { + "accuracy": 0.916234, + "f1": 0.914866, + "f1_weighted": 0.914866 + }, + { + "accuracy": 0.916558, + "f1": 0.915769, + "f1_weighted": 0.915769 + }, + { + "accuracy": 0.91526, + "f1": 0.914493, + "f1_weighted": 0.914493 + }, + { + "accuracy": 0.911688, + "f1": 0.91047, + "f1_weighted": 0.91047 + }, + { + "accuracy": 0.90487, + "f1": 0.902653, + "f1_weighted": 0.902653 + }, + { + "accuracy": 0.905195, + "f1": 0.901001, + "f1_weighted": 0.901001 + }, + { + "accuracy": 0.915584, + "f1": 0.914242, + "f1_weighted": 0.914242 + } + ], + "main_score": 0.912468, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 42.07764482498169, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/BiorxivClusteringP2P.v2.json b/results/ByteDance-Seed__Seed-Embedding/1/BiorxivClusteringP2P.v2.json new file mode 100644 index 0000000000..de5483b4bb --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/BiorxivClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "f5dbc242e11dd8e24def4c4268607a49e02946dc", + "task_name": "BiorxivClusteringP2P.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.561504, + 0.557192, + 0.561424, + 0.560246, + 0.550013, + 0.563429, + 0.552817, + 0.560374, + 0.566841, + 0.557839 + ] + }, + "v_measure": 0.559168, + "v_measure_std": 0.004691, + "main_score": 0.559168, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11.150334119796753, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/CLSClusteringP2P.json b/results/ByteDance-Seed__Seed-Embedding/1/CLSClusteringP2P.json new file mode 100644 index 0000000000..be2273ebd7 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/CLSClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476", + "task_name": "CLSClusteringP2P", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "v_measure": 0.514046, + "v_measure_std": 0.015248, + "v_measures": [ + 0.521281, + 0.538393, + 0.510385, + 0.523693, + 0.493318, + 0.489151, + 0.517012, + 0.526729, + 0.523003, + 0.497492 + ], + "main_score": 0.514046, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 134.91166996955872, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/CLSClusteringS2S.json b/results/ByteDance-Seed__Seed-Embedding/1/CLSClusteringS2S.json new file mode 100644 index 0000000000..6c2e5ce9f4 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/CLSClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f", + "task_name": "CLSClusteringS2S", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "v_measure": 0.572582, + "v_measure_std": 0.015953, + "v_measures": [ + 0.572429, + 0.569374, + 0.577527, + 0.57303, + 0.565517, + 0.585961, + 0.56842, + 0.553318, + 0.609945, + 0.550299 + ], + "main_score": 0.572582, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 79.66315746307373, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/CMedQAv1-reranking.json b/results/ByteDance-Seed__Seed-Embedding/1/CMedQAv1-reranking.json new file mode 100644 index 0000000000..b7afcbe3cb --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/CMedQAv1-reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "8d7f1e942507dac42dc58017c1a001c3717da7df", + "task_name": "CMedQAv1-reranking", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "map": 0.869013, + "mrr": 0.892866, + "nAUC_map_max": 0.593634, + "nAUC_map_std": 0.230368, + "nAUC_map_diff1": 0.607619, + "nAUC_mrr_max": 0.708248, + "nAUC_mrr_std": 0.343957, + "nAUC_mrr_diff1": 0.717343, + "main_score": 0.869013, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 65.89924955368042, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/CMedQAv2-reranking.json b/results/ByteDance-Seed__Seed-Embedding/1/CMedQAv2-reranking.json new file mode 100644 index 0000000000..f263c04e67 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/CMedQAv2-reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "23d186750531a14a0357ca22cd92d712fd512ea0", + "task_name": "CMedQAv2-reranking", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "map": 0.862406, + "mrr": 0.888294, + "nAUC_map_max": 0.642646, + "nAUC_map_std": 0.168837, + "nAUC_map_diff1": 0.645659, + "nAUC_mrr_max": 0.761593, + "nAUC_mrr_std": 0.28314, + "nAUC_mrr_diff1": 0.735789, + "main_score": 0.862406, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 169.12384462356567, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/CQADupstackGamingRetrieval.json b/results/ByteDance-Seed__Seed-Embedding/1/CQADupstackGamingRetrieval.json new file mode 100644 index 0000000000..319b4b382a --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/CQADupstackGamingRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4885aa143210c98657558c04aaf3dc47cfb54340", + "task_name": "CQADupstackGamingRetrieval", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.57304, + "ndcg_at_3": 0.65432, + "ndcg_at_5": 0.68641, + "ndcg_at_10": 0.71373, + "ndcg_at_20": 0.72619, + "ndcg_at_100": 0.73961, + "ndcg_at_1000": 0.73961, + "map_at_1": 0.49967, + "map_at_3": 0.61039, + "map_at_5": 0.63348, + "map_at_10": 0.64913, + "map_at_20": 0.65445, + "map_at_100": 0.65754, + "map_at_1000": 0.65754, + "recall_at_1": 0.49967, + "recall_at_3": 0.71086, + "recall_at_5": 0.78945, + "recall_at_10": 0.86555, + "recall_at_20": 0.91013, + "recall_at_100": 0.96913, + "recall_at_1000": 0.96913, + "precision_at_1": 0.57304, + "precision_at_3": 0.29112, + "precision_at_5": 0.199, + "precision_at_10": 0.11354, + "precision_at_20": 0.06135, + "precision_at_100": 0.01353, + "precision_at_1000": 0.00135, + "mrr_at_1": 0.573041, + "mrr_at_3": 0.660711, + "mrr_at_5": 0.676917, + "mrr_at_10": 0.684712, + "mrr_at_20": 0.687223, + "mrr_at_100": 0.688573, + "mrr_at_1000": 0.688573, + "nauc_ndcg_at_1_max": 0.211132, + "nauc_ndcg_at_1_std": 0.023022, + "nauc_ndcg_at_1_diff1": 0.556816, + "nauc_ndcg_at_3_max": 0.237673, + "nauc_ndcg_at_3_std": 0.065828, + "nauc_ndcg_at_3_diff1": 0.524955, + "nauc_ndcg_at_5_max": 0.250063, + "nauc_ndcg_at_5_std": 0.089708, + "nauc_ndcg_at_5_diff1": 0.524523, + "nauc_ndcg_at_10_max": 0.266034, + "nauc_ndcg_at_10_std": 0.123381, + "nauc_ndcg_at_10_diff1": 0.528784, + "nauc_ndcg_at_20_max": 0.264261, + "nauc_ndcg_at_20_std": 0.116198, + "nauc_ndcg_at_20_diff1": 0.526552, + "nauc_ndcg_at_100_max": 0.260132, + "nauc_ndcg_at_100_std": 0.105998, + "nauc_ndcg_at_100_diff1": 0.531493, + "nauc_ndcg_at_1000_max": 0.260132, + "nauc_ndcg_at_1000_std": 0.105998, + "nauc_ndcg_at_1000_diff1": 0.531493, + "nauc_map_at_1_max": 0.139645, + "nauc_map_at_1_std": -0.059805, + "nauc_map_at_1_diff1": 0.554878, + "nauc_map_at_3_max": 0.20313, + "nauc_map_at_3_std": 0.015276, + "nauc_map_at_3_diff1": 0.530649, + "nauc_map_at_5_max": 0.215189, + "nauc_map_at_5_std": 0.039036, + "nauc_map_at_5_diff1": 0.531067, + "nauc_map_at_10_max": 0.227327, + "nauc_map_at_10_std": 0.062066, + "nauc_map_at_10_diff1": 0.534018, + "nauc_map_at_20_max": 0.230966, + "nauc_map_at_20_std": 0.065678, + "nauc_map_at_20_diff1": 0.533715, + "nauc_map_at_100_max": 0.231982, + "nauc_map_at_100_std": 0.066081, + "nauc_map_at_100_diff1": 0.534761, + "nauc_map_at_1000_max": 0.231982, + "nauc_map_at_1000_std": 0.066081, + "nauc_map_at_1000_diff1": 0.534761, + "nauc_recall_at_1_max": 0.139645, + "nauc_recall_at_1_std": -0.059805, + "nauc_recall_at_1_diff1": 0.554878, + "nauc_recall_at_3_max": 0.24274, + "nauc_recall_at_3_std": 0.079031, + "nauc_recall_at_3_diff1": 0.471301, + "nauc_recall_at_5_max": 0.282581, + "nauc_recall_at_5_std": 0.169228, + "nauc_recall_at_5_diff1": 0.451369, + "nauc_recall_at_10_max": 0.376176, + "nauc_recall_at_10_std": 0.371972, + "nauc_recall_at_10_diff1": 0.455207, + "nauc_recall_at_20_max": 0.425234, + "nauc_recall_at_20_std": 0.41989, + "nauc_recall_at_20_diff1": 0.42209, + "nauc_recall_at_100_max": 0.578556, + "nauc_recall_at_100_std": 0.617542, + "nauc_recall_at_100_diff1": 0.441287, + "nauc_recall_at_1000_max": 0.578556, + "nauc_recall_at_1000_std": 0.617542, + "nauc_recall_at_1000_diff1": 0.441287, + "nauc_precision_at_1_max": 0.211132, + "nauc_precision_at_1_std": 0.023022, + "nauc_precision_at_1_diff1": 0.556816, + "nauc_precision_at_3_max": 0.29122, + "nauc_precision_at_3_std": 0.24271, + "nauc_precision_at_3_diff1": 0.237195, + "nauc_precision_at_5_max": 0.296589, + "nauc_precision_at_5_std": 0.337675, + "nauc_precision_at_5_diff1": 0.129178, + "nauc_precision_at_10_max": 0.287196, + "nauc_precision_at_10_std": 0.403674, + "nauc_precision_at_10_diff1": 0.018802, + "nauc_precision_at_20_max": 0.273475, + "nauc_precision_at_20_std": 0.393492, + "nauc_precision_at_20_diff1": -0.047526, + "nauc_precision_at_100_max": 0.217003, + "nauc_precision_at_100_std": 0.323044, + "nauc_precision_at_100_diff1": -0.103055, + "nauc_precision_at_1000_max": 0.217003, + "nauc_precision_at_1000_std": 0.323044, + "nauc_precision_at_1000_diff1": -0.103055, + "nauc_mrr_at_1_max": 0.211132, + "nauc_mrr_at_1_std": 0.023022, + "nauc_mrr_at_1_diff1": 0.556816, + "nauc_mrr_at_3_max": 0.250594, + "nauc_mrr_at_3_std": 0.075573, + "nauc_mrr_at_3_diff1": 0.537284, + "nauc_mrr_at_5_max": 0.251421, + "nauc_mrr_at_5_std": 0.084193, + "nauc_mrr_at_5_diff1": 0.535215, + "nauc_mrr_at_10_max": 0.25223, + "nauc_mrr_at_10_std": 0.089193, + "nauc_mrr_at_10_diff1": 0.536242, + "nauc_mrr_at_20_max": 0.250983, + "nauc_mrr_at_20_std": 0.085522, + "nauc_mrr_at_20_diff1": 0.536088, + "nauc_mrr_at_100_max": 0.250057, + "nauc_mrr_at_100_std": 0.084342, + "nauc_mrr_at_100_diff1": 0.536674, + "nauc_mrr_at_1000_max": 0.250057, + "nauc_mrr_at_1000_std": 0.084342, + "nauc_mrr_at_1000_diff1": 0.536674, + "main_score": 0.71373, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 29.369771718978882, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/CQADupstackUnixRetrieval.json b/results/ByteDance-Seed__Seed-Embedding/1/CQADupstackUnixRetrieval.json new file mode 100644 index 0000000000..8f7cc0b51c --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/CQADupstackUnixRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53", + "task_name": "CQADupstackUnixRetrieval", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.4375, + "ndcg_at_3": 0.50487, + "ndcg_at_5": 0.53621, + "ndcg_at_10": 0.57451, + "ndcg_at_20": 0.59288, + "ndcg_at_100": 0.61586, + "ndcg_at_1000": 0.61586, + "map_at_1": 0.36777, + "map_at_3": 0.46215, + "map_at_5": 0.48557, + "map_at_10": 0.50583, + "map_at_20": 0.51219, + "map_at_100": 0.51652, + "map_at_1000": 0.51652, + "recall_at_1": 0.36777, + "recall_at_3": 0.55016, + "recall_at_5": 0.62973, + "recall_at_10": 0.73974, + "recall_at_20": 0.80373, + "recall_at_100": 0.90855, + "recall_at_1000": 0.90855, + "precision_at_1": 0.4375, + "precision_at_3": 0.23383, + "precision_at_5": 0.16549, + "precision_at_10": 0.10028, + "precision_at_20": 0.05592, + "precision_at_100": 0.01326, + "precision_at_1000": 0.00133, + "mrr_at_1": 0.4375, + "mrr_at_3": 0.516947, + "mrr_at_5": 0.533644, + "mrr_at_10": 0.546948, + "mrr_at_20": 0.55099, + "mrr_at_100": 0.553285, + "mrr_at_1000": 0.553285, + "nauc_ndcg_at_1_max": 0.355973, + "nauc_ndcg_at_1_std": 0.249063, + "nauc_ndcg_at_1_diff1": 0.506743, + "nauc_ndcg_at_3_max": 0.337312, + "nauc_ndcg_at_3_std": 0.228865, + "nauc_ndcg_at_3_diff1": 0.459019, + "nauc_ndcg_at_5_max": 0.346607, + "nauc_ndcg_at_5_std": 0.267002, + "nauc_ndcg_at_5_diff1": 0.443642, + "nauc_ndcg_at_10_max": 0.351131, + "nauc_ndcg_at_10_std": 0.295231, + "nauc_ndcg_at_10_diff1": 0.443823, + "nauc_ndcg_at_20_max": 0.359987, + "nauc_ndcg_at_20_std": 0.2973, + "nauc_ndcg_at_20_diff1": 0.450046, + "nauc_ndcg_at_100_max": 0.355292, + "nauc_ndcg_at_100_std": 0.286342, + "nauc_ndcg_at_100_diff1": 0.447614, + "nauc_ndcg_at_1000_max": 0.355292, + "nauc_ndcg_at_1000_std": 0.286342, + "nauc_ndcg_at_1000_diff1": 0.447614, + "nauc_map_at_1_max": 0.283646, + "nauc_map_at_1_std": 0.151053, + "nauc_map_at_1_diff1": 0.5291, + "nauc_map_at_3_max": 0.318538, + "nauc_map_at_3_std": 0.201527, + "nauc_map_at_3_diff1": 0.477618, + "nauc_map_at_5_max": 0.332112, + "nauc_map_at_5_std": 0.234828, + "nauc_map_at_5_diff1": 0.46827, + "nauc_map_at_10_max": 0.334639, + "nauc_map_at_10_std": 0.252369, + "nauc_map_at_10_diff1": 0.463841, + "nauc_map_at_20_max": 0.338473, + "nauc_map_at_20_std": 0.254478, + "nauc_map_at_20_diff1": 0.465597, + "nauc_map_at_100_max": 0.338352, + "nauc_map_at_100_std": 0.252972, + "nauc_map_at_100_diff1": 0.464978, + "nauc_map_at_1000_max": 0.338352, + "nauc_map_at_1000_std": 0.252972, + "nauc_map_at_1000_diff1": 0.464978, + "nauc_recall_at_1_max": 0.283646, + "nauc_recall_at_1_std": 0.151053, + "nauc_recall_at_1_diff1": 0.5291, + "nauc_recall_at_3_max": 0.313619, + "nauc_recall_at_3_std": 0.204859, + "nauc_recall_at_3_diff1": 0.420296, + "nauc_recall_at_5_max": 0.338024, + "nauc_recall_at_5_std": 0.301061, + "nauc_recall_at_5_diff1": 0.3712, + "nauc_recall_at_10_max": 0.352022, + "nauc_recall_at_10_std": 0.410858, + "nauc_recall_at_10_diff1": 0.357107, + "nauc_recall_at_20_max": 0.395157, + "nauc_recall_at_20_std": 0.44442, + "nauc_recall_at_20_diff1": 0.377462, + "nauc_recall_at_100_max": 0.373281, + "nauc_recall_at_100_std": 0.460297, + "nauc_recall_at_100_diff1": 0.315804, + "nauc_recall_at_1000_max": 0.373281, + "nauc_recall_at_1000_std": 0.460297, + "nauc_recall_at_1000_diff1": 0.315804, + "nauc_precision_at_1_max": 0.355973, + "nauc_precision_at_1_std": 0.249063, + "nauc_precision_at_1_diff1": 0.506743, + "nauc_precision_at_3_max": 0.336537, + "nauc_precision_at_3_std": 0.305508, + "nauc_precision_at_3_diff1": 0.241725, + "nauc_precision_at_5_max": 0.317635, + "nauc_precision_at_5_std": 0.36642, + "nauc_precision_at_5_diff1": 0.131995, + "nauc_precision_at_10_max": 0.252598, + "nauc_precision_at_10_std": 0.383539, + "nauc_precision_at_10_diff1": 0.028773, + "nauc_precision_at_20_max": 0.234428, + "nauc_precision_at_20_std": 0.345699, + "nauc_precision_at_20_diff1": -0.014836, + "nauc_precision_at_100_max": 0.110356, + "nauc_precision_at_100_std": 0.190717, + "nauc_precision_at_100_diff1": -0.130708, + "nauc_precision_at_1000_max": 0.110356, + "nauc_precision_at_1000_std": 0.190717, + "nauc_precision_at_1000_diff1": -0.130708, + "nauc_mrr_at_1_max": 0.355973, + "nauc_mrr_at_1_std": 0.249063, + "nauc_mrr_at_1_diff1": 0.506743, + "nauc_mrr_at_3_max": 0.361759, + "nauc_mrr_at_3_std": 0.261839, + "nauc_mrr_at_3_diff1": 0.464115, + "nauc_mrr_at_5_max": 0.365465, + "nauc_mrr_at_5_std": 0.277284, + "nauc_mrr_at_5_diff1": 0.45393, + "nauc_mrr_at_10_max": 0.366139, + "nauc_mrr_at_10_std": 0.282609, + "nauc_mrr_at_10_diff1": 0.458948, + "nauc_mrr_at_20_max": 0.366723, + "nauc_mrr_at_20_std": 0.281238, + "nauc_mrr_at_20_diff1": 0.460643, + "nauc_mrr_at_100_max": 0.365828, + "nauc_mrr_at_100_std": 0.279873, + "nauc_mrr_at_100_diff1": 0.460285, + "nauc_mrr_at_1000_max": 0.365828, + "nauc_mrr_at_1000_std": 0.279873, + "nauc_mrr_at_1000_diff1": 0.460285, + "main_score": 0.57451, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 24.48278522491455, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/ClimateFEVERHardNegatives.json b/results/ByteDance-Seed__Seed-Embedding/1/ClimateFEVERHardNegatives.json new file mode 100644 index 0000000000..1e9e519392 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/ClimateFEVERHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "3a309e201f3c2c4b13bd4a367a8f37eee2ec1d21", + "task_name": "ClimateFEVERHardNegatives", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.504, + "ndcg_at_3": 0.41397, + "ndcg_at_5": 0.43611, + "ndcg_at_10": 0.47554, + "ndcg_at_20": 0.50652, + "ndcg_at_100": 0.55001, + "ndcg_at_1000": 0.55001, + "map_at_1": 0.21843, + "map_at_3": 0.31456, + "map_at_5": 0.34628, + "map_at_10": 0.37018, + "map_at_20": 0.38288, + "map_at_100": 0.39308, + "map_at_1000": 0.39308, + "recall_at_1": 0.21843, + "recall_at_3": 0.3661, + "recall_at_5": 0.43888, + "recall_at_10": 0.52362, + "recall_at_20": 0.60923, + "recall_at_100": 0.77123, + "recall_at_1000": 0.77123, + "precision_at_1": 0.504, + "precision_at_3": 0.30767, + "precision_at_5": 0.231, + "precision_at_10": 0.1428, + "precision_at_20": 0.0852, + "precision_at_100": 0.02249, + "precision_at_1000": 0.00225, + "mrr_at_1": 0.504, + "mrr_at_3": 0.598333, + "mrr_at_5": 0.612183, + "mrr_at_10": 0.620017, + "mrr_at_20": 0.624378, + "mrr_at_100": 0.625984, + "mrr_at_1000": 0.625984, + "nauc_ndcg_at_1_max": 0.361363, + "nauc_ndcg_at_1_std": 0.280609, + "nauc_ndcg_at_1_diff1": 0.330934, + "nauc_ndcg_at_3_max": 0.394164, + "nauc_ndcg_at_3_std": 0.354972, + "nauc_ndcg_at_3_diff1": 0.221943, + "nauc_ndcg_at_5_max": 0.415585, + "nauc_ndcg_at_5_std": 0.397465, + "nauc_ndcg_at_5_diff1": 0.227053, + "nauc_ndcg_at_10_max": 0.416809, + "nauc_ndcg_at_10_std": 0.420652, + "nauc_ndcg_at_10_diff1": 0.213965, + "nauc_ndcg_at_20_max": 0.413938, + "nauc_ndcg_at_20_std": 0.428754, + "nauc_ndcg_at_20_diff1": 0.212231, + "nauc_ndcg_at_100_max": 0.424224, + "nauc_ndcg_at_100_std": 0.43472, + "nauc_ndcg_at_100_diff1": 0.222924, + "nauc_ndcg_at_1000_max": 0.424224, + "nauc_ndcg_at_1000_std": 0.43472, + "nauc_ndcg_at_1000_diff1": 0.222924, + "nauc_map_at_1_max": 0.370261, + "nauc_map_at_1_std": 0.234891, + "nauc_map_at_1_diff1": 0.345897, + "nauc_map_at_3_max": 0.387165, + "nauc_map_at_3_std": 0.327082, + "nauc_map_at_3_diff1": 0.228785, + "nauc_map_at_5_max": 0.40006, + "nauc_map_at_5_std": 0.367756, + "nauc_map_at_5_diff1": 0.226156, + "nauc_map_at_10_max": 0.402611, + "nauc_map_at_10_std": 0.387924, + "nauc_map_at_10_diff1": 0.216349, + "nauc_map_at_20_max": 0.402502, + "nauc_map_at_20_std": 0.393906, + "nauc_map_at_20_diff1": 0.213408, + "nauc_map_at_100_max": 0.40522, + "nauc_map_at_100_std": 0.396372, + "nauc_map_at_100_diff1": 0.214583, + "nauc_map_at_1000_max": 0.40522, + "nauc_map_at_1000_std": 0.396372, + "nauc_map_at_1000_diff1": 0.214583, + "nauc_recall_at_1_max": 0.370261, + "nauc_recall_at_1_std": 0.234891, + "nauc_recall_at_1_diff1": 0.345897, + "nauc_recall_at_3_max": 0.380698, + "nauc_recall_at_3_std": 0.343048, + "nauc_recall_at_3_diff1": 0.1751, + "nauc_recall_at_5_max": 0.39257, + "nauc_recall_at_5_std": 0.405247, + "nauc_recall_at_5_diff1": 0.165824, + "nauc_recall_at_10_max": 0.381686, + "nauc_recall_at_10_std": 0.435708, + "nauc_recall_at_10_diff1": 0.136457, + "nauc_recall_at_20_max": 0.361641, + "nauc_recall_at_20_std": 0.444964, + "nauc_recall_at_20_diff1": 0.125086, + "nauc_recall_at_100_max": 0.389216, + "nauc_recall_at_100_std": 0.473524, + "nauc_recall_at_100_diff1": 0.147336, + "nauc_recall_at_1000_max": 0.389216, + "nauc_recall_at_1000_std": 0.473524, + "nauc_recall_at_1000_diff1": 0.147336, + "nauc_precision_at_1_max": 0.361363, + "nauc_precision_at_1_std": 0.280609, + "nauc_precision_at_1_diff1": 0.330934, + "nauc_precision_at_3_max": 0.315386, + "nauc_precision_at_3_std": 0.378428, + "nauc_precision_at_3_diff1": 0.07174, + "nauc_precision_at_5_max": 0.305201, + "nauc_precision_at_5_std": 0.427065, + "nauc_precision_at_5_diff1": 0.05649, + "nauc_precision_at_10_max": 0.242069, + "nauc_precision_at_10_std": 0.404578, + "nauc_precision_at_10_diff1": 0.001816, + "nauc_precision_at_20_max": 0.171233, + "nauc_precision_at_20_std": 0.361633, + "nauc_precision_at_20_diff1": -0.03156, + "nauc_precision_at_100_max": 0.076596, + "nauc_precision_at_100_std": 0.237218, + "nauc_precision_at_100_diff1": -0.047571, + "nauc_precision_at_1000_max": 0.076596, + "nauc_precision_at_1000_std": 0.237218, + "nauc_precision_at_1000_diff1": -0.047571, + "nauc_mrr_at_1_max": 0.361363, + "nauc_mrr_at_1_std": 0.280609, + "nauc_mrr_at_1_diff1": 0.330934, + "nauc_mrr_at_3_max": 0.398253, + "nauc_mrr_at_3_std": 0.341189, + "nauc_mrr_at_3_diff1": 0.288613, + "nauc_mrr_at_5_max": 0.407612, + "nauc_mrr_at_5_std": 0.355423, + "nauc_mrr_at_5_diff1": 0.293943, + "nauc_mrr_at_10_max": 0.406799, + "nauc_mrr_at_10_std": 0.355426, + "nauc_mrr_at_10_diff1": 0.292778, + "nauc_mrr_at_20_max": 0.404467, + "nauc_mrr_at_20_std": 0.352577, + "nauc_mrr_at_20_diff1": 0.294741, + "nauc_mrr_at_100_max": 0.404091, + "nauc_mrr_at_100_std": 0.351467, + "nauc_mrr_at_100_diff1": 0.295475, + "nauc_mrr_at_1000_max": 0.404091, + "nauc_mrr_at_1000_std": 0.351467, + "nauc_mrr_at_1000_diff1": 0.295475, + "main_score": 0.47554, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 26.074533700942993, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/CmedqaRetrieval.json b/results/ByteDance-Seed__Seed-Embedding/1/CmedqaRetrieval.json new file mode 100644 index 0000000000..5c2efc729a --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/CmedqaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301", + "task_name": "CmedqaRetrieval", + "mteb_version": "1.35.1", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.47437, + "ndcg_at_3": 0.4676, + "ndcg_at_5": 0.48802, + "ndcg_at_10": 0.52019, + "ndcg_at_20": 0.54604, + "ndcg_at_100": 0.58667, + "ndcg_at_1000": 0.58667, + "map_at_1": 0.31577, + "map_at_3": 0.40857, + "map_at_5": 0.43342, + "map_at_10": 0.45335, + "map_at_20": 0.46328, + "map_at_100": 0.47095, + "map_at_1000": 0.47095, + "recall_at_1": 0.31577, + "recall_at_3": 0.46095, + "recall_at_5": 0.52802, + "recall_at_10": 0.62438, + "recall_at_20": 0.71141, + "recall_at_100": 0.89913, + "recall_at_1000": 0.89913, + "precision_at_1": 0.47437, + "precision_at_3": 0.25948, + "precision_at_5": 0.18455, + "precision_at_10": 0.11158, + "precision_at_20": 0.06464, + "precision_at_100": 0.0166, + "precision_at_1000": 0.00166, + "mrr_at_1": 0.474369, + "mrr_at_3": 0.525173, + "mrr_at_5": 0.537589, + "mrr_at_10": 0.548357, + "mrr_at_20": 0.553334, + "mrr_at_100": 0.556929, + "mrr_at_1000": 0.556929, + "nauc_ndcg_at_1_max": 0.618366, + "nauc_ndcg_at_1_std": 0.41969, + "nauc_ndcg_at_1_diff1": 0.70306, + "nauc_ndcg_at_3_max": 0.533521, + "nauc_ndcg_at_3_std": 0.420879, + "nauc_ndcg_at_3_diff1": 0.58921, + "nauc_ndcg_at_5_max": 0.524008, + "nauc_ndcg_at_5_std": 0.433724, + "nauc_ndcg_at_5_diff1": 0.584547, + "nauc_ndcg_at_10_max": 0.523398, + "nauc_ndcg_at_10_std": 0.455381, + "nauc_ndcg_at_10_diff1": 0.577583, + "nauc_ndcg_at_20_max": 0.530766, + "nauc_ndcg_at_20_std": 0.471045, + "nauc_ndcg_at_20_diff1": 0.577674, + "nauc_ndcg_at_100_max": 0.54497, + "nauc_ndcg_at_100_std": 0.466413, + "nauc_ndcg_at_100_diff1": 0.587536, + "nauc_ndcg_at_1000_max": 0.54497, + "nauc_ndcg_at_1000_std": 0.466413, + "nauc_ndcg_at_1000_diff1": 0.587536, + "nauc_map_at_1_max": 0.383189, + "nauc_map_at_1_std": 0.232585, + "nauc_map_at_1_diff1": 0.581097, + "nauc_map_at_3_max": 0.469226, + "nauc_map_at_3_std": 0.357125, + "nauc_map_at_3_diff1": 0.571181, + "nauc_map_at_5_max": 0.492116, + "nauc_map_at_5_std": 0.393623, + "nauc_map_at_5_diff1": 0.574248, + "nauc_map_at_10_max": 0.504238, + "nauc_map_at_10_std": 0.416715, + "nauc_map_at_10_diff1": 0.573204, + "nauc_map_at_20_max": 0.510835, + "nauc_map_at_20_std": 0.425829, + "nauc_map_at_20_diff1": 0.57439, + "nauc_map_at_100_max": 0.514489, + "nauc_map_at_100_std": 0.426347, + "nauc_map_at_100_diff1": 0.576418, + "nauc_map_at_1000_max": 0.514489, + "nauc_map_at_1000_std": 0.426347, + "nauc_map_at_1000_diff1": 0.576418, + "nauc_recall_at_1_max": 0.383189, + "nauc_recall_at_1_std": 0.232585, + "nauc_recall_at_1_diff1": 0.581097, + "nauc_recall_at_3_max": 0.421844, + "nauc_recall_at_3_std": 0.362055, + "nauc_recall_at_3_diff1": 0.506809, + "nauc_recall_at_5_max": 0.426902, + "nauc_recall_at_5_std": 0.415092, + "nauc_recall_at_5_diff1": 0.489725, + "nauc_recall_at_10_max": 0.419766, + "nauc_recall_at_10_std": 0.482072, + "nauc_recall_at_10_diff1": 0.445447, + "nauc_recall_at_20_max": 0.418559, + "nauc_recall_at_20_std": 0.544913, + "nauc_recall_at_20_diff1": 0.417079, + "nauc_recall_at_100_max": 0.443203, + "nauc_recall_at_100_std": 0.617722, + "nauc_recall_at_100_diff1": 0.355905, + "nauc_recall_at_1000_max": 0.443203, + "nauc_recall_at_1000_std": 0.617722, + "nauc_recall_at_1000_diff1": 0.355905, + "nauc_precision_at_1_max": 0.618366, + "nauc_precision_at_1_std": 0.41969, + "nauc_precision_at_1_diff1": 0.70306, + "nauc_precision_at_3_max": 0.553784, + "nauc_precision_at_3_std": 0.493372, + "nauc_precision_at_3_diff1": 0.412752, + "nauc_precision_at_5_max": 0.527251, + "nauc_precision_at_5_std": 0.51323, + "nauc_precision_at_5_diff1": 0.340615, + "nauc_precision_at_10_max": 0.48073, + "nauc_precision_at_10_std": 0.508549, + "nauc_precision_at_10_diff1": 0.257162, + "nauc_precision_at_20_max": 0.438408, + "nauc_precision_at_20_std": 0.481263, + "nauc_precision_at_20_diff1": 0.191747, + "nauc_precision_at_100_max": 0.339466, + "nauc_precision_at_100_std": 0.323784, + "nauc_precision_at_100_diff1": 0.083706, + "nauc_precision_at_1000_max": 0.339466, + "nauc_precision_at_1000_std": 0.323784, + "nauc_precision_at_1000_diff1": 0.083706, + "nauc_mrr_at_1_max": 0.618366, + "nauc_mrr_at_1_std": 0.41969, + "nauc_mrr_at_1_diff1": 0.70306, + "nauc_mrr_at_3_max": 0.603435, + "nauc_mrr_at_3_std": 0.442006, + "nauc_mrr_at_3_diff1": 0.673028, + "nauc_mrr_at_5_max": 0.601593, + "nauc_mrr_at_5_std": 0.44853, + "nauc_mrr_at_5_diff1": 0.669691, + "nauc_mrr_at_10_max": 0.600867, + "nauc_mrr_at_10_std": 0.452586, + "nauc_mrr_at_10_diff1": 0.666411, + "nauc_mrr_at_20_max": 0.600507, + "nauc_mrr_at_20_std": 0.452884, + "nauc_mrr_at_20_diff1": 0.665948, + "nauc_mrr_at_100_max": 0.601298, + "nauc_mrr_at_100_std": 0.452039, + "nauc_mrr_at_100_diff1": 0.666824, + "nauc_mrr_at_1000_max": 0.601298, + "nauc_mrr_at_1000_std": 0.452039, + "nauc_mrr_at_1000_diff1": 0.666824, + "main_score": 0.52019, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 75.74386143684387, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/Cmnli.json b/results/ByteDance-Seed__Seed-Embedding/1/Cmnli.json new file mode 100644 index 0000000000..50048a3b7b --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/Cmnli.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "41bc36f332156f7adc9e38f53777c959b2ae9766", + "task_name": "Cmnli", + "mteb_version": "1.35.1", + "scores": { + "validation": [ + { + "similarity_accuracy": 0.866627, + "similarity_accuracy_threshold": 0.602409, + "similarity_f1": 0.87278, + "similarity_f1_threshold": 0.569875, + "similarity_precision": 0.835543, + "similarity_recall": 0.913491, + "similarity_ap": 0.931254, + "cosine_accuracy": 0.866627, + "cosine_accuracy_threshold": 0.602409, + "cosine_f1": 0.87278, + "cosine_f1_threshold": 0.569875, + "cosine_precision": 0.835543, + "cosine_recall": 0.913491, + "cosine_ap": 0.931254, + "manhattan_accuracy": 0.860373, + "manhattan_accuracy_threshold": 31.017099, + "manhattan_f1": 0.867664, + "manhattan_f1_threshold": 32.076721, + "manhattan_precision": 0.841723, + "manhattan_recall": 0.895254, + "manhattan_ap": 0.928004, + "euclidean_accuracy": 0.867108, + "euclidean_accuracy_threshold": 0.893193, + "euclidean_f1": 0.872922, + "euclidean_f1_threshold": 0.928544, + "euclidean_precision": 0.834827, + "euclidean_recall": 0.91466, + "euclidean_ap": 0.931286, + "dot_accuracy": 0.866506, + "dot_accuracy_threshold": 0.602343, + "dot_f1": 0.87295, + "dot_f1_threshold": 0.56879, + "dot_precision": 0.834684, + "dot_recall": 0.914894, + "dot_ap": 0.931114, + "max_accuracy": 0.867108, + "max_f1": 0.87295, + "max_precision": 0.841723, + "max_recall": 0.914894, + "max_ap": 0.931286, + "main_score": 0.931286, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 10.506418704986572, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/CovidRetrieval.json b/results/ByteDance-Seed__Seed-Embedding/1/CovidRetrieval.json new file mode 100644 index 0000000000..3ce4629ba8 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/CovidRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "1271c7809071a13532e05f25fb53511ffce77117", + "task_name": "CovidRetrieval", + "mteb_version": "1.35.1", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.76923, + "ndcg_at_3": 0.84751, + "ndcg_at_5": 0.86285, + "ndcg_at_10": 0.87305, + "ndcg_at_20": 0.87705, + "ndcg_at_100": 0.87945, + "ndcg_at_1000": 0.87945, + "map_at_1": 0.76712, + "map_at_3": 0.82903, + "map_at_5": 0.83763, + "map_at_10": 0.84218, + "map_at_20": 0.84327, + "map_at_100": 0.84362, + "map_at_1000": 0.84362, + "recall_at_1": 0.76712, + "recall_at_3": 0.89989, + "recall_at_5": 0.93678, + "recall_at_10": 0.96733, + "recall_at_20": 0.98314, + "recall_at_100": 0.99579, + "recall_at_1000": 0.99579, + "precision_at_1": 0.76923, + "precision_at_3": 0.30172, + "precision_at_5": 0.18904, + "precision_at_10": 0.09779, + "precision_at_20": 0.04968, + "precision_at_100": 0.01006, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.769231, + "mrr_at_3": 0.829645, + "mrr_at_5": 0.837917, + "mrr_at_10": 0.842003, + "mrr_at_20": 0.843093, + "mrr_at_100": 0.843447, + "mrr_at_1000": 0.843447, + "nauc_ndcg_at_1_max": 0.441231, + "nauc_ndcg_at_1_std": -0.032512, + "nauc_ndcg_at_1_diff1": 0.879457, + "nauc_ndcg_at_3_max": 0.51261, + "nauc_ndcg_at_3_std": 0.032805, + "nauc_ndcg_at_3_diff1": 0.849011, + "nauc_ndcg_at_5_max": 0.511498, + "nauc_ndcg_at_5_std": 0.021379, + "nauc_ndcg_at_5_diff1": 0.845453, + "nauc_ndcg_at_10_max": 0.502187, + "nauc_ndcg_at_10_std": 0.020179, + "nauc_ndcg_at_10_diff1": 0.85245, + "nauc_ndcg_at_20_max": 0.501198, + "nauc_ndcg_at_20_std": 0.03011, + "nauc_ndcg_at_20_diff1": 0.85281, + "nauc_ndcg_at_100_max": 0.495664, + "nauc_ndcg_at_100_std": 0.022178, + "nauc_ndcg_at_100_diff1": 0.856699, + "nauc_ndcg_at_1000_max": 0.495664, + "nauc_ndcg_at_1000_std": 0.022178, + "nauc_ndcg_at_1000_diff1": 0.856699, + "nauc_map_at_1_max": 0.443308, + "nauc_map_at_1_std": -0.033356, + "nauc_map_at_1_diff1": 0.882453, + "nauc_map_at_3_max": 0.491683, + "nauc_map_at_3_std": 0.011795, + "nauc_map_at_3_diff1": 0.858158, + "nauc_map_at_5_max": 0.490308, + "nauc_map_at_5_std": 0.005958, + "nauc_map_at_5_diff1": 0.856997, + "nauc_map_at_10_max": 0.487128, + "nauc_map_at_10_std": 0.00807, + "nauc_map_at_10_diff1": 0.859661, + "nauc_map_at_20_max": 0.486793, + "nauc_map_at_20_std": 0.010191, + "nauc_map_at_20_diff1": 0.859804, + "nauc_map_at_100_max": 0.486104, + "nauc_map_at_100_std": 0.009105, + "nauc_map_at_100_diff1": 0.860208, + "nauc_map_at_1000_max": 0.486104, + "nauc_map_at_1000_std": 0.009105, + "nauc_map_at_1000_diff1": 0.860208, + "nauc_recall_at_1_max": 0.443308, + "nauc_recall_at_1_std": -0.033356, + "nauc_recall_at_1_diff1": 0.882453, + "nauc_recall_at_3_max": 0.604697, + "nauc_recall_at_3_std": 0.124347, + "nauc_recall_at_3_diff1": 0.80794, + "nauc_recall_at_5_max": 0.64866, + "nauc_recall_at_5_std": 0.104929, + "nauc_recall_at_5_diff1": 0.76793, + "nauc_recall_at_10_max": 0.669242, + "nauc_recall_at_10_std": 0.135746, + "nauc_recall_at_10_diff1": 0.770759, + "nauc_recall_at_20_max": 0.799241, + "nauc_recall_at_20_std": 0.505874, + "nauc_recall_at_20_diff1": 0.703225, + "nauc_recall_at_100_max": 0.930542, + "nauc_recall_at_100_std": 0.897858, + "nauc_recall_at_100_diff1": 0.828401, + "nauc_recall_at_1000_max": 0.930542, + "nauc_recall_at_1000_std": 0.897858, + "nauc_recall_at_1000_diff1": 0.828401, + "nauc_precision_at_1_max": 0.441231, + "nauc_precision_at_1_std": -0.032512, + "nauc_precision_at_1_diff1": 0.879457, + "nauc_precision_at_3_max": 0.545903, + "nauc_precision_at_3_std": 0.126299, + "nauc_precision_at_3_diff1": 0.714866, + "nauc_precision_at_5_max": 0.511634, + "nauc_precision_at_5_std": 0.153685, + "nauc_precision_at_5_diff1": 0.544442, + "nauc_precision_at_10_max": 0.376548, + "nauc_precision_at_10_std": 0.248692, + "nauc_precision_at_10_diff1": 0.301023, + "nauc_precision_at_20_max": 0.307333, + "nauc_precision_at_20_std": 0.476859, + "nauc_precision_at_20_diff1": 0.057568, + "nauc_precision_at_100_max": 0.055806, + "nauc_precision_at_100_std": 0.533909, + "nauc_precision_at_100_diff1": -0.281399, + "nauc_precision_at_1000_max": 0.055806, + "nauc_precision_at_1000_std": 0.533909, + "nauc_precision_at_1000_diff1": -0.281399, + "nauc_mrr_at_1_max": 0.441231, + "nauc_mrr_at_1_std": -0.032512, + "nauc_mrr_at_1_diff1": 0.879457, + "nauc_mrr_at_3_max": 0.493188, + "nauc_mrr_at_3_std": 0.014174, + "nauc_mrr_at_3_diff1": 0.857828, + "nauc_mrr_at_5_max": 0.491355, + "nauc_mrr_at_5_std": 0.008508, + "nauc_mrr_at_5_diff1": 0.856504, + "nauc_mrr_at_10_max": 0.487055, + "nauc_mrr_at_10_std": 0.005943, + "nauc_mrr_at_10_diff1": 0.859676, + "nauc_mrr_at_20_max": 0.486721, + "nauc_mrr_at_20_std": 0.008045, + "nauc_mrr_at_20_diff1": 0.859819, + "nauc_mrr_at_100_max": 0.486033, + "nauc_mrr_at_100_std": 0.006955, + "nauc_mrr_at_100_diff1": 0.860222, + "nauc_mrr_at_1000_max": 0.486033, + "nauc_mrr_at_1000_std": 0.006955, + "nauc_mrr_at_1000_diff1": 0.860222, + "main_score": 0.87305, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 31.084179401397705, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/DuRetrieval.json b/results/ByteDance-Seed__Seed-Embedding/1/DuRetrieval.json new file mode 100644 index 0000000000..925cc711a7 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/DuRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced", + "task_name": "DuRetrieval", + "mteb_version": "1.35.1", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.9565, + "ndcg_at_3": 0.9367, + "ndcg_at_5": 0.92493, + "ndcg_at_10": 0.93812, + "ndcg_at_20": 0.94658, + "ndcg_at_100": 0.95332, + "ndcg_at_1000": 0.95332, + "map_at_1": 0.29084, + "map_at_3": 0.62868, + "map_at_5": 0.78848, + "map_at_10": 0.88765, + "map_at_20": 0.90618, + "map_at_100": 0.91093, + "map_at_1000": 0.91093, + "recall_at_1": 0.29084, + "recall_at_3": 0.64195, + "recall_at_5": 0.81584, + "recall_at_10": 0.93932, + "recall_at_20": 0.97168, + "recall_at_100": 0.99298, + "recall_at_1000": 0.99298, + "precision_at_1": 0.9565, + "precision_at_3": 0.83583, + "precision_at_5": 0.7017, + "precision_at_10": 0.4395, + "precision_at_20": 0.23583, + "precision_at_100": 0.04884, + "precision_at_1000": 0.00488, + "mrr_at_1": 0.9565, + "mrr_at_3": 0.96975, + "mrr_at_5": 0.9711, + "mrr_at_10": 0.971773, + "mrr_at_20": 0.971798, + "mrr_at_100": 0.971855, + "mrr_at_1000": 0.971855, + "nauc_ndcg_at_1_max": 0.842338, + "nauc_ndcg_at_1_std": 0.760585, + "nauc_ndcg_at_1_diff1": 0.518991, + "nauc_ndcg_at_3_max": 0.58065, + "nauc_ndcg_at_3_std": 0.655718, + "nauc_ndcg_at_3_diff1": 0.035422, + "nauc_ndcg_at_5_max": 0.464316, + "nauc_ndcg_at_5_std": 0.504273, + "nauc_ndcg_at_5_diff1": 0.061758, + "nauc_ndcg_at_10_max": 0.503836, + "nauc_ndcg_at_10_std": 0.525954, + "nauc_ndcg_at_10_diff1": 0.114847, + "nauc_ndcg_at_20_max": 0.554843, + "nauc_ndcg_at_20_std": 0.574009, + "nauc_ndcg_at_20_diff1": 0.134899, + "nauc_ndcg_at_100_max": 0.578013, + "nauc_ndcg_at_100_std": 0.611558, + "nauc_ndcg_at_100_diff1": 0.126225, + "nauc_ndcg_at_1000_max": 0.578013, + "nauc_ndcg_at_1000_std": 0.611558, + "nauc_ndcg_at_1000_diff1": 0.126225, + "nauc_map_at_1_max": -0.101423, + "nauc_map_at_1_std": -0.498174, + "nauc_map_at_1_diff1": 0.546857, + "nauc_map_at_3_max": 0.028285, + "nauc_map_at_3_std": -0.444817, + "nauc_map_at_3_diff1": 0.357639, + "nauc_map_at_5_max": 0.178536, + "nauc_map_at_5_std": -0.214205, + "nauc_map_at_5_diff1": 0.233507, + "nauc_map_at_10_max": 0.398026, + "nauc_map_at_10_std": 0.274077, + "nauc_map_at_10_diff1": 0.119197, + "nauc_map_at_20_max": 0.455091, + "nauc_map_at_20_std": 0.451933, + "nauc_map_at_20_diff1": 0.079663, + "nauc_map_at_100_max": 0.46157, + "nauc_map_at_100_std": 0.485574, + "nauc_map_at_100_diff1": 0.066027, + "nauc_map_at_1000_max": 0.46157, + "nauc_map_at_1000_std": 0.485574, + "nauc_map_at_1000_diff1": 0.066027, + "nauc_recall_at_1_max": -0.101423, + "nauc_recall_at_1_std": -0.498174, + "nauc_recall_at_1_diff1": 0.546857, + "nauc_recall_at_3_max": -0.013889, + "nauc_recall_at_3_std": -0.493275, + "nauc_recall_at_3_diff1": 0.351511, + "nauc_recall_at_5_max": 0.094454, + "nauc_recall_at_5_std": -0.35158, + "nauc_recall_at_5_diff1": 0.241063, + "nauc_recall_at_10_max": 0.324701, + "nauc_recall_at_10_std": 0.088238, + "nauc_recall_at_10_diff1": 0.164564, + "nauc_recall_at_20_max": 0.505416, + "nauc_recall_at_20_std": 0.4543, + "nauc_recall_at_20_diff1": 0.145233, + "nauc_recall_at_100_max": 0.729, + "nauc_recall_at_100_std": 0.727355, + "nauc_recall_at_100_diff1": 0.016835, + "nauc_recall_at_1000_max": 0.729, + "nauc_recall_at_1000_std": 0.727355, + "nauc_recall_at_1000_diff1": 0.016835, + "nauc_precision_at_1_max": 0.842338, + "nauc_precision_at_1_std": 0.760585, + "nauc_precision_at_1_diff1": 0.518991, + "nauc_precision_at_3_max": 0.415822, + "nauc_precision_at_3_std": 0.80641, + "nauc_precision_at_3_diff1": -0.552513, + "nauc_precision_at_5_max": 0.293658, + "nauc_precision_at_5_std": 0.774375, + "nauc_precision_at_5_diff1": -0.482437, + "nauc_precision_at_10_max": 0.197379, + "nauc_precision_at_10_std": 0.722468, + "nauc_precision_at_10_diff1": -0.390379, + "nauc_precision_at_20_max": 0.147201, + "nauc_precision_at_20_std": 0.678128, + "nauc_precision_at_20_diff1": -0.368012, + "nauc_precision_at_100_max": 0.115973, + "nauc_precision_at_100_std": 0.636015, + "nauc_precision_at_100_diff1": -0.363192, + "nauc_precision_at_1000_max": 0.115973, + "nauc_precision_at_1000_std": 0.636015, + "nauc_precision_at_1000_diff1": -0.363192, + "nauc_mrr_at_1_max": 0.842338, + "nauc_mrr_at_1_std": 0.760585, + "nauc_mrr_at_1_diff1": 0.518991, + "nauc_mrr_at_3_max": 0.878897, + "nauc_mrr_at_3_std": 0.814271, + "nauc_mrr_at_3_diff1": 0.54115, + "nauc_mrr_at_5_max": 0.874767, + "nauc_mrr_at_5_std": 0.808268, + "nauc_mrr_at_5_diff1": 0.539202, + "nauc_mrr_at_10_max": 0.872554, + "nauc_mrr_at_10_std": 0.804802, + "nauc_mrr_at_10_diff1": 0.535272, + "nauc_mrr_at_20_max": 0.872441, + "nauc_mrr_at_20_std": 0.804629, + "nauc_mrr_at_20_diff1": 0.53486, + "nauc_mrr_at_100_max": 0.872213, + "nauc_mrr_at_100_std": 0.804263, + "nauc_mrr_at_100_diff1": 0.534628, + "nauc_mrr_at_1000_max": 0.872213, + "nauc_mrr_at_1000_std": 0.804263, + "nauc_mrr_at_1000_diff1": 0.534628, + "main_score": 0.93812, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 43.78036093711853, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/EcomRetrieval.json b/results/ByteDance-Seed__Seed-Embedding/1/EcomRetrieval.json new file mode 100644 index 0000000000..fbe51675db --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/EcomRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9", + "task_name": "EcomRetrieval", + "mteb_version": "1.35.1", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.576, + "ndcg_at_3": 0.68197, + "ndcg_at_5": 0.70668, + "ndcg_at_10": 0.72919, + "ndcg_at_20": 0.73997, + "ndcg_at_100": 0.74701, + "ndcg_at_1000": 0.74701, + "map_at_1": 0.576, + "map_at_3": 0.65633, + "map_at_5": 0.67028, + "map_at_10": 0.67969, + "map_at_20": 0.68273, + "map_at_100": 0.68378, + "map_at_1000": 0.68378, + "recall_at_1": 0.576, + "recall_at_3": 0.756, + "recall_at_5": 0.815, + "recall_at_10": 0.884, + "recall_at_20": 0.926, + "recall_at_100": 0.963, + "recall_at_1000": 0.963, + "precision_at_1": 0.576, + "precision_at_3": 0.252, + "precision_at_5": 0.163, + "precision_at_10": 0.0884, + "precision_at_20": 0.0463, + "precision_at_100": 0.00963, + "precision_at_1000": 0.00096, + "mrr_at_1": 0.576, + "mrr_at_3": 0.656333, + "mrr_at_5": 0.670283, + "mrr_at_10": 0.679688, + "mrr_at_20": 0.682734, + "mrr_at_100": 0.683779, + "mrr_at_1000": 0.683779, + "nauc_ndcg_at_1_max": 0.363944, + "nauc_ndcg_at_1_std": -0.003755, + "nauc_ndcg_at_1_diff1": 0.77604, + "nauc_ndcg_at_3_max": 0.401604, + "nauc_ndcg_at_3_std": 0.062506, + "nauc_ndcg_at_3_diff1": 0.741914, + "nauc_ndcg_at_5_max": 0.413997, + "nauc_ndcg_at_5_std": 0.085254, + "nauc_ndcg_at_5_diff1": 0.737666, + "nauc_ndcg_at_10_max": 0.426132, + "nauc_ndcg_at_10_std": 0.107268, + "nauc_ndcg_at_10_diff1": 0.73606, + "nauc_ndcg_at_20_max": 0.426806, + "nauc_ndcg_at_20_std": 0.11161, + "nauc_ndcg_at_20_diff1": 0.740999, + "nauc_ndcg_at_100_max": 0.426027, + "nauc_ndcg_at_100_std": 0.102984, + "nauc_ndcg_at_100_diff1": 0.744469, + "nauc_ndcg_at_1000_max": 0.426027, + "nauc_ndcg_at_1000_std": 0.102984, + "nauc_ndcg_at_1000_diff1": 0.744469, + "nauc_map_at_1_max": 0.363944, + "nauc_map_at_1_std": -0.003755, + "nauc_map_at_1_diff1": 0.77604, + "nauc_map_at_3_max": 0.392976, + "nauc_map_at_3_std": 0.044392, + "nauc_map_at_3_diff1": 0.750299, + "nauc_map_at_5_max": 0.398841, + "nauc_map_at_5_std": 0.055112, + "nauc_map_at_5_diff1": 0.748462, + "nauc_map_at_10_max": 0.40267, + "nauc_map_at_10_std": 0.061963, + "nauc_map_at_10_diff1": 0.748238, + "nauc_map_at_20_max": 0.402493, + "nauc_map_at_20_std": 0.062147, + "nauc_map_at_20_diff1": 0.74954, + "nauc_map_at_100_max": 0.402374, + "nauc_map_at_100_std": 0.061306, + "nauc_map_at_100_diff1": 0.750023, + "nauc_map_at_1000_max": 0.402374, + "nauc_map_at_1000_std": 0.061306, + "nauc_map_at_1000_diff1": 0.750023, + "nauc_recall_at_1_max": 0.363944, + "nauc_recall_at_1_std": -0.003755, + "nauc_recall_at_1_diff1": 0.77604, + "nauc_recall_at_3_max": 0.432224, + "nauc_recall_at_3_std": 0.128668, + "nauc_recall_at_3_diff1": 0.711795, + "nauc_recall_at_5_max": 0.482791, + "nauc_recall_at_5_std": 0.223779, + "nauc_recall_at_5_diff1": 0.689658, + "nauc_recall_at_10_max": 0.593187, + "nauc_recall_at_10_std": 0.431457, + "nauc_recall_at_10_diff1": 0.654394, + "nauc_recall_at_20_max": 0.692974, + "nauc_recall_at_20_std": 0.664244, + "nauc_recall_at_20_diff1": 0.665979, + "nauc_recall_at_100_max": 0.928647, + "nauc_recall_at_100_std": 0.925114, + "nauc_recall_at_100_diff1": 0.688874, + "nauc_recall_at_1000_max": 0.928647, + "nauc_recall_at_1000_std": 0.925114, + "nauc_recall_at_1000_diff1": 0.688874, + "nauc_precision_at_1_max": 0.363944, + "nauc_precision_at_1_std": -0.003755, + "nauc_precision_at_1_diff1": 0.77604, + "nauc_precision_at_3_max": 0.432224, + "nauc_precision_at_3_std": 0.128668, + "nauc_precision_at_3_diff1": 0.711795, + "nauc_precision_at_5_max": 0.482791, + "nauc_precision_at_5_std": 0.223779, + "nauc_precision_at_5_diff1": 0.689658, + "nauc_precision_at_10_max": 0.593187, + "nauc_precision_at_10_std": 0.431457, + "nauc_precision_at_10_diff1": 0.654394, + "nauc_precision_at_20_max": 0.692974, + "nauc_precision_at_20_std": 0.664244, + "nauc_precision_at_20_diff1": 0.665979, + "nauc_precision_at_100_max": 0.928647, + "nauc_precision_at_100_std": 0.925114, + "nauc_precision_at_100_diff1": 0.688874, + "nauc_precision_at_1000_max": 0.928647, + "nauc_precision_at_1000_std": 0.925114, + "nauc_precision_at_1000_diff1": 0.688874, + "nauc_mrr_at_1_max": 0.363944, + "nauc_mrr_at_1_std": -0.003755, + "nauc_mrr_at_1_diff1": 0.77604, + "nauc_mrr_at_3_max": 0.392976, + "nauc_mrr_at_3_std": 0.044392, + "nauc_mrr_at_3_diff1": 0.750299, + "nauc_mrr_at_5_max": 0.398841, + "nauc_mrr_at_5_std": 0.055112, + "nauc_mrr_at_5_diff1": 0.748462, + "nauc_mrr_at_10_max": 0.40267, + "nauc_mrr_at_10_std": 0.061963, + "nauc_mrr_at_10_diff1": 0.748238, + "nauc_mrr_at_20_max": 0.402493, + "nauc_mrr_at_20_std": 0.062147, + "nauc_mrr_at_20_diff1": 0.74954, + "nauc_mrr_at_100_max": 0.402374, + "nauc_mrr_at_100_std": 0.061306, + "nauc_mrr_at_100_diff1": 0.750023, + "nauc_mrr_at_1000_max": 0.402374, + "nauc_mrr_at_1000_std": 0.061306, + "nauc_mrr_at_1000_diff1": 0.750023, + "main_score": 0.72919, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 26.247467756271362, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/FEVERHardNegatives.json b/results/ByteDance-Seed__Seed-Embedding/1/FEVERHardNegatives.json new file mode 100644 index 0000000000..7c3f6824d0 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/FEVERHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "080c9ed6267b65029207906e815d44a9240bafca", + "task_name": "FEVERHardNegatives", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.917, + "ndcg_at_3": 0.93514, + "ndcg_at_5": 0.9417, + "ndcg_at_10": 0.94604, + "ndcg_at_20": 0.94751, + "ndcg_at_100": 0.94965, + "ndcg_at_1000": 0.94965, + "map_at_1": 0.86205, + "map_at_3": 0.91528, + "map_at_5": 0.92168, + "map_at_10": 0.92494, + "map_at_20": 0.92579, + "map_at_100": 0.92634, + "map_at_1000": 0.92634, + "recall_at_1": 0.86205, + "recall_at_3": 0.95104, + "recall_at_5": 0.97006, + "recall_at_10": 0.98205, + "recall_at_20": 0.98646, + "recall_at_100": 0.99471, + "recall_at_1000": 0.99471, + "precision_at_1": 0.917, + "precision_at_3": 0.348, + "precision_at_5": 0.2152, + "precision_at_10": 0.1105, + "precision_at_20": 0.056, + "precision_at_100": 0.01149, + "precision_at_1000": 0.00115, + "mrr_at_1": 0.917, + "mrr_at_3": 0.948333, + "mrr_at_5": 0.951033, + "mrr_at_10": 0.951492, + "mrr_at_20": 0.951492, + "mrr_at_100": 0.951505, + "mrr_at_1000": 0.951505, + "nauc_ndcg_at_1_max": 0.297245, + "nauc_ndcg_at_1_std": -0.373111, + "nauc_ndcg_at_1_diff1": 0.860231, + "nauc_ndcg_at_3_max": 0.409842, + "nauc_ndcg_at_3_std": -0.056805, + "nauc_ndcg_at_3_diff1": 0.672537, + "nauc_ndcg_at_5_max": 0.392266, + "nauc_ndcg_at_5_std": -0.060548, + "nauc_ndcg_at_5_diff1": 0.648357, + "nauc_ndcg_at_10_max": 0.415152, + "nauc_ndcg_at_10_std": -0.048932, + "nauc_ndcg_at_10_diff1": 0.668733, + "nauc_ndcg_at_20_max": 0.412406, + "nauc_ndcg_at_20_std": -0.063126, + "nauc_ndcg_at_20_diff1": 0.68106, + "nauc_ndcg_at_100_max": 0.39349, + "nauc_ndcg_at_100_std": -0.100608, + "nauc_ndcg_at_100_diff1": 0.692383, + "nauc_ndcg_at_1000_max": 0.39349, + "nauc_ndcg_at_1000_std": -0.100608, + "nauc_ndcg_at_1000_diff1": 0.692383, + "nauc_map_at_1_max": 0.341401, + "nauc_map_at_1_std": -0.15483, + "nauc_map_at_1_diff1": 0.698022, + "nauc_map_at_3_max": 0.40506, + "nauc_map_at_3_std": -0.045033, + "nauc_map_at_3_diff1": 0.657237, + "nauc_map_at_5_max": 0.391571, + "nauc_map_at_5_std": -0.06017, + "nauc_map_at_5_diff1": 0.651566, + "nauc_map_at_10_max": 0.401293, + "nauc_map_at_10_std": -0.056888, + "nauc_map_at_10_diff1": 0.661192, + "nauc_map_at_20_max": 0.399358, + "nauc_map_at_20_std": -0.06207, + "nauc_map_at_20_diff1": 0.665791, + "nauc_map_at_100_max": 0.396128, + "nauc_map_at_100_std": -0.068147, + "nauc_map_at_100_diff1": 0.667832, + "nauc_map_at_1000_max": 0.396128, + "nauc_map_at_1000_std": -0.068147, + "nauc_map_at_1000_diff1": 0.667832, + "nauc_recall_at_1_max": 0.341401, + "nauc_recall_at_1_std": -0.15483, + "nauc_recall_at_1_diff1": 0.698022, + "nauc_recall_at_3_max": 0.524571, + "nauc_recall_at_3_std": 0.256012, + "nauc_recall_at_3_diff1": 0.490489, + "nauc_recall_at_5_max": 0.470495, + "nauc_recall_at_5_std": 0.328528, + "nauc_recall_at_5_diff1": 0.322017, + "nauc_recall_at_10_max": 0.678272, + "nauc_recall_at_10_std": 0.660064, + "nauc_recall_at_10_diff1": 0.266198, + "nauc_recall_at_20_max": 0.747072, + "nauc_recall_at_20_std": 0.763583, + "nauc_recall_at_20_diff1": 0.266248, + "nauc_recall_at_100_max": 0.576884, + "nauc_recall_at_100_std": 0.65015, + "nauc_recall_at_100_diff1": 0.033763, + "nauc_recall_at_1000_max": 0.576884, + "nauc_recall_at_1000_std": 0.65015, + "nauc_recall_at_1000_diff1": 0.033763, + "nauc_precision_at_1_max": 0.297245, + "nauc_precision_at_1_std": -0.373111, + "nauc_precision_at_1_diff1": 0.860231, + "nauc_precision_at_3_max": -0.096533, + "nauc_precision_at_3_std": 0.002257, + "nauc_precision_at_3_diff1": -0.168001, + "nauc_precision_at_5_max": -0.202186, + "nauc_precision_at_5_std": -0.035494, + "nauc_precision_at_5_diff1": -0.283903, + "nauc_precision_at_10_max": -0.191778, + "nauc_precision_at_10_std": -0.024516, + "nauc_precision_at_10_diff1": -0.267616, + "nauc_precision_at_20_max": -0.205964, + "nauc_precision_at_20_std": -0.0473, + "nauc_precision_at_20_diff1": -0.247938, + "nauc_precision_at_100_max": -0.260963, + "nauc_precision_at_100_std": -0.115057, + "nauc_precision_at_100_diff1": -0.258358, + "nauc_precision_at_1000_max": -0.260963, + "nauc_precision_at_1000_std": -0.115057, + "nauc_precision_at_1000_diff1": -0.258358, + "nauc_mrr_at_1_max": 0.297245, + "nauc_mrr_at_1_std": -0.373111, + "nauc_mrr_at_1_diff1": 0.860231, + "nauc_mrr_at_3_max": 0.371367, + "nauc_mrr_at_3_std": -0.296478, + "nauc_mrr_at_3_diff1": 0.863149, + "nauc_mrr_at_5_max": 0.350946, + "nauc_mrr_at_5_std": -0.321253, + "nauc_mrr_at_5_diff1": 0.861292, + "nauc_mrr_at_10_max": 0.349482, + "nauc_mrr_at_10_std": -0.319845, + "nauc_mrr_at_10_diff1": 0.86185, + "nauc_mrr_at_20_max": 0.349482, + "nauc_mrr_at_20_std": -0.319845, + "nauc_mrr_at_20_diff1": 0.86185, + "nauc_mrr_at_100_max": 0.349305, + "nauc_mrr_at_100_std": -0.320167, + "nauc_mrr_at_100_diff1": 0.861813, + "nauc_mrr_at_1000_max": 0.349305, + "nauc_mrr_at_1000_std": -0.320167, + "nauc_mrr_at_1000_diff1": 0.861813, + "main_score": 0.94604, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 40.42910552024841, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/FiQA2018.json b/results/ByteDance-Seed__Seed-Embedding/1/FiQA2018.json new file mode 100644 index 0000000000..1f8e32bad8 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/FiQA2018.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "27a168819829fe9bcd655c2df245fb19452e8e06", + "task_name": "FiQA2018", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.6713, + "ndcg_at_3": 0.62644, + "ndcg_at_5": 0.63765, + "ndcg_at_10": 0.67161, + "ndcg_at_20": 0.69574, + "ndcg_at_100": 0.72354, + "ndcg_at_1000": 0.72354, + "map_at_1": 0.35703, + "map_at_3": 0.51718, + "map_at_5": 0.55515, + "map_at_10": 0.58941, + "map_at_20": 0.60248, + "map_at_100": 0.60949, + "map_at_1000": 0.60949, + "recall_at_1": 0.35703, + "recall_at_3": 0.56539, + "recall_at_5": 0.63812, + "recall_at_10": 0.74403, + "recall_at_20": 0.8149, + "recall_at_100": 0.92544, + "recall_at_1000": 0.92544, + "precision_at_1": 0.6713, + "precision_at_3": 0.41718, + "precision_at_5": 0.30062, + "precision_at_10": 0.18395, + "precision_at_20": 0.10378, + "precision_at_100": 0.02414, + "precision_at_1000": 0.00241, + "mrr_at_1": 0.671296, + "mrr_at_3": 0.725309, + "mrr_at_5": 0.735262, + "mrr_at_10": 0.742963, + "mrr_at_20": 0.744853, + "mrr_at_100": 0.746249, + "mrr_at_1000": 0.746249, + "nauc_ndcg_at_1_max": 0.575794, + "nauc_ndcg_at_1_std": 0.392811, + "nauc_ndcg_at_1_diff1": 0.619862, + "nauc_ndcg_at_3_max": 0.478911, + "nauc_ndcg_at_3_std": 0.343662, + "nauc_ndcg_at_3_diff1": 0.468246, + "nauc_ndcg_at_5_max": 0.457473, + "nauc_ndcg_at_5_std": 0.334903, + "nauc_ndcg_at_5_diff1": 0.465707, + "nauc_ndcg_at_10_max": 0.481723, + "nauc_ndcg_at_10_std": 0.37374, + "nauc_ndcg_at_10_diff1": 0.481582, + "nauc_ndcg_at_20_max": 0.491179, + "nauc_ndcg_at_20_std": 0.397563, + "nauc_ndcg_at_20_diff1": 0.490564, + "nauc_ndcg_at_100_max": 0.507219, + "nauc_ndcg_at_100_std": 0.403484, + "nauc_ndcg_at_100_diff1": 0.489462, + "nauc_ndcg_at_1000_max": 0.507219, + "nauc_ndcg_at_1000_std": 0.403484, + "nauc_ndcg_at_1000_diff1": 0.489462, + "nauc_map_at_1_max": 0.169902, + "nauc_map_at_1_std": 0.020108, + "nauc_map_at_1_diff1": 0.498101, + "nauc_map_at_3_max": 0.345345, + "nauc_map_at_3_std": 0.173834, + "nauc_map_at_3_diff1": 0.478286, + "nauc_map_at_5_max": 0.392413, + "nauc_map_at_5_std": 0.246544, + "nauc_map_at_5_diff1": 0.466661, + "nauc_map_at_10_max": 0.436877, + "nauc_map_at_10_std": 0.31447, + "nauc_map_at_10_diff1": 0.462924, + "nauc_map_at_20_max": 0.44422, + "nauc_map_at_20_std": 0.332782, + "nauc_map_at_20_diff1": 0.463157, + "nauc_map_at_100_max": 0.448435, + "nauc_map_at_100_std": 0.336135, + "nauc_map_at_100_diff1": 0.46202, + "nauc_map_at_1000_max": 0.448435, + "nauc_map_at_1000_std": 0.336135, + "nauc_map_at_1000_diff1": 0.46202, + "nauc_recall_at_1_max": 0.169902, + "nauc_recall_at_1_std": 0.020108, + "nauc_recall_at_1_diff1": 0.498101, + "nauc_recall_at_3_max": 0.308569, + "nauc_recall_at_3_std": 0.150871, + "nauc_recall_at_3_diff1": 0.436134, + "nauc_recall_at_5_max": 0.329424, + "nauc_recall_at_5_std": 0.215295, + "nauc_recall_at_5_diff1": 0.402475, + "nauc_recall_at_10_max": 0.399496, + "nauc_recall_at_10_std": 0.356304, + "nauc_recall_at_10_diff1": 0.381022, + "nauc_recall_at_20_max": 0.406635, + "nauc_recall_at_20_std": 0.442853, + "nauc_recall_at_20_diff1": 0.392445, + "nauc_recall_at_100_max": 0.475183, + "nauc_recall_at_100_std": 0.543495, + "nauc_recall_at_100_diff1": 0.318193, + "nauc_recall_at_1000_max": 0.475183, + "nauc_recall_at_1000_std": 0.543495, + "nauc_recall_at_1000_diff1": 0.318193, + "nauc_precision_at_1_max": 0.575794, + "nauc_precision_at_1_std": 0.392811, + "nauc_precision_at_1_diff1": 0.619862, + "nauc_precision_at_3_max": 0.462555, + "nauc_precision_at_3_std": 0.439525, + "nauc_precision_at_3_diff1": 0.128691, + "nauc_precision_at_5_max": 0.431131, + "nauc_precision_at_5_std": 0.481187, + "nauc_precision_at_5_diff1": 0.029207, + "nauc_precision_at_10_max": 0.39996, + "nauc_precision_at_10_std": 0.517079, + "nauc_precision_at_10_diff1": -0.050148, + "nauc_precision_at_20_max": 0.340464, + "nauc_precision_at_20_std": 0.496084, + "nauc_precision_at_20_diff1": -0.093378, + "nauc_precision_at_100_max": 0.267794, + "nauc_precision_at_100_std": 0.401012, + "nauc_precision_at_100_diff1": -0.164759, + "nauc_precision_at_1000_max": 0.267794, + "nauc_precision_at_1000_std": 0.401012, + "nauc_precision_at_1000_diff1": -0.164759, + "nauc_mrr_at_1_max": 0.575794, + "nauc_mrr_at_1_std": 0.392811, + "nauc_mrr_at_1_diff1": 0.619862, + "nauc_mrr_at_3_max": 0.589633, + "nauc_mrr_at_3_std": 0.423796, + "nauc_mrr_at_3_diff1": 0.61227, + "nauc_mrr_at_5_max": 0.588397, + "nauc_mrr_at_5_std": 0.424623, + "nauc_mrr_at_5_diff1": 0.609805, + "nauc_mrr_at_10_max": 0.589991, + "nauc_mrr_at_10_std": 0.431719, + "nauc_mrr_at_10_diff1": 0.607834, + "nauc_mrr_at_20_max": 0.589665, + "nauc_mrr_at_20_std": 0.431076, + "nauc_mrr_at_20_diff1": 0.609845, + "nauc_mrr_at_100_max": 0.590314, + "nauc_mrr_at_100_std": 0.431188, + "nauc_mrr_at_100_diff1": 0.610026, + "nauc_mrr_at_1000_max": 0.590314, + "nauc_mrr_at_1000_std": 0.431188, + "nauc_mrr_at_1000_diff1": 0.610026, + "main_score": 0.67161, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 18.795050859451294, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/HotpotQAHardNegatives.json b/results/ByteDance-Seed__Seed-Embedding/1/HotpotQAHardNegatives.json new file mode 100644 index 0000000000..746f51e1d5 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/HotpotQAHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "617612fa63afcb60e3b134bed8b7216a99707c37", + "task_name": "HotpotQAHardNegatives", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.906, + "ndcg_at_3": 0.83175, + "ndcg_at_5": 0.85665, + "ndcg_at_10": 0.87208, + "ndcg_at_20": 0.88092, + "ndcg_at_100": 0.89045, + "ndcg_at_1000": 0.89045, + "map_at_1": 0.453, + "map_at_3": 0.78825, + "map_at_5": 0.8092, + "map_at_10": 0.81905, + "map_at_20": 0.82291, + "map_at_100": 0.82511, + "map_at_1000": 0.82511, + "recall_at_1": 0.453, + "recall_at_3": 0.823, + "recall_at_5": 0.8725, + "recall_at_10": 0.9115, + "recall_at_20": 0.9395, + "recall_at_100": 0.9805, + "recall_at_1000": 0.9805, + "precision_at_1": 0.906, + "precision_at_3": 0.54867, + "precision_at_5": 0.349, + "precision_at_10": 0.1823, + "precision_at_20": 0.09395, + "precision_at_100": 0.01961, + "precision_at_1000": 0.00196, + "mrr_at_1": 0.906, + "mrr_at_3": 0.933833, + "mrr_at_5": 0.936833, + "mrr_at_10": 0.937836, + "mrr_at_20": 0.938311, + "mrr_at_100": 0.938434, + "mrr_at_1000": 0.938434, + "nauc_ndcg_at_1_max": 0.478838, + "nauc_ndcg_at_1_std": 0.267984, + "nauc_ndcg_at_1_diff1": 0.604322, + "nauc_ndcg_at_3_max": 0.452525, + "nauc_ndcg_at_3_std": 0.482133, + "nauc_ndcg_at_3_diff1": 0.062625, + "nauc_ndcg_at_5_max": 0.462091, + "nauc_ndcg_at_5_std": 0.505727, + "nauc_ndcg_at_5_diff1": 0.080685, + "nauc_ndcg_at_10_max": 0.458398, + "nauc_ndcg_at_10_std": 0.520604, + "nauc_ndcg_at_10_diff1": 0.077764, + "nauc_ndcg_at_20_max": 0.458079, + "nauc_ndcg_at_20_std": 0.515427, + "nauc_ndcg_at_20_diff1": 0.090456, + "nauc_ndcg_at_100_max": 0.455094, + "nauc_ndcg_at_100_std": 0.495248, + "nauc_ndcg_at_100_diff1": 0.104224, + "nauc_ndcg_at_1000_max": 0.455094, + "nauc_ndcg_at_1000_std": 0.495248, + "nauc_ndcg_at_1000_diff1": 0.104224, + "nauc_map_at_1_max": 0.478838, + "nauc_map_at_1_std": 0.267984, + "nauc_map_at_1_diff1": 0.604322, + "nauc_map_at_3_max": 0.422948, + "nauc_map_at_3_std": 0.463043, + "nauc_map_at_3_diff1": 0.018883, + "nauc_map_at_5_max": 0.431718, + "nauc_map_at_5_std": 0.48156, + "nauc_map_at_5_diff1": 0.033299, + "nauc_map_at_10_max": 0.429281, + "nauc_map_at_10_std": 0.489469, + "nauc_map_at_10_diff1": 0.031132, + "nauc_map_at_20_max": 0.429096, + "nauc_map_at_20_std": 0.48728, + "nauc_map_at_20_diff1": 0.034354, + "nauc_map_at_100_max": 0.428263, + "nauc_map_at_100_std": 0.48368, + "nauc_map_at_100_diff1": 0.035986, + "nauc_map_at_1000_max": 0.428263, + "nauc_map_at_1000_std": 0.48368, + "nauc_map_at_1000_diff1": 0.035986, + "nauc_recall_at_1_max": 0.478838, + "nauc_recall_at_1_std": 0.267984, + "nauc_recall_at_1_diff1": 0.604322, + "nauc_recall_at_3_max": 0.458946, + "nauc_recall_at_3_std": 0.53881, + "nauc_recall_at_3_diff1": -0.036257, + "nauc_recall_at_5_max": 0.476495, + "nauc_recall_at_5_std": 0.601345, + "nauc_recall_at_5_diff1": -0.025488, + "nauc_recall_at_10_max": 0.46599, + "nauc_recall_at_10_std": 0.676027, + "nauc_recall_at_10_diff1": -0.07515, + "nauc_recall_at_20_max": 0.470122, + "nauc_recall_at_20_std": 0.715511, + "nauc_recall_at_20_diff1": -0.068782, + "nauc_recall_at_100_max": 0.502849, + "nauc_recall_at_100_std": 0.788336, + "nauc_recall_at_100_diff1": -0.098207, + "nauc_recall_at_1000_max": 0.502849, + "nauc_recall_at_1000_std": 0.788336, + "nauc_recall_at_1000_diff1": -0.098207, + "nauc_precision_at_1_max": 0.478838, + "nauc_precision_at_1_std": 0.267984, + "nauc_precision_at_1_diff1": 0.604322, + "nauc_precision_at_3_max": 0.458946, + "nauc_precision_at_3_std": 0.53881, + "nauc_precision_at_3_diff1": -0.036257, + "nauc_precision_at_5_max": 0.476495, + "nauc_precision_at_5_std": 0.601345, + "nauc_precision_at_5_diff1": -0.025488, + "nauc_precision_at_10_max": 0.46599, + "nauc_precision_at_10_std": 0.676027, + "nauc_precision_at_10_diff1": -0.07515, + "nauc_precision_at_20_max": 0.470122, + "nauc_precision_at_20_std": 0.715511, + "nauc_precision_at_20_diff1": -0.068782, + "nauc_precision_at_100_max": 0.502849, + "nauc_precision_at_100_std": 0.788336, + "nauc_precision_at_100_diff1": -0.098207, + "nauc_precision_at_1000_max": 0.502849, + "nauc_precision_at_1000_std": 0.788336, + "nauc_precision_at_1000_diff1": -0.098207, + "nauc_mrr_at_1_max": 0.478838, + "nauc_mrr_at_1_std": 0.267984, + "nauc_mrr_at_1_diff1": 0.604322, + "nauc_mrr_at_3_max": 0.572693, + "nauc_mrr_at_3_std": 0.384841, + "nauc_mrr_at_3_diff1": 0.613469, + "nauc_mrr_at_5_max": 0.556875, + "nauc_mrr_at_5_std": 0.3643, + "nauc_mrr_at_5_diff1": 0.603873, + "nauc_mrr_at_10_max": 0.553665, + "nauc_mrr_at_10_std": 0.360396, + "nauc_mrr_at_10_diff1": 0.602799, + "nauc_mrr_at_20_max": 0.551356, + "nauc_mrr_at_20_std": 0.358071, + "nauc_mrr_at_20_diff1": 0.605951, + "nauc_mrr_at_100_max": 0.550496, + "nauc_mrr_at_100_std": 0.356826, + "nauc_mrr_at_100_diff1": 0.605733, + "nauc_mrr_at_1000_max": 0.550496, + "nauc_mrr_at_1000_std": 0.356826, + "nauc_mrr_at_1000_diff1": 0.605733, + "main_score": 0.87208, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 47.27411389350891, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/IFlyTek.json b/results/ByteDance-Seed__Seed-Embedding/1/IFlyTek.json new file mode 100644 index 0000000000..c3d95d68ae --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/IFlyTek.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "421605374b29664c5fc098418fe20ada9bd55f8a", + "task_name": "IFlyTek", + "mteb_version": "1.35.1", + "scores": { + "validation": [ + { + "accuracy": 0.526818, + "f1": 0.36599, + "f1_weighted": 0.510214, + "scores_per_experiment": [ + { + "accuracy": 0.522124, + "f1": 0.367167, + "f1_weighted": 0.504403 + }, + { + "accuracy": 0.521354, + "f1": 0.361476, + "f1_weighted": 0.501622 + }, + { + "accuracy": 0.532897, + "f1": 0.371412, + "f1_weighted": 0.518739 + }, + { + "accuracy": 0.523278, + "f1": 0.36519, + "f1_weighted": 0.505261 + }, + { + "accuracy": 0.534436, + "f1": 0.364705, + "f1_weighted": 0.521043 + } + ], + "main_score": 0.526818, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 58.80963611602783, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/ImdbClassification.json b/results/ByteDance-Seed__Seed-Embedding/1/ImdbClassification.json new file mode 100644 index 0000000000..2dca6f2355 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/ImdbClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7", + "task_name": "ImdbClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.970796, + "f1": 0.97079, + "f1_weighted": 0.97079, + "ap": 0.957268, + "ap_weighted": 0.957268, + "scores_per_experiment": [ + { + "accuracy": 0.96784, + "f1": 0.967829, + "f1_weighted": 0.967829, + "ap": 0.961006, + "ap_weighted": 0.961006 + }, + { + "accuracy": 0.9632, + "f1": 0.963176, + "f1_weighted": 0.963176, + "ap": 0.95777, + "ap_weighted": 0.95777 + }, + { + "accuracy": 0.97144, + "f1": 0.971435, + "f1_weighted": 0.971435, + "ap": 0.952327, + "ap_weighted": 0.952327 + }, + { + "accuracy": 0.97316, + "f1": 0.97316, + "f1_weighted": 0.97316, + "ap": 0.962102, + "ap_weighted": 0.962102 + }, + { + "accuracy": 0.97316, + "f1": 0.973159, + "f1_weighted": 0.973159, + "ap": 0.962576, + "ap_weighted": 0.962576 + }, + { + "accuracy": 0.97372, + "f1": 0.973719, + "f1_weighted": 0.973719, + "ap": 0.958417, + "ap_weighted": 0.958417 + }, + { + "accuracy": 0.96952, + "f1": 0.969511, + "f1_weighted": 0.969511, + "ap": 0.94801, + "ap_weighted": 0.94801 + }, + { + "accuracy": 0.97248, + "f1": 0.972477, + "f1_weighted": 0.972477, + "ap": 0.954689, + "ap_weighted": 0.954689 + }, + { + "accuracy": 0.97236, + "f1": 0.972356, + "f1_weighted": 0.972356, + "ap": 0.954228, + "ap_weighted": 0.954228 + }, + { + "accuracy": 0.97108, + "f1": 0.971078, + "f1_weighted": 0.971078, + "ap": 0.961561, + "ap_weighted": 0.961561 + } + ], + "main_score": 0.970796, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 29.73388981819153, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/JDReview.json b/results/ByteDance-Seed__Seed-Embedding/1/JDReview.json new file mode 100644 index 0000000000..d4444701b1 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/JDReview.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b", + "task_name": "JDReview", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "accuracy": 0.906004, + "f1": 0.861582, + "f1_weighted": 0.910858, + "ap": 0.64244, + "ap_weighted": 0.64244, + "scores_per_experiment": [ + { + "accuracy": 0.909944, + "f1": 0.866097, + "f1_weighted": 0.914256, + "ap": 0.651135, + "ap_weighted": 0.651135 + }, + { + "accuracy": 0.900563, + "f1": 0.855345, + "f1_weighted": 0.906177, + "ap": 0.630057, + "ap_weighted": 0.630057 + }, + { + "accuracy": 0.904315, + "f1": 0.859071, + "f1_weighted": 0.909259, + "ap": 0.63672, + "ap_weighted": 0.63672 + }, + { + "accuracy": 0.904315, + "f1": 0.85818, + "f1_weighted": 0.90902, + "ap": 0.634224, + "ap_weighted": 0.634224 + }, + { + "accuracy": 0.904315, + "f1": 0.859946, + "f1_weighted": 0.909492, + "ap": 0.639237, + "ap_weighted": 0.639237 + }, + { + "accuracy": 0.896811, + "f1": 0.852559, + "f1_weighted": 0.903327, + "ap": 0.626494, + "ap_weighted": 0.626494 + }, + { + "accuracy": 0.917448, + "f1": 0.873995, + "f1_weighted": 0.920503, + "ap": 0.667103, + "ap_weighted": 0.667103 + }, + { + "accuracy": 0.91182, + "f1": 0.867613, + "f1_weighted": 0.915695, + "ap": 0.653763, + "ap_weighted": 0.653763 + }, + { + "accuracy": 0.904315, + "f1": 0.859071, + "f1_weighted": 0.909259, + "ap": 0.63672, + "ap_weighted": 0.63672 + }, + { + "accuracy": 0.906191, + "f1": 0.863947, + "f1_weighted": 0.911596, + "ap": 0.648948, + "ap_weighted": 0.648948 + } + ], + "main_score": 0.906004, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 15.881753921508789, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/LCQMC.json b/results/ByteDance-Seed__Seed-Embedding/1/LCQMC.json new file mode 100644 index 0000000000..44d9a460ed --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/LCQMC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "17f9b096f80380fce5ed12a9be8be7784b337daf", + "task_name": "LCQMC", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "pearson": 0.786317, + "spearman": 0.80582, + "cosine_pearson": 0.786317, + "cosine_spearman": 0.80582, + "manhattan_pearson": 0.814792, + "manhattan_spearman": 0.805513, + "euclidean_pearson": 0.815561, + "euclidean_spearman": 0.8058, + "main_score": 0.80582, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 20.375383377075195, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/MMarcoReranking.json b/results/ByteDance-Seed__Seed-Embedding/1/MMarcoReranking.json new file mode 100644 index 0000000000..a543067703 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/MMarcoReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6", + "task_name": "MMarcoReranking", + "mteb_version": "1.35.1", + "scores": { + "dev": [ + { + "map": 0.375899, + "mrr": 0.36527, + "nAUC_map_max": 0.134607, + "nAUC_map_std": -0.297926, + "nAUC_map_diff1": 0.323064, + "nAUC_mrr_max": 0.139683, + "nAUC_mrr_std": -0.29336, + "nAUC_mrr_diff1": 0.313012, + "main_score": 0.375899, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 298.74765515327454, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/MMarcoRetrieval.json b/results/ByteDance-Seed__Seed-Embedding/1/MMarcoRetrieval.json new file mode 100644 index 0000000000..3c6a6bf588 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/MMarcoRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "539bbde593d947e2a124ba72651aafc09eb33fc2", + "task_name": "MMarcoRetrieval", + "mteb_version": "1.35.1", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.80559, + "ndcg_at_3": 0.86298, + "ndcg_at_5": 0.87763, + "ndcg_at_10": 0.88683, + "ndcg_at_20": 0.89059, + "ndcg_at_100": 0.89288, + "ndcg_at_1000": 0.89288, + "map_at_1": 0.7804, + "map_at_3": 0.84635, + "map_at_5": 0.85543, + "map_at_10": 0.85956, + "map_at_20": 0.86073, + "map_at_100": 0.8611, + "map_at_1000": 0.8611, + "recall_at_1": 0.7804, + "recall_at_3": 0.90399, + "recall_at_5": 0.9385, + "recall_at_10": 0.9658, + "recall_at_20": 0.97999, + "recall_at_100": 0.99157, + "recall_at_1000": 0.99157, + "precision_at_1": 0.80559, + "precision_at_3": 0.31877, + "precision_at_5": 0.19934, + "precision_at_10": 0.10272, + "precision_at_20": 0.05218, + "precision_at_100": 0.01057, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.805587, + "mrr_at_3": 0.853032, + "mrr_at_5": 0.860081, + "mrr_at_10": 0.863668, + "mrr_at_20": 0.864604, + "mrr_at_100": 0.864922, + "mrr_at_1000": 0.864922, + "nauc_ndcg_at_1_max": 0.389295, + "nauc_ndcg_at_1_std": -0.272704, + "nauc_ndcg_at_1_diff1": 0.852371, + "nauc_ndcg_at_3_max": 0.447528, + "nauc_ndcg_at_3_std": -0.196047, + "nauc_ndcg_at_3_diff1": 0.830663, + "nauc_ndcg_at_5_max": 0.463046, + "nauc_ndcg_at_5_std": -0.160181, + "nauc_ndcg_at_5_diff1": 0.835047, + "nauc_ndcg_at_10_max": 0.463885, + "nauc_ndcg_at_10_std": -0.148753, + "nauc_ndcg_at_10_diff1": 0.837548, + "nauc_ndcg_at_20_max": 0.457724, + "nauc_ndcg_at_20_std": -0.158757, + "nauc_ndcg_at_20_diff1": 0.839244, + "nauc_ndcg_at_100_max": 0.449355, + "nauc_ndcg_at_100_std": -0.175352, + "nauc_ndcg_at_100_diff1": 0.839751, + "nauc_ndcg_at_1000_max": 0.449355, + "nauc_ndcg_at_1000_std": -0.175352, + "nauc_ndcg_at_1000_diff1": 0.839751, + "nauc_map_at_1_max": 0.340899, + "nauc_map_at_1_std": -0.307688, + "nauc_map_at_1_diff1": 0.845826, + "nauc_map_at_3_max": 0.425792, + "nauc_map_at_3_std": -0.227272, + "nauc_map_at_3_diff1": 0.834764, + "nauc_map_at_5_max": 0.434325, + "nauc_map_at_5_std": -0.208809, + "nauc_map_at_5_diff1": 0.837326, + "nauc_map_at_10_max": 0.434609, + "nauc_map_at_10_std": -0.205047, + "nauc_map_at_10_diff1": 0.838674, + "nauc_map_at_20_max": 0.432986, + "nauc_map_at_20_std": -0.207573, + "nauc_map_at_20_diff1": 0.839222, + "nauc_map_at_100_max": 0.431876, + "nauc_map_at_100_std": -0.209699, + "nauc_map_at_100_diff1": 0.839279, + "nauc_map_at_1000_max": 0.431876, + "nauc_map_at_1000_std": -0.209699, + "nauc_map_at_1000_diff1": 0.839279, + "nauc_recall_at_1_max": 0.340899, + "nauc_recall_at_1_std": -0.307688, + "nauc_recall_at_1_diff1": 0.845826, + "nauc_recall_at_3_max": 0.519009, + "nauc_recall_at_3_std": -0.081582, + "nauc_recall_at_3_diff1": 0.796724, + "nauc_recall_at_5_max": 0.629235, + "nauc_recall_at_5_std": 0.139154, + "nauc_recall_at_5_diff1": 0.803614, + "nauc_recall_at_10_max": 0.76316, + "nauc_recall_at_10_std": 0.470125, + "nauc_recall_at_10_diff1": 0.803646, + "nauc_recall_at_20_max": 0.855274, + "nauc_recall_at_20_std": 0.703193, + "nauc_recall_at_20_diff1": 0.815753, + "nauc_recall_at_100_max": 0.890678, + "nauc_recall_at_100_std": 0.855712, + "nauc_recall_at_100_diff1": 0.819583, + "nauc_recall_at_1000_max": 0.890678, + "nauc_recall_at_1000_std": 0.855712, + "nauc_recall_at_1000_diff1": 0.819583, + "nauc_precision_at_1_max": 0.389295, + "nauc_precision_at_1_std": -0.272704, + "nauc_precision_at_1_diff1": 0.852371, + "nauc_precision_at_3_max": 0.33168, + "nauc_precision_at_3_std": 0.109594, + "nauc_precision_at_3_diff1": 0.267008, + "nauc_precision_at_5_max": 0.286688, + "nauc_precision_at_5_std": 0.250295, + "nauc_precision_at_5_diff1": 0.090763, + "nauc_precision_at_10_max": 0.225488, + "nauc_precision_at_10_std": 0.349698, + "nauc_precision_at_10_diff1": -0.07369, + "nauc_precision_at_20_max": 0.158977, + "nauc_precision_at_20_std": 0.355254, + "nauc_precision_at_20_diff1": -0.176181, + "nauc_precision_at_100_max": 0.079126, + "nauc_precision_at_100_std": 0.312464, + "nauc_precision_at_100_diff1": -0.273349, + "nauc_precision_at_1000_max": 0.079126, + "nauc_precision_at_1000_std": 0.312464, + "nauc_precision_at_1000_diff1": -0.273349, + "nauc_mrr_at_1_max": 0.389295, + "nauc_mrr_at_1_std": -0.272704, + "nauc_mrr_at_1_diff1": 0.852371, + "nauc_mrr_at_3_max": 0.437736, + "nauc_mrr_at_3_std": -0.211573, + "nauc_mrr_at_3_diff1": 0.84179, + "nauc_mrr_at_5_max": 0.441515, + "nauc_mrr_at_5_std": -0.199492, + "nauc_mrr_at_5_diff1": 0.843787, + "nauc_mrr_at_10_max": 0.440545, + "nauc_mrr_at_10_std": -0.197376, + "nauc_mrr_at_10_diff1": 0.844282, + "nauc_mrr_at_20_max": 0.438992, + "nauc_mrr_at_20_std": -0.200122, + "nauc_mrr_at_20_diff1": 0.844528, + "nauc_mrr_at_100_max": 0.437971, + "nauc_mrr_at_100_std": -0.202097, + "nauc_mrr_at_100_diff1": 0.844569, + "nauc_mrr_at_1000_max": 0.437971, + "nauc_mrr_at_1000_std": -0.202097, + "nauc_mrr_at_1000_diff1": 0.844569, + "main_score": 0.88683, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 96.76973414421082, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/MTOPDomainClassification.json b/results/ByteDance-Seed__Seed-Embedding/1/MTOPDomainClassification.json new file mode 100644 index 0000000000..e7e6c648bb --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/MTOPDomainClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf", + "task_name": "MTOPDomainClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.992909, + "f1": 0.991758, + "f1_weighted": 0.99293, + "scores_per_experiment": [ + { + "accuracy": 0.993844, + "f1": 0.99298, + "f1_weighted": 0.993857 + }, + { + "accuracy": 0.99316, + "f1": 0.991825, + "f1_weighted": 0.993183 + }, + { + "accuracy": 0.992932, + "f1": 0.991652, + "f1_weighted": 0.992955 + }, + { + "accuracy": 0.992476, + "f1": 0.991032, + "f1_weighted": 0.992513 + }, + { + "accuracy": 0.992248, + "f1": 0.990864, + "f1_weighted": 0.992276 + }, + { + "accuracy": 0.992704, + "f1": 0.991545, + "f1_weighted": 0.992725 + }, + { + "accuracy": 0.99316, + "f1": 0.992188, + "f1_weighted": 0.993176 + }, + { + "accuracy": 0.992704, + "f1": 0.991491, + "f1_weighted": 0.992727 + }, + { + "accuracy": 0.992704, + "f1": 0.991697, + "f1_weighted": 0.992719 + }, + { + "accuracy": 0.99316, + "f1": 0.992308, + "f1_weighted": 0.993172 + } + ], + "main_score": 0.992909, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 19.291016578674316, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/MassiveIntentClassification.json b/results/ByteDance-Seed__Seed-Embedding/1/MassiveIntentClassification.json new file mode 100644 index 0000000000..7dc777ddfc --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/MassiveIntentClassification.json @@ -0,0 +1,135 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "task_name": "MassiveIntentClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.863517, + "f1": 0.833505, + "f1_weighted": 0.853087, + "scores_per_experiment": [ + { + "accuracy": 0.867182, + "f1": 0.832364, + "f1_weighted": 0.857703 + }, + { + "accuracy": 0.866846, + "f1": 0.846703, + "f1_weighted": 0.856773 + }, + { + "accuracy": 0.854741, + "f1": 0.83106, + "f1_weighted": 0.840921 + }, + { + "accuracy": 0.86382, + "f1": 0.831648, + "f1_weighted": 0.852523 + }, + { + "accuracy": 0.848352, + "f1": 0.814638, + "f1_weighted": 0.834354 + }, + { + "accuracy": 0.870545, + "f1": 0.842881, + "f1_weighted": 0.864304 + }, + { + "accuracy": 0.861802, + "f1": 0.833052, + "f1_weighted": 0.848606 + }, + { + "accuracy": 0.859112, + "f1": 0.823551, + "f1_weighted": 0.847796 + }, + { + "accuracy": 0.870545, + "f1": 0.839725, + "f1_weighted": 0.862495 + }, + { + "accuracy": 0.872226, + "f1": 0.839428, + "f1_weighted": 0.86539 + } + ], + "main_score": 0.863517, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.826698, + "f1": 0.790039, + "f1_weighted": 0.814572, + "scores_per_experiment": [ + { + "accuracy": 0.833221, + "f1": 0.787581, + "f1_weighted": 0.820758 + }, + { + "accuracy": 0.827841, + "f1": 0.796587, + "f1_weighted": 0.814893 + }, + { + "accuracy": 0.81809, + "f1": 0.783208, + "f1_weighted": 0.80415 + }, + { + "accuracy": 0.827841, + "f1": 0.791901, + "f1_weighted": 0.817397 + }, + { + "accuracy": 0.813046, + "f1": 0.772874, + "f1_weighted": 0.796535 + }, + { + "accuracy": 0.829859, + "f1": 0.797409, + "f1_weighted": 0.82121 + }, + { + "accuracy": 0.826833, + "f1": 0.79147, + "f1_weighted": 0.812158 + }, + { + "accuracy": 0.825151, + "f1": 0.781464, + "f1_weighted": 0.814138 + }, + { + "accuracy": 0.830868, + "f1": 0.789058, + "f1_weighted": 0.821285 + }, + { + "accuracy": 0.83423, + "f1": 0.808839, + "f1_weighted": 0.823198 + } + ], + "main_score": 0.826698, + "hf_subset": "zh-CN", + "languages": [ + "cmo-Hans" + ] + } + ] + }, + "evaluation_time": 23.778774738311768, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/MassiveScenarioClassification.json b/results/ByteDance-Seed__Seed-Embedding/1/MassiveScenarioClassification.json new file mode 100644 index 0000000000..245f739533 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/MassiveScenarioClassification.json @@ -0,0 +1,135 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "task_name": "MassiveScenarioClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.921015, + "f1": 0.911745, + "f1_weighted": 0.918137, + "scores_per_experiment": [ + { + "accuracy": 0.921318, + "f1": 0.913157, + "f1_weighted": 0.91882 + }, + { + "accuracy": 0.920982, + "f1": 0.91131, + "f1_weighted": 0.918021 + }, + { + "accuracy": 0.917283, + "f1": 0.906097, + "f1_weighted": 0.913655 + }, + { + "accuracy": 0.918292, + "f1": 0.90799, + "f1_weighted": 0.915185 + }, + { + "accuracy": 0.921318, + "f1": 0.911873, + "f1_weighted": 0.917986 + }, + { + "accuracy": 0.924344, + "f1": 0.916062, + "f1_weighted": 0.92148 + }, + { + "accuracy": 0.918292, + "f1": 0.909794, + "f1_weighted": 0.915412 + }, + { + "accuracy": 0.920646, + "f1": 0.910449, + "f1_weighted": 0.917772 + }, + { + "accuracy": 0.925353, + "f1": 0.916817, + "f1_weighted": 0.922639 + }, + { + "accuracy": 0.922327, + "f1": 0.913902, + "f1_weighted": 0.920398 + } + ], + "main_score": 0.921015, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.90464, + "f1": 0.895682, + "f1_weighted": 0.900277, + "scores_per_experiment": [ + { + "accuracy": 0.905178, + "f1": 0.90006, + "f1_weighted": 0.902574 + }, + { + "accuracy": 0.905178, + "f1": 0.895935, + "f1_weighted": 0.902211 + }, + { + "accuracy": 0.906523, + "f1": 0.896834, + "f1_weighted": 0.901178 + }, + { + "accuracy": 0.901816, + "f1": 0.890816, + "f1_weighted": 0.896117 + }, + { + "accuracy": 0.905514, + "f1": 0.89548, + "f1_weighted": 0.900748 + }, + { + "accuracy": 0.907532, + "f1": 0.899589, + "f1_weighted": 0.904181 + }, + { + "accuracy": 0.897781, + "f1": 0.886564, + "f1_weighted": 0.890561 + }, + { + "accuracy": 0.904506, + "f1": 0.896082, + "f1_weighted": 0.900172 + }, + { + "accuracy": 0.908541, + "f1": 0.900922, + "f1_weighted": 0.904819 + }, + { + "accuracy": 0.903833, + "f1": 0.894534, + "f1_weighted": 0.900205 + } + ], + "main_score": 0.90464, + "hf_subset": "zh-CN", + "languages": [ + "cmo-Hans" + ] + } + ] + }, + "evaluation_time": 123.09003067016602, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/MedicalRetrieval.json b/results/ByteDance-Seed__Seed-Embedding/1/MedicalRetrieval.json new file mode 100644 index 0000000000..6567f29004 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/MedicalRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6", + "task_name": "MedicalRetrieval", + "mteb_version": "1.35.1", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.62, + "ndcg_at_3": 0.67753, + "ndcg_at_5": 0.68984, + "ndcg_at_10": 0.70358, + "ndcg_at_20": 0.71327, + "ndcg_at_100": 0.7278, + "ndcg_at_1000": 0.7278, + "map_at_1": 0.62, + "map_at_3": 0.66383, + "map_at_5": 0.67063, + "map_at_10": 0.6762, + "map_at_20": 0.67891, + "map_at_100": 0.68093, + "map_at_1000": 0.68093, + "recall_at_1": 0.62, + "recall_at_3": 0.717, + "recall_at_5": 0.747, + "recall_at_10": 0.79, + "recall_at_20": 0.828, + "recall_at_100": 0.906, + "recall_at_1000": 0.906, + "precision_at_1": 0.62, + "precision_at_3": 0.239, + "precision_at_5": 0.1494, + "precision_at_10": 0.079, + "precision_at_20": 0.0414, + "precision_at_100": 0.00906, + "precision_at_1000": 0.00091, + "mrr_at_1": 0.62, + "mrr_at_3": 0.663833, + "mrr_at_5": 0.670633, + "mrr_at_10": 0.676202, + "mrr_at_20": 0.678907, + "mrr_at_100": 0.680926, + "mrr_at_1000": 0.680926, + "nauc_ndcg_at_1_max": 0.796857, + "nauc_ndcg_at_1_std": 0.618781, + "nauc_ndcg_at_1_diff1": 0.86264, + "nauc_ndcg_at_3_max": 0.790248, + "nauc_ndcg_at_3_std": 0.662494, + "nauc_ndcg_at_3_diff1": 0.828483, + "nauc_ndcg_at_5_max": 0.783236, + "nauc_ndcg_at_5_std": 0.667395, + "nauc_ndcg_at_5_diff1": 0.81997, + "nauc_ndcg_at_10_max": 0.77985, + "nauc_ndcg_at_10_std": 0.676117, + "nauc_ndcg_at_10_diff1": 0.816594, + "nauc_ndcg_at_20_max": 0.778896, + "nauc_ndcg_at_20_std": 0.676266, + "nauc_ndcg_at_20_diff1": 0.817197, + "nauc_ndcg_at_100_max": 0.77816, + "nauc_ndcg_at_100_std": 0.671824, + "nauc_ndcg_at_100_diff1": 0.818452, + "nauc_ndcg_at_1000_max": 0.77816, + "nauc_ndcg_at_1000_std": 0.671824, + "nauc_ndcg_at_1000_diff1": 0.818452, + "nauc_map_at_1_max": 0.796857, + "nauc_map_at_1_std": 0.618781, + "nauc_map_at_1_diff1": 0.86264, + "nauc_map_at_3_max": 0.791818, + "nauc_map_at_3_std": 0.652511, + "nauc_map_at_3_diff1": 0.836757, + "nauc_map_at_5_max": 0.788109, + "nauc_map_at_5_std": 0.654969, + "nauc_map_at_5_diff1": 0.832325, + "nauc_map_at_10_max": 0.786887, + "nauc_map_at_10_std": 0.657785, + "nauc_map_at_10_diff1": 0.83128, + "nauc_map_at_20_max": 0.786655, + "nauc_map_at_20_std": 0.657565, + "nauc_map_at_20_diff1": 0.831435, + "nauc_map_at_100_max": 0.786641, + "nauc_map_at_100_std": 0.656997, + "nauc_map_at_100_diff1": 0.83173, + "nauc_map_at_1000_max": 0.786641, + "nauc_map_at_1000_std": 0.656997, + "nauc_map_at_1000_diff1": 0.83173, + "nauc_recall_at_1_max": 0.796857, + "nauc_recall_at_1_std": 0.618781, + "nauc_recall_at_1_diff1": 0.86264, + "nauc_recall_at_3_max": 0.78522, + "nauc_recall_at_3_std": 0.694261, + "nauc_recall_at_3_diff1": 0.801903, + "nauc_recall_at_5_max": 0.765076, + "nauc_recall_at_5_std": 0.710927, + "nauc_recall_at_5_diff1": 0.77569, + "nauc_recall_at_10_max": 0.748486, + "nauc_recall_at_10_std": 0.755419, + "nauc_recall_at_10_diff1": 0.754543, + "nauc_recall_at_20_max": 0.737415, + "nauc_recall_at_20_std": 0.772965, + "nauc_recall_at_20_diff1": 0.747513, + "nauc_recall_at_100_max": 0.696759, + "nauc_recall_at_100_std": 0.781542, + "nauc_recall_at_100_diff1": 0.711197, + "nauc_recall_at_1000_max": 0.696759, + "nauc_recall_at_1000_std": 0.781542, + "nauc_recall_at_1000_diff1": 0.711197, + "nauc_precision_at_1_max": 0.796857, + "nauc_precision_at_1_std": 0.618781, + "nauc_precision_at_1_diff1": 0.86264, + "nauc_precision_at_3_max": 0.78522, + "nauc_precision_at_3_std": 0.694261, + "nauc_precision_at_3_diff1": 0.801903, + "nauc_precision_at_5_max": 0.765076, + "nauc_precision_at_5_std": 0.710927, + "nauc_precision_at_5_diff1": 0.77569, + "nauc_precision_at_10_max": 0.748486, + "nauc_precision_at_10_std": 0.755419, + "nauc_precision_at_10_diff1": 0.754543, + "nauc_precision_at_20_max": 0.737415, + "nauc_precision_at_20_std": 0.772965, + "nauc_precision_at_20_diff1": 0.747513, + "nauc_precision_at_100_max": 0.696759, + "nauc_precision_at_100_std": 0.781542, + "nauc_precision_at_100_diff1": 0.711197, + "nauc_precision_at_1000_max": 0.696759, + "nauc_precision_at_1000_std": 0.781542, + "nauc_precision_at_1000_diff1": 0.711197, + "nauc_mrr_at_1_max": 0.796857, + "nauc_mrr_at_1_std": 0.618781, + "nauc_mrr_at_1_diff1": 0.86264, + "nauc_mrr_at_3_max": 0.791818, + "nauc_mrr_at_3_std": 0.652511, + "nauc_mrr_at_3_diff1": 0.836757, + "nauc_mrr_at_5_max": 0.788109, + "nauc_mrr_at_5_std": 0.654969, + "nauc_mrr_at_5_diff1": 0.832325, + "nauc_mrr_at_10_max": 0.786887, + "nauc_mrr_at_10_std": 0.657785, + "nauc_mrr_at_10_diff1": 0.83128, + "nauc_mrr_at_20_max": 0.786655, + "nauc_mrr_at_20_std": 0.657565, + "nauc_mrr_at_20_diff1": 0.831435, + "nauc_mrr_at_100_max": 0.786641, + "nauc_mrr_at_100_std": 0.656997, + "nauc_mrr_at_100_diff1": 0.83173, + "nauc_mrr_at_1000_max": 0.786641, + "nauc_mrr_at_1000_std": 0.656997, + "nauc_mrr_at_1000_diff1": 0.83173, + "main_score": 0.70358, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 673.0681726932526, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/MedrxivClusteringP2P.v2.json b/results/ByteDance-Seed__Seed-Embedding/1/MedrxivClusteringP2P.v2.json new file mode 100644 index 0000000000..0b71086950 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/MedrxivClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "task_name": "MedrxivClusteringP2P.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.507201, + 0.522836, + 0.51016, + 0.506487, + 0.511975, + 0.513209, + 0.519803, + 0.514015, + 0.526014, + 0.510814 + ] + }, + "v_measure": 0.514251, + "v_measure_std": 0.006227, + "main_score": 0.514251, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11.216255903244019, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/MedrxivClusteringS2S.v2.json b/results/ByteDance-Seed__Seed-Embedding/1/MedrxivClusteringS2S.v2.json new file mode 100644 index 0000000000..3eec5723f4 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/MedrxivClusteringS2S.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "task_name": "MedrxivClusteringS2S.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.501532, + 0.49995, + 0.516034, + 0.49678, + 0.514673, + 0.50266, + 0.51342, + 0.510737, + 0.521541, + 0.507143 + ] + }, + "v_measure": 0.508447, + "v_measure_std": 0.007675, + "main_score": 0.508447, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.548509359359741, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/MindSmallReranking.json b/results/ByteDance-Seed__Seed-Embedding/1/MindSmallReranking.json new file mode 100644 index 0000000000..281f750a36 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/MindSmallReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "59042f120c80e8afa9cdbb224f67076cec0fc9a7", + "task_name": "MindSmallReranking", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "map": 0.317914, + "mrr": 0.329086, + "nAUC_map_max": -0.233635, + "nAUC_map_std": -0.076562, + "nAUC_map_diff1": 0.14612, + "nAUC_mrr_max": -0.183038, + "nAUC_mrr_std": -0.054845, + "nAUC_mrr_diff1": 0.133983, + "main_score": 0.317914, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 136.08986639976501, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/MultilingualSentiment.json b/results/ByteDance-Seed__Seed-Embedding/1/MultilingualSentiment.json new file mode 100644 index 0000000000..cc4f0d22fc --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/MultilingualSentiment.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a", + "task_name": "MultilingualSentiment", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "accuracy": 0.828, + "f1": 0.828001, + "f1_weighted": 0.828001, + "scores_per_experiment": [ + { + "accuracy": 0.826, + "f1": 0.825954, + "f1_weighted": 0.825954 + }, + { + "accuracy": 0.828667, + "f1": 0.826868, + "f1_weighted": 0.826868 + }, + { + "accuracy": 0.832, + "f1": 0.831412, + "f1_weighted": 0.831412 + }, + { + "accuracy": 0.83, + "f1": 0.830747, + "f1_weighted": 0.830747 + }, + { + "accuracy": 0.824333, + "f1": 0.825362, + "f1_weighted": 0.825362 + }, + { + "accuracy": 0.82, + "f1": 0.820523, + "f1_weighted": 0.820523 + }, + { + "accuracy": 0.834667, + "f1": 0.835025, + "f1_weighted": 0.835025 + }, + { + "accuracy": 0.829667, + "f1": 0.828028, + "f1_weighted": 0.828028 + }, + { + "accuracy": 0.823333, + "f1": 0.824147, + "f1_weighted": 0.824147 + }, + { + "accuracy": 0.831333, + "f1": 0.831947, + "f1_weighted": 0.831947 + } + ], + "main_score": 0.828, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 39.28332757949829, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/Ocnli.json b/results/ByteDance-Seed__Seed-Embedding/1/Ocnli.json new file mode 100644 index 0000000000..351c0730f7 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/Ocnli.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "66e76a618a34d6d565d5538088562851e6daa7ec", + "task_name": "Ocnli", + "mteb_version": "1.35.1", + "scores": { + "validation": [ + { + "similarity_accuracy": 0.822956, + "similarity_accuracy_threshold": 0.573201, + "similarity_f1": 0.836292, + "similarity_f1_threshold": 0.556233, + "similarity_precision": 0.784459, + "similarity_recall": 0.895459, + "similarity_ap": 0.860088, + "cosine_accuracy": 0.822956, + "cosine_accuracy_threshold": 0.573201, + "cosine_f1": 0.836292, + "cosine_f1_threshold": 0.556233, + "cosine_precision": 0.784459, + "cosine_recall": 0.895459, + "cosine_ap": 0.860088, + "manhattan_accuracy": 0.815376, + "manhattan_accuracy_threshold": 32.198013, + "manhattan_f1": 0.827241, + "manhattan_f1_threshold": 32.464794, + "manhattan_precision": 0.786667, + "manhattan_recall": 0.872228, + "manhattan_ap": 0.854782, + "euclidean_accuracy": 0.822956, + "euclidean_accuracy_threshold": 0.923687, + "euclidean_f1": 0.83622, + "euclidean_f1_threshold": 0.94009, + "euclidean_precision": 0.786778, + "euclidean_recall": 0.892291, + "euclidean_ap": 0.859887, + "dot_accuracy": 0.824039, + "dot_accuracy_threshold": 0.571308, + "dot_f1": 0.837256, + "dot_f1_threshold": 0.564057, + "dot_precision": 0.79619, + "dot_recall": 0.882788, + "dot_ap": 0.860567, + "max_accuracy": 0.824039, + "max_f1": 0.837256, + "max_precision": 0.79619, + "max_recall": 0.895459, + "max_ap": 0.860567, + "main_score": 0.860567, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 3.561516284942627, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/OnlineShopping.json b/results/ByteDance-Seed__Seed-Embedding/1/OnlineShopping.json new file mode 100644 index 0000000000..d1233318aa --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/OnlineShopping.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e610f2ebd179a8fda30ae534c3878750a96db120", + "task_name": "OnlineShopping", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "accuracy": 0.9596, + "f1": 0.959523, + "f1_weighted": 0.959606, + "ap": 0.946888, + "ap_weighted": 0.946888, + "scores_per_experiment": [ + { + "accuracy": 0.959, + "f1": 0.958885, + "f1_weighted": 0.958989, + "ap": 0.941664, + "ap_weighted": 0.941664 + }, + { + "accuracy": 0.958, + "f1": 0.957945, + "f1_weighted": 0.958018, + "ap": 0.947686, + "ap_weighted": 0.947686 + }, + { + "accuracy": 0.963, + "f1": 0.962911, + "f1_weighted": 0.962998, + "ap": 0.948702, + "ap_weighted": 0.948702 + }, + { + "accuracy": 0.96, + "f1": 0.9599, + "f1_weighted": 0.959996, + "ap": 0.944264, + "ap_weighted": 0.944264 + }, + { + "accuracy": 0.961, + "f1": 0.960914, + "f1_weighted": 0.961002, + "ap": 0.94689, + "ap_weighted": 0.94689 + }, + { + "accuracy": 0.957, + "f1": 0.956959, + "f1_weighted": 0.957023, + "ap": 0.948575, + "ap_weighted": 0.948575 + }, + { + "accuracy": 0.957, + "f1": 0.956953, + "f1_weighted": 0.957021, + "ap": 0.947674, + "ap_weighted": 0.947674 + }, + { + "accuracy": 0.961, + "f1": 0.960947, + "f1_weighted": 0.961016, + "ap": 0.951302, + "ap_weighted": 0.951302 + }, + { + "accuracy": 0.959, + "f1": 0.958876, + "f1_weighted": 0.958984, + "ap": 0.940826, + "ap_weighted": 0.940826 + }, + { + "accuracy": 0.961, + "f1": 0.960947, + "f1_weighted": 0.961016, + "ap": 0.951302, + "ap_weighted": 0.951302 + } + ], + "main_score": 0.9596, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 17.254213333129883, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/PAWSX.json b/results/ByteDance-Seed__Seed-Embedding/1/PAWSX.json new file mode 100644 index 0000000000..77ff0b6744 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/PAWSX.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1", + "task_name": "PAWSX", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "pearson": 0.558472, + "spearman": 0.614902, + "cosine_pearson": 0.558472, + "cosine_spearman": 0.614902, + "manhattan_pearson": 0.598272, + "manhattan_spearman": 0.614181, + "euclidean_pearson": 0.598859, + "euclidean_spearman": 0.614929, + "main_score": 0.614902, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 3.6350340843200684, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/QBQTC.json b/results/ByteDance-Seed__Seed-Embedding/1/QBQTC.json new file mode 100644 index 0000000000..938bd2f95d --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/QBQTC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7", + "task_name": "QBQTC", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "pearson": 0.514771, + "spearman": 0.518706, + "cosine_pearson": 0.514771, + "cosine_spearman": 0.518706, + "manhattan_pearson": 0.518701, + "manhattan_spearman": 0.518503, + "euclidean_pearson": 0.518781, + "euclidean_spearman": 0.518678, + "main_score": 0.518706, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 8.448439121246338, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/SCIDOCS.json b/results/ByteDance-Seed__Seed-Embedding/1/SCIDOCS.json new file mode 100644 index 0000000000..452fd136b8 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/SCIDOCS.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88", + "task_name": "SCIDOCS", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.268, + "ndcg_at_3": 0.2315, + "ndcg_at_5": 0.2099, + "ndcg_at_10": 0.26003, + "ndcg_at_20": 0.3008, + "ndcg_at_100": 0.37031, + "ndcg_at_1000": 0.37031, + "map_at_1": 0.05443, + "map_at_3": 0.10548, + "map_at_5": 0.1317, + "map_at_10": 0.15929, + "map_at_20": 0.17573, + "map_at_100": 0.19195, + "map_at_1000": 0.19195, + "recall_at_1": 0.05443, + "recall_at_3": 0.13408, + "recall_at_5": 0.19353, + "recall_at_10": 0.28545, + "recall_at_20": 0.38175, + "recall_at_100": 0.60628, + "recall_at_1000": 0.60628, + "precision_at_1": 0.268, + "precision_at_3": 0.22033, + "precision_at_5": 0.1908, + "precision_at_10": 0.1408, + "precision_at_20": 0.0941, + "precision_at_100": 0.0299, + "precision_at_1000": 0.00299, + "mrr_at_1": 0.268, + "mrr_at_3": 0.361, + "mrr_at_5": 0.3861, + "mrr_at_10": 0.404329, + "mrr_at_20": 0.411015, + "mrr_at_100": 0.414564, + "mrr_at_1000": 0.414564, + "nauc_ndcg_at_1_max": 0.183094, + "nauc_ndcg_at_1_std": 0.223546, + "nauc_ndcg_at_1_diff1": 0.200427, + "nauc_ndcg_at_3_max": 0.206106, + "nauc_ndcg_at_3_std": 0.315677, + "nauc_ndcg_at_3_diff1": 0.167723, + "nauc_ndcg_at_5_max": 0.227304, + "nauc_ndcg_at_5_std": 0.36081, + "nauc_ndcg_at_5_diff1": 0.145147, + "nauc_ndcg_at_10_max": 0.230009, + "nauc_ndcg_at_10_std": 0.404886, + "nauc_ndcg_at_10_diff1": 0.128981, + "nauc_ndcg_at_20_max": 0.244799, + "nauc_ndcg_at_20_std": 0.439764, + "nauc_ndcg_at_20_diff1": 0.138587, + "nauc_ndcg_at_100_max": 0.245875, + "nauc_ndcg_at_100_std": 0.445492, + "nauc_ndcg_at_100_diff1": 0.124179, + "nauc_ndcg_at_1000_max": 0.245875, + "nauc_ndcg_at_1000_std": 0.445492, + "nauc_ndcg_at_1000_diff1": 0.124179, + "nauc_map_at_1_max": 0.184548, + "nauc_map_at_1_std": 0.224195, + "nauc_map_at_1_diff1": 0.201166, + "nauc_map_at_3_max": 0.212403, + "nauc_map_at_3_std": 0.325186, + "nauc_map_at_3_diff1": 0.168885, + "nauc_map_at_5_max": 0.234198, + "nauc_map_at_5_std": 0.368691, + "nauc_map_at_5_diff1": 0.151636, + "nauc_map_at_10_max": 0.242176, + "nauc_map_at_10_std": 0.407775, + "nauc_map_at_10_diff1": 0.138518, + "nauc_map_at_20_max": 0.249165, + "nauc_map_at_20_std": 0.432186, + "nauc_map_at_20_diff1": 0.14297, + "nauc_map_at_100_max": 0.24869, + "nauc_map_at_100_std": 0.438885, + "nauc_map_at_100_diff1": 0.135687, + "nauc_map_at_1000_max": 0.24869, + "nauc_map_at_1000_std": 0.438885, + "nauc_map_at_1000_diff1": 0.135687, + "nauc_recall_at_1_max": 0.184548, + "nauc_recall_at_1_std": 0.224195, + "nauc_recall_at_1_diff1": 0.201166, + "nauc_recall_at_3_max": 0.213348, + "nauc_recall_at_3_std": 0.346617, + "nauc_recall_at_3_diff1": 0.153036, + "nauc_recall_at_5_max": 0.232811, + "nauc_recall_at_5_std": 0.393578, + "nauc_recall_at_5_diff1": 0.107208, + "nauc_recall_at_10_max": 0.215937, + "nauc_recall_at_10_std": 0.436532, + "nauc_recall_at_10_diff1": 0.074911, + "nauc_recall_at_20_max": 0.23198, + "nauc_recall_at_20_std": 0.476918, + "nauc_recall_at_20_diff1": 0.090473, + "nauc_recall_at_100_max": 0.208683, + "nauc_recall_at_100_std": 0.43484, + "nauc_recall_at_100_diff1": 0.041724, + "nauc_recall_at_1000_max": 0.208683, + "nauc_recall_at_1000_std": 0.43484, + "nauc_recall_at_1000_diff1": 0.041724, + "nauc_precision_at_1_max": 0.183094, + "nauc_precision_at_1_std": 0.223546, + "nauc_precision_at_1_diff1": 0.200427, + "nauc_precision_at_3_max": 0.213645, + "nauc_precision_at_3_std": 0.348578, + "nauc_precision_at_3_diff1": 0.152468, + "nauc_precision_at_5_max": 0.233552, + "nauc_precision_at_5_std": 0.394617, + "nauc_precision_at_5_diff1": 0.106956, + "nauc_precision_at_10_max": 0.216031, + "nauc_precision_at_10_std": 0.436177, + "nauc_precision_at_10_diff1": 0.074495, + "nauc_precision_at_20_max": 0.233845, + "nauc_precision_at_20_std": 0.474555, + "nauc_precision_at_20_diff1": 0.089282, + "nauc_precision_at_100_max": 0.208356, + "nauc_precision_at_100_std": 0.42697, + "nauc_precision_at_100_diff1": 0.037101, + "nauc_precision_at_1000_max": 0.208356, + "nauc_precision_at_1000_std": 0.42697, + "nauc_precision_at_1000_diff1": 0.037101, + "nauc_mrr_at_1_max": 0.183094, + "nauc_mrr_at_1_std": 0.223546, + "nauc_mrr_at_1_diff1": 0.200427, + "nauc_mrr_at_3_max": 0.189414, + "nauc_mrr_at_3_std": 0.261157, + "nauc_mrr_at_3_diff1": 0.181386, + "nauc_mrr_at_5_max": 0.195156, + "nauc_mrr_at_5_std": 0.271933, + "nauc_mrr_at_5_diff1": 0.172393, + "nauc_mrr_at_10_max": 0.194597, + "nauc_mrr_at_10_std": 0.276458, + "nauc_mrr_at_10_diff1": 0.171221, + "nauc_mrr_at_20_max": 0.198161, + "nauc_mrr_at_20_std": 0.275992, + "nauc_mrr_at_20_diff1": 0.17476, + "nauc_mrr_at_100_max": 0.196921, + "nauc_mrr_at_100_std": 0.274636, + "nauc_mrr_at_100_diff1": 0.174406, + "nauc_mrr_at_1000_max": 0.196921, + "nauc_mrr_at_1000_std": 0.274636, + "nauc_mrr_at_1000_diff1": 0.174406, + "main_score": 0.26003, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 128.1468005180359, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/SICK-R.json b/results/ByteDance-Seed__Seed-Embedding/1/SICK-R.json new file mode 100644 index 0000000000..3bc35dd425 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/SICK-R.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "20a6d6f312dd54037fe07a32d58e5e168867909d", + "task_name": "SICK-R", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.874091, + "spearman": 0.842368, + "cosine_pearson": 0.874091, + "cosine_spearman": 0.842368, + "manhattan_pearson": 0.857464, + "manhattan_spearman": 0.843541, + "euclidean_pearson": 0.853954, + "euclidean_spearman": 0.8424, + "main_score": 0.842368, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.270653486251831, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/STS12.json b/results/ByteDance-Seed__Seed-Embedding/1/STS12.json new file mode 100644 index 0000000000..8b8dead853 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/STS12.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384", + "task_name": "STS12", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.903202, + "spearman": 0.850144, + "cosine_pearson": 0.903202, + "cosine_spearman": 0.850144, + "manhattan_pearson": 0.896183, + "manhattan_spearman": 0.851508, + "euclidean_pearson": 0.895238, + "euclidean_spearman": 0.85018, + "main_score": 0.850144, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.626424789428711, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/STS13.json b/results/ByteDance-Seed__Seed-Embedding/1/STS13.json new file mode 100644 index 0000000000..e8953679c4 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/STS13.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca", + "task_name": "STS13", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.908614, + "spearman": 0.913553, + "cosine_pearson": 0.908614, + "cosine_spearman": 0.913553, + "manhattan_pearson": 0.910515, + "manhattan_spearman": 0.912277, + "euclidean_pearson": 0.911889, + "euclidean_spearman": 0.913533, + "main_score": 0.913553, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.9195587635040283, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/STS14.json b/results/ByteDance-Seed__Seed-Embedding/1/STS14.json new file mode 100644 index 0000000000..a5cb2b56fa --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/STS14.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375", + "task_name": "STS14", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.899403, + "spearman": 0.885275, + "cosine_pearson": 0.899403, + "cosine_spearman": 0.885275, + "manhattan_pearson": 0.897066, + "manhattan_spearman": 0.884209, + "euclidean_pearson": 0.898228, + "euclidean_spearman": 0.885261, + "main_score": 0.885275, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 280.6405167579651, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/STS15.json b/results/ByteDance-Seed__Seed-Embedding/1/STS15.json new file mode 100644 index 0000000000..320efeb853 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/STS15.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3", + "task_name": "STS15", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.901696, + "spearman": 0.913311, + "cosine_pearson": 0.901696, + "cosine_spearman": 0.913311, + "manhattan_pearson": 0.90656, + "manhattan_spearman": 0.911364, + "euclidean_pearson": 0.908393, + "euclidean_spearman": 0.913288, + "main_score": 0.913311, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 3.160160779953003, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/STS17.json b/results/ByteDance-Seed__Seed-Embedding/1/STS17.json new file mode 100644 index 0000000000..7e39702357 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/STS17.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "faeb762787bd10488a50c8b5be4a3b82e411949c", + "task_name": "STS17", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.920016, + "spearman": 0.917152, + "cosine_pearson": 0.920016, + "cosine_spearman": 0.917152, + "manhattan_pearson": 0.919211, + "manhattan_spearman": 0.917188, + "euclidean_pearson": 0.919553, + "euclidean_spearman": 0.917181, + "main_score": 0.917152, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 0.2377007007598877, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/STS22.v2.json b/results/ByteDance-Seed__Seed-Embedding/1/STS22.v2.json new file mode 100644 index 0000000000..35343ce40e --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/STS22.v2.json @@ -0,0 +1,41 @@ +{ + "dataset_revision": "d31f33a128469b20e357535c39b82fb3c3f6f2bd", + "task_name": "STS22.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.709882, + "spearman": 0.708484, + "cosine_pearson": 0.709882, + "cosine_spearman": 0.708484, + "manhattan_pearson": 0.716826, + "manhattan_spearman": 0.707172, + "euclidean_pearson": 0.717671, + "euclidean_spearman": 0.708159, + "main_score": 0.708484, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "pearson": 0.725524, + "spearman": 0.728327, + "cosine_pearson": 0.725524, + "cosine_spearman": 0.728327, + "manhattan_pearson": 0.707727, + "manhattan_spearman": 0.730864, + "euclidean_pearson": 0.705987, + "euclidean_spearman": 0.728682, + "main_score": 0.728327, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 95.78716564178467, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/STSB.json b/results/ByteDance-Seed__Seed-Embedding/1/STSB.json new file mode 100644 index 0000000000..bfad9d6378 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/STSB.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0", + "task_name": "STSB", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "pearson": 0.836307, + "spearman": 0.85175, + "cosine_pearson": 0.836307, + "cosine_spearman": 0.85175, + "manhattan_pearson": 0.840145, + "manhattan_spearman": 0.851219, + "euclidean_pearson": 0.840589, + "euclidean_spearman": 0.85167, + "main_score": 0.85175, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 3.1008989810943604, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/STSBenchmark.json b/results/ByteDance-Seed__Seed-Embedding/1/STSBenchmark.json new file mode 100644 index 0000000000..4662156b93 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/STSBenchmark.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831", + "task_name": "STSBenchmark", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.888887, + "spearman": 0.910884, + "cosine_pearson": 0.888887, + "cosine_spearman": 0.910884, + "manhattan_pearson": 0.902903, + "manhattan_spearman": 0.910015, + "euclidean_pearson": 0.904206, + "euclidean_spearman": 0.910969, + "main_score": 0.910884, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1.1382956504821777, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/SprintDuplicateQuestions.json b/results/ByteDance-Seed__Seed-Embedding/1/SprintDuplicateQuestions.json new file mode 100644 index 0000000000..7f17f91fbf --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/SprintDuplicateQuestions.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", + "task_name": "SprintDuplicateQuestions", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "similarity_accuracy": 0.998861, + "similarity_accuracy_threshold": 0.927353, + "similarity_f1": 0.941356, + "similarity_f1_threshold": 0.926653, + "similarity_precision": 0.960458, + "similarity_recall": 0.923, + "similarity_ap": 0.971965, + "cosine_accuracy": 0.998861, + "cosine_accuracy_threshold": 0.927353, + "cosine_f1": 0.941356, + "cosine_f1_threshold": 0.926653, + "cosine_precision": 0.960458, + "cosine_recall": 0.923, + "cosine_ap": 0.971965, + "manhattan_accuracy": 0.998861, + "manhattan_accuracy_threshold": 13.792428, + "manhattan_f1": 0.941356, + "manhattan_f1_threshold": 13.792428, + "manhattan_precision": 0.960458, + "manhattan_recall": 0.923, + "manhattan_ap": 0.972059, + "euclidean_accuracy": 0.998871, + "euclidean_accuracy_threshold": 0.382899, + "euclidean_f1": 0.941837, + "euclidean_f1_threshold": 0.382899, + "euclidean_precision": 0.961458, + "euclidean_recall": 0.923, + "euclidean_ap": 0.971997, + "dot_accuracy": 0.998832, + "dot_accuracy_threshold": 0.927605, + "dot_f1": 0.939673, + "dot_f1_threshold": 0.927605, + "dot_precision": 0.961297, + "dot_recall": 0.919, + "dot_ap": 0.971617, + "max_accuracy": 0.998871, + "max_f1": 0.941837, + "max_precision": 0.961458, + "max_recall": 0.923, + "max_ap": 0.972059, + "main_score": 0.972059, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.131572008132935, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/StackExchangeClustering.v2.json b/results/ByteDance-Seed__Seed-Embedding/1/StackExchangeClustering.v2.json new file mode 100644 index 0000000000..a42eadb759 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/StackExchangeClustering.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "task_name": "StackExchangeClustering.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.796877, + 0.807386, + 0.802627, + 0.799506, + 0.800045, + 0.796323, + 0.796674, + 0.798111, + 0.79961, + 0.790931 + ] + }, + "v_measure": 0.798809, + "v_measure_std": 0.004097, + "main_score": 0.798809, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 16.241570711135864, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/StackExchangeClusteringP2P.v2.json b/results/ByteDance-Seed__Seed-Embedding/1/StackExchangeClusteringP2P.v2.json new file mode 100644 index 0000000000..0d44cbdc0e --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/StackExchangeClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "task_name": "StackExchangeClusteringP2P.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.50572, + 0.513721, + 0.516578, + 0.519407, + 0.512086, + 0.515701, + 0.520822, + 0.517258, + 0.514599, + 0.515694 + ] + }, + "v_measure": 0.515159, + "v_measure_std": 0.003975, + "main_score": 0.515159, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 31.18744206428528, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/SummEvalSummarization.v2.json b/results/ByteDance-Seed__Seed-Embedding/1/SummEvalSummarization.v2.json new file mode 100644 index 0000000000..8c086690db --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/SummEvalSummarization.v2.json @@ -0,0 +1,24 @@ +{ + "dataset_revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c", + "task_name": "SummEvalSummarization.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.39061, + "spearman": 0.341341, + "cosine_spearman": 0.341341, + "cosine_pearson": 0.39061, + "dot_spearman": 0.328747, + "dot_pearson": 0.386257, + "main_score": 0.341341, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 56.90901780128479, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/T2Reranking.json b/results/ByteDance-Seed__Seed-Embedding/1/T2Reranking.json new file mode 100644 index 0000000000..5be5282c2b --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/T2Reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "76631901a18387f85eaa53e5450019b87ad58ef9", + "task_name": "T2Reranking", + "mteb_version": "1.35.1", + "scores": { + "dev": [ + { + "map": 0.667318, + "mrr": 0.765873, + "nAUC_map_max": 0.27736, + "nAUC_map_std": 0.025183, + "nAUC_map_diff1": -0.095255, + "nAUC_mrr_max": 0.198903, + "nAUC_mrr_std": -0.014698, + "nAUC_mrr_diff1": -0.088916, + "main_score": 0.667318, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 320.76477909088135, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/T2Retrieval.json b/results/ByteDance-Seed__Seed-Embedding/1/T2Retrieval.json new file mode 100644 index 0000000000..2dc1ee40a4 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/T2Retrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "8731a845f1bf500a4f111cf1070785c793d10e64", + "task_name": "T2Retrieval", + "mteb_version": "1.35.1", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.931, + "ndcg_at_3": 0.90302, + "ndcg_at_5": 0.89267, + "ndcg_at_10": 0.89365, + "ndcg_at_20": 0.90746, + "ndcg_at_100": 0.91993, + "ndcg_at_1000": 0.91993, + "map_at_1": 0.2917, + "map_at_3": 0.58334, + "map_at_5": 0.71879, + "map_at_10": 0.83172, + "map_at_20": 0.8582, + "map_at_100": 0.86469, + "map_at_1000": 0.86469, + "recall_at_1": 0.2917, + "recall_at_3": 0.597, + "recall_at_5": 0.74558, + "recall_at_10": 0.88304, + "recall_at_20": 0.93289, + "recall_at_100": 0.97201, + "recall_at_1000": 0.97201, + "precision_at_1": 0.931, + "precision_at_3": 0.78881, + "precision_at_5": 0.66363, + "precision_at_10": 0.44247, + "precision_at_20": 0.24269, + "precision_at_100": 0.05099, + "precision_at_1000": 0.0051, + "mrr_at_1": 0.931001, + "mrr_at_3": 0.946446, + "mrr_at_5": 0.948171, + "mrr_at_10": 0.949292, + "mrr_at_20": 0.9496, + "mrr_at_100": 0.94979, + "mrr_at_1000": 0.94979, + "nauc_ndcg_at_1_max": 0.806868, + "nauc_ndcg_at_1_std": 0.689349, + "nauc_ndcg_at_1_diff1": 0.49506, + "nauc_ndcg_at_3_max": 0.672699, + "nauc_ndcg_at_3_std": 0.649299, + "nauc_ndcg_at_3_diff1": 0.105843, + "nauc_ndcg_at_5_max": 0.597023, + "nauc_ndcg_at_5_std": 0.590361, + "nauc_ndcg_at_5_diff1": 0.110792, + "nauc_ndcg_at_10_max": 0.522403, + "nauc_ndcg_at_10_std": 0.511055, + "nauc_ndcg_at_10_diff1": 0.148279, + "nauc_ndcg_at_20_max": 0.56915, + "nauc_ndcg_at_20_std": 0.571318, + "nauc_ndcg_at_20_diff1": 0.151154, + "nauc_ndcg_at_100_max": 0.617933, + "nauc_ndcg_at_100_std": 0.618944, + "nauc_ndcg_at_100_diff1": 0.155341, + "nauc_ndcg_at_1000_max": 0.617933, + "nauc_ndcg_at_1000_std": 0.618944, + "nauc_ndcg_at_1000_diff1": 0.155341, + "nauc_map_at_1_max": -0.265043, + "nauc_map_at_1_std": -0.475821, + "nauc_map_at_1_diff1": 0.552186, + "nauc_map_at_3_max": -0.163245, + "nauc_map_at_3_std": -0.418898, + "nauc_map_at_3_diff1": 0.398304, + "nauc_map_at_5_max": -0.004731, + "nauc_map_at_5_std": -0.229543, + "nauc_map_at_5_diff1": 0.312199, + "nauc_map_at_10_max": 0.350389, + "nauc_map_at_10_std": 0.275076, + "nauc_map_at_10_diff1": 0.160018, + "nauc_map_at_20_max": 0.482819, + "nauc_map_at_20_std": 0.476911, + "nauc_map_at_20_diff1": 0.113275, + "nauc_map_at_100_max": 0.507063, + "nauc_map_at_100_std": 0.508536, + "nauc_map_at_100_diff1": 0.107442, + "nauc_map_at_1000_max": 0.507063, + "nauc_map_at_1000_std": 0.508536, + "nauc_map_at_1000_diff1": 0.107442, + "nauc_recall_at_1_max": -0.265043, + "nauc_recall_at_1_std": -0.475821, + "nauc_recall_at_1_diff1": 0.552186, + "nauc_recall_at_3_max": -0.204601, + "nauc_recall_at_3_std": -0.45848, + "nauc_recall_at_3_diff1": 0.385103, + "nauc_recall_at_5_max": -0.092515, + "nauc_recall_at_5_std": -0.325789, + "nauc_recall_at_5_diff1": 0.309503, + "nauc_recall_at_10_max": 0.231574, + "nauc_recall_at_10_std": 0.154981, + "nauc_recall_at_10_diff1": 0.154308, + "nauc_recall_at_20_max": 0.433711, + "nauc_recall_at_20_std": 0.487248, + "nauc_recall_at_20_diff1": 0.082358, + "nauc_recall_at_100_max": 0.567832, + "nauc_recall_at_100_std": 0.655583, + "nauc_recall_at_100_diff1": 0.063779, + "nauc_recall_at_1000_max": 0.567832, + "nauc_recall_at_1000_std": 0.655583, + "nauc_recall_at_1000_diff1": 0.063779, + "nauc_precision_at_1_max": 0.806868, + "nauc_precision_at_1_std": 0.689349, + "nauc_precision_at_1_diff1": 0.49506, + "nauc_precision_at_3_max": 0.631272, + "nauc_precision_at_3_std": 0.779895, + "nauc_precision_at_3_diff1": -0.435023, + "nauc_precision_at_5_max": 0.567889, + "nauc_precision_at_5_std": 0.788999, + "nauc_precision_at_5_diff1": -0.434078, + "nauc_precision_at_10_max": 0.503762, + "nauc_precision_at_10_std": 0.773743, + "nauc_precision_at_10_diff1": -0.390847, + "nauc_precision_at_20_max": 0.473492, + "nauc_precision_at_20_std": 0.756265, + "nauc_precision_at_20_diff1": -0.375369, + "nauc_precision_at_100_max": 0.448247, + "nauc_precision_at_100_std": 0.723237, + "nauc_precision_at_100_diff1": -0.369583, + "nauc_precision_at_1000_max": 0.448247, + "nauc_precision_at_1000_std": 0.723237, + "nauc_precision_at_1000_diff1": -0.369583, + "nauc_mrr_at_1_max": 0.806868, + "nauc_mrr_at_1_std": 0.689349, + "nauc_mrr_at_1_diff1": 0.49506, + "nauc_mrr_at_3_max": 0.83929, + "nauc_mrr_at_3_std": 0.73653, + "nauc_mrr_at_3_diff1": 0.495132, + "nauc_mrr_at_5_max": 0.840175, + "nauc_mrr_at_5_std": 0.737844, + "nauc_mrr_at_5_diff1": 0.494772, + "nauc_mrr_at_10_max": 0.839247, + "nauc_mrr_at_10_std": 0.736966, + "nauc_mrr_at_10_diff1": 0.494976, + "nauc_mrr_at_20_max": 0.838699, + "nauc_mrr_at_20_std": 0.736385, + "nauc_mrr_at_20_diff1": 0.495517, + "nauc_mrr_at_100_max": 0.838277, + "nauc_mrr_at_100_std": 0.735702, + "nauc_mrr_at_100_diff1": 0.495495, + "nauc_mrr_at_1000_max": 0.838277, + "nauc_mrr_at_1000_std": 0.735702, + "nauc_mrr_at_1000_diff1": 0.495495, + "main_score": 0.89365, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 649.0593817234039, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/TNews.json b/results/ByteDance-Seed__Seed-Embedding/1/TNews.json new file mode 100644 index 0000000000..3ffc4c9af7 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/TNews.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "317f262bf1e6126357bbe89e875451e4b0938fe4", + "task_name": "TNews", + "mteb_version": "1.35.1", + "scores": { + "validation": [ + { + "accuracy": 0.58315, + "f1": 0.562544, + "f1_weighted": 0.579285, + "scores_per_experiment": [ + { + "accuracy": 0.5826, + "f1": 0.561596, + "f1_weighted": 0.579602 + }, + { + "accuracy": 0.5842, + "f1": 0.559198, + "f1_weighted": 0.574592 + }, + { + "accuracy": 0.583, + "f1": 0.563117, + "f1_weighted": 0.578847 + }, + { + "accuracy": 0.5887, + "f1": 0.569155, + "f1_weighted": 0.585963 + }, + { + "accuracy": 0.5835, + "f1": 0.565522, + "f1_weighted": 0.582977 + }, + { + "accuracy": 0.5852, + "f1": 0.564922, + "f1_weighted": 0.580204 + }, + { + "accuracy": 0.5833, + "f1": 0.563476, + "f1_weighted": 0.582277 + }, + { + "accuracy": 0.5797, + "f1": 0.558291, + "f1_weighted": 0.573397 + }, + { + "accuracy": 0.5762, + "f1": 0.552731, + "f1_weighted": 0.568881 + }, + { + "accuracy": 0.5851, + "f1": 0.567433, + "f1_weighted": 0.586114 + } + ], + "main_score": 0.58315, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 51.33865189552307, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/TRECCOVID.json b/results/ByteDance-Seed__Seed-Embedding/1/TRECCOVID.json new file mode 100644 index 0000000000..e88e182682 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/TRECCOVID.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bb9466bac8153a0349341eb1b22e06409e78ef4e", + "task_name": "TRECCOVID", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.89, + "ndcg_at_3": 0.89531, + "ndcg_at_5": 0.88626, + "ndcg_at_10": 0.86393, + "ndcg_at_20": 0.83649, + "ndcg_at_100": 0.69218, + "ndcg_at_1000": 0.27893, + "map_at_1": 0.00234, + "map_at_3": 0.00701, + "map_at_5": 0.01169, + "map_at_10": 0.02287, + "map_at_20": 0.04305, + "map_at_100": 0.14883, + "map_at_1000": 0.14883, + "recall_at_1": 0.00234, + "recall_at_3": 0.0073, + "recall_at_5": 0.01227, + "recall_at_10": 0.0243, + "recall_at_20": 0.04613, + "recall_at_100": 0.17464, + "recall_at_1000": 0.17464, + "precision_at_1": 0.92, + "precision_at_3": 0.93333, + "precision_at_5": 0.936, + "precision_at_10": 0.92, + "precision_at_20": 0.885, + "precision_at_100": 0.721, + "precision_at_1000": 0.0721, + "mrr_at_1": 0.92, + "mrr_at_3": 0.956667, + "mrr_at_5": 0.956667, + "mrr_at_10": 0.956667, + "mrr_at_20": 0.956667, + "mrr_at_100": 0.956667, + "mrr_at_1000": 0.956667, + "nauc_ndcg_at_1_max": 0.354926, + "nauc_ndcg_at_1_std": 0.535171, + "nauc_ndcg_at_1_diff1": -0.062038, + "nauc_ndcg_at_3_max": 0.313566, + "nauc_ndcg_at_3_std": 0.277834, + "nauc_ndcg_at_3_diff1": -0.17879, + "nauc_ndcg_at_5_max": 0.282493, + "nauc_ndcg_at_5_std": 0.249465, + "nauc_ndcg_at_5_diff1": -0.262039, + "nauc_ndcg_at_10_max": 0.249245, + "nauc_ndcg_at_10_std": 0.243678, + "nauc_ndcg_at_10_diff1": -0.235193, + "nauc_ndcg_at_20_max": 0.239284, + "nauc_ndcg_at_20_std": 0.332304, + "nauc_ndcg_at_20_diff1": -0.268679, + "nauc_ndcg_at_100_max": 0.250948, + "nauc_ndcg_at_100_std": -0.040726, + "nauc_ndcg_at_100_diff1": -0.247243, + "nauc_ndcg_at_1000_max": -0.034077, + "nauc_ndcg_at_1000_std": 0.475848, + "nauc_ndcg_at_1000_diff1": -0.076226, + "nauc_map_at_1_max": -0.011905, + "nauc_map_at_1_std": 0.45959, + "nauc_map_at_1_diff1": 0.005408, + "nauc_map_at_3_max": -0.06835, + "nauc_map_at_3_std": 0.442061, + "nauc_map_at_3_diff1": -0.086956, + "nauc_map_at_5_max": -0.076288, + "nauc_map_at_5_std": 0.438641, + "nauc_map_at_5_diff1": -0.118691, + "nauc_map_at_10_max": -0.110018, + "nauc_map_at_10_std": 0.491763, + "nauc_map_at_10_diff1": -0.131543, + "nauc_map_at_20_max": -0.094184, + "nauc_map_at_20_std": 0.564412, + "nauc_map_at_20_diff1": -0.135849, + "nauc_map_at_100_max": 0.018266, + "nauc_map_at_100_std": 0.445042, + "nauc_map_at_100_diff1": -0.115166, + "nauc_map_at_1000_max": 0.018266, + "nauc_map_at_1000_std": 0.445042, + "nauc_map_at_1000_diff1": -0.115166, + "nauc_recall_at_1_max": -0.011905, + "nauc_recall_at_1_std": 0.45959, + "nauc_recall_at_1_diff1": 0.005408, + "nauc_recall_at_3_max": -0.13339, + "nauc_recall_at_3_std": 0.415226, + "nauc_recall_at_3_diff1": -0.092255, + "nauc_recall_at_5_max": -0.134273, + "nauc_recall_at_5_std": 0.403028, + "nauc_recall_at_5_diff1": -0.100144, + "nauc_recall_at_10_max": -0.166237, + "nauc_recall_at_10_std": 0.437187, + "nauc_recall_at_10_diff1": -0.091566, + "nauc_recall_at_20_max": -0.139168, + "nauc_recall_at_20_std": 0.51046, + "nauc_recall_at_20_diff1": -0.090063, + "nauc_recall_at_100_max": -0.076768, + "nauc_recall_at_100_std": 0.393076, + "nauc_recall_at_100_diff1": -0.047481, + "nauc_recall_at_1000_max": -0.076768, + "nauc_recall_at_1000_std": 0.393076, + "nauc_recall_at_1000_diff1": -0.047481, + "nauc_precision_at_1_max": 0.786415, + "nauc_precision_at_1_std": 0.525093, + "nauc_precision_at_1_diff1": 0.233427, + "nauc_precision_at_3_max": 0.604729, + "nauc_precision_at_3_std": 0.192069, + "nauc_precision_at_3_diff1": -0.178276, + "nauc_precision_at_5_max": 0.613882, + "nauc_precision_at_5_std": 0.225649, + "nauc_precision_at_5_diff1": -0.342055, + "nauc_precision_at_10_max": 0.41263, + "nauc_precision_at_10_std": 0.433556, + "nauc_precision_at_10_diff1": -0.293664, + "nauc_precision_at_20_max": 0.36348, + "nauc_precision_at_20_std": 0.519378, + "nauc_precision_at_20_diff1": -0.232842, + "nauc_precision_at_100_max": 0.332336, + "nauc_precision_at_100_std": -0.04515, + "nauc_precision_at_100_diff1": -0.172338, + "nauc_precision_at_1000_max": 0.332336, + "nauc_precision_at_1000_std": -0.04515, + "nauc_precision_at_1000_diff1": -0.172338, + "nauc_mrr_at_1_max": 0.786415, + "nauc_mrr_at_1_std": 0.525093, + "nauc_mrr_at_1_diff1": 0.233427, + "nauc_mrr_at_3_max": 0.792789, + "nauc_mrr_at_3_std": 0.527329, + "nauc_mrr_at_3_diff1": 0.202291, + "nauc_mrr_at_5_max": 0.792789, + "nauc_mrr_at_5_std": 0.527329, + "nauc_mrr_at_5_diff1": 0.202291, + "nauc_mrr_at_10_max": 0.792789, + "nauc_mrr_at_10_std": 0.527329, + "nauc_mrr_at_10_diff1": 0.202291, + "nauc_mrr_at_20_max": 0.792789, + "nauc_mrr_at_20_std": 0.527329, + "nauc_mrr_at_20_diff1": 0.202291, + "nauc_mrr_at_100_max": 0.792789, + "nauc_mrr_at_100_std": 0.527329, + "nauc_mrr_at_100_diff1": 0.202291, + "nauc_mrr_at_1000_max": 0.792789, + "nauc_mrr_at_1000_std": 0.527329, + "nauc_mrr_at_1000_diff1": 0.202291, + "main_score": 0.86393, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 36.017045974731445, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/ThuNewsClusteringP2P.json b/results/ByteDance-Seed__Seed-Embedding/1/ThuNewsClusteringP2P.json new file mode 100644 index 0000000000..ae08c75130 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/ThuNewsClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "5798586b105c0434e4f0fe5e767abe619442cf93", + "task_name": "ThuNewsClusteringP2P", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "v_measure": 0.8492, + "v_measure_std": 0.01218, + "v_measures": [ + 0.852145, + 0.869699, + 0.858401, + 0.838885, + 0.867323, + 0.841358, + 0.832914, + 0.852352, + 0.837382, + 0.841536 + ], + "main_score": 0.8492, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 110.71957588195801, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/ThuNewsClusteringS2S.json b/results/ByteDance-Seed__Seed-Embedding/1/ThuNewsClusteringS2S.json new file mode 100644 index 0000000000..90efa72072 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/ThuNewsClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d", + "task_name": "ThuNewsClusteringS2S", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "v_measure": 0.857612, + "v_measure_std": 0.01339, + "v_measures": [ + 0.875158, + 0.828455, + 0.852891, + 0.84437, + 0.874209, + 0.857316, + 0.858995, + 0.854518, + 0.869463, + 0.860746 + ], + "main_score": 0.857612, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 82.52749848365784, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/Touche2020Retrieval.v3.json b/results/ByteDance-Seed__Seed-Embedding/1/Touche2020Retrieval.v3.json new file mode 100644 index 0000000000..d420260be7 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/Touche2020Retrieval.v3.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "431886eaecc48f067a3975b70d0949ea2862463c", + "task_name": "Touche2020Retrieval.v3", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.80612, + "ndcg_at_3": 0.77706, + "ndcg_at_5": 0.72175, + "ndcg_at_10": 0.65418, + "ndcg_at_20": 0.5723, + "ndcg_at_100": 0.63108, + "ndcg_at_1000": 0.63108, + "map_at_1": 0.02888, + "map_at_3": 0.07904, + "map_at_5": 0.11668, + "map_at_10": 0.19944, + "map_at_20": 0.28055, + "map_at_100": 0.38749, + "map_at_1000": 0.38749, + "recall_at_1": 0.02888, + "recall_at_3": 0.08582, + "recall_at_5": 0.12966, + "recall_at_10": 0.23273, + "recall_at_20": 0.35362, + "recall_at_100": 0.63848, + "recall_at_1000": 0.63848, + "precision_at_1": 0.81633, + "precision_at_3": 0.80952, + "precision_at_5": 0.74286, + "precision_at_10": 0.66122, + "precision_at_20": 0.51735, + "precision_at_100": 0.20224, + "precision_at_1000": 0.02022, + "mrr_at_1": 0.816327, + "mrr_at_3": 0.904762, + "mrr_at_5": 0.904762, + "mrr_at_10": 0.904762, + "mrr_at_20": 0.904762, + "mrr_at_100": 0.904762, + "mrr_at_1000": 0.904762, + "nauc_ndcg_at_1_max": 0.132239, + "nauc_ndcg_at_1_std": 0.239, + "nauc_ndcg_at_1_diff1": 0.171041, + "nauc_ndcg_at_3_max": 0.127912, + "nauc_ndcg_at_3_std": 0.347307, + "nauc_ndcg_at_3_diff1": 0.141072, + "nauc_ndcg_at_5_max": 0.124638, + "nauc_ndcg_at_5_std": 0.382064, + "nauc_ndcg_at_5_diff1": 0.023495, + "nauc_ndcg_at_10_max": 0.09518, + "nauc_ndcg_at_10_std": 0.643508, + "nauc_ndcg_at_10_diff1": 0.229012, + "nauc_ndcg_at_20_max": 0.070965, + "nauc_ndcg_at_20_std": 0.732965, + "nauc_ndcg_at_20_diff1": 0.292787, + "nauc_ndcg_at_100_max": 0.024034, + "nauc_ndcg_at_100_std": 0.823702, + "nauc_ndcg_at_100_diff1": 0.31938, + "nauc_ndcg_at_1000_max": 0.024034, + "nauc_ndcg_at_1000_std": 0.823702, + "nauc_ndcg_at_1000_diff1": 0.31938, + "nauc_map_at_1_max": -0.101674, + "nauc_map_at_1_std": 0.329454, + "nauc_map_at_1_diff1": 0.274781, + "nauc_map_at_3_max": -0.065127, + "nauc_map_at_3_std": 0.579092, + "nauc_map_at_3_diff1": 0.273286, + "nauc_map_at_5_max": -0.037034, + "nauc_map_at_5_std": 0.671108, + "nauc_map_at_5_diff1": 0.225173, + "nauc_map_at_10_max": 0.016102, + "nauc_map_at_10_std": 0.811872, + "nauc_map_at_10_diff1": 0.307742, + "nauc_map_at_20_max": 0.00409, + "nauc_map_at_20_std": 0.828783, + "nauc_map_at_20_diff1": 0.327096, + "nauc_map_at_100_max": 0.031165, + "nauc_map_at_100_std": 0.836105, + "nauc_map_at_100_diff1": 0.33393, + "nauc_map_at_1000_max": 0.031165, + "nauc_map_at_1000_std": 0.836105, + "nauc_map_at_1000_diff1": 0.33393, + "nauc_recall_at_1_max": -0.101674, + "nauc_recall_at_1_std": 0.329454, + "nauc_recall_at_1_diff1": 0.274781, + "nauc_recall_at_3_max": -0.12902, + "nauc_recall_at_3_std": 0.569354, + "nauc_recall_at_3_diff1": 0.226798, + "nauc_recall_at_5_max": -0.095336, + "nauc_recall_at_5_std": 0.656014, + "nauc_recall_at_5_diff1": 0.166545, + "nauc_recall_at_10_max": -0.043069, + "nauc_recall_at_10_std": 0.784123, + "nauc_recall_at_10_diff1": 0.247227, + "nauc_recall_at_20_max": -0.060467, + "nauc_recall_at_20_std": 0.787288, + "nauc_recall_at_20_diff1": 0.249283, + "nauc_recall_at_100_max": -0.105838, + "nauc_recall_at_100_std": 0.67899, + "nauc_recall_at_100_diff1": 0.228136, + "nauc_recall_at_1000_max": -0.105838, + "nauc_recall_at_1000_std": 0.67899, + "nauc_recall_at_1000_diff1": 0.228136, + "nauc_precision_at_1_max": 0.239493, + "nauc_precision_at_1_std": 0.261043, + "nauc_precision_at_1_diff1": 0.280284, + "nauc_precision_at_3_max": 0.034533, + "nauc_precision_at_3_std": 0.44298, + "nauc_precision_at_3_diff1": 0.330073, + "nauc_precision_at_5_max": 0.078854, + "nauc_precision_at_5_std": 0.469099, + "nauc_precision_at_5_diff1": 0.078035, + "nauc_precision_at_10_max": 0.077869, + "nauc_precision_at_10_std": 0.769787, + "nauc_precision_at_10_diff1": 0.319325, + "nauc_precision_at_20_max": 0.072713, + "nauc_precision_at_20_std": 0.600312, + "nauc_precision_at_20_diff1": 0.315156, + "nauc_precision_at_100_max": 0.115896, + "nauc_precision_at_100_std": -0.122032, + "nauc_precision_at_100_diff1": 0.084595, + "nauc_precision_at_1000_max": 0.115896, + "nauc_precision_at_1000_std": -0.122032, + "nauc_precision_at_1000_diff1": 0.084595, + "nauc_mrr_at_1_max": 0.239493, + "nauc_mrr_at_1_std": 0.261043, + "nauc_mrr_at_1_diff1": 0.280284, + "nauc_mrr_at_3_max": 0.251304, + "nauc_mrr_at_3_std": 0.20853, + "nauc_mrr_at_3_diff1": 0.283029, + "nauc_mrr_at_5_max": 0.251304, + "nauc_mrr_at_5_std": 0.20853, + "nauc_mrr_at_5_diff1": 0.283029, + "nauc_mrr_at_10_max": 0.251304, + "nauc_mrr_at_10_std": 0.20853, + "nauc_mrr_at_10_diff1": 0.283029, + "nauc_mrr_at_20_max": 0.251304, + "nauc_mrr_at_20_std": 0.20853, + "nauc_mrr_at_20_diff1": 0.283029, + "nauc_mrr_at_100_max": 0.251304, + "nauc_mrr_at_100_std": 0.20853, + "nauc_mrr_at_100_diff1": 0.283029, + "nauc_mrr_at_1000_max": 0.251304, + "nauc_mrr_at_1000_std": 0.20853, + "nauc_mrr_at_1000_diff1": 0.283029, + "main_score": 0.65418, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 82.49586725234985, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/ToxicConversationsClassification.json b/results/ByteDance-Seed__Seed-Embedding/1/ToxicConversationsClassification.json new file mode 100644 index 0000000000..270e0b6301 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/ToxicConversationsClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "edfaf9da55d3dd50d43143d90c1ac476895ae6de", + "task_name": "ToxicConversationsClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.893701, + "f1": 0.75818, + "f1_weighted": 0.910515, + "ap": 0.392288, + "ap_weighted": 0.392288, + "scores_per_experiment": [ + { + "accuracy": 0.883301, + "f1": 0.742258, + "f1_weighted": 0.902758, + "ap": 0.366863, + "ap_weighted": 0.366863 + }, + { + "accuracy": 0.915039, + "f1": 0.787748, + "f1_weighted": 0.926115, + "ap": 0.435456, + "ap_weighted": 0.435456 + }, + { + "accuracy": 0.894043, + "f1": 0.757406, + "f1_weighted": 0.910667, + "ap": 0.389527, + "ap_weighted": 0.389527 + }, + { + "accuracy": 0.869141, + "f1": 0.725291, + "f1_weighted": 0.89263, + "ap": 0.345564, + "ap_weighted": 0.345564 + }, + { + "accuracy": 0.903809, + "f1": 0.77126, + "f1_weighted": 0.917837, + "ap": 0.410199, + "ap_weighted": 0.410199 + }, + { + "accuracy": 0.878906, + "f1": 0.737964, + "f1_weighted": 0.899738, + "ap": 0.363231, + "ap_weighted": 0.363231 + }, + { + "accuracy": 0.92334, + "f1": 0.80131, + "f1_weighted": 0.932388, + "ap": 0.458489, + "ap_weighted": 0.458489 + }, + { + "accuracy": 0.865723, + "f1": 0.721818, + "f1_weighted": 0.890244, + "ap": 0.342152, + "ap_weighted": 0.342152 + }, + { + "accuracy": 0.896973, + "f1": 0.760896, + "f1_weighted": 0.912738, + "ap": 0.393678, + "ap_weighted": 0.393678 + }, + { + "accuracy": 0.906738, + "f1": 0.775845, + "f1_weighted": 0.920037, + "ap": 0.41772, + "ap_weighted": 0.41772 + } + ], + "main_score": 0.893701, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 17.44493865966797, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/TweetSentimentExtractionClassification.json b/results/ByteDance-Seed__Seed-Embedding/1/TweetSentimentExtractionClassification.json new file mode 100644 index 0000000000..e3ed5eef60 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/TweetSentimentExtractionClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a", + "task_name": "TweetSentimentExtractionClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.740323, + "f1": 0.743937, + "f1_weighted": 0.737087, + "scores_per_experiment": [ + { + "accuracy": 0.740804, + "f1": 0.743266, + "f1_weighted": 0.735177 + }, + { + "accuracy": 0.733447, + "f1": 0.737668, + "f1_weighted": 0.730659 + }, + { + "accuracy": 0.74618, + "f1": 0.750095, + "f1_weighted": 0.743739 + }, + { + "accuracy": 0.746746, + "f1": 0.750452, + "f1_weighted": 0.743458 + }, + { + "accuracy": 0.75382, + "f1": 0.756859, + "f1_weighted": 0.751087 + }, + { + "accuracy": 0.742784, + "f1": 0.746869, + "f1_weighted": 0.739372 + }, + { + "accuracy": 0.719298, + "f1": 0.722191, + "f1_weighted": 0.716162 + }, + { + "accuracy": 0.756933, + "f1": 0.760627, + "f1_weighted": 0.754528 + }, + { + "accuracy": 0.728636, + "f1": 0.731771, + "f1_weighted": 0.723671 + }, + { + "accuracy": 0.734578, + "f1": 0.739574, + "f1_weighted": 0.733015 + } + ], + "main_score": 0.740323, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 19.25722908973694, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/TwentyNewsgroupsClustering.v2.json b/results/ByteDance-Seed__Seed-Embedding/1/TwentyNewsgroupsClustering.v2.json new file mode 100644 index 0000000000..5c4d188f9c --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/TwentyNewsgroupsClustering.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "task_name": "TwentyNewsgroupsClustering.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.648023, + 0.637351, + 0.662935, + 0.641048, + 0.647821, + 0.656199, + 0.639937, + 0.641925, + 0.635745, + 0.636892 + ] + }, + "v_measure": 0.644788, + "v_measure_std": 0.008511, + "main_score": 0.644788, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 8.093946695327759, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/TwitterSemEval2015.json b/results/ByteDance-Seed__Seed-Embedding/1/TwitterSemEval2015.json new file mode 100644 index 0000000000..46a082e388 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/TwitterSemEval2015.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", + "task_name": "TwitterSemEval2015", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "similarity_accuracy": 0.883829, + "similarity_accuracy_threshold": 0.901033, + "similarity_f1": 0.733282, + "similarity_f1_threshold": 0.891321, + "similarity_precision": 0.712649, + "similarity_recall": 0.755145, + "similarity_ap": 0.802586, + "cosine_accuracy": 0.883829, + "cosine_accuracy_threshold": 0.901033, + "cosine_f1": 0.733282, + "cosine_f1_threshold": 0.891321, + "cosine_precision": 0.712649, + "cosine_recall": 0.755145, + "cosine_ap": 0.802586, + "manhattan_accuracy": 0.883054, + "manhattan_accuracy_threshold": 15.896994, + "manhattan_f1": 0.731967, + "manhattan_f1_threshold": 16.622101, + "manhattan_precision": 0.703307, + "manhattan_recall": 0.763061, + "manhattan_ap": 0.802462, + "euclidean_accuracy": 0.883471, + "euclidean_accuracy_threshold": 0.444181, + "euclidean_f1": 0.733419, + "euclidean_f1_threshold": 0.465228, + "euclidean_precision": 0.715038, + "euclidean_recall": 0.75277, + "euclidean_ap": 0.802578, + "dot_accuracy": 0.883948, + "dot_accuracy_threshold": 0.901307, + "dot_f1": 0.732478, + "dot_f1_threshold": 0.892735, + "dot_precision": 0.719522, + "dot_recall": 0.74591, + "dot_ap": 0.802187, + "max_accuracy": 0.883948, + "max_f1": 0.733419, + "max_precision": 0.719522, + "max_recall": 0.763061, + "max_ap": 0.802586, + "main_score": 0.802586, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 6.735085487365723, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/TwitterURLCorpus.json b/results/ByteDance-Seed__Seed-Embedding/1/TwitterURLCorpus.json new file mode 100644 index 0000000000..b8b4f53fb9 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/TwitterURLCorpus.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", + "task_name": "TwitterURLCorpus", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "similarity_accuracy": 0.896845, + "similarity_accuracy_threshold": 0.899358, + "similarity_f1": 0.796911, + "similarity_f1_threshold": 0.893445, + "similarity_precision": 0.780187, + "similarity_recall": 0.814367, + "similarity_ap": 0.873309, + "cosine_accuracy": 0.896845, + "cosine_accuracy_threshold": 0.899358, + "cosine_f1": 0.796911, + "cosine_f1_threshold": 0.893445, + "cosine_precision": 0.780187, + "cosine_recall": 0.814367, + "cosine_ap": 0.873309, + "manhattan_accuracy": 0.896903, + "manhattan_accuracy_threshold": 16.024261, + "manhattan_f1": 0.796493, + "manhattan_f1_threshold": 16.601732, + "manhattan_precision": 0.772599, + "manhattan_recall": 0.821913, + "manhattan_ap": 0.873153, + "euclidean_accuracy": 0.896767, + "euclidean_accuracy_threshold": 0.450287, + "euclidean_f1": 0.797063, + "euclidean_f1_threshold": 0.461865, + "euclidean_precision": 0.779915, + "euclidean_recall": 0.814983, + "euclidean_ap": 0.873312, + "dot_accuracy": 0.897233, + "dot_accuracy_threshold": 0.899412, + "dot_f1": 0.796162, + "dot_f1_threshold": 0.892925, + "dot_precision": 0.775612, + "dot_recall": 0.817832, + "dot_ap": 0.87341, + "max_accuracy": 0.897233, + "max_f1": 0.797063, + "max_precision": 0.780187, + "max_recall": 0.821913, + "max_ap": 0.87341, + "main_score": 0.87341, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 11.048704862594604, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/VideoRetrieval.json b/results/ByteDance-Seed__Seed-Embedding/1/VideoRetrieval.json new file mode 100644 index 0000000000..033644d130 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/VideoRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "58c2597a5943a2ba48f4668c3b90d796283c5639", + "task_name": "VideoRetrieval", + "mteb_version": "1.35.1", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.688, + "ndcg_at_3": 0.78077, + "ndcg_at_5": 0.79747, + "ndcg_at_10": 0.81106, + "ndcg_at_20": 0.81711, + "ndcg_at_100": 0.82328, + "ndcg_at_1000": 0.82328, + "map_at_1": 0.688, + "map_at_3": 0.7585, + "map_at_5": 0.7679, + "map_at_10": 0.77351, + "map_at_20": 0.77516, + "map_at_100": 0.77609, + "map_at_1000": 0.77609, + "recall_at_1": 0.688, + "recall_at_3": 0.845, + "recall_at_5": 0.885, + "recall_at_10": 0.927, + "recall_at_20": 0.951, + "recall_at_100": 0.983, + "recall_at_1000": 0.983, + "precision_at_1": 0.688, + "precision_at_3": 0.28167, + "precision_at_5": 0.177, + "precision_at_10": 0.0927, + "precision_at_20": 0.04755, + "precision_at_100": 0.00983, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.688, + "mrr_at_3": 0.7585, + "mrr_at_5": 0.7679, + "mrr_at_10": 0.773512, + "mrr_at_20": 0.775165, + "mrr_at_100": 0.776092, + "mrr_at_1000": 0.776092, + "nauc_ndcg_at_1_max": 0.113285, + "nauc_ndcg_at_1_std": -0.359756, + "nauc_ndcg_at_1_diff1": 0.798296, + "nauc_ndcg_at_3_max": 0.124522, + "nauc_ndcg_at_3_std": -0.355954, + "nauc_ndcg_at_3_diff1": 0.773091, + "nauc_ndcg_at_5_max": 0.14004, + "nauc_ndcg_at_5_std": -0.321199, + "nauc_ndcg_at_5_diff1": 0.762746, + "nauc_ndcg_at_10_max": 0.134951, + "nauc_ndcg_at_10_std": -0.304831, + "nauc_ndcg_at_10_diff1": 0.763609, + "nauc_ndcg_at_20_max": 0.139786, + "nauc_ndcg_at_20_std": -0.294293, + "nauc_ndcg_at_20_diff1": 0.766023, + "nauc_ndcg_at_100_max": 0.140095, + "nauc_ndcg_at_100_std": -0.302203, + "nauc_ndcg_at_100_diff1": 0.771735, + "nauc_ndcg_at_1000_max": 0.140095, + "nauc_ndcg_at_1000_std": -0.302203, + "nauc_ndcg_at_1000_diff1": 0.771735, + "nauc_map_at_1_max": 0.113285, + "nauc_map_at_1_std": -0.359756, + "nauc_map_at_1_diff1": 0.798296, + "nauc_map_at_3_max": 0.119785, + "nauc_map_at_3_std": -0.359006, + "nauc_map_at_3_diff1": 0.779375, + "nauc_map_at_5_max": 0.127454, + "nauc_map_at_5_std": -0.341798, + "nauc_map_at_5_diff1": 0.774637, + "nauc_map_at_10_max": 0.125574, + "nauc_map_at_10_std": -0.335971, + "nauc_map_at_10_diff1": 0.774978, + "nauc_map_at_20_max": 0.126591, + "nauc_map_at_20_std": -0.333441, + "nauc_map_at_20_diff1": 0.775532, + "nauc_map_at_100_max": 0.126667, + "nauc_map_at_100_std": -0.334416, + "nauc_map_at_100_diff1": 0.776248, + "nauc_map_at_1000_max": 0.126667, + "nauc_map_at_1000_std": -0.334416, + "nauc_map_at_1000_diff1": 0.776248, + "nauc_recall_at_1_max": 0.113285, + "nauc_recall_at_1_std": -0.359756, + "nauc_recall_at_1_diff1": 0.798296, + "nauc_recall_at_3_max": 0.145103, + "nauc_recall_at_3_std": -0.34208, + "nauc_recall_at_3_diff1": 0.747636, + "nauc_recall_at_5_max": 0.210498, + "nauc_recall_at_5_std": -0.203482, + "nauc_recall_at_5_diff1": 0.698542, + "nauc_recall_at_10_max": 0.208933, + "nauc_recall_at_10_std": -0.03119, + "nauc_recall_at_10_diff1": 0.673574, + "nauc_recall_at_20_max": 0.308581, + "nauc_recall_at_20_std": 0.228558, + "nauc_recall_at_20_diff1": 0.663856, + "nauc_recall_at_100_max": 0.627039, + "nauc_recall_at_100_std": 0.810018, + "nauc_recall_at_100_diff1": 0.747268, + "nauc_recall_at_1000_max": 0.627039, + "nauc_recall_at_1000_std": 0.810018, + "nauc_recall_at_1000_diff1": 0.747268, + "nauc_precision_at_1_max": 0.113285, + "nauc_precision_at_1_std": -0.359756, + "nauc_precision_at_1_diff1": 0.798296, + "nauc_precision_at_3_max": 0.145103, + "nauc_precision_at_3_std": -0.34208, + "nauc_precision_at_3_diff1": 0.747636, + "nauc_precision_at_5_max": 0.210498, + "nauc_precision_at_5_std": -0.203482, + "nauc_precision_at_5_diff1": 0.698542, + "nauc_precision_at_10_max": 0.208933, + "nauc_precision_at_10_std": -0.03119, + "nauc_precision_at_10_diff1": 0.673574, + "nauc_precision_at_20_max": 0.308581, + "nauc_precision_at_20_std": 0.228558, + "nauc_precision_at_20_diff1": 0.663856, + "nauc_precision_at_100_max": 0.627039, + "nauc_precision_at_100_std": 0.810018, + "nauc_precision_at_100_diff1": 0.747268, + "nauc_precision_at_1000_max": 0.627039, + "nauc_precision_at_1000_std": 0.810018, + "nauc_precision_at_1000_diff1": 0.747268, + "nauc_mrr_at_1_max": 0.113285, + "nauc_mrr_at_1_std": -0.359756, + "nauc_mrr_at_1_diff1": 0.798296, + "nauc_mrr_at_3_max": 0.119785, + "nauc_mrr_at_3_std": -0.359006, + "nauc_mrr_at_3_diff1": 0.779375, + "nauc_mrr_at_5_max": 0.127454, + "nauc_mrr_at_5_std": -0.341798, + "nauc_mrr_at_5_diff1": 0.774637, + "nauc_mrr_at_10_max": 0.125574, + "nauc_mrr_at_10_std": -0.335971, + "nauc_mrr_at_10_diff1": 0.774978, + "nauc_mrr_at_20_max": 0.126591, + "nauc_mrr_at_20_std": -0.333441, + "nauc_mrr_at_20_diff1": 0.775532, + "nauc_mrr_at_100_max": 0.126667, + "nauc_mrr_at_100_std": -0.334416, + "nauc_mrr_at_100_diff1": 0.776248, + "nauc_mrr_at_1000_max": 0.126667, + "nauc_mrr_at_1000_std": -0.334416, + "nauc_mrr_at_1000_diff1": 0.776248, + "main_score": 0.81106, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 89.88959312438965, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed-Embedding/1/Waimai.json b/results/ByteDance-Seed__Seed-Embedding/1/Waimai.json new file mode 100644 index 0000000000..80c963a982 --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/Waimai.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "339287def212450dcaa9df8c22bf93e9980c7023", + "task_name": "Waimai", + "mteb_version": "1.35.1", + "scores": { + "test": [ + { + "accuracy": 0.9094, + "f1": 0.897798, + "f1_weighted": 0.909832, + "ap": 0.785184, + "ap_weighted": 0.785184, + "scores_per_experiment": [ + { + "accuracy": 0.913, + "f1": 0.902149, + "f1_weighted": 0.913554, + "ap": 0.791635, + "ap_weighted": 0.791635 + }, + { + "accuracy": 0.916, + "f1": 0.903652, + "f1_weighted": 0.915724, + "ap": 0.801554, + "ap_weighted": 0.801554 + }, + { + "accuracy": 0.902, + "f1": 0.891281, + "f1_weighted": 0.903229, + "ap": 0.767723, + "ap_weighted": 0.767723 + }, + { + "accuracy": 0.912, + "f1": 0.90095, + "f1_weighted": 0.912529, + "ap": 0.78963, + "ap_weighted": 0.78963 + }, + { + "accuracy": 0.896, + "f1": 0.88541, + "f1_weighted": 0.897602, + "ap": 0.755651, + "ap_weighted": 0.755651 + }, + { + "accuracy": 0.911, + "f1": 0.898, + "f1_weighted": 0.910745, + "ap": 0.790419, + "ap_weighted": 0.790419 + }, + { + "accuracy": 0.912, + "f1": 0.899392, + "f1_weighted": 0.911858, + "ap": 0.792139, + "ap_weighted": 0.792139 + }, + { + "accuracy": 0.911, + "f1": 0.898656, + "f1_weighted": 0.911035, + "ap": 0.789267, + "ap_weighted": 0.789267 + }, + { + "accuracy": 0.906, + "f1": 0.895126, + "f1_weighted": 0.906946, + "ap": 0.776203, + "ap_weighted": 0.776203 + }, + { + "accuracy": 0.915, + "f1": 0.903364, + "f1_weighted": 0.915101, + "ap": 0.797623, + "ap_weighted": 0.797623 + } + ], + "main_score": 0.9094, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 17.048202753067017, + "kg_co2_emissions": null +} \ No newline at end of file From 0d44f77016ca1c69121cdacfe19ef381f74a4eb1 Mon Sep 17 00:00:00 2001 From: namespace-Pt Date: Fri, 25 Apr 2025 02:48:28 +0800 Subject: [PATCH 2/2] update model_meta --- results/ByteDance-Seed__Seed-Embedding/1/model_meta.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 results/ByteDance-Seed__Seed-Embedding/1/model_meta.json diff --git a/results/ByteDance-Seed__Seed-Embedding/1/model_meta.json b/results/ByteDance-Seed__Seed-Embedding/1/model_meta.json new file mode 100644 index 0000000000..0d6d0eadfe --- /dev/null +++ b/results/ByteDance-Seed__Seed-Embedding/1/model_meta.json @@ -0,0 +1 @@ +{"name": "ByteDance-Seed/Seed-Embedding", "revision": "1", "release_date": "2025-04-25", "languages": ["eng-Latn", "zho-Hans"], "n_parameters": null, "memory_usage_mb": null, "max_tokens": 32768, "embed_dim": 2048, "license": null, "open_weights": false, "public_training_code": null, "public_training_data": null, "framework": ["API"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": null, "adapted_from": null, "superseded_by": null, "modalities": ["text"], "loader": null} \ No newline at end of file