diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/model_meta.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/1/model_meta.json deleted file mode 100644 index 8264ccdc3f..0000000000 --- a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "ByteDance-Seed/Doubao-1.5-Embedding", "revision": "1", "release_date": "2025-04-25", "languages": ["eng-Latn","zho-Hans"], "n_parameters": null, "memory_usage_mb": null, "max_tokens": 32768, "embed_dim": 2048, "license": null, "open_weights": false, "public_training_code": null, "public_training_data": null, "framework": ["API"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": null, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/model_meta.json b/results/ByteDance-Seed__Doubao-1.5-Embedding/2/model_meta.json deleted file mode 100644 index 8d94ddab24..0000000000 --- a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/model_meta.json +++ /dev/null @@ -1 +0,0 @@ -{"name": "ByteDance-Seed/Doubao-1.5-Embedding", "revision": "2", "release_date": "2025-04-25", "languages": ["eng-Latn","zho-Hans"], "n_parameters": null, "memory_usage_mb": null, "max_tokens": 32768, "embed_dim": 2048, "license": null, "open_weights": false, "public_training_code": null, "public_training_data": null, "framework": ["API"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": null, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AFQMC.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/AFQMC.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/AFQMC.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/AFQMC.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ATEC.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/ATEC.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/ATEC.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/ATEC.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AmazonCounterfactualClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/AmazonCounterfactualClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/AmazonCounterfactualClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/AmazonCounterfactualClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AmazonReviewsClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/AmazonReviewsClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/AmazonReviewsClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/AmazonReviewsClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArXivHierarchicalClusteringP2P.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/ArXivHierarchicalClusteringP2P.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArXivHierarchicalClusteringP2P.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/ArXivHierarchicalClusteringP2P.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArXivHierarchicalClusteringS2S.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/ArXivHierarchicalClusteringS2S.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArXivHierarchicalClusteringS2S.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/ArXivHierarchicalClusteringS2S.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArguAna.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/ArguAna.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/ArguAna.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/ArguAna.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/AskUbuntuDupQuestions.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/AskUbuntuDupQuestions.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/AskUbuntuDupQuestions.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/AskUbuntuDupQuestions.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/BIOSSES.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/BIOSSES.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/BIOSSES.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/BIOSSES.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/BQ.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/BQ.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/BQ.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/BQ.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Banking77Classification.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/Banking77Classification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/Banking77Classification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/Banking77Classification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/BiorxivClusteringP2P.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/BiorxivClusteringP2P.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/BiorxivClusteringP2P.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/BiorxivClusteringP2P.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CLSClusteringP2P.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/CLSClusteringP2P.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/CLSClusteringP2P.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/CLSClusteringP2P.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CLSClusteringS2S.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/CLSClusteringS2S.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/CLSClusteringS2S.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/CLSClusteringS2S.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CMedQAv1-reranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/CMedQAv1-reranking.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/CMedQAv1-reranking.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/CMedQAv1-reranking.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CMedQAv2-reranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/CMedQAv2-reranking.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/CMedQAv2-reranking.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/CMedQAv2-reranking.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CQADupstackGamingRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/CQADupstackGamingRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/CQADupstackGamingRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/CQADupstackGamingRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CQADupstackUnixRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/CQADupstackUnixRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/CQADupstackUnixRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/CQADupstackUnixRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ClimateFEVERHardNegatives.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/ClimateFEVERHardNegatives.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/ClimateFEVERHardNegatives.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/ClimateFEVERHardNegatives.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CmedqaRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/CmedqaRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/CmedqaRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/CmedqaRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Cmnli.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/Cmnli.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/Cmnli.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/Cmnli.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/CovidRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/CovidRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/CovidRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/CovidRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/DuRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/DuRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/DuRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/DuRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/EcomRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/EcomRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/EcomRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/EcomRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/FEVERHardNegatives.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/FEVERHardNegatives.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/FEVERHardNegatives.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/FEVERHardNegatives.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/FiQA2018.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/FiQA2018.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/FiQA2018.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/FiQA2018.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/HotpotQAHardNegatives.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/HotpotQAHardNegatives.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/HotpotQAHardNegatives.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/HotpotQAHardNegatives.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/IFlyTek.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/IFlyTek.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/IFlyTek.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/IFlyTek.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ImdbClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/ImdbClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/ImdbClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/ImdbClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/JDReview.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/JDReview.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/JDReview.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/JDReview.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/LCQMC.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/LCQMC.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/LCQMC.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/LCQMC.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MMarcoReranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/MMarcoReranking.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/MMarcoReranking.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/MMarcoReranking.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MMarcoRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/MMarcoRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/MMarcoRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/MMarcoRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MTOPDomainClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/MTOPDomainClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/MTOPDomainClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/MTOPDomainClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MassiveIntentClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/MassiveIntentClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/MassiveIntentClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/MassiveIntentClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MassiveScenarioClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/MassiveScenarioClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/MassiveScenarioClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/MassiveScenarioClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedicalRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/MedicalRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedicalRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/MedicalRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedrxivClusteringP2P.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/MedrxivClusteringP2P.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedrxivClusteringP2P.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/MedrxivClusteringP2P.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedrxivClusteringS2S.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/MedrxivClusteringS2S.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/MedrxivClusteringS2S.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/MedrxivClusteringS2S.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MindSmallReranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/MindSmallReranking.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/MindSmallReranking.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/MindSmallReranking.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/MultilingualSentiment.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/MultilingualSentiment.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/MultilingualSentiment.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/MultilingualSentiment.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Ocnli.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/Ocnli.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/Ocnli.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/Ocnli.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/OnlineShopping.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/OnlineShopping.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/OnlineShopping.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/OnlineShopping.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/PAWSX.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/PAWSX.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/PAWSX.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/PAWSX.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/QBQTC.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/QBQTC.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/QBQTC.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/QBQTC.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SCIDOCS.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/SCIDOCS.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/SCIDOCS.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/SCIDOCS.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SICK-R.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/SICK-R.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/SICK-R.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/SICK-R.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS12.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/STS12.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS12.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/STS12.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS13.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/STS13.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS13.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/STS13.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS14.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/STS14.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS14.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/STS14.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS15.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/STS15.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS15.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/STS15.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS17.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/STS17.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS17.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/STS17.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS22.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/STS22.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/STS22.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/STS22.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STSB.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/STSB.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/STSB.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/STSB.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/STSBenchmark.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/STSBenchmark.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/STSBenchmark.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/STSBenchmark.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SprintDuplicateQuestions.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/SprintDuplicateQuestions.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/SprintDuplicateQuestions.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/SprintDuplicateQuestions.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/StackExchangeClustering.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/StackExchangeClustering.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/StackExchangeClustering.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/StackExchangeClustering.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/StackExchangeClusteringP2P.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/StackExchangeClusteringP2P.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/StackExchangeClusteringP2P.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/StackExchangeClusteringP2P.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/SummEvalSummarization.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/SummEvalSummarization.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/SummEvalSummarization.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/SummEvalSummarization.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/T2Reranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/T2Reranking.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/T2Reranking.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/T2Reranking.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/T2Retrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/T2Retrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/T2Retrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/T2Retrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TNews.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/TNews.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/TNews.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/TNews.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TRECCOVID.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/TRECCOVID.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/TRECCOVID.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/TRECCOVID.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ThuNewsClusteringP2P.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/ThuNewsClusteringP2P.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/ThuNewsClusteringP2P.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/ThuNewsClusteringP2P.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ThuNewsClusteringS2S.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/ThuNewsClusteringS2S.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/ThuNewsClusteringS2S.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/ThuNewsClusteringS2S.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Touche2020Retrieval.v3.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/Touche2020Retrieval.v3.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/Touche2020Retrieval.v3.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/Touche2020Retrieval.v3.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/ToxicConversationsClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/ToxicConversationsClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/ToxicConversationsClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/ToxicConversationsClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TweetSentimentExtractionClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/TweetSentimentExtractionClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/TweetSentimentExtractionClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/TweetSentimentExtractionClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwentyNewsgroupsClustering.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/TwentyNewsgroupsClustering.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwentyNewsgroupsClustering.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/TwentyNewsgroupsClustering.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwitterSemEval2015.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/TwitterSemEval2015.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwitterSemEval2015.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/TwitterSemEval2015.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwitterURLCorpus.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/TwitterURLCorpus.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/TwitterURLCorpus.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/TwitterURLCorpus.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/VideoRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/VideoRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/VideoRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/VideoRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/1/Waimai.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/Waimai.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/1/Waimai.json rename to results/ByteDance-Seed__Seed1.5-Embedding/1/Waimai.json diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/1/model_meta.json b/results/ByteDance-Seed__Seed1.5-Embedding/1/model_meta.json new file mode 100644 index 0000000000..a3f2293617 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/1/model_meta.json @@ -0,0 +1 @@ +{"name": "ByteDance-Seed/Seed1.5-Embedding", "revision": "1", "release_date": "2025-04-25", "languages": ["eng-Latn","zho-Hans"], "n_parameters": null, "memory_usage_mb": null, "max_tokens": 32768, "embed_dim": 2048, "license": null, "open_weights": false, "public_training_code": null, "public_training_data": null, "framework": ["API"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": null, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/AFQMC.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/AFQMC.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/AFQMC.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/AFQMC.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/ATEC.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/ATEC.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/ATEC.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/ATEC.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/AmazonCounterfactualClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/AmazonCounterfactualClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/AmazonCounterfactualClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/AmazonCounterfactualClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/AmazonReviewsClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/AmazonReviewsClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/AmazonReviewsClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/AmazonReviewsClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/ArXivHierarchicalClusteringP2P.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/ArXivHierarchicalClusteringP2P.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/ArXivHierarchicalClusteringP2P.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/ArXivHierarchicalClusteringP2P.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/ArXivHierarchicalClusteringS2S.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/ArXivHierarchicalClusteringS2S.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/ArXivHierarchicalClusteringS2S.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/ArXivHierarchicalClusteringS2S.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/ArguAna.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/ArguAna.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/ArguAna.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/ArguAna.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/AskUbuntuDupQuestions.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/AskUbuntuDupQuestions.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/AskUbuntuDupQuestions.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/AskUbuntuDupQuestions.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/BIOSSES.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/BIOSSES.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/BIOSSES.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/BIOSSES.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/BQ.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/BQ.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/BQ.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/BQ.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/Banking77Classification.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/Banking77Classification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/Banking77Classification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/Banking77Classification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/BiorxivClusteringP2P.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/BiorxivClusteringP2P.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/BiorxivClusteringP2P.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/BiorxivClusteringP2P.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/BrightRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/BrightRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/BrightRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/BrightRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/CLSClusteringP2P.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/CLSClusteringP2P.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/CLSClusteringP2P.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/CLSClusteringP2P.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/CLSClusteringS2S.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/CLSClusteringS2S.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/CLSClusteringS2S.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/CLSClusteringS2S.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/CMedQAv1-reranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/CMedQAv1-reranking.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/CMedQAv1-reranking.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/CMedQAv1-reranking.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/CMedQAv2-reranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/CMedQAv2-reranking.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/CMedQAv2-reranking.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/CMedQAv2-reranking.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/CQADupstackGamingRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/CQADupstackGamingRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/CQADupstackGamingRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/CQADupstackGamingRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/CQADupstackUnixRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/CQADupstackUnixRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/CQADupstackUnixRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/CQADupstackUnixRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/ClimateFEVERHardNegatives.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/ClimateFEVERHardNegatives.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/ClimateFEVERHardNegatives.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/ClimateFEVERHardNegatives.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/CmedqaRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/CmedqaRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/CmedqaRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/CmedqaRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/Cmnli.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/Cmnli.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/Cmnli.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/Cmnli.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/CovidRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/CovidRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/CovidRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/CovidRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/DuRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/DuRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/DuRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/DuRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/EcomRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/EcomRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/EcomRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/EcomRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/FEVERHardNegatives.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/FEVERHardNegatives.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/FEVERHardNegatives.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/FEVERHardNegatives.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/FiQA2018.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/FiQA2018.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/FiQA2018.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/FiQA2018.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/HotpotQAHardNegatives.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/HotpotQAHardNegatives.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/HotpotQAHardNegatives.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/HotpotQAHardNegatives.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/IFlyTek.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/IFlyTek.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/IFlyTek.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/IFlyTek.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/ImdbClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/ImdbClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/ImdbClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/ImdbClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/JDReview.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/JDReview.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/JDReview.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/JDReview.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/LCQMC.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/LCQMC.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/LCQMC.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/LCQMC.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/MMarcoReranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/MMarcoReranking.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/MMarcoReranking.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/MMarcoReranking.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/MMarcoRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/MMarcoRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/MMarcoRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/MMarcoRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/MTOPDomainClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/MTOPDomainClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/MTOPDomainClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/MTOPDomainClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/MassiveIntentClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/MassiveIntentClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/MassiveIntentClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/MassiveIntentClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/MassiveScenarioClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/MassiveScenarioClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/MassiveScenarioClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/MassiveScenarioClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/MedicalRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/MedicalRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/MedicalRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/MedicalRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/MedrxivClusteringP2P.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/MedrxivClusteringP2P.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/MedrxivClusteringP2P.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/MedrxivClusteringP2P.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/MedrxivClusteringS2S.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/MedrxivClusteringS2S.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/MedrxivClusteringS2S.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/MedrxivClusteringS2S.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/MindSmallReranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/MindSmallReranking.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/MindSmallReranking.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/MindSmallReranking.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/MultilingualSentiment.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/MultilingualSentiment.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/MultilingualSentiment.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/MultilingualSentiment.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/Ocnli.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/Ocnli.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/Ocnli.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/Ocnli.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/OnlineShopping.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/OnlineShopping.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/OnlineShopping.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/OnlineShopping.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/PAWSX.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/PAWSX.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/PAWSX.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/PAWSX.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/QBQTC.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/QBQTC.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/QBQTC.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/QBQTC.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/SCIDOCS.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/SCIDOCS.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/SCIDOCS.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/SCIDOCS.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/SICK-R.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/SICK-R.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/SICK-R.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/SICK-R.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/STS12.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/STS12.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/STS12.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/STS12.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/STS13.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/STS13.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/STS13.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/STS13.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/STS14.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/STS14.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/STS14.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/STS14.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/STS15.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/STS15.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/STS15.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/STS15.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/STS17.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/STS17.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/STS17.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/STS17.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/STS22.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/STS22.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/STS22.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/STS22.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/STSB.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/STSB.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/STSB.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/STSB.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/STSBenchmark.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/STSBenchmark.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/STSBenchmark.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/STSBenchmark.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/SprintDuplicateQuestions.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/SprintDuplicateQuestions.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/SprintDuplicateQuestions.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/SprintDuplicateQuestions.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/StackExchangeClustering.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/StackExchangeClustering.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/StackExchangeClustering.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/StackExchangeClustering.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/StackExchangeClusteringP2P.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/StackExchangeClusteringP2P.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/StackExchangeClusteringP2P.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/StackExchangeClusteringP2P.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/SummEvalSummarization.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/SummEvalSummarization.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/SummEvalSummarization.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/SummEvalSummarization.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/T2Reranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/T2Reranking.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/T2Reranking.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/T2Reranking.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/T2Retrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/T2Retrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/T2Retrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/T2Retrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/TNews.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/TNews.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/TNews.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/TNews.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/TRECCOVID.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/TRECCOVID.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/TRECCOVID.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/TRECCOVID.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/ThuNewsClusteringP2P.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/ThuNewsClusteringP2P.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/ThuNewsClusteringP2P.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/ThuNewsClusteringP2P.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/ThuNewsClusteringS2S.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/ThuNewsClusteringS2S.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/ThuNewsClusteringS2S.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/ThuNewsClusteringS2S.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/Touche2020Retrieval.v3.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/Touche2020Retrieval.v3.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/Touche2020Retrieval.v3.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/Touche2020Retrieval.v3.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/ToxicConversationsClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/ToxicConversationsClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/ToxicConversationsClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/ToxicConversationsClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/TweetSentimentExtractionClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/TweetSentimentExtractionClassification.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/TweetSentimentExtractionClassification.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/TweetSentimentExtractionClassification.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/TwentyNewsgroupsClustering.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/TwentyNewsgroupsClustering.v2.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/TwentyNewsgroupsClustering.v2.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/TwentyNewsgroupsClustering.v2.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/TwitterSemEval2015.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/TwitterSemEval2015.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/TwitterSemEval2015.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/TwitterSemEval2015.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/TwitterURLCorpus.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/TwitterURLCorpus.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/TwitterURLCorpus.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/TwitterURLCorpus.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/VideoRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/VideoRetrieval.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/VideoRetrieval.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/VideoRetrieval.json diff --git a/results/ByteDance-Seed__Doubao-1.5-Embedding/2/Waimai.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/Waimai.json similarity index 100% rename from results/ByteDance-Seed__Doubao-1.5-Embedding/2/Waimai.json rename to results/ByteDance-Seed__Seed1.5-Embedding/2/Waimai.json diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/2/model_meta.json b/results/ByteDance-Seed__Seed1.5-Embedding/2/model_meta.json new file mode 100644 index 0000000000..09111cfeff --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/2/model_meta.json @@ -0,0 +1 @@ +{"name": "ByteDance-Seed/Seed1.5-Embedding", "revision": "2", "release_date": "2025-04-25", "languages": ["eng-Latn","zho-Hans"], "n_parameters": null, "memory_usage_mb": null, "max_tokens": 32768, "embed_dim": 2048, "license": null, "open_weights": false, "public_training_code": null, "public_training_data": null, "framework": ["API"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": null, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["text"], "loader": null} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/AFQMC.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/AFQMC.json new file mode 100644 index 0000000000..9290dee145 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/AFQMC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "b44c3b011063adb25877c13823db83bb193913c4", + "task_name": "AFQMC", + "mteb_version": "1.36.5", + "scores": { + "validation": [ + { + "pearson": 0.526944, + "spearman": 0.559333, + "cosine_pearson": 0.526944, + "cosine_spearman": 0.559333, + "manhattan_pearson": 0.539891, + "manhattan_spearman": 0.557663, + "euclidean_pearson": 0.541654, + "euclidean_spearman": 0.559357, + "main_score": 0.559333, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 14.07524037361145, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/ATEC.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/ATEC.json new file mode 100644 index 0000000000..0f4cb66eff --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/ATEC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "0f319b1142f28d00e055a6770f3f726ae9b7d865", + "task_name": "ATEC", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.515076, + "spearman": 0.545954, + "cosine_pearson": 0.515076, + "cosine_spearman": 0.545954, + "manhattan_pearson": 0.560066, + "manhattan_spearman": 0.54492, + "euclidean_pearson": 0.561606, + "euclidean_spearman": 0.545949, + "main_score": 0.545954, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 23.127310514450073, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/AmazonCounterfactualClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/AmazonCounterfactualClassification.json new file mode 100644 index 0000000000..908f6ae35d --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/AmazonCounterfactualClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e8379541af4e31359cca9fbcf4b00f2671dba205", + "task_name": "AmazonCounterfactualClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.91403, + "f1": 0.878321, + "f1_weighted": 0.918457, + "ap": 0.684074, + "ap_weighted": 0.684074, + "scores_per_experiment": [ + { + "accuracy": 0.91194, + "f1": 0.875918, + "f1_weighted": 0.91663, + "ap": 0.679038, + "ap_weighted": 0.679038 + }, + { + "accuracy": 0.91194, + "f1": 0.875918, + "f1_weighted": 0.91663, + "ap": 0.679038, + "ap_weighted": 0.679038 + }, + { + "accuracy": 0.91791, + "f1": 0.882722, + "f1_weighted": 0.921842, + "ap": 0.692855, + "ap_weighted": 0.692855 + }, + { + "accuracy": 0.91194, + "f1": 0.875918, + "f1_weighted": 0.91663, + "ap": 0.679038, + "ap_weighted": 0.679038 + }, + { + "accuracy": 0.913433, + "f1": 0.877743, + "f1_weighted": 0.917968, + "ap": 0.682824, + "ap_weighted": 0.682824 + }, + { + "accuracy": 0.91791, + "f1": 0.882722, + "f1_weighted": 0.921842, + "ap": 0.692855, + "ap_weighted": 0.692855 + }, + { + "accuracy": 0.91791, + "f1": 0.882722, + "f1_weighted": 0.921842, + "ap": 0.692855, + "ap_weighted": 0.692855 + }, + { + "accuracy": 0.922388, + "f1": 0.8878, + "f1_weighted": 0.925735, + "ap": 0.703488, + "ap_weighted": 0.703488 + }, + { + "accuracy": 0.914925, + "f1": 0.879577, + "f1_weighted": 0.919307, + "ap": 0.686653, + "ap_weighted": 0.686653 + }, + { + "accuracy": 0.9, + "f1": 0.862173, + "f1_weighted": 0.906143, + "ap": 0.652095, + "ap_weighted": 0.652095 + } + ], + "main_score": 0.91403, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 13.539509296417236, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/AmazonReviewsClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/AmazonReviewsClassification.json new file mode 100644 index 0000000000..c13ac02f0c --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/AmazonReviewsClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "1399c76144fd37290681b995c656ef9b2e06e26d", + "task_name": "AmazonReviewsClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.58904, + "f1": 0.574923, + "f1_weighted": 0.574923, + "scores_per_experiment": [ + { + "accuracy": 0.5892, + "f1": 0.58641, + "f1_weighted": 0.58641 + }, + { + "accuracy": 0.5666, + "f1": 0.533806, + "f1_weighted": 0.533806 + }, + { + "accuracy": 0.5864, + "f1": 0.568436, + "f1_weighted": 0.568436 + }, + { + "accuracy": 0.6062, + "f1": 0.59882, + "f1_weighted": 0.59882 + }, + { + "accuracy": 0.5808, + "f1": 0.557392, + "f1_weighted": 0.557392 + }, + { + "accuracy": 0.594, + "f1": 0.578833, + "f1_weighted": 0.578833 + }, + { + "accuracy": 0.5898, + "f1": 0.585487, + "f1_weighted": 0.585487 + }, + { + "accuracy": 0.5876, + "f1": 0.572296, + "f1_weighted": 0.572296 + }, + { + "accuracy": 0.5974, + "f1": 0.590304, + "f1_weighted": 0.590304 + }, + { + "accuracy": 0.5924, + "f1": 0.577443, + "f1_weighted": 0.577443 + } + ], + "main_score": 0.58904, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 27.29578471183777, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/ArXivHierarchicalClusteringP2P.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/ArXivHierarchicalClusteringP2P.json new file mode 100644 index 0000000000..e37141b05b --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/ArXivHierarchicalClusteringP2P.json @@ -0,0 +1,46 @@ +{ + "dataset_revision": "0bbdb47bcbe3a90093699aefeed338a0f28a7ee8", + "task_name": "ArXivHierarchicalClusteringP2P", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.698214, + 0.720434, + 0.717838, + 0.719626, + 0.756369, + 0.692329, + 0.667222, + 0.729426, + 0.686439, + 0.725508 + ], + "Level 1": [ + 0.570393, + 0.599181, + 0.592785, + 0.604224, + 0.596483, + 0.591464, + 0.624237, + 0.58823, + 0.591593, + 0.607302 + ] + }, + "v_measure": 0.653965, + "v_measure_std": 0.060603, + "main_score": 0.653965, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 50.32472491264343, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/ArXivHierarchicalClusteringS2S.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/ArXivHierarchicalClusteringS2S.json new file mode 100644 index 0000000000..e499c01534 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/ArXivHierarchicalClusteringS2S.json @@ -0,0 +1,46 @@ +{ + "dataset_revision": "b73bd54100e5abfa6e3a23dcafb46fe4d2438dc3", + "task_name": "ArXivHierarchicalClusteringS2S", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.679147, + 0.665077, + 0.668014, + 0.66496, + 0.708301, + 0.681688, + 0.687764, + 0.693356, + 0.68039, + 0.690421 + ], + "Level 1": [ + 0.596373, + 0.598845, + 0.598289, + 0.596597, + 0.611598, + 0.60093, + 0.60979, + 0.580738, + 0.603515, + 0.592527 + ] + }, + "v_measure": 0.640416, + "v_measure_std": 0.042915, + "main_score": 0.640416, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 44.01299571990967, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/ArguAna.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/ArguAna.json new file mode 100644 index 0000000000..9804472b8b --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/ArguAna.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a", + "task_name": "ArguAna", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.5441, + "ndcg_at_3": 0.69019, + "ndcg_at_5": 0.72707, + "ndcg_at_10": 0.75104, + "ndcg_at_20": 0.75852, + "ndcg_at_100": 0.76038, + "ndcg_at_1000": 0.76038, + "map_at_1": 0.5441, + "map_at_3": 0.65375, + "map_at_5": 0.67419, + "map_at_10": 0.68424, + "map_at_20": 0.68645, + "map_at_100": 0.68677, + "map_at_1000": 0.68677, + "recall_at_1": 0.5441, + "recall_at_3": 0.79587, + "recall_at_5": 0.88549, + "recall_at_10": 0.95875, + "recall_at_20": 0.9872, + "recall_at_100": 0.99644, + "recall_at_1000": 0.99644, + "precision_at_1": 0.5441, + "precision_at_3": 0.26529, + "precision_at_5": 0.1771, + "precision_at_10": 0.09587, + "precision_at_20": 0.04936, + "precision_at_100": 0.00996, + "precision_at_1000": 0.001, + "mrr_at_1": 0.544097, + "mrr_at_3": 0.653746, + "mrr_at_5": 0.674194, + "mrr_at_10": 0.684244, + "mrr_at_20": 0.686452, + "mrr_at_100": 0.686768, + "mrr_at_1000": 0.686768, + "nauc_ndcg_at_1_max": -0.038382, + "nauc_ndcg_at_1_std": -0.190293, + "nauc_ndcg_at_1_diff1": 0.279499, + "nauc_ndcg_at_3_max": -0.045598, + "nauc_ndcg_at_3_std": -0.200647, + "nauc_ndcg_at_3_diff1": 0.221828, + "nauc_ndcg_at_5_max": -0.028078, + "nauc_ndcg_at_5_std": -0.189219, + "nauc_ndcg_at_5_diff1": 0.231643, + "nauc_ndcg_at_10_max": -0.041761, + "nauc_ndcg_at_10_std": -0.194865, + "nauc_ndcg_at_10_diff1": 0.230347, + "nauc_ndcg_at_20_max": -0.037333, + "nauc_ndcg_at_20_std": -0.184716, + "nauc_ndcg_at_20_diff1": 0.237313, + "nauc_ndcg_at_100_max": -0.041849, + "nauc_ndcg_at_100_std": -0.191357, + "nauc_ndcg_at_100_diff1": 0.236397, + "nauc_ndcg_at_1000_max": -0.041849, + "nauc_ndcg_at_1000_std": -0.191357, + "nauc_ndcg_at_1000_diff1": 0.236397, + "nauc_map_at_1_max": -0.038382, + "nauc_map_at_1_std": -0.190293, + "nauc_map_at_1_diff1": 0.279499, + "nauc_map_at_3_max": -0.047446, + "nauc_map_at_3_std": -0.200179, + "nauc_map_at_3_diff1": 0.23388, + "nauc_map_at_5_max": -0.039998, + "nauc_map_at_5_std": -0.194778, + "nauc_map_at_5_diff1": 0.239102, + "nauc_map_at_10_max": -0.045081, + "nauc_map_at_10_std": -0.19736, + "nauc_map_at_10_diff1": 0.239314, + "nauc_map_at_20_max": -0.044167, + "nauc_map_at_20_std": -0.194997, + "nauc_map_at_20_diff1": 0.240762, + "nauc_map_at_100_max": -0.044725, + "nauc_map_at_100_std": -0.195842, + "nauc_map_at_100_diff1": 0.240712, + "nauc_map_at_1000_max": -0.044725, + "nauc_map_at_1000_std": -0.195842, + "nauc_map_at_1000_diff1": 0.240712, + "nauc_recall_at_1_max": -0.038382, + "nauc_recall_at_1_std": -0.190293, + "nauc_recall_at_1_diff1": 0.279499, + "nauc_recall_at_3_max": -0.035539, + "nauc_recall_at_3_std": -0.201056, + "nauc_recall_at_3_diff1": 0.174606, + "nauc_recall_at_5_max": 0.062194, + "nauc_recall_at_5_std": -0.146124, + "nauc_recall_at_5_diff1": 0.191685, + "nauc_recall_at_10_max": 0.018558, + "nauc_recall_at_10_std": -0.139401, + "nauc_recall_at_10_diff1": 0.092361, + "nauc_recall_at_20_max": 0.412161, + "nauc_recall_at_20_std": 0.547346, + "nauc_recall_at_20_diff1": 0.211007, + "nauc_recall_at_100_max": 0.344399, + "nauc_recall_at_100_std": 0.661144, + "nauc_recall_at_100_diff1": -0.167167, + "nauc_recall_at_1000_max": 0.344399, + "nauc_recall_at_1000_std": 0.661144, + "nauc_recall_at_1000_diff1": -0.167167, + "nauc_precision_at_1_max": -0.038382, + "nauc_precision_at_1_std": -0.190293, + "nauc_precision_at_1_diff1": 0.279499, + "nauc_precision_at_3_max": -0.035539, + "nauc_precision_at_3_std": -0.201056, + "nauc_precision_at_3_diff1": 0.174606, + "nauc_precision_at_5_max": 0.062194, + "nauc_precision_at_5_std": -0.146124, + "nauc_precision_at_5_diff1": 0.191685, + "nauc_precision_at_10_max": 0.018558, + "nauc_precision_at_10_std": -0.139401, + "nauc_precision_at_10_diff1": 0.092361, + "nauc_precision_at_20_max": 0.412161, + "nauc_precision_at_20_std": 0.547346, + "nauc_precision_at_20_diff1": 0.211007, + "nauc_precision_at_100_max": 0.344399, + "nauc_precision_at_100_std": 0.661144, + "nauc_precision_at_100_diff1": -0.167167, + "nauc_precision_at_1000_max": 0.344399, + "nauc_precision_at_1000_std": 0.661144, + "nauc_precision_at_1000_diff1": -0.167167, + "nauc_mrr_at_1_max": -0.038382, + "nauc_mrr_at_1_std": -0.190293, + "nauc_mrr_at_1_diff1": 0.279499, + "nauc_mrr_at_3_max": -0.047446, + "nauc_mrr_at_3_std": -0.200179, + "nauc_mrr_at_3_diff1": 0.23388, + "nauc_mrr_at_5_max": -0.039998, + "nauc_mrr_at_5_std": -0.194778, + "nauc_mrr_at_5_diff1": 0.239102, + "nauc_mrr_at_10_max": -0.045081, + "nauc_mrr_at_10_std": -0.19736, + "nauc_mrr_at_10_diff1": 0.239314, + "nauc_mrr_at_20_max": -0.044167, + "nauc_mrr_at_20_std": -0.194997, + "nauc_mrr_at_20_diff1": 0.240762, + "nauc_mrr_at_100_max": -0.044725, + "nauc_mrr_at_100_std": -0.195842, + "nauc_mrr_at_100_diff1": 0.240712, + "nauc_mrr_at_1000_max": -0.044725, + "nauc_mrr_at_1000_std": -0.195842, + "nauc_mrr_at_1000_diff1": 0.240712, + "main_score": 0.75104, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 140.33609533309937, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/AskUbuntuDupQuestions.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/AskUbuntuDupQuestions.json new file mode 100644 index 0000000000..aa75d200ae --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/AskUbuntuDupQuestions.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "2000358ca161889fa9c082cb41daa8dcfb161a54", + "task_name": "AskUbuntuDupQuestions", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "map": 0.688206, + "mrr": 0.823317, + "nAUC_map_max": 0.309587, + "nAUC_map_std": 0.211267, + "nAUC_map_diff1": 0.066175, + "nAUC_mrr_max": 0.452276, + "nAUC_mrr_std": 0.295109, + "nAUC_mrr_diff1": 0.073983, + "main_score": 0.688206, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.710558652877808, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/BIOSSES.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/BIOSSES.json new file mode 100644 index 0000000000..1040425117 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/BIOSSES.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "d3fb88f8f02e40887cd149695127462bbcf29b4a", + "task_name": "BIOSSES", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.845587, + "spearman": 0.834965, + "cosine_pearson": 0.845587, + "cosine_spearman": 0.834965, + "manhattan_pearson": 0.836672, + "manhattan_spearman": 0.836001, + "euclidean_pearson": 0.83409, + "euclidean_spearman": 0.835483, + "main_score": 0.834965, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.979576110839844, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/BQ.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/BQ.json new file mode 100644 index 0000000000..51c25d84e1 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/BQ.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "e3dda5e115e487b39ec7e618c0c6a29137052a55", + "task_name": "BQ", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.676437, + "spearman": 0.697975, + "cosine_pearson": 0.676437, + "cosine_spearman": 0.697975, + "manhattan_pearson": 0.671023, + "manhattan_spearman": 0.695794, + "euclidean_pearson": 0.673622, + "euclidean_spearman": 0.698, + "main_score": 0.697975, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 17.909261465072632, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/Banking77Classification.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/Banking77Classification.json new file mode 100644 index 0000000000..ee397c50dc --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/Banking77Classification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "0fd18e25b25c072e09e0d92ab615fda904d66300", + "task_name": "Banking77Classification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.917045, + "f1": 0.916194, + "f1_weighted": 0.916194, + "scores_per_experiment": [ + { + "accuracy": 0.920779, + "f1": 0.920177, + "f1_weighted": 0.920177 + }, + { + "accuracy": 0.916234, + "f1": 0.915092, + "f1_weighted": 0.915092 + }, + { + "accuracy": 0.921753, + "f1": 0.920853, + "f1_weighted": 0.920853 + }, + { + "accuracy": 0.919481, + "f1": 0.918548, + "f1_weighted": 0.918548 + }, + { + "accuracy": 0.918831, + "f1": 0.918275, + "f1_weighted": 0.918275 + }, + { + "accuracy": 0.912013, + "f1": 0.911079, + "f1_weighted": 0.911079 + }, + { + "accuracy": 0.920455, + "f1": 0.919748, + "f1_weighted": 0.919748 + }, + { + "accuracy": 0.912662, + "f1": 0.911768, + "f1_weighted": 0.911768 + }, + { + "accuracy": 0.912987, + "f1": 0.911936, + "f1_weighted": 0.911936 + }, + { + "accuracy": 0.91526, + "f1": 0.914466, + "f1_weighted": 0.914466 + } + ], + "main_score": 0.917045, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 22.45311188697815, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/BiorxivClusteringP2P.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/BiorxivClusteringP2P.v2.json new file mode 100644 index 0000000000..d609cb4cf0 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/BiorxivClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "f5dbc242e11dd8e24def4c4268607a49e02946dc", + "task_name": "BiorxivClusteringP2P.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.56151, + 0.567383, + 0.571734, + 0.561726, + 0.573255, + 0.562778, + 0.564215, + 0.560089, + 0.563702, + 0.555539 + ] + }, + "v_measure": 0.564193, + "v_measure_std": 0.005064, + "main_score": 0.564193, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 18.583871364593506, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/BrightRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/BrightRetrieval.json new file mode 100644 index 0000000000..ed2359164b --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/BrightRetrieval.json @@ -0,0 +1,1775 @@ +{ + "dataset_revision": "a75a0eb", + "task_name": "BrightRetrieval", + "mteb_version": "1.36.5", + "scores": { + "standard": [ + { + "ndcg_at_1": 0.14414, + "ndcg_at_3": 0.10572, + "ndcg_at_5": 0.11443, + "ndcg_at_10": 0.12208, + "ndcg_at_20": 0.14753, + "ndcg_at_100": 0.17042, + "ndcg_at_1000": 0.17042, + "map_at_1": 0.03571, + "map_at_3": 0.05255, + "map_at_5": 0.06734, + "map_at_10": 0.07776, + "map_at_20": 0.08674, + "map_at_100": 0.09039, + "map_at_1000": 0.09039, + "recall_at_1": 0.03571, + "recall_at_3": 0.06655, + "recall_at_5": 0.1014, + "recall_at_10": 0.13787, + "recall_at_20": 0.19916, + "recall_at_100": 0.27928, + "recall_at_1000": 0.27928, + "precision_at_1": 0.14414, + "precision_at_3": 0.08709, + "precision_at_5": 0.08288, + "precision_at_10": 0.05676, + "precision_at_20": 0.04414, + "precision_at_100": 0.01189, + "precision_at_1000": 0.00119, + "mrr_at_1": 0.144144, + "mrr_at_3": 0.177177, + "mrr_at_5": 0.189339, + "mrr_at_10": 0.195317, + "mrr_at_20": 0.203403, + "mrr_at_100": 0.207499, + "mrr_at_1000": 0.207499, + "nauc_ndcg_at_1_max": -0.034342, + "nauc_ndcg_at_1_std": 0.007492, + "nauc_ndcg_at_1_diff1": 0.231109, + "nauc_ndcg_at_3_max": -0.013274, + "nauc_ndcg_at_3_std": -0.035902, + "nauc_ndcg_at_3_diff1": 0.235096, + "nauc_ndcg_at_5_max": -0.042764, + "nauc_ndcg_at_5_std": -0.06606, + "nauc_ndcg_at_5_diff1": 0.251904, + "nauc_ndcg_at_10_max": -0.086391, + "nauc_ndcg_at_10_std": -0.076415, + "nauc_ndcg_at_10_diff1": 0.211015, + "nauc_ndcg_at_20_max": -0.084811, + "nauc_ndcg_at_20_std": -0.036244, + "nauc_ndcg_at_20_diff1": 0.201519, + "nauc_ndcg_at_100_max": -0.057705, + "nauc_ndcg_at_100_std": 0.004973, + "nauc_ndcg_at_100_diff1": 0.196774, + "nauc_ndcg_at_1000_max": -0.057705, + "nauc_ndcg_at_1000_std": 0.004973, + "nauc_ndcg_at_1000_diff1": 0.196774, + "nauc_map_at_1_max": -0.068966, + "nauc_map_at_1_std": -0.056086, + "nauc_map_at_1_diff1": 0.079157, + "nauc_map_at_3_max": -0.038235, + "nauc_map_at_3_std": -0.076699, + "nauc_map_at_3_diff1": 0.119937, + "nauc_map_at_5_max": -0.043307, + "nauc_map_at_5_std": -0.080708, + "nauc_map_at_5_diff1": 0.192102, + "nauc_map_at_10_max": -0.0698, + "nauc_map_at_10_std": -0.073165, + "nauc_map_at_10_diff1": 0.196272, + "nauc_map_at_20_max": -0.0698, + "nauc_map_at_20_std": -0.060318, + "nauc_map_at_20_diff1": 0.208098, + "nauc_map_at_100_max": -0.062594, + "nauc_map_at_100_std": -0.049284, + "nauc_map_at_100_diff1": 0.203256, + "nauc_map_at_1000_max": -0.062594, + "nauc_map_at_1000_std": -0.049284, + "nauc_map_at_1000_diff1": 0.203256, + "nauc_recall_at_1_max": -0.068966, + "nauc_recall_at_1_std": -0.056086, + "nauc_recall_at_1_diff1": 0.079157, + "nauc_recall_at_3_max": -0.032072, + "nauc_recall_at_3_std": -0.120036, + "nauc_recall_at_3_diff1": 0.085634, + "nauc_recall_at_5_max": -0.054159, + "nauc_recall_at_5_std": -0.095688, + "nauc_recall_at_5_diff1": 0.218306, + "nauc_recall_at_10_max": -0.119139, + "nauc_recall_at_10_std": -0.080627, + "nauc_recall_at_10_diff1": 0.19008, + "nauc_recall_at_20_max": -0.11724, + "nauc_recall_at_20_std": 0.001897, + "nauc_recall_at_20_diff1": 0.162756, + "nauc_recall_at_100_max": -0.051897, + "nauc_recall_at_100_std": 0.100025, + "nauc_recall_at_100_diff1": 0.145726, + "nauc_recall_at_1000_max": -0.051897, + "nauc_recall_at_1000_std": 0.100025, + "nauc_recall_at_1000_diff1": 0.145726, + "nauc_precision_at_1_max": -0.034342, + "nauc_precision_at_1_std": 0.007492, + "nauc_precision_at_1_diff1": 0.231109, + "nauc_precision_at_3_max": -0.013735, + "nauc_precision_at_3_std": -0.049085, + "nauc_precision_at_3_diff1": 0.27343, + "nauc_precision_at_5_max": -0.05813, + "nauc_precision_at_5_std": -0.071165, + "nauc_precision_at_5_diff1": 0.358543, + "nauc_precision_at_10_max": -0.090928, + "nauc_precision_at_10_std": -0.042963, + "nauc_precision_at_10_diff1": 0.331685, + "nauc_precision_at_20_max": -0.055038, + "nauc_precision_at_20_std": 0.04662, + "nauc_precision_at_20_diff1": 0.270293, + "nauc_precision_at_100_max": 0.020058, + "nauc_precision_at_100_std": 0.136708, + "nauc_precision_at_100_diff1": 0.249342, + "nauc_precision_at_1000_max": 0.020058, + "nauc_precision_at_1000_std": 0.136708, + "nauc_precision_at_1000_diff1": 0.249342, + "nauc_mrr_at_1_max": -0.034342, + "nauc_mrr_at_1_std": 0.007492, + "nauc_mrr_at_1_diff1": 0.231109, + "nauc_mrr_at_3_max": -0.017226, + "nauc_mrr_at_3_std": -0.046775, + "nauc_mrr_at_3_diff1": 0.211695, + "nauc_mrr_at_5_max": -0.035774, + "nauc_mrr_at_5_std": -0.042263, + "nauc_mrr_at_5_diff1": 0.232453, + "nauc_mrr_at_10_max": -0.049026, + "nauc_mrr_at_10_std": -0.050769, + "nauc_mrr_at_10_diff1": 0.21907, + "nauc_mrr_at_20_max": -0.048479, + "nauc_mrr_at_20_std": -0.040532, + "nauc_mrr_at_20_diff1": 0.210436, + "nauc_mrr_at_100_max": -0.04202, + "nauc_mrr_at_100_std": -0.033545, + "nauc_mrr_at_100_diff1": 0.21157, + "nauc_mrr_at_1000_max": -0.04202, + "nauc_mrr_at_1000_std": -0.033545, + "nauc_mrr_at_1000_diff1": 0.21157, + "main_score": 0.12208, + "hf_subset": "aops", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.36893, + "ndcg_at_3": 0.32288, + "ndcg_at_5": 0.32283, + "ndcg_at_10": 0.34806, + "ndcg_at_20": 0.38794, + "ndcg_at_100": 0.44402, + "ndcg_at_1000": 0.44402, + "map_at_1": 0.11367, + "map_at_3": 0.20127, + "map_at_5": 0.23032, + "map_at_10": 0.26111, + "map_at_20": 0.28154, + "map_at_100": 0.29594, + "map_at_1000": 0.29594, + "recall_at_1": 0.11367, + "recall_at_3": 0.23334, + "recall_at_5": 0.29733, + "recall_at_10": 0.3822, + "recall_at_20": 0.49388, + "recall_at_100": 0.70462, + "recall_at_1000": 0.70462, + "precision_at_1": 0.36893, + "precision_at_3": 0.27508, + "precision_at_5": 0.21359, + "precision_at_10": 0.14369, + "precision_at_20": 0.09563, + "precision_at_100": 0.0265, + "precision_at_1000": 0.00265, + "mrr_at_1": 0.368932, + "mrr_at_3": 0.427184, + "mrr_at_5": 0.442718, + "mrr_at_10": 0.451117, + "mrr_at_20": 0.457404, + "mrr_at_100": 0.460578, + "mrr_at_1000": 0.460578, + "nauc_ndcg_at_1_max": 0.134472, + "nauc_ndcg_at_1_std": 0.130287, + "nauc_ndcg_at_1_diff1": 0.164914, + "nauc_ndcg_at_3_max": 0.147173, + "nauc_ndcg_at_3_std": 0.185052, + "nauc_ndcg_at_3_diff1": 0.190515, + "nauc_ndcg_at_5_max": 0.124109, + "nauc_ndcg_at_5_std": 0.116536, + "nauc_ndcg_at_5_diff1": 0.173095, + "nauc_ndcg_at_10_max": 0.131486, + "nauc_ndcg_at_10_std": 0.133504, + "nauc_ndcg_at_10_diff1": 0.164098, + "nauc_ndcg_at_20_max": 0.119694, + "nauc_ndcg_at_20_std": 0.162054, + "nauc_ndcg_at_20_diff1": 0.161136, + "nauc_ndcg_at_100_max": 0.101663, + "nauc_ndcg_at_100_std": 0.191308, + "nauc_ndcg_at_100_diff1": 0.142491, + "nauc_ndcg_at_1000_max": 0.101663, + "nauc_ndcg_at_1000_std": 0.191308, + "nauc_ndcg_at_1000_diff1": 0.142491, + "nauc_map_at_1_max": 0.160645, + "nauc_map_at_1_std": 0.030675, + "nauc_map_at_1_diff1": 0.137594, + "nauc_map_at_3_max": 0.17339, + "nauc_map_at_3_std": 0.111912, + "nauc_map_at_3_diff1": 0.231352, + "nauc_map_at_5_max": 0.151762, + "nauc_map_at_5_std": 0.092398, + "nauc_map_at_5_diff1": 0.221001, + "nauc_map_at_10_max": 0.140547, + "nauc_map_at_10_std": 0.117936, + "nauc_map_at_10_diff1": 0.202149, + "nauc_map_at_20_max": 0.130143, + "nauc_map_at_20_std": 0.141943, + "nauc_map_at_20_diff1": 0.194651, + "nauc_map_at_100_max": 0.123245, + "nauc_map_at_100_std": 0.154745, + "nauc_map_at_100_diff1": 0.186714, + "nauc_map_at_1000_max": 0.123245, + "nauc_map_at_1000_std": 0.154745, + "nauc_map_at_1000_diff1": 0.186714, + "nauc_recall_at_1_max": 0.160645, + "nauc_recall_at_1_std": 0.030675, + "nauc_recall_at_1_diff1": 0.137594, + "nauc_recall_at_3_max": 0.223479, + "nauc_recall_at_3_std": 0.162395, + "nauc_recall_at_3_diff1": 0.233809, + "nauc_recall_at_5_max": 0.15335, + "nauc_recall_at_5_std": 0.082208, + "nauc_recall_at_5_diff1": 0.182958, + "nauc_recall_at_10_max": 0.137412, + "nauc_recall_at_10_std": 0.12111, + "nauc_recall_at_10_diff1": 0.142135, + "nauc_recall_at_20_max": 0.078166, + "nauc_recall_at_20_std": 0.174135, + "nauc_recall_at_20_diff1": 0.115309, + "nauc_recall_at_100_max": -0.025721, + "nauc_recall_at_100_std": 0.2852, + "nauc_recall_at_100_diff1": 0.03211, + "nauc_recall_at_1000_max": -0.025721, + "nauc_recall_at_1000_std": 0.2852, + "nauc_recall_at_1000_diff1": 0.03211, + "nauc_precision_at_1_max": 0.134472, + "nauc_precision_at_1_std": 0.130287, + "nauc_precision_at_1_diff1": 0.164914, + "nauc_precision_at_3_max": 0.09297, + "nauc_precision_at_3_std": 0.210478, + "nauc_precision_at_3_diff1": 0.141339, + "nauc_precision_at_5_max": 0.018101, + "nauc_precision_at_5_std": 0.133387, + "nauc_precision_at_5_diff1": 0.096337, + "nauc_precision_at_10_max": 0.002023, + "nauc_precision_at_10_std": 0.155554, + "nauc_precision_at_10_diff1": 0.075522, + "nauc_precision_at_20_max": -0.028143, + "nauc_precision_at_20_std": 0.1967, + "nauc_precision_at_20_diff1": 0.037996, + "nauc_precision_at_100_max": -0.082528, + "nauc_precision_at_100_std": 0.209477, + "nauc_precision_at_100_diff1": -0.017283, + "nauc_precision_at_1000_max": -0.082528, + "nauc_precision_at_1000_std": 0.209477, + "nauc_precision_at_1000_diff1": -0.017283, + "nauc_mrr_at_1_max": 0.134472, + "nauc_mrr_at_1_std": 0.130287, + "nauc_mrr_at_1_diff1": 0.164914, + "nauc_mrr_at_3_max": 0.178991, + "nauc_mrr_at_3_std": 0.194805, + "nauc_mrr_at_3_diff1": 0.154422, + "nauc_mrr_at_5_max": 0.15751, + "nauc_mrr_at_5_std": 0.170214, + "nauc_mrr_at_5_diff1": 0.138381, + "nauc_mrr_at_10_max": 0.162712, + "nauc_mrr_at_10_std": 0.16714, + "nauc_mrr_at_10_diff1": 0.145802, + "nauc_mrr_at_20_max": 0.15652, + "nauc_mrr_at_20_std": 0.162084, + "nauc_mrr_at_20_diff1": 0.146942, + "nauc_mrr_at_100_max": 0.154986, + "nauc_mrr_at_100_std": 0.164975, + "nauc_mrr_at_100_diff1": 0.144303, + "nauc_mrr_at_1000_max": 0.154986, + "nauc_mrr_at_1000_std": 0.164975, + "nauc_mrr_at_1000_diff1": 0.144303, + "main_score": 0.34806, + "hf_subset": "biology", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.4569, + "ndcg_at_3": 0.45688, + "ndcg_at_5": 0.46634, + "ndcg_at_10": 0.46916, + "ndcg_at_20": 0.49348, + "ndcg_at_100": 0.54209, + "ndcg_at_1000": 0.54209, + "map_at_1": 0.17879, + "map_at_3": 0.28479, + "map_at_5": 0.33966, + "map_at_10": 0.36457, + "map_at_20": 0.38304, + "map_at_100": 0.39922, + "map_at_1000": 0.39922, + "recall_at_1": 0.17879, + "recall_at_3": 0.33543, + "recall_at_5": 0.43877, + "recall_at_10": 0.50006, + "recall_at_20": 0.58093, + "recall_at_100": 0.73285, + "recall_at_1000": 0.73285, + "precision_at_1": 0.4569, + "precision_at_3": 0.3592, + "precision_at_5": 0.30345, + "precision_at_10": 0.18966, + "precision_at_20": 0.11897, + "precision_at_100": 0.03362, + "precision_at_1000": 0.00336, + "mrr_at_1": 0.456897, + "mrr_at_3": 0.557471, + "mrr_at_5": 0.573851, + "mrr_at_10": 0.579502, + "mrr_at_20": 0.581846, + "mrr_at_100": 0.583811, + "mrr_at_1000": 0.583811, + "nauc_ndcg_at_1_max": 0.348589, + "nauc_ndcg_at_1_std": 0.197427, + "nauc_ndcg_at_1_diff1": 0.459481, + "nauc_ndcg_at_3_max": 0.295108, + "nauc_ndcg_at_3_std": 0.22705, + "nauc_ndcg_at_3_diff1": 0.381515, + "nauc_ndcg_at_5_max": 0.295448, + "nauc_ndcg_at_5_std": 0.267625, + "nauc_ndcg_at_5_diff1": 0.342264, + "nauc_ndcg_at_10_max": 0.267108, + "nauc_ndcg_at_10_std": 0.250806, + "nauc_ndcg_at_10_diff1": 0.338535, + "nauc_ndcg_at_20_max": 0.285449, + "nauc_ndcg_at_20_std": 0.26871, + "nauc_ndcg_at_20_diff1": 0.343829, + "nauc_ndcg_at_100_max": 0.28452, + "nauc_ndcg_at_100_std": 0.297217, + "nauc_ndcg_at_100_diff1": 0.321211, + "nauc_ndcg_at_1000_max": 0.28452, + "nauc_ndcg_at_1000_std": 0.297217, + "nauc_ndcg_at_1000_diff1": 0.321211, + "nauc_map_at_1_max": 0.214557, + "nauc_map_at_1_std": 0.096914, + "nauc_map_at_1_diff1": 0.511221, + "nauc_map_at_3_max": 0.277323, + "nauc_map_at_3_std": 0.202344, + "nauc_map_at_3_diff1": 0.464144, + "nauc_map_at_5_max": 0.305164, + "nauc_map_at_5_std": 0.23943, + "nauc_map_at_5_diff1": 0.414835, + "nauc_map_at_10_max": 0.290973, + "nauc_map_at_10_std": 0.245189, + "nauc_map_at_10_diff1": 0.387685, + "nauc_map_at_20_max": 0.301782, + "nauc_map_at_20_std": 0.255265, + "nauc_map_at_20_diff1": 0.384041, + "nauc_map_at_100_max": 0.30436, + "nauc_map_at_100_std": 0.266723, + "nauc_map_at_100_diff1": 0.376755, + "nauc_map_at_1000_max": 0.30436, + "nauc_map_at_1000_std": 0.266723, + "nauc_map_at_1000_diff1": 0.376755, + "nauc_recall_at_1_max": 0.214557, + "nauc_recall_at_1_std": 0.096914, + "nauc_recall_at_1_diff1": 0.511221, + "nauc_recall_at_3_max": 0.196553, + "nauc_recall_at_3_std": 0.182456, + "nauc_recall_at_3_diff1": 0.360517, + "nauc_recall_at_5_max": 0.22619, + "nauc_recall_at_5_std": 0.252689, + "nauc_recall_at_5_diff1": 0.281595, + "nauc_recall_at_10_max": 0.18731, + "nauc_recall_at_10_std": 0.245981, + "nauc_recall_at_10_diff1": 0.229308, + "nauc_recall_at_20_max": 0.222439, + "nauc_recall_at_20_std": 0.284421, + "nauc_recall_at_20_diff1": 0.197109, + "nauc_recall_at_100_max": 0.176517, + "nauc_recall_at_100_std": 0.389725, + "nauc_recall_at_100_diff1": 0.084546, + "nauc_recall_at_1000_max": 0.176517, + "nauc_recall_at_1000_std": 0.389725, + "nauc_recall_at_1000_diff1": 0.084546, + "nauc_precision_at_1_max": 0.348589, + "nauc_precision_at_1_std": 0.197427, + "nauc_precision_at_1_diff1": 0.459481, + "nauc_precision_at_3_max": 0.268934, + "nauc_precision_at_3_std": 0.254617, + "nauc_precision_at_3_diff1": 0.183447, + "nauc_precision_at_5_max": 0.227913, + "nauc_precision_at_5_std": 0.256818, + "nauc_precision_at_5_diff1": 0.061038, + "nauc_precision_at_10_max": 0.143331, + "nauc_precision_at_10_std": 0.204826, + "nauc_precision_at_10_diff1": -0.020851, + "nauc_precision_at_20_max": 0.106184, + "nauc_precision_at_20_std": 0.168973, + "nauc_precision_at_20_diff1": -0.048573, + "nauc_precision_at_100_max": 0.037741, + "nauc_precision_at_100_std": 0.134932, + "nauc_precision_at_100_diff1": -0.128181, + "nauc_precision_at_1000_max": 0.037741, + "nauc_precision_at_1000_std": 0.134932, + "nauc_precision_at_1000_diff1": -0.128181, + "nauc_mrr_at_1_max": 0.348589, + "nauc_mrr_at_1_std": 0.197427, + "nauc_mrr_at_1_diff1": 0.459481, + "nauc_mrr_at_3_max": 0.284183, + "nauc_mrr_at_3_std": 0.19121, + "nauc_mrr_at_3_diff1": 0.381561, + "nauc_mrr_at_5_max": 0.29276, + "nauc_mrr_at_5_std": 0.213008, + "nauc_mrr_at_5_diff1": 0.396166, + "nauc_mrr_at_10_max": 0.286579, + "nauc_mrr_at_10_std": 0.204953, + "nauc_mrr_at_10_diff1": 0.396008, + "nauc_mrr_at_20_max": 0.286533, + "nauc_mrr_at_20_std": 0.207658, + "nauc_mrr_at_20_diff1": 0.395955, + "nauc_mrr_at_100_max": 0.286547, + "nauc_mrr_at_100_std": 0.208101, + "nauc_mrr_at_100_diff1": 0.393837, + "nauc_mrr_at_1000_max": 0.286547, + "nauc_mrr_at_1000_std": 0.208101, + "nauc_mrr_at_1000_diff1": 0.393837, + "main_score": 0.46916, + "hf_subset": "earth_science", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.17476, + "ndcg_at_3": 0.19258, + "ndcg_at_5": 0.20932, + "ndcg_at_10": 0.2339, + "ndcg_at_20": 0.25726, + "ndcg_at_100": 0.32931, + "ndcg_at_1000": 0.32931, + "map_at_1": 0.05818, + "map_at_3": 0.09918, + "map_at_5": 0.11981, + "map_at_10": 0.14244, + "map_at_20": 0.15769, + "map_at_100": 0.19021, + "map_at_1000": 0.19021, + "recall_at_1": 0.05818, + "recall_at_3": 0.14092, + "recall_at_5": 0.19222, + "recall_at_10": 0.28531, + "recall_at_20": 0.35322, + "recall_at_100": 0.63254, + "recall_at_1000": 0.63254, + "precision_at_1": 0.17476, + "precision_at_3": 0.1521, + "precision_at_5": 0.14369, + "precision_at_10": 0.1165, + "precision_at_20": 0.09466, + "precision_at_100": 0.04573, + "precision_at_1000": 0.00457, + "mrr_at_1": 0.174757, + "mrr_at_3": 0.226537, + "mrr_at_5": 0.24644, + "mrr_at_10": 0.262494, + "mrr_at_20": 0.267064, + "mrr_at_100": 0.2724, + "mrr_at_1000": 0.2724, + "nauc_ndcg_at_1_max": 0.090132, + "nauc_ndcg_at_1_std": 0.0251, + "nauc_ndcg_at_1_diff1": 0.113061, + "nauc_ndcg_at_3_max": 0.108356, + "nauc_ndcg_at_3_std": 0.028404, + "nauc_ndcg_at_3_diff1": 0.151501, + "nauc_ndcg_at_5_max": 0.138411, + "nauc_ndcg_at_5_std": 0.048975, + "nauc_ndcg_at_5_diff1": 0.189362, + "nauc_ndcg_at_10_max": 0.172821, + "nauc_ndcg_at_10_std": 0.092721, + "nauc_ndcg_at_10_diff1": 0.242428, + "nauc_ndcg_at_20_max": 0.187069, + "nauc_ndcg_at_20_std": 0.064595, + "nauc_ndcg_at_20_diff1": 0.262895, + "nauc_ndcg_at_100_max": 0.210358, + "nauc_ndcg_at_100_std": 0.051701, + "nauc_ndcg_at_100_diff1": 0.26856, + "nauc_ndcg_at_1000_max": 0.210358, + "nauc_ndcg_at_1000_std": 0.051701, + "nauc_ndcg_at_1000_diff1": 0.26856, + "nauc_map_at_1_max": -0.149001, + "nauc_map_at_1_std": -0.062812, + "nauc_map_at_1_diff1": 0.159731, + "nauc_map_at_3_max": -0.014819, + "nauc_map_at_3_std": 0.002007, + "nauc_map_at_3_diff1": 0.17013, + "nauc_map_at_5_max": 0.009449, + "nauc_map_at_5_std": 0.028013, + "nauc_map_at_5_diff1": 0.179964, + "nauc_map_at_10_max": 0.053332, + "nauc_map_at_10_std": 0.049503, + "nauc_map_at_10_diff1": 0.19356, + "nauc_map_at_20_max": 0.07942, + "nauc_map_at_20_std": 0.048111, + "nauc_map_at_20_diff1": 0.204822, + "nauc_map_at_100_max": 0.156869, + "nauc_map_at_100_std": 0.076731, + "nauc_map_at_100_diff1": 0.249642, + "nauc_map_at_1000_max": 0.156869, + "nauc_map_at_1000_std": 0.076731, + "nauc_map_at_1000_diff1": 0.249642, + "nauc_recall_at_1_max": -0.149001, + "nauc_recall_at_1_std": -0.062812, + "nauc_recall_at_1_diff1": 0.159731, + "nauc_recall_at_3_max": 0.065065, + "nauc_recall_at_3_std": 0.089813, + "nauc_recall_at_3_diff1": 0.184374, + "nauc_recall_at_5_max": 0.127763, + "nauc_recall_at_5_std": 0.129521, + "nauc_recall_at_5_diff1": 0.227703, + "nauc_recall_at_10_max": 0.172843, + "nauc_recall_at_10_std": 0.17113, + "nauc_recall_at_10_diff1": 0.263538, + "nauc_recall_at_20_max": 0.191535, + "nauc_recall_at_20_std": 0.121269, + "nauc_recall_at_20_diff1": 0.292218, + "nauc_recall_at_100_max": 0.314138, + "nauc_recall_at_100_std": 0.127628, + "nauc_recall_at_100_diff1": 0.28713, + "nauc_recall_at_1000_max": 0.314138, + "nauc_recall_at_1000_std": 0.127628, + "nauc_recall_at_1000_diff1": 0.28713, + "nauc_precision_at_1_max": 0.090132, + "nauc_precision_at_1_std": 0.0251, + "nauc_precision_at_1_diff1": 0.113061, + "nauc_precision_at_3_max": 0.149074, + "nauc_precision_at_3_std": 0.014466, + "nauc_precision_at_3_diff1": 0.129178, + "nauc_precision_at_5_max": 0.193341, + "nauc_precision_at_5_std": 0.008257, + "nauc_precision_at_5_diff1": 0.127064, + "nauc_precision_at_10_max": 0.272103, + "nauc_precision_at_10_std": 0.058857, + "nauc_precision_at_10_diff1": 0.144841, + "nauc_precision_at_20_max": 0.322254, + "nauc_precision_at_20_std": 0.043143, + "nauc_precision_at_20_diff1": 0.188417, + "nauc_precision_at_100_max": 0.371963, + "nauc_precision_at_100_std": 0.022072, + "nauc_precision_at_100_diff1": 0.231166, + "nauc_precision_at_1000_max": 0.371963, + "nauc_precision_at_1000_std": 0.022072, + "nauc_precision_at_1000_diff1": 0.231166, + "nauc_mrr_at_1_max": 0.090132, + "nauc_mrr_at_1_std": 0.0251, + "nauc_mrr_at_1_diff1": 0.113061, + "nauc_mrr_at_3_max": 0.109204, + "nauc_mrr_at_3_std": -0.001196, + "nauc_mrr_at_3_diff1": 0.169469, + "nauc_mrr_at_5_max": 0.127295, + "nauc_mrr_at_5_std": -0.002047, + "nauc_mrr_at_5_diff1": 0.173412, + "nauc_mrr_at_10_max": 0.124984, + "nauc_mrr_at_10_std": -0.005627, + "nauc_mrr_at_10_diff1": 0.175228, + "nauc_mrr_at_20_max": 0.125548, + "nauc_mrr_at_20_std": -0.009193, + "nauc_mrr_at_20_diff1": 0.180928, + "nauc_mrr_at_100_max": 0.127297, + "nauc_mrr_at_100_std": -0.009734, + "nauc_mrr_at_100_diff1": 0.181454, + "nauc_mrr_at_1000_max": 0.127297, + "nauc_mrr_at_1000_std": -0.009734, + "nauc_mrr_at_1000_diff1": 0.181454, + "main_score": 0.2339, + "hf_subset": "economics", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.28713, + "ndcg_at_3": 0.27448, + "ndcg_at_5": 0.29157, + "ndcg_at_10": 0.31602, + "ndcg_at_20": 0.34107, + "ndcg_at_100": 0.39457, + "ndcg_at_1000": 0.39457, + "map_at_1": 0.11846, + "map_at_3": 0.17495, + "map_at_5": 0.19984, + "map_at_10": 0.23256, + "map_at_20": 0.25422, + "map_at_100": 0.2692, + "map_at_1000": 0.2692, + "recall_at_1": 0.11846, + "recall_at_3": 0.20551, + "recall_at_5": 0.26138, + "recall_at_10": 0.35864, + "recall_at_20": 0.4734, + "recall_at_100": 0.68348, + "recall_at_1000": 0.68348, + "precision_at_1": 0.28713, + "precision_at_3": 0.19802, + "precision_at_5": 0.17624, + "precision_at_10": 0.13762, + "precision_at_20": 0.09406, + "precision_at_100": 0.03307, + "precision_at_1000": 0.00331, + "mrr_at_1": 0.287129, + "mrr_at_3": 0.321782, + "mrr_at_5": 0.337624, + "mrr_at_10": 0.351025, + "mrr_at_20": 0.358863, + "mrr_at_100": 0.363396, + "mrr_at_1000": 0.363396, + "nauc_ndcg_at_1_max": 0.189017, + "nauc_ndcg_at_1_std": 0.259005, + "nauc_ndcg_at_1_diff1": 0.316923, + "nauc_ndcg_at_3_max": 0.295867, + "nauc_ndcg_at_3_std": 0.33165, + "nauc_ndcg_at_3_diff1": 0.299145, + "nauc_ndcg_at_5_max": 0.264012, + "nauc_ndcg_at_5_std": 0.307916, + "nauc_ndcg_at_5_diff1": 0.268527, + "nauc_ndcg_at_10_max": 0.276717, + "nauc_ndcg_at_10_std": 0.309932, + "nauc_ndcg_at_10_diff1": 0.217709, + "nauc_ndcg_at_20_max": 0.299357, + "nauc_ndcg_at_20_std": 0.347935, + "nauc_ndcg_at_20_diff1": 0.216208, + "nauc_ndcg_at_100_max": 0.289966, + "nauc_ndcg_at_100_std": 0.342306, + "nauc_ndcg_at_100_diff1": 0.235788, + "nauc_ndcg_at_1000_max": 0.289966, + "nauc_ndcg_at_1000_std": 0.342306, + "nauc_ndcg_at_1000_diff1": 0.235788, + "nauc_map_at_1_max": 0.176192, + "nauc_map_at_1_std": 0.186878, + "nauc_map_at_1_diff1": 0.348812, + "nauc_map_at_3_max": 0.290672, + "nauc_map_at_3_std": 0.262583, + "nauc_map_at_3_diff1": 0.312355, + "nauc_map_at_5_max": 0.27454, + "nauc_map_at_5_std": 0.283631, + "nauc_map_at_5_diff1": 0.296691, + "nauc_map_at_10_max": 0.281756, + "nauc_map_at_10_std": 0.319356, + "nauc_map_at_10_diff1": 0.268299, + "nauc_map_at_20_max": 0.284614, + "nauc_map_at_20_std": 0.355823, + "nauc_map_at_20_diff1": 0.261418, + "nauc_map_at_100_max": 0.293666, + "nauc_map_at_100_std": 0.358526, + "nauc_map_at_100_diff1": 0.261045, + "nauc_map_at_1000_max": 0.293666, + "nauc_map_at_1000_std": 0.358526, + "nauc_map_at_1000_diff1": 0.261045, + "nauc_recall_at_1_max": 0.176192, + "nauc_recall_at_1_std": 0.186878, + "nauc_recall_at_1_diff1": 0.348812, + "nauc_recall_at_3_max": 0.33128, + "nauc_recall_at_3_std": 0.256816, + "nauc_recall_at_3_diff1": 0.275958, + "nauc_recall_at_5_max": 0.264212, + "nauc_recall_at_5_std": 0.250267, + "nauc_recall_at_5_diff1": 0.245734, + "nauc_recall_at_10_max": 0.277144, + "nauc_recall_at_10_std": 0.248758, + "nauc_recall_at_10_diff1": 0.128997, + "nauc_recall_at_20_max": 0.318667, + "nauc_recall_at_20_std": 0.342332, + "nauc_recall_at_20_diff1": 0.060568, + "nauc_recall_at_100_max": 0.333353, + "nauc_recall_at_100_std": 0.403705, + "nauc_recall_at_100_diff1": 0.111065, + "nauc_recall_at_1000_max": 0.333353, + "nauc_recall_at_1000_std": 0.403705, + "nauc_recall_at_1000_diff1": 0.111065, + "nauc_precision_at_1_max": 0.189017, + "nauc_precision_at_1_std": 0.259005, + "nauc_precision_at_1_diff1": 0.316923, + "nauc_precision_at_3_max": 0.275261, + "nauc_precision_at_3_std": 0.358209, + "nauc_precision_at_3_diff1": 0.228778, + "nauc_precision_at_5_max": 0.188109, + "nauc_precision_at_5_std": 0.305244, + "nauc_precision_at_5_diff1": 0.151674, + "nauc_precision_at_10_max": 0.172742, + "nauc_precision_at_10_std": 0.317539, + "nauc_precision_at_10_diff1": 0.064348, + "nauc_precision_at_20_max": 0.17847, + "nauc_precision_at_20_std": 0.360606, + "nauc_precision_at_20_diff1": 0.057792, + "nauc_precision_at_100_max": 0.142362, + "nauc_precision_at_100_std": 0.187854, + "nauc_precision_at_100_diff1": 0.021978, + "nauc_precision_at_1000_max": 0.142362, + "nauc_precision_at_1000_std": 0.187854, + "nauc_precision_at_1000_diff1": 0.021978, + "nauc_mrr_at_1_max": 0.189017, + "nauc_mrr_at_1_std": 0.259005, + "nauc_mrr_at_1_diff1": 0.316923, + "nauc_mrr_at_3_max": 0.235765, + "nauc_mrr_at_3_std": 0.282722, + "nauc_mrr_at_3_diff1": 0.308274, + "nauc_mrr_at_5_max": 0.222147, + "nauc_mrr_at_5_std": 0.263458, + "nauc_mrr_at_5_diff1": 0.296748, + "nauc_mrr_at_10_max": 0.21855, + "nauc_mrr_at_10_std": 0.251757, + "nauc_mrr_at_10_diff1": 0.279546, + "nauc_mrr_at_20_max": 0.223227, + "nauc_mrr_at_20_std": 0.258301, + "nauc_mrr_at_20_diff1": 0.275005, + "nauc_mrr_at_100_max": 0.224049, + "nauc_mrr_at_100_std": 0.263364, + "nauc_mrr_at_100_diff1": 0.278341, + "nauc_mrr_at_1000_max": 0.224049, + "nauc_mrr_at_1000_std": 0.263364, + "nauc_mrr_at_1000_diff1": 0.278341, + "main_score": 0.31602, + "hf_subset": "psychology", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.18812, + "ndcg_at_3": 0.17571, + "ndcg_at_5": 0.17239, + "ndcg_at_10": 0.19096, + "ndcg_at_20": 0.22022, + "ndcg_at_100": 0.28085, + "ndcg_at_1000": 0.28085, + "map_at_1": 0.0791, + "map_at_3": 0.10896, + "map_at_5": 0.11726, + "map_at_10": 0.13075, + "map_at_20": 0.14392, + "map_at_100": 0.15733, + "map_at_1000": 0.15733, + "recall_at_1": 0.0791, + "recall_at_3": 0.12819, + "recall_at_5": 0.1538, + "recall_at_10": 0.22411, + "recall_at_20": 0.33365, + "recall_at_100": 0.55843, + "recall_at_1000": 0.55843, + "precision_at_1": 0.18812, + "precision_at_3": 0.12211, + "precision_at_5": 0.09505, + "precision_at_10": 0.07624, + "precision_at_20": 0.0604, + "precision_at_100": 0.02446, + "precision_at_1000": 0.00245, + "mrr_at_1": 0.188119, + "mrr_at_3": 0.219472, + "mrr_at_5": 0.230363, + "mrr_at_10": 0.246912, + "mrr_at_20": 0.254528, + "mrr_at_100": 0.259062, + "mrr_at_1000": 0.259062, + "nauc_ndcg_at_1_max": 0.163635, + "nauc_ndcg_at_1_std": 0.069106, + "nauc_ndcg_at_1_diff1": 0.276247, + "nauc_ndcg_at_3_max": 0.227138, + "nauc_ndcg_at_3_std": 0.16873, + "nauc_ndcg_at_3_diff1": 0.321947, + "nauc_ndcg_at_5_max": 0.235198, + "nauc_ndcg_at_5_std": 0.17977, + "nauc_ndcg_at_5_diff1": 0.300628, + "nauc_ndcg_at_10_max": 0.244317, + "nauc_ndcg_at_10_std": 0.173967, + "nauc_ndcg_at_10_diff1": 0.27078, + "nauc_ndcg_at_20_max": 0.255962, + "nauc_ndcg_at_20_std": 0.192828, + "nauc_ndcg_at_20_diff1": 0.254658, + "nauc_ndcg_at_100_max": 0.246908, + "nauc_ndcg_at_100_std": 0.204345, + "nauc_ndcg_at_100_diff1": 0.253221, + "nauc_ndcg_at_1000_max": 0.246908, + "nauc_ndcg_at_1000_std": 0.204345, + "nauc_ndcg_at_1000_diff1": 0.253221, + "nauc_map_at_1_max": 0.269368, + "nauc_map_at_1_std": 0.056196, + "nauc_map_at_1_diff1": 0.383603, + "nauc_map_at_3_max": 0.279143, + "nauc_map_at_3_std": 0.148988, + "nauc_map_at_3_diff1": 0.345256, + "nauc_map_at_5_max": 0.275577, + "nauc_map_at_5_std": 0.168798, + "nauc_map_at_5_diff1": 0.322575, + "nauc_map_at_10_max": 0.271685, + "nauc_map_at_10_std": 0.165733, + "nauc_map_at_10_diff1": 0.316126, + "nauc_map_at_20_max": 0.274098, + "nauc_map_at_20_std": 0.180189, + "nauc_map_at_20_diff1": 0.313501, + "nauc_map_at_100_max": 0.271425, + "nauc_map_at_100_std": 0.18098, + "nauc_map_at_100_diff1": 0.319492, + "nauc_map_at_1000_max": 0.271425, + "nauc_map_at_1000_std": 0.18098, + "nauc_map_at_1000_diff1": 0.319492, + "nauc_recall_at_1_max": 0.269368, + "nauc_recall_at_1_std": 0.056196, + "nauc_recall_at_1_diff1": 0.383603, + "nauc_recall_at_3_max": 0.281018, + "nauc_recall_at_3_std": 0.203989, + "nauc_recall_at_3_diff1": 0.287741, + "nauc_recall_at_5_max": 0.291536, + "nauc_recall_at_5_std": 0.244781, + "nauc_recall_at_5_diff1": 0.221505, + "nauc_recall_at_10_max": 0.22329, + "nauc_recall_at_10_std": 0.179269, + "nauc_recall_at_10_diff1": 0.13137, + "nauc_recall_at_20_max": 0.214445, + "nauc_recall_at_20_std": 0.207507, + "nauc_recall_at_20_diff1": 0.090596, + "nauc_recall_at_100_max": 0.166524, + "nauc_recall_at_100_std": 0.289593, + "nauc_recall_at_100_diff1": 0.064992, + "nauc_recall_at_1000_max": 0.166524, + "nauc_recall_at_1000_std": 0.289593, + "nauc_recall_at_1000_diff1": 0.064992, + "nauc_precision_at_1_max": 0.163635, + "nauc_precision_at_1_std": 0.069106, + "nauc_precision_at_1_diff1": 0.276247, + "nauc_precision_at_3_max": 0.20512, + "nauc_precision_at_3_std": 0.225103, + "nauc_precision_at_3_diff1": 0.285726, + "nauc_precision_at_5_max": 0.184638, + "nauc_precision_at_5_std": 0.229523, + "nauc_precision_at_5_diff1": 0.257122, + "nauc_precision_at_10_max": 0.193635, + "nauc_precision_at_10_std": 0.206974, + "nauc_precision_at_10_diff1": 0.262353, + "nauc_precision_at_20_max": 0.183852, + "nauc_precision_at_20_std": 0.22345, + "nauc_precision_at_20_diff1": 0.274323, + "nauc_precision_at_100_max": 0.067423, + "nauc_precision_at_100_std": 0.084686, + "nauc_precision_at_100_diff1": 0.155119, + "nauc_precision_at_1000_max": 0.067423, + "nauc_precision_at_1000_std": 0.084686, + "nauc_precision_at_1000_diff1": 0.155119, + "nauc_mrr_at_1_max": 0.163635, + "nauc_mrr_at_1_std": 0.069106, + "nauc_mrr_at_1_diff1": 0.276247, + "nauc_mrr_at_3_max": 0.162737, + "nauc_mrr_at_3_std": 0.100034, + "nauc_mrr_at_3_diff1": 0.237731, + "nauc_mrr_at_5_max": 0.1752, + "nauc_mrr_at_5_std": 0.108039, + "nauc_mrr_at_5_diff1": 0.236425, + "nauc_mrr_at_10_max": 0.16947, + "nauc_mrr_at_10_std": 0.104715, + "nauc_mrr_at_10_diff1": 0.220346, + "nauc_mrr_at_20_max": 0.16615, + "nauc_mrr_at_20_std": 0.099424, + "nauc_mrr_at_20_diff1": 0.227532, + "nauc_mrr_at_100_max": 0.16636, + "nauc_mrr_at_100_std": 0.102245, + "nauc_mrr_at_100_diff1": 0.225625, + "nauc_mrr_at_1000_max": 0.16636, + "nauc_mrr_at_1000_std": 0.102245, + "nauc_mrr_at_1000_diff1": 0.225625, + "main_score": 0.19096, + "hf_subset": "robotics", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.2735, + "ndcg_at_3": 0.21864, + "ndcg_at_5": 0.22748, + "ndcg_at_10": 0.25413, + "ndcg_at_20": 0.28046, + "ndcg_at_100": 0.35518, + "ndcg_at_1000": 0.35518, + "map_at_1": 0.07665, + "map_at_3": 0.12568, + "map_at_5": 0.14435, + "map_at_10": 0.16929, + "map_at_20": 0.1811, + "map_at_100": 0.20246, + "map_at_1000": 0.20246, + "recall_at_1": 0.07665, + "recall_at_3": 0.14642, + "recall_at_5": 0.18849, + "recall_at_10": 0.28981, + "recall_at_20": 0.3721, + "recall_at_100": 0.61959, + "recall_at_1000": 0.61959, + "precision_at_1": 0.2735, + "precision_at_3": 0.17664, + "precision_at_5": 0.15385, + "precision_at_10": 0.12051, + "precision_at_20": 0.08376, + "precision_at_100": 0.0353, + "precision_at_1000": 0.00353, + "mrr_at_1": 0.273504, + "mrr_at_3": 0.307692, + "mrr_at_5": 0.326496, + "mrr_at_10": 0.3442, + "mrr_at_20": 0.354145, + "mrr_at_100": 0.358174, + "mrr_at_1000": 0.358174, + "nauc_ndcg_at_1_max": 0.009922, + "nauc_ndcg_at_1_std": 0.065902, + "nauc_ndcg_at_1_diff1": 0.070884, + "nauc_ndcg_at_3_max": 0.066509, + "nauc_ndcg_at_3_std": 0.090646, + "nauc_ndcg_at_3_diff1": 0.030583, + "nauc_ndcg_at_5_max": 0.077696, + "nauc_ndcg_at_5_std": 0.137612, + "nauc_ndcg_at_5_diff1": 0.02524, + "nauc_ndcg_at_10_max": 0.05176, + "nauc_ndcg_at_10_std": 0.142785, + "nauc_ndcg_at_10_diff1": 0.017501, + "nauc_ndcg_at_20_max": 0.033397, + "nauc_ndcg_at_20_std": 0.144205, + "nauc_ndcg_at_20_diff1": 0.041218, + "nauc_ndcg_at_100_max": 0.079436, + "nauc_ndcg_at_100_std": 0.192234, + "nauc_ndcg_at_100_diff1": 0.045949, + "nauc_ndcg_at_1000_max": 0.079436, + "nauc_ndcg_at_1000_std": 0.192234, + "nauc_ndcg_at_1000_diff1": 0.045949, + "nauc_map_at_1_max": 0.079092, + "nauc_map_at_1_std": 0.156315, + "nauc_map_at_1_diff1": 0.173359, + "nauc_map_at_3_max": 0.14285, + "nauc_map_at_3_std": 0.141887, + "nauc_map_at_3_diff1": 0.170421, + "nauc_map_at_5_max": 0.120422, + "nauc_map_at_5_std": 0.154686, + "nauc_map_at_5_diff1": 0.141693, + "nauc_map_at_10_max": 0.095084, + "nauc_map_at_10_std": 0.154204, + "nauc_map_at_10_diff1": 0.12559, + "nauc_map_at_20_max": 0.08543, + "nauc_map_at_20_std": 0.147928, + "nauc_map_at_20_diff1": 0.11918, + "nauc_map_at_100_max": 0.086413, + "nauc_map_at_100_std": 0.158867, + "nauc_map_at_100_diff1": 0.108405, + "nauc_map_at_1000_max": 0.086413, + "nauc_map_at_1000_std": 0.158867, + "nauc_map_at_1000_diff1": 0.108405, + "nauc_recall_at_1_max": 0.079092, + "nauc_recall_at_1_std": 0.156315, + "nauc_recall_at_1_diff1": 0.173359, + "nauc_recall_at_3_max": 0.15105, + "nauc_recall_at_3_std": 0.177819, + "nauc_recall_at_3_diff1": 0.103443, + "nauc_recall_at_5_max": 0.102501, + "nauc_recall_at_5_std": 0.186236, + "nauc_recall_at_5_diff1": 0.065915, + "nauc_recall_at_10_max": 0.00961, + "nauc_recall_at_10_std": 0.166232, + "nauc_recall_at_10_diff1": -0.026098, + "nauc_recall_at_20_max": 0.006438, + "nauc_recall_at_20_std": 0.174972, + "nauc_recall_at_20_diff1": 0.011252, + "nauc_recall_at_100_max": 0.130018, + "nauc_recall_at_100_std": 0.312065, + "nauc_recall_at_100_diff1": 0.026461, + "nauc_recall_at_1000_max": 0.130018, + "nauc_recall_at_1000_std": 0.312065, + "nauc_recall_at_1000_diff1": 0.026461, + "nauc_precision_at_1_max": 0.009922, + "nauc_precision_at_1_std": 0.065902, + "nauc_precision_at_1_diff1": 0.070884, + "nauc_precision_at_3_max": 0.052359, + "nauc_precision_at_3_std": 0.069695, + "nauc_precision_at_3_diff1": 0.024652, + "nauc_precision_at_5_max": 0.040736, + "nauc_precision_at_5_std": 0.115615, + "nauc_precision_at_5_diff1": -0.039386, + "nauc_precision_at_10_max": 0.020719, + "nauc_precision_at_10_std": 0.067818, + "nauc_precision_at_10_diff1": -0.13075, + "nauc_precision_at_20_max": -0.03959, + "nauc_precision_at_20_std": 0.019335, + "nauc_precision_at_20_diff1": -0.156956, + "nauc_precision_at_100_max": -0.061473, + "nauc_precision_at_100_std": -0.001579, + "nauc_precision_at_100_diff1": -0.116278, + "nauc_precision_at_1000_max": -0.061473, + "nauc_precision_at_1000_std": -0.001579, + "nauc_precision_at_1000_diff1": -0.116278, + "nauc_mrr_at_1_max": 0.009922, + "nauc_mrr_at_1_std": 0.065902, + "nauc_mrr_at_1_diff1": 0.070884, + "nauc_mrr_at_3_max": 0.006462, + "nauc_mrr_at_3_std": 0.101662, + "nauc_mrr_at_3_diff1": 0.025118, + "nauc_mrr_at_5_max": -0.001629, + "nauc_mrr_at_5_std": 0.099911, + "nauc_mrr_at_5_diff1": 0.011651, + "nauc_mrr_at_10_max": -0.0063, + "nauc_mrr_at_10_std": 0.094784, + "nauc_mrr_at_10_diff1": -0.013188, + "nauc_mrr_at_20_max": -0.005532, + "nauc_mrr_at_20_std": 0.098312, + "nauc_mrr_at_20_diff1": -0.005801, + "nauc_mrr_at_100_max": -0.002711, + "nauc_mrr_at_100_std": 0.098218, + "nauc_mrr_at_100_diff1": -0.003241, + "nauc_mrr_at_1000_max": -0.002711, + "nauc_mrr_at_1000_std": 0.098218, + "nauc_mrr_at_1000_diff1": -0.003241, + "main_score": 0.25413, + "hf_subset": "stackoverflow", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.13889, + "ndcg_at_3": 0.17562, + "ndcg_at_5": 0.18722, + "ndcg_at_10": 0.2102, + "ndcg_at_20": 0.25522, + "ndcg_at_100": 0.31639, + "ndcg_at_1000": 0.31639, + "map_at_1": 0.0545, + "map_at_3": 0.10967, + "map_at_5": 0.12646, + "map_at_10": 0.14775, + "map_at_20": 0.1669, + "map_at_100": 0.18277, + "map_at_1000": 0.18277, + "recall_at_1": 0.0545, + "recall_at_3": 0.15545, + "recall_at_5": 0.20169, + "recall_at_10": 0.27486, + "recall_at_20": 0.41328, + "recall_at_100": 0.61708, + "recall_at_1000": 0.61708, + "precision_at_1": 0.13889, + "precision_at_3": 0.14506, + "precision_at_5": 0.12222, + "precision_at_10": 0.09167, + "precision_at_20": 0.07269, + "precision_at_100": 0.0275, + "precision_at_1000": 0.00275, + "mrr_at_1": 0.138889, + "mrr_at_3": 0.208333, + "mrr_at_5": 0.226389, + "mrr_at_10": 0.236714, + "mrr_at_20": 0.248031, + "mrr_at_100": 0.252467, + "mrr_at_1000": 0.252467, + "nauc_ndcg_at_1_max": 0.301347, + "nauc_ndcg_at_1_std": 0.257799, + "nauc_ndcg_at_1_diff1": 0.351481, + "nauc_ndcg_at_3_max": 0.253299, + "nauc_ndcg_at_3_std": 0.237548, + "nauc_ndcg_at_3_diff1": 0.307942, + "nauc_ndcg_at_5_max": 0.275571, + "nauc_ndcg_at_5_std": 0.253464, + "nauc_ndcg_at_5_diff1": 0.27904, + "nauc_ndcg_at_10_max": 0.267982, + "nauc_ndcg_at_10_std": 0.262976, + "nauc_ndcg_at_10_diff1": 0.242268, + "nauc_ndcg_at_20_max": 0.254364, + "nauc_ndcg_at_20_std": 0.283455, + "nauc_ndcg_at_20_diff1": 0.201747, + "nauc_ndcg_at_100_max": 0.311879, + "nauc_ndcg_at_100_std": 0.318527, + "nauc_ndcg_at_100_diff1": 0.243696, + "nauc_ndcg_at_1000_max": 0.311879, + "nauc_ndcg_at_1000_std": 0.318527, + "nauc_ndcg_at_1000_diff1": 0.243696, + "nauc_map_at_1_max": 0.314438, + "nauc_map_at_1_std": 0.349607, + "nauc_map_at_1_diff1": 0.533335, + "nauc_map_at_3_max": 0.264649, + "nauc_map_at_3_std": 0.314805, + "nauc_map_at_3_diff1": 0.377791, + "nauc_map_at_5_max": 0.273727, + "nauc_map_at_5_std": 0.306239, + "nauc_map_at_5_diff1": 0.342012, + "nauc_map_at_10_max": 0.269196, + "nauc_map_at_10_std": 0.296953, + "nauc_map_at_10_diff1": 0.300877, + "nauc_map_at_20_max": 0.268827, + "nauc_map_at_20_std": 0.291695, + "nauc_map_at_20_diff1": 0.272365, + "nauc_map_at_100_max": 0.285021, + "nauc_map_at_100_std": 0.306071, + "nauc_map_at_100_diff1": 0.285316, + "nauc_map_at_1000_max": 0.285021, + "nauc_map_at_1000_std": 0.306071, + "nauc_map_at_1000_diff1": 0.285316, + "nauc_recall_at_1_max": 0.314438, + "nauc_recall_at_1_std": 0.349607, + "nauc_recall_at_1_diff1": 0.533335, + "nauc_recall_at_3_max": 0.231884, + "nauc_recall_at_3_std": 0.292603, + "nauc_recall_at_3_diff1": 0.303341, + "nauc_recall_at_5_max": 0.254534, + "nauc_recall_at_5_std": 0.299027, + "nauc_recall_at_5_diff1": 0.207588, + "nauc_recall_at_10_max": 0.222371, + "nauc_recall_at_10_std": 0.26784, + "nauc_recall_at_10_diff1": 0.147093, + "nauc_recall_at_20_max": 0.17931, + "nauc_recall_at_20_std": 0.284983, + "nauc_recall_at_20_diff1": 0.045199, + "nauc_recall_at_100_max": 0.395199, + "nauc_recall_at_100_std": 0.404949, + "nauc_recall_at_100_diff1": 0.194397, + "nauc_recall_at_1000_max": 0.395199, + "nauc_recall_at_1000_std": 0.404949, + "nauc_recall_at_1000_diff1": 0.194397, + "nauc_precision_at_1_max": 0.301347, + "nauc_precision_at_1_std": 0.257799, + "nauc_precision_at_1_diff1": 0.351481, + "nauc_precision_at_3_max": 0.196636, + "nauc_precision_at_3_std": 0.173919, + "nauc_precision_at_3_diff1": 0.197737, + "nauc_precision_at_5_max": 0.228473, + "nauc_precision_at_5_std": 0.121463, + "nauc_precision_at_5_diff1": 0.170536, + "nauc_precision_at_10_max": 0.230604, + "nauc_precision_at_10_std": 0.100368, + "nauc_precision_at_10_diff1": 0.104812, + "nauc_precision_at_20_max": 0.201264, + "nauc_precision_at_20_std": 0.101982, + "nauc_precision_at_20_diff1": 0.041411, + "nauc_precision_at_100_max": 0.127633, + "nauc_precision_at_100_std": 0.08685, + "nauc_precision_at_100_diff1": 0.046244, + "nauc_precision_at_1000_max": 0.127633, + "nauc_precision_at_1000_std": 0.08685, + "nauc_precision_at_1000_diff1": 0.046244, + "nauc_mrr_at_1_max": 0.301347, + "nauc_mrr_at_1_std": 0.257799, + "nauc_mrr_at_1_diff1": 0.351481, + "nauc_mrr_at_3_max": 0.258628, + "nauc_mrr_at_3_std": 0.229253, + "nauc_mrr_at_3_diff1": 0.266325, + "nauc_mrr_at_5_max": 0.27534, + "nauc_mrr_at_5_std": 0.233215, + "nauc_mrr_at_5_diff1": 0.256801, + "nauc_mrr_at_10_max": 0.268237, + "nauc_mrr_at_10_std": 0.220736, + "nauc_mrr_at_10_diff1": 0.247114, + "nauc_mrr_at_20_max": 0.268048, + "nauc_mrr_at_20_std": 0.228085, + "nauc_mrr_at_20_diff1": 0.243593, + "nauc_mrr_at_100_max": 0.274477, + "nauc_mrr_at_100_std": 0.23534, + "nauc_mrr_at_100_diff1": 0.247093, + "nauc_mrr_at_1000_max": 0.274477, + "nauc_mrr_at_1000_std": 0.23534, + "nauc_mrr_at_1000_diff1": 0.247093, + "main_score": 0.2102, + "hf_subset": "sustainable_living", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.08036, + "ndcg_at_3": 0.06514, + "ndcg_at_5": 0.05672, + "ndcg_at_10": 0.04932, + "ndcg_at_20": 0.04978, + "ndcg_at_100": 0.09629, + "ndcg_at_1000": 0.09629, + "map_at_1": 0.00532, + "map_at_3": 0.00957, + "map_at_5": 0.01151, + "map_at_10": 0.01377, + "map_at_20": 0.01565, + "map_at_100": 0.02179, + "map_at_1000": 0.02179, + "recall_at_1": 0.00532, + "recall_at_3": 0.01108, + "recall_at_5": 0.0165, + "recall_at_10": 0.02574, + "recall_at_20": 0.0404, + "recall_at_100": 0.13945, + "recall_at_1000": 0.13945, + "precision_at_1": 0.08036, + "precision_at_3": 0.05952, + "precision_at_5": 0.05, + "precision_at_10": 0.04196, + "precision_at_20": 0.0375, + "precision_at_100": 0.02857, + "precision_at_1000": 0.00286, + "mrr_at_1": 0.080357, + "mrr_at_3": 0.105655, + "mrr_at_5": 0.116815, + "mrr_at_10": 0.131955, + "mrr_at_20": 0.143401, + "mrr_at_100": 0.152607, + "mrr_at_1000": 0.152607, + "nauc_ndcg_at_1_max": 0.144804, + "nauc_ndcg_at_1_std": 0.091717, + "nauc_ndcg_at_1_diff1": 0.058905, + "nauc_ndcg_at_3_max": 0.245378, + "nauc_ndcg_at_3_std": 0.169867, + "nauc_ndcg_at_3_diff1": 0.03807, + "nauc_ndcg_at_5_max": 0.277354, + "nauc_ndcg_at_5_std": 0.231918, + "nauc_ndcg_at_5_diff1": 0.089208, + "nauc_ndcg_at_10_max": 0.262113, + "nauc_ndcg_at_10_std": 0.249227, + "nauc_ndcg_at_10_diff1": 0.05236, + "nauc_ndcg_at_20_max": 0.249364, + "nauc_ndcg_at_20_std": 0.240078, + "nauc_ndcg_at_20_diff1": 0.075804, + "nauc_ndcg_at_100_max": 0.201205, + "nauc_ndcg_at_100_std": 0.187971, + "nauc_ndcg_at_100_diff1": 0.029022, + "nauc_ndcg_at_1000_max": 0.201205, + "nauc_ndcg_at_1000_std": 0.187971, + "nauc_ndcg_at_1000_diff1": 0.029022, + "nauc_map_at_1_max": 0.183379, + "nauc_map_at_1_std": 0.176537, + "nauc_map_at_1_diff1": 0.031815, + "nauc_map_at_3_max": 0.275608, + "nauc_map_at_3_std": 0.266502, + "nauc_map_at_3_diff1": 0.017634, + "nauc_map_at_5_max": 0.305043, + "nauc_map_at_5_std": 0.35145, + "nauc_map_at_5_diff1": 0.066593, + "nauc_map_at_10_max": 0.284364, + "nauc_map_at_10_std": 0.367724, + "nauc_map_at_10_diff1": 0.051497, + "nauc_map_at_20_max": 0.283311, + "nauc_map_at_20_std": 0.350968, + "nauc_map_at_20_diff1": 0.06125, + "nauc_map_at_100_max": 0.26147, + "nauc_map_at_100_std": 0.307339, + "nauc_map_at_100_diff1": 0.034731, + "nauc_map_at_1000_max": 0.26147, + "nauc_map_at_1000_std": 0.307339, + "nauc_map_at_1000_diff1": 0.034731, + "nauc_recall_at_1_max": 0.183379, + "nauc_recall_at_1_std": 0.176537, + "nauc_recall_at_1_diff1": 0.031815, + "nauc_recall_at_3_max": 0.300565, + "nauc_recall_at_3_std": 0.254386, + "nauc_recall_at_3_diff1": 0.022078, + "nauc_recall_at_5_max": 0.372703, + "nauc_recall_at_5_std": 0.410167, + "nauc_recall_at_5_diff1": 0.172195, + "nauc_recall_at_10_max": 0.290352, + "nauc_recall_at_10_std": 0.365075, + "nauc_recall_at_10_diff1": 0.077822, + "nauc_recall_at_20_max": 0.252747, + "nauc_recall_at_20_std": 0.276598, + "nauc_recall_at_20_diff1": 0.104066, + "nauc_recall_at_100_max": 0.151598, + "nauc_recall_at_100_std": 0.163491, + "nauc_recall_at_100_diff1": 0.023257, + "nauc_recall_at_1000_max": 0.151598, + "nauc_recall_at_1000_std": 0.163491, + "nauc_recall_at_1000_diff1": 0.023257, + "nauc_precision_at_1_max": 0.144804, + "nauc_precision_at_1_std": 0.091717, + "nauc_precision_at_1_diff1": 0.058905, + "nauc_precision_at_3_max": 0.28273, + "nauc_precision_at_3_std": 0.192269, + "nauc_precision_at_3_diff1": 0.044956, + "nauc_precision_at_5_max": 0.334518, + "nauc_precision_at_5_std": 0.298597, + "nauc_precision_at_5_diff1": 0.136806, + "nauc_precision_at_10_max": 0.273969, + "nauc_precision_at_10_std": 0.288257, + "nauc_precision_at_10_diff1": 0.035833, + "nauc_precision_at_20_max": 0.216387, + "nauc_precision_at_20_std": 0.180191, + "nauc_precision_at_20_diff1": 0.072216, + "nauc_precision_at_100_max": 0.114596, + "nauc_precision_at_100_std": 0.077941, + "nauc_precision_at_100_diff1": -0.015369, + "nauc_precision_at_1000_max": 0.114596, + "nauc_precision_at_1000_std": 0.077941, + "nauc_precision_at_1000_diff1": -0.015369, + "nauc_mrr_at_1_max": 0.144804, + "nauc_mrr_at_1_std": 0.091717, + "nauc_mrr_at_1_diff1": 0.058905, + "nauc_mrr_at_3_max": 0.191588, + "nauc_mrr_at_3_std": 0.110199, + "nauc_mrr_at_3_diff1": 0.029126, + "nauc_mrr_at_5_max": 0.213341, + "nauc_mrr_at_5_std": 0.135857, + "nauc_mrr_at_5_diff1": 0.064151, + "nauc_mrr_at_10_max": 0.197962, + "nauc_mrr_at_10_std": 0.124133, + "nauc_mrr_at_10_diff1": 0.035247, + "nauc_mrr_at_20_max": 0.198451, + "nauc_mrr_at_20_std": 0.118903, + "nauc_mrr_at_20_diff1": 0.050332, + "nauc_mrr_at_100_max": 0.193167, + "nauc_mrr_at_100_std": 0.113923, + "nauc_mrr_at_100_diff1": 0.047035, + "nauc_mrr_at_1000_max": 0.193167, + "nauc_mrr_at_1000_std": 0.113923, + "nauc_mrr_at_1000_diff1": 0.047035, + "main_score": 0.04932, + "hf_subset": "pony", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.39437, + "ndcg_at_3": 0.38345, + "ndcg_at_5": 0.40184, + "ndcg_at_10": 0.43192, + "ndcg_at_20": 0.4673, + "ndcg_at_100": 0.49559, + "ndcg_at_1000": 0.49559, + "map_at_1": 0.23122, + "map_at_3": 0.3151, + "map_at_5": 0.33801, + "map_at_10": 0.35396, + "map_at_20": 0.36612, + "map_at_100": 0.37162, + "map_at_1000": 0.37162, + "recall_at_1": 0.23122, + "recall_at_3": 0.3865, + "recall_at_5": 0.4439, + "recall_at_10": 0.51913, + "recall_at_20": 0.6392, + "recall_at_100": 0.76878, + "recall_at_1000": 0.76878, + "precision_at_1": 0.39437, + "precision_at_3": 0.23239, + "precision_at_5": 0.16338, + "precision_at_10": 0.09718, + "precision_at_20": 0.05986, + "precision_at_100": 0.01451, + "precision_at_1000": 0.00145, + "mrr_at_1": 0.394366, + "mrr_at_3": 0.483568, + "mrr_at_5": 0.491667, + "mrr_at_10": 0.500447, + "mrr_at_20": 0.507099, + "mrr_at_100": 0.5089, + "mrr_at_1000": 0.5089, + "nauc_ndcg_at_1_max": 0.288343, + "nauc_ndcg_at_1_std": 0.26545, + "nauc_ndcg_at_1_diff1": 0.275726, + "nauc_ndcg_at_3_max": 0.26264, + "nauc_ndcg_at_3_std": 0.234856, + "nauc_ndcg_at_3_diff1": 0.204717, + "nauc_ndcg_at_5_max": 0.21567, + "nauc_ndcg_at_5_std": 0.183853, + "nauc_ndcg_at_5_diff1": 0.189961, + "nauc_ndcg_at_10_max": 0.222382, + "nauc_ndcg_at_10_std": 0.189984, + "nauc_ndcg_at_10_diff1": 0.17507, + "nauc_ndcg_at_20_max": 0.248167, + "nauc_ndcg_at_20_std": 0.230532, + "nauc_ndcg_at_20_diff1": 0.177408, + "nauc_ndcg_at_100_max": 0.243363, + "nauc_ndcg_at_100_std": 0.224494, + "nauc_ndcg_at_100_diff1": 0.165421, + "nauc_ndcg_at_1000_max": 0.243363, + "nauc_ndcg_at_1000_std": 0.224494, + "nauc_ndcg_at_1000_diff1": 0.165421, + "nauc_map_at_1_max": 0.17318, + "nauc_map_at_1_std": 0.116191, + "nauc_map_at_1_diff1": 0.237586, + "nauc_map_at_3_max": 0.221018, + "nauc_map_at_3_std": 0.18059, + "nauc_map_at_3_diff1": 0.209163, + "nauc_map_at_5_max": 0.208084, + "nauc_map_at_5_std": 0.173451, + "nauc_map_at_5_diff1": 0.199585, + "nauc_map_at_10_max": 0.218356, + "nauc_map_at_10_std": 0.184456, + "nauc_map_at_10_diff1": 0.193259, + "nauc_map_at_20_max": 0.229681, + "nauc_map_at_20_std": 0.200776, + "nauc_map_at_20_diff1": 0.193484, + "nauc_map_at_100_max": 0.230743, + "nauc_map_at_100_std": 0.202023, + "nauc_map_at_100_diff1": 0.191953, + "nauc_map_at_1000_max": 0.230743, + "nauc_map_at_1000_std": 0.202023, + "nauc_map_at_1000_diff1": 0.191953, + "nauc_recall_at_1_max": 0.17318, + "nauc_recall_at_1_std": 0.116191, + "nauc_recall_at_1_diff1": 0.237586, + "nauc_recall_at_3_max": 0.171818, + "nauc_recall_at_3_std": 0.144629, + "nauc_recall_at_3_diff1": 0.16395, + "nauc_recall_at_5_max": 0.157815, + "nauc_recall_at_5_std": 0.122264, + "nauc_recall_at_5_diff1": 0.144459, + "nauc_recall_at_10_max": 0.150374, + "nauc_recall_at_10_std": 0.111651, + "nauc_recall_at_10_diff1": 0.093922, + "nauc_recall_at_20_max": 0.210894, + "nauc_recall_at_20_std": 0.22091, + "nauc_recall_at_20_diff1": 0.078379, + "nauc_recall_at_100_max": 0.14073, + "nauc_recall_at_100_std": 0.139056, + "nauc_recall_at_100_diff1": -0.034541, + "nauc_recall_at_1000_max": 0.14073, + "nauc_recall_at_1000_std": 0.139056, + "nauc_recall_at_1000_diff1": -0.034541, + "nauc_precision_at_1_max": 0.288343, + "nauc_precision_at_1_std": 0.26545, + "nauc_precision_at_1_diff1": 0.275726, + "nauc_precision_at_3_max": 0.328136, + "nauc_precision_at_3_std": 0.359259, + "nauc_precision_at_3_diff1": 0.10115, + "nauc_precision_at_5_max": 0.262719, + "nauc_precision_at_5_std": 0.291358, + "nauc_precision_at_5_diff1": 0.077042, + "nauc_precision_at_10_max": 0.264829, + "nauc_precision_at_10_std": 0.28576, + "nauc_precision_at_10_diff1": 0.04395, + "nauc_precision_at_20_max": 0.310977, + "nauc_precision_at_20_std": 0.372329, + "nauc_precision_at_20_diff1": 0.037483, + "nauc_precision_at_100_max": 0.275693, + "nauc_precision_at_100_std": 0.336466, + "nauc_precision_at_100_diff1": 0.004351, + "nauc_precision_at_1000_max": 0.275693, + "nauc_precision_at_1000_std": 0.336466, + "nauc_precision_at_1000_diff1": 0.004351, + "nauc_mrr_at_1_max": 0.288343, + "nauc_mrr_at_1_std": 0.26545, + "nauc_mrr_at_1_diff1": 0.275726, + "nauc_mrr_at_3_max": 0.273246, + "nauc_mrr_at_3_std": 0.25554, + "nauc_mrr_at_3_diff1": 0.226368, + "nauc_mrr_at_5_max": 0.280918, + "nauc_mrr_at_5_std": 0.255186, + "nauc_mrr_at_5_diff1": 0.225419, + "nauc_mrr_at_10_max": 0.276576, + "nauc_mrr_at_10_std": 0.252011, + "nauc_mrr_at_10_diff1": 0.216455, + "nauc_mrr_at_20_max": 0.278725, + "nauc_mrr_at_20_std": 0.258419, + "nauc_mrr_at_20_diff1": 0.217967, + "nauc_mrr_at_100_max": 0.276405, + "nauc_mrr_at_100_std": 0.255861, + "nauc_mrr_at_100_diff1": 0.216452, + "nauc_mrr_at_1000_max": 0.276405, + "nauc_mrr_at_1000_std": 0.255861, + "nauc_mrr_at_1000_diff1": 0.216452, + "main_score": 0.43192, + "hf_subset": "leetcode", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.26154, + "ndcg_at_3": 0.20993, + "ndcg_at_5": 0.28324, + "ndcg_at_10": 0.30536, + "ndcg_at_20": 0.3332, + "ndcg_at_100": 0.39121, + "ndcg_at_1000": 0.39121, + "map_at_1": 0.14282, + "map_at_3": 0.17573, + "map_at_5": 0.22744, + "map_at_10": 0.2411, + "map_at_20": 0.25099, + "map_at_100": 0.26093, + "map_at_1000": 0.26093, + "recall_at_1": 0.14282, + "recall_at_3": 0.1941, + "recall_at_5": 0.35154, + "recall_at_10": 0.40462, + "recall_at_20": 0.49615, + "recall_at_100": 0.77282, + "recall_at_1000": 0.77282, + "precision_at_1": 0.26154, + "precision_at_3": 0.12308, + "precision_at_5": 0.13538, + "precision_at_10": 0.08154, + "precision_at_20": 0.05077, + "precision_at_100": 0.01477, + "precision_at_1000": 0.00148, + "mrr_at_1": 0.261538, + "mrr_at_3": 0.287179, + "mrr_at_5": 0.319487, + "mrr_at_10": 0.323608, + "mrr_at_20": 0.328895, + "mrr_at_100": 0.335475, + "mrr_at_1000": 0.335475, + "nauc_ndcg_at_1_max": 0.290721, + "nauc_ndcg_at_1_std": 0.31373, + "nauc_ndcg_at_1_diff1": 0.473405, + "nauc_ndcg_at_3_max": 0.305541, + "nauc_ndcg_at_3_std": 0.328759, + "nauc_ndcg_at_3_diff1": 0.410644, + "nauc_ndcg_at_5_max": 0.254402, + "nauc_ndcg_at_5_std": 0.371724, + "nauc_ndcg_at_5_diff1": 0.348555, + "nauc_ndcg_at_10_max": 0.258988, + "nauc_ndcg_at_10_std": 0.388646, + "nauc_ndcg_at_10_diff1": 0.375271, + "nauc_ndcg_at_20_max": 0.211136, + "nauc_ndcg_at_20_std": 0.340454, + "nauc_ndcg_at_20_diff1": 0.352083, + "nauc_ndcg_at_100_max": 0.237947, + "nauc_ndcg_at_100_std": 0.369761, + "nauc_ndcg_at_100_diff1": 0.351162, + "nauc_ndcg_at_1000_max": 0.237947, + "nauc_ndcg_at_1000_std": 0.369761, + "nauc_ndcg_at_1000_diff1": 0.351162, + "nauc_map_at_1_max": 0.270201, + "nauc_map_at_1_std": 0.267319, + "nauc_map_at_1_diff1": 0.540107, + "nauc_map_at_3_max": 0.327581, + "nauc_map_at_3_std": 0.347261, + "nauc_map_at_3_diff1": 0.44612, + "nauc_map_at_5_max": 0.294881, + "nauc_map_at_5_std": 0.375069, + "nauc_map_at_5_diff1": 0.392761, + "nauc_map_at_10_max": 0.303697, + "nauc_map_at_10_std": 0.387632, + "nauc_map_at_10_diff1": 0.412975, + "nauc_map_at_20_max": 0.28405, + "nauc_map_at_20_std": 0.367035, + "nauc_map_at_20_diff1": 0.401736, + "nauc_map_at_100_max": 0.285198, + "nauc_map_at_100_std": 0.37059, + "nauc_map_at_100_diff1": 0.399342, + "nauc_map_at_1000_max": 0.285198, + "nauc_map_at_1000_std": 0.37059, + "nauc_map_at_1000_diff1": 0.399342, + "nauc_recall_at_1_max": 0.270201, + "nauc_recall_at_1_std": 0.267319, + "nauc_recall_at_1_diff1": 0.540107, + "nauc_recall_at_3_max": 0.302905, + "nauc_recall_at_3_std": 0.322844, + "nauc_recall_at_3_diff1": 0.373664, + "nauc_recall_at_5_max": 0.168365, + "nauc_recall_at_5_std": 0.37127, + "nauc_recall_at_5_diff1": 0.21562, + "nauc_recall_at_10_max": 0.183771, + "nauc_recall_at_10_std": 0.423482, + "nauc_recall_at_10_diff1": 0.28454, + "nauc_recall_at_20_max": 0.054831, + "nauc_recall_at_20_std": 0.303276, + "nauc_recall_at_20_diff1": 0.235376, + "nauc_recall_at_100_max": 0.193938, + "nauc_recall_at_100_std": 0.553116, + "nauc_recall_at_100_diff1": 0.17237, + "nauc_recall_at_1000_max": 0.193938, + "nauc_recall_at_1000_std": 0.553116, + "nauc_recall_at_1000_diff1": 0.17237, + "nauc_precision_at_1_max": 0.290721, + "nauc_precision_at_1_std": 0.31373, + "nauc_precision_at_1_diff1": 0.473405, + "nauc_precision_at_3_max": 0.384543, + "nauc_precision_at_3_std": 0.41302, + "nauc_precision_at_3_diff1": 0.311298, + "nauc_precision_at_5_max": 0.230733, + "nauc_precision_at_5_std": 0.376454, + "nauc_precision_at_5_diff1": 0.187078, + "nauc_precision_at_10_max": 0.20844, + "nauc_precision_at_10_std": 0.366403, + "nauc_precision_at_10_diff1": 0.217116, + "nauc_precision_at_20_max": 0.081956, + "nauc_precision_at_20_std": 0.228673, + "nauc_precision_at_20_diff1": 0.144994, + "nauc_precision_at_100_max": 0.048123, + "nauc_precision_at_100_std": 0.135151, + "nauc_precision_at_100_diff1": 0.058873, + "nauc_precision_at_1000_max": 0.048123, + "nauc_precision_at_1000_std": 0.135151, + "nauc_precision_at_1000_diff1": 0.058873, + "nauc_mrr_at_1_max": 0.290721, + "nauc_mrr_at_1_std": 0.31373, + "nauc_mrr_at_1_diff1": 0.473405, + "nauc_mrr_at_3_max": 0.274445, + "nauc_mrr_at_3_std": 0.311232, + "nauc_mrr_at_3_diff1": 0.41253, + "nauc_mrr_at_5_max": 0.240042, + "nauc_mrr_at_5_std": 0.317317, + "nauc_mrr_at_5_diff1": 0.383411, + "nauc_mrr_at_10_max": 0.23666, + "nauc_mrr_at_10_std": 0.317946, + "nauc_mrr_at_10_diff1": 0.387206, + "nauc_mrr_at_20_max": 0.228963, + "nauc_mrr_at_20_std": 0.310239, + "nauc_mrr_at_20_diff1": 0.386102, + "nauc_mrr_at_100_max": 0.235254, + "nauc_mrr_at_100_std": 0.315648, + "nauc_mrr_at_100_diff1": 0.386326, + "nauc_mrr_at_1000_max": 0.235254, + "nauc_mrr_at_1000_std": 0.315648, + "nauc_mrr_at_1000_diff1": 0.386326, + "main_score": 0.30536, + "hf_subset": "theoremqa_theorems", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.33659, + "ndcg_at_3": 0.31038, + "ndcg_at_5": 0.32336, + "ndcg_at_10": 0.3331, + "ndcg_at_20": 0.34644, + "ndcg_at_100": 0.35937, + "ndcg_at_1000": 0.35937, + "map_at_1": 0.18496, + "map_at_3": 0.2748, + "map_at_5": 0.28972, + "map_at_10": 0.29534, + "map_at_20": 0.29942, + "map_at_100": 0.302, + "map_at_1000": 0.302, + "recall_at_1": 0.18496, + "recall_at_3": 0.30122, + "recall_at_5": 0.3435, + "recall_at_10": 0.36667, + "recall_at_20": 0.4122, + "recall_at_100": 0.46806, + "recall_at_1000": 0.46806, + "precision_at_1": 0.33659, + "precision_at_3": 0.20813, + "precision_at_5": 0.14049, + "precision_at_10": 0.0761, + "precision_at_20": 0.04195, + "precision_at_100": 0.00951, + "precision_at_1000": 0.00095, + "mrr_at_1": 0.336585, + "mrr_at_3": 0.362602, + "mrr_at_5": 0.371138, + "mrr_at_10": 0.37477, + "mrr_at_20": 0.37839, + "mrr_at_100": 0.380022, + "mrr_at_1000": 0.380022, + "nauc_ndcg_at_1_max": 0.164795, + "nauc_ndcg_at_1_std": 0.325969, + "nauc_ndcg_at_1_diff1": 0.51941, + "nauc_ndcg_at_3_max": 0.158501, + "nauc_ndcg_at_3_std": 0.314561, + "nauc_ndcg_at_3_diff1": 0.504109, + "nauc_ndcg_at_5_max": 0.152499, + "nauc_ndcg_at_5_std": 0.315947, + "nauc_ndcg_at_5_diff1": 0.483883, + "nauc_ndcg_at_10_max": 0.132603, + "nauc_ndcg_at_10_std": 0.317209, + "nauc_ndcg_at_10_diff1": 0.484288, + "nauc_ndcg_at_20_max": 0.11184, + "nauc_ndcg_at_20_std": 0.31154, + "nauc_ndcg_at_20_diff1": 0.470405, + "nauc_ndcg_at_100_max": 0.117397, + "nauc_ndcg_at_100_std": 0.322782, + "nauc_ndcg_at_100_diff1": 0.465457, + "nauc_ndcg_at_1000_max": 0.117397, + "nauc_ndcg_at_1000_std": 0.322782, + "nauc_ndcg_at_1000_diff1": 0.465457, + "nauc_map_at_1_max": 0.15084, + "nauc_map_at_1_std": 0.283413, + "nauc_map_at_1_diff1": 0.613631, + "nauc_map_at_3_max": 0.169787, + "nauc_map_at_3_std": 0.307124, + "nauc_map_at_3_diff1": 0.503477, + "nauc_map_at_5_max": 0.16448, + "nauc_map_at_5_std": 0.309099, + "nauc_map_at_5_diff1": 0.484916, + "nauc_map_at_10_max": 0.152396, + "nauc_map_at_10_std": 0.311914, + "nauc_map_at_10_diff1": 0.483127, + "nauc_map_at_20_max": 0.145786, + "nauc_map_at_20_std": 0.310625, + "nauc_map_at_20_diff1": 0.479578, + "nauc_map_at_100_max": 0.145048, + "nauc_map_at_100_std": 0.312726, + "nauc_map_at_100_diff1": 0.479688, + "nauc_map_at_1000_max": 0.145048, + "nauc_map_at_1000_std": 0.312726, + "nauc_map_at_1000_diff1": 0.479688, + "nauc_recall_at_1_max": 0.15084, + "nauc_recall_at_1_std": 0.283413, + "nauc_recall_at_1_diff1": 0.613631, + "nauc_recall_at_3_max": 0.151103, + "nauc_recall_at_3_std": 0.318457, + "nauc_recall_at_3_diff1": 0.511541, + "nauc_recall_at_5_max": 0.13183, + "nauc_recall_at_5_std": 0.304711, + "nauc_recall_at_5_diff1": 0.438774, + "nauc_recall_at_10_max": 0.084919, + "nauc_recall_at_10_std": 0.300616, + "nauc_recall_at_10_diff1": 0.442494, + "nauc_recall_at_20_max": 0.016589, + "nauc_recall_at_20_std": 0.278731, + "nauc_recall_at_20_diff1": 0.385752, + "nauc_recall_at_100_max": 0.043839, + "nauc_recall_at_100_std": 0.319464, + "nauc_recall_at_100_diff1": 0.346279, + "nauc_recall_at_1000_max": 0.043839, + "nauc_recall_at_1000_std": 0.319464, + "nauc_recall_at_1000_diff1": 0.346279, + "nauc_precision_at_1_max": 0.164795, + "nauc_precision_at_1_std": 0.325969, + "nauc_precision_at_1_diff1": 0.51941, + "nauc_precision_at_3_max": 0.148387, + "nauc_precision_at_3_std": 0.266438, + "nauc_precision_at_3_diff1": 0.297905, + "nauc_precision_at_5_max": 0.122664, + "nauc_precision_at_5_std": 0.258934, + "nauc_precision_at_5_diff1": 0.256742, + "nauc_precision_at_10_max": 0.071914, + "nauc_precision_at_10_std": 0.25913, + "nauc_precision_at_10_diff1": 0.246767, + "nauc_precision_at_20_max": 0.020864, + "nauc_precision_at_20_std": 0.231071, + "nauc_precision_at_20_diff1": 0.214267, + "nauc_precision_at_100_max": 0.042621, + "nauc_precision_at_100_std": 0.265246, + "nauc_precision_at_100_diff1": 0.183215, + "nauc_precision_at_1000_max": 0.042621, + "nauc_precision_at_1000_std": 0.265246, + "nauc_precision_at_1000_diff1": 0.183215, + "nauc_mrr_at_1_max": 0.164795, + "nauc_mrr_at_1_std": 0.325969, + "nauc_mrr_at_1_diff1": 0.51941, + "nauc_mrr_at_3_max": 0.153758, + "nauc_mrr_at_3_std": 0.330636, + "nauc_mrr_at_3_diff1": 0.520648, + "nauc_mrr_at_5_max": 0.150047, + "nauc_mrr_at_5_std": 0.324835, + "nauc_mrr_at_5_diff1": 0.509525, + "nauc_mrr_at_10_max": 0.144874, + "nauc_mrr_at_10_std": 0.327471, + "nauc_mrr_at_10_diff1": 0.510844, + "nauc_mrr_at_20_max": 0.139758, + "nauc_mrr_at_20_std": 0.325758, + "nauc_mrr_at_20_diff1": 0.506645, + "nauc_mrr_at_100_max": 0.142023, + "nauc_mrr_at_100_std": 0.326769, + "nauc_mrr_at_100_diff1": 0.505808, + "nauc_mrr_at_1000_max": 0.142023, + "nauc_mrr_at_1000_std": 0.326769, + "nauc_mrr_at_1000_diff1": 0.505808, + "main_score": 0.3331, + "hf_subset": "theoremqa_questions", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 588.8510918617249, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/CLSClusteringP2P.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/CLSClusteringP2P.json new file mode 100644 index 0000000000..ad37f1cfd5 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/CLSClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "4b6227591c6c1a73bc76b1055f3b7f3588e72476", + "task_name": "CLSClusteringP2P", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measure": 0.566625, + "v_measure_std": 0.012436, + "v_measures": [ + 0.554467, + 0.57713, + 0.578521, + 0.577968, + 0.582639, + 0.544184, + 0.55508, + 0.557264, + 0.572717, + 0.56628 + ], + "main_score": 0.566625, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 74.90198755264282, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/CLSClusteringS2S.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/CLSClusteringS2S.json new file mode 100644 index 0000000000..2f805f86d6 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/CLSClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "e458b3f5414b62b7f9f83499ac1f5497ae2e869f", + "task_name": "CLSClusteringS2S", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measure": 0.607148, + "v_measure_std": 0.014696, + "v_measures": [ + 0.61686, + 0.639059, + 0.623421, + 0.594706, + 0.603135, + 0.589141, + 0.607821, + 0.595907, + 0.606948, + 0.594483 + ], + "main_score": 0.607148, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 41.29370975494385, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/CMedQAv1-reranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/CMedQAv1-reranking.json new file mode 100644 index 0000000000..26383eba20 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/CMedQAv1-reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "8d7f1e942507dac42dc58017c1a001c3717da7df", + "task_name": "CMedQAv1-reranking", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "map": 0.875769, + "mrr": 0.897024, + "nAUC_map_max": 0.591451, + "nAUC_map_std": 0.193072, + "nAUC_map_diff1": 0.575015, + "nAUC_mrr_max": 0.709184, + "nAUC_mrr_std": 0.300826, + "nAUC_mrr_diff1": 0.694331, + "main_score": 0.875769, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 37.65912485122681, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/CMedQAv2-reranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/CMedQAv2-reranking.json new file mode 100644 index 0000000000..a511ff7a86 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/CMedQAv2-reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "23d186750531a14a0357ca22cd92d712fd512ea0", + "task_name": "CMedQAv2-reranking", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "map": 0.880226, + "mrr": 0.904164, + "nAUC_map_max": 0.640356, + "nAUC_map_std": 0.161182, + "nAUC_map_diff1": 0.609791, + "nAUC_mrr_max": 0.751781, + "nAUC_mrr_std": 0.258882, + "nAUC_mrr_diff1": 0.705551, + "main_score": 0.880226, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 42.145179271698, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/CQADupstackGamingRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/CQADupstackGamingRetrieval.json new file mode 100644 index 0000000000..a0600241de --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/CQADupstackGamingRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4885aa143210c98657558c04aaf3dc47cfb54340", + "task_name": "CQADupstackGamingRetrieval", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.57555, + "ndcg_at_3": 0.65968, + "ndcg_at_5": 0.68844, + "ndcg_at_10": 0.71536, + "ndcg_at_20": 0.72929, + "ndcg_at_100": 0.74312, + "ndcg_at_1000": 0.74312, + "map_at_1": 0.50342, + "map_at_3": 0.61643, + "map_at_5": 0.63877, + "map_at_10": 0.65354, + "map_at_20": 0.65963, + "map_at_100": 0.66267, + "map_at_1000": 0.66267, + "recall_at_1": 0.50342, + "recall_at_3": 0.71415, + "recall_at_5": 0.78456, + "recall_at_10": 0.86125, + "recall_at_20": 0.91017, + "recall_at_100": 0.97296, + "recall_at_1000": 0.97296, + "precision_at_1": 0.57555, + "precision_at_3": 0.29342, + "precision_at_5": 0.19912, + "precision_at_10": 0.11273, + "precision_at_20": 0.06154, + "precision_at_100": 0.01357, + "precision_at_1000": 0.00136, + "mrr_at_1": 0.575549, + "mrr_at_3": 0.664054, + "mrr_at_5": 0.677001, + "mrr_at_10": 0.686465, + "mrr_at_20": 0.689059, + "mrr_at_100": 0.690391, + "mrr_at_1000": 0.690391, + "nauc_ndcg_at_1_max": 0.247121, + "nauc_ndcg_at_1_std": 0.063795, + "nauc_ndcg_at_1_diff1": 0.566867, + "nauc_ndcg_at_3_max": 0.264585, + "nauc_ndcg_at_3_std": 0.089427, + "nauc_ndcg_at_3_diff1": 0.53241, + "nauc_ndcg_at_5_max": 0.290123, + "nauc_ndcg_at_5_std": 0.122763, + "nauc_ndcg_at_5_diff1": 0.526848, + "nauc_ndcg_at_10_max": 0.290489, + "nauc_ndcg_at_10_std": 0.137495, + "nauc_ndcg_at_10_diff1": 0.524537, + "nauc_ndcg_at_20_max": 0.291839, + "nauc_ndcg_at_20_std": 0.141151, + "nauc_ndcg_at_20_diff1": 0.534573, + "nauc_ndcg_at_100_max": 0.290528, + "nauc_ndcg_at_100_std": 0.132454, + "nauc_ndcg_at_100_diff1": 0.538333, + "nauc_ndcg_at_1000_max": 0.290528, + "nauc_ndcg_at_1000_std": 0.132454, + "nauc_ndcg_at_1000_diff1": 0.538333, + "nauc_map_at_1_max": 0.165503, + "nauc_map_at_1_std": -0.024007, + "nauc_map_at_1_diff1": 0.563866, + "nauc_map_at_3_max": 0.229542, + "nauc_map_at_3_std": 0.041148, + "nauc_map_at_3_diff1": 0.539102, + "nauc_map_at_5_max": 0.25122, + "nauc_map_at_5_std": 0.074725, + "nauc_map_at_5_diff1": 0.534857, + "nauc_map_at_10_max": 0.257037, + "nauc_map_at_10_std": 0.090742, + "nauc_map_at_10_diff1": 0.534908, + "nauc_map_at_20_max": 0.261471, + "nauc_map_at_20_std": 0.096645, + "nauc_map_at_20_diff1": 0.538319, + "nauc_map_at_100_max": 0.263008, + "nauc_map_at_100_std": 0.097621, + "nauc_map_at_100_diff1": 0.539005, + "nauc_map_at_1000_max": 0.263008, + "nauc_map_at_1000_std": 0.097621, + "nauc_map_at_1000_diff1": 0.539005, + "nauc_recall_at_1_max": 0.165503, + "nauc_recall_at_1_std": -0.024007, + "nauc_recall_at_1_diff1": 0.563866, + "nauc_recall_at_3_max": 0.258568, + "nauc_recall_at_3_std": 0.0723, + "nauc_recall_at_3_diff1": 0.48525, + "nauc_recall_at_5_max": 0.329825, + "nauc_recall_at_5_std": 0.185186, + "nauc_recall_at_5_diff1": 0.451245, + "nauc_recall_at_10_max": 0.347574, + "nauc_recall_at_10_std": 0.28414, + "nauc_recall_at_10_diff1": 0.410677, + "nauc_recall_at_20_max": 0.409223, + "nauc_recall_at_20_std": 0.40471, + "nauc_recall_at_20_diff1": 0.461299, + "nauc_recall_at_100_max": 0.579193, + "nauc_recall_at_100_std": 0.551771, + "nauc_recall_at_100_diff1": 0.529997, + "nauc_recall_at_1000_max": 0.579193, + "nauc_recall_at_1000_std": 0.551771, + "nauc_recall_at_1000_diff1": 0.529997, + "nauc_precision_at_1_max": 0.247121, + "nauc_precision_at_1_std": 0.063795, + "nauc_precision_at_1_diff1": 0.566867, + "nauc_precision_at_3_max": 0.293804, + "nauc_precision_at_3_std": 0.246961, + "nauc_precision_at_3_diff1": 0.219725, + "nauc_precision_at_5_max": 0.312845, + "nauc_precision_at_5_std": 0.339458, + "nauc_precision_at_5_diff1": 0.106771, + "nauc_precision_at_10_max": 0.293943, + "nauc_precision_at_10_std": 0.400381, + "nauc_precision_at_10_diff1": -0.000471, + "nauc_precision_at_20_max": 0.268924, + "nauc_precision_at_20_std": 0.39635, + "nauc_precision_at_20_diff1": -0.044255, + "nauc_precision_at_100_max": 0.219335, + "nauc_precision_at_100_std": 0.342461, + "nauc_precision_at_100_diff1": -0.108277, + "nauc_precision_at_1000_max": 0.219335, + "nauc_precision_at_1000_std": 0.342461, + "nauc_precision_at_1000_diff1": -0.108277, + "nauc_mrr_at_1_max": 0.247121, + "nauc_mrr_at_1_std": 0.063795, + "nauc_mrr_at_1_diff1": 0.566867, + "nauc_mrr_at_3_max": 0.288319, + "nauc_mrr_at_3_std": 0.105212, + "nauc_mrr_at_3_diff1": 0.548129, + "nauc_mrr_at_5_max": 0.292502, + "nauc_mrr_at_5_std": 0.116151, + "nauc_mrr_at_5_diff1": 0.543734, + "nauc_mrr_at_10_max": 0.289451, + "nauc_mrr_at_10_std": 0.117062, + "nauc_mrr_at_10_diff1": 0.542954, + "nauc_mrr_at_20_max": 0.288985, + "nauc_mrr_at_20_std": 0.11671, + "nauc_mrr_at_20_diff1": 0.545109, + "nauc_mrr_at_100_max": 0.288605, + "nauc_mrr_at_100_std": 0.115419, + "nauc_mrr_at_100_diff1": 0.54552, + "nauc_mrr_at_1000_max": 0.288605, + "nauc_mrr_at_1000_std": 0.115419, + "nauc_mrr_at_1000_diff1": 0.54552, + "main_score": 0.71536, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 99.81649327278137, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/CQADupstackUnixRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/CQADupstackUnixRetrieval.json new file mode 100644 index 0000000000..b6e7e50491 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/CQADupstackUnixRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53", + "task_name": "CQADupstackUnixRetrieval", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.42351, + "ndcg_at_3": 0.49556, + "ndcg_at_5": 0.53038, + "ndcg_at_10": 0.56556, + "ndcg_at_20": 0.58404, + "ndcg_at_100": 0.60988, + "ndcg_at_1000": 0.60988, + "map_at_1": 0.35593, + "map_at_3": 0.45237, + "map_at_5": 0.47757, + "map_at_10": 0.49626, + "map_at_20": 0.50269, + "map_at_100": 0.50731, + "map_at_1000": 0.50731, + "recall_at_1": 0.35593, + "recall_at_3": 0.54407, + "recall_at_5": 0.6307, + "recall_at_10": 0.73214, + "recall_at_20": 0.79592, + "recall_at_100": 0.91574, + "recall_at_1000": 0.91574, + "precision_at_1": 0.42351, + "precision_at_3": 0.23103, + "precision_at_5": 0.1666, + "precision_at_10": 0.09981, + "precision_at_20": 0.05578, + "precision_at_100": 0.01347, + "precision_at_1000": 0.00135, + "mrr_at_1": 0.423507, + "mrr_at_3": 0.506996, + "mrr_at_5": 0.525326, + "mrr_at_10": 0.537268, + "mrr_at_20": 0.541288, + "mrr_at_100": 0.543959, + "mrr_at_1000": 0.543959, + "nauc_ndcg_at_1_max": 0.287544, + "nauc_ndcg_at_1_std": 0.156946, + "nauc_ndcg_at_1_diff1": 0.494336, + "nauc_ndcg_at_3_max": 0.295264, + "nauc_ndcg_at_3_std": 0.163928, + "nauc_ndcg_at_3_diff1": 0.445315, + "nauc_ndcg_at_5_max": 0.303249, + "nauc_ndcg_at_5_std": 0.201106, + "nauc_ndcg_at_5_diff1": 0.434636, + "nauc_ndcg_at_10_max": 0.315135, + "nauc_ndcg_at_10_std": 0.232554, + "nauc_ndcg_at_10_diff1": 0.438519, + "nauc_ndcg_at_20_max": 0.32884, + "nauc_ndcg_at_20_std": 0.240819, + "nauc_ndcg_at_20_diff1": 0.441467, + "nauc_ndcg_at_100_max": 0.319002, + "nauc_ndcg_at_100_std": 0.224241, + "nauc_ndcg_at_100_diff1": 0.44282, + "nauc_ndcg_at_1000_max": 0.319002, + "nauc_ndcg_at_1000_std": 0.224241, + "nauc_ndcg_at_1000_diff1": 0.44282, + "nauc_map_at_1_max": 0.22608, + "nauc_map_at_1_std": 0.069362, + "nauc_map_at_1_diff1": 0.51161, + "nauc_map_at_3_max": 0.277413, + "nauc_map_at_3_std": 0.134123, + "nauc_map_at_3_diff1": 0.463816, + "nauc_map_at_5_max": 0.290747, + "nauc_map_at_5_std": 0.167269, + "nauc_map_at_5_diff1": 0.45657, + "nauc_map_at_10_max": 0.297074, + "nauc_map_at_10_std": 0.188419, + "nauc_map_at_10_diff1": 0.455443, + "nauc_map_at_20_max": 0.301805, + "nauc_map_at_20_std": 0.191827, + "nauc_map_at_20_diff1": 0.455999, + "nauc_map_at_100_max": 0.301284, + "nauc_map_at_100_std": 0.189978, + "nauc_map_at_100_diff1": 0.456128, + "nauc_map_at_1000_max": 0.301284, + "nauc_map_at_1000_std": 0.189978, + "nauc_map_at_1000_diff1": 0.456128, + "nauc_recall_at_1_max": 0.22608, + "nauc_recall_at_1_std": 0.069362, + "nauc_recall_at_1_diff1": 0.51161, + "nauc_recall_at_3_max": 0.281548, + "nauc_recall_at_3_std": 0.146797, + "nauc_recall_at_3_diff1": 0.402365, + "nauc_recall_at_5_max": 0.301689, + "nauc_recall_at_5_std": 0.243787, + "nauc_recall_at_5_diff1": 0.365866, + "nauc_recall_at_10_max": 0.337302, + "nauc_recall_at_10_std": 0.358567, + "nauc_recall_at_10_diff1": 0.36024, + "nauc_recall_at_20_max": 0.416898, + "nauc_recall_at_20_std": 0.431044, + "nauc_recall_at_20_diff1": 0.37026, + "nauc_recall_at_100_max": 0.364827, + "nauc_recall_at_100_std": 0.416617, + "nauc_recall_at_100_diff1": 0.330984, + "nauc_recall_at_1000_max": 0.364827, + "nauc_recall_at_1000_std": 0.416617, + "nauc_recall_at_1000_diff1": 0.330984, + "nauc_precision_at_1_max": 0.287544, + "nauc_precision_at_1_std": 0.156946, + "nauc_precision_at_1_diff1": 0.494336, + "nauc_precision_at_3_max": 0.323399, + "nauc_precision_at_3_std": 0.274659, + "nauc_precision_at_3_diff1": 0.236528, + "nauc_precision_at_5_max": 0.304642, + "nauc_precision_at_5_std": 0.352523, + "nauc_precision_at_5_diff1": 0.12472, + "nauc_precision_at_10_max": 0.276618, + "nauc_precision_at_10_std": 0.403821, + "nauc_precision_at_10_diff1": 0.048418, + "nauc_precision_at_20_max": 0.25659, + "nauc_precision_at_20_std": 0.373056, + "nauc_precision_at_20_diff1": -0.008565, + "nauc_precision_at_100_max": 0.119934, + "nauc_precision_at_100_std": 0.211982, + "nauc_precision_at_100_diff1": -0.122859, + "nauc_precision_at_1000_max": 0.119934, + "nauc_precision_at_1000_std": 0.211982, + "nauc_precision_at_1000_diff1": -0.122859, + "nauc_mrr_at_1_max": 0.287544, + "nauc_mrr_at_1_std": 0.156946, + "nauc_mrr_at_1_diff1": 0.494336, + "nauc_mrr_at_3_max": 0.305333, + "nauc_mrr_at_3_std": 0.186666, + "nauc_mrr_at_3_diff1": 0.452912, + "nauc_mrr_at_5_max": 0.308443, + "nauc_mrr_at_5_std": 0.20123, + "nauc_mrr_at_5_diff1": 0.446851, + "nauc_mrr_at_10_max": 0.311664, + "nauc_mrr_at_10_std": 0.206783, + "nauc_mrr_at_10_diff1": 0.450819, + "nauc_mrr_at_20_max": 0.313883, + "nauc_mrr_at_20_std": 0.207261, + "nauc_mrr_at_20_diff1": 0.452511, + "nauc_mrr_at_100_max": 0.312299, + "nauc_mrr_at_100_std": 0.205163, + "nauc_mrr_at_100_diff1": 0.452764, + "nauc_mrr_at_1000_max": 0.312299, + "nauc_mrr_at_1000_std": 0.205163, + "nauc_mrr_at_1000_diff1": 0.452764, + "main_score": 0.56556, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 127.69204425811768, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/ClimateFEVERHardNegatives.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/ClimateFEVERHardNegatives.json new file mode 100644 index 0000000000..4c290152ea --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/ClimateFEVERHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "3a309e201f3c2c4b13bd4a367a8f37eee2ec1d21", + "task_name": "ClimateFEVERHardNegatives", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.506, + "ndcg_at_3": 0.42431, + "ndcg_at_5": 0.44086, + "ndcg_at_10": 0.48223, + "ndcg_at_20": 0.51133, + "ndcg_at_100": 0.55442, + "ndcg_at_1000": 0.55442, + "map_at_1": 0.22017, + "map_at_3": 0.3247, + "map_at_5": 0.3525, + "map_at_10": 0.37753, + "map_at_20": 0.39018, + "map_at_100": 0.39993, + "map_at_1000": 0.39993, + "recall_at_1": 0.22017, + "recall_at_3": 0.37818, + "recall_at_5": 0.44185, + "recall_at_10": 0.53078, + "recall_at_20": 0.61088, + "recall_at_100": 0.77068, + "recall_at_1000": 0.77068, + "precision_at_1": 0.506, + "precision_at_3": 0.318, + "precision_at_5": 0.2314, + "precision_at_10": 0.1443, + "precision_at_20": 0.0852, + "precision_at_100": 0.02248, + "precision_at_1000": 0.00225, + "mrr_at_1": 0.506, + "mrr_at_3": 0.604, + "mrr_at_5": 0.6155, + "mrr_at_10": 0.623798, + "mrr_at_20": 0.626894, + "mrr_at_100": 0.628905, + "mrr_at_1000": 0.628905, + "nauc_ndcg_at_1_max": 0.351465, + "nauc_ndcg_at_1_std": 0.226626, + "nauc_ndcg_at_1_diff1": 0.304025, + "nauc_ndcg_at_3_max": 0.374246, + "nauc_ndcg_at_3_std": 0.33497, + "nauc_ndcg_at_3_diff1": 0.203182, + "nauc_ndcg_at_5_max": 0.383908, + "nauc_ndcg_at_5_std": 0.371864, + "nauc_ndcg_at_5_diff1": 0.197778, + "nauc_ndcg_at_10_max": 0.395224, + "nauc_ndcg_at_10_std": 0.390525, + "nauc_ndcg_at_10_diff1": 0.18532, + "nauc_ndcg_at_20_max": 0.39554, + "nauc_ndcg_at_20_std": 0.397655, + "nauc_ndcg_at_20_diff1": 0.173056, + "nauc_ndcg_at_100_max": 0.404011, + "nauc_ndcg_at_100_std": 0.3995, + "nauc_ndcg_at_100_diff1": 0.181442, + "nauc_ndcg_at_1000_max": 0.404011, + "nauc_ndcg_at_1000_std": 0.3995, + "nauc_ndcg_at_1000_diff1": 0.181442, + "nauc_map_at_1_max": 0.361825, + "nauc_map_at_1_std": 0.203361, + "nauc_map_at_1_diff1": 0.344902, + "nauc_map_at_3_max": 0.371427, + "nauc_map_at_3_std": 0.310962, + "nauc_map_at_3_diff1": 0.220264, + "nauc_map_at_5_max": 0.374683, + "nauc_map_at_5_std": 0.343212, + "nauc_map_at_5_diff1": 0.204449, + "nauc_map_at_10_max": 0.380548, + "nauc_map_at_10_std": 0.359426, + "nauc_map_at_10_diff1": 0.193742, + "nauc_map_at_20_max": 0.381712, + "nauc_map_at_20_std": 0.364215, + "nauc_map_at_20_diff1": 0.187688, + "nauc_map_at_100_max": 0.384136, + "nauc_map_at_100_std": 0.365821, + "nauc_map_at_100_diff1": 0.188362, + "nauc_map_at_1000_max": 0.384136, + "nauc_map_at_1000_std": 0.365821, + "nauc_map_at_1000_diff1": 0.188362, + "nauc_recall_at_1_max": 0.361825, + "nauc_recall_at_1_std": 0.203361, + "nauc_recall_at_1_diff1": 0.344902, + "nauc_recall_at_3_max": 0.354789, + "nauc_recall_at_3_std": 0.341395, + "nauc_recall_at_3_diff1": 0.166596, + "nauc_recall_at_5_max": 0.353567, + "nauc_recall_at_5_std": 0.394619, + "nauc_recall_at_5_diff1": 0.138211, + "nauc_recall_at_10_max": 0.365014, + "nauc_recall_at_10_std": 0.416442, + "nauc_recall_at_10_diff1": 0.106051, + "nauc_recall_at_20_max": 0.354807, + "nauc_recall_at_20_std": 0.427216, + "nauc_recall_at_20_diff1": 0.06895, + "nauc_recall_at_100_max": 0.385331, + "nauc_recall_at_100_std": 0.452808, + "nauc_recall_at_100_diff1": 0.075813, + "nauc_recall_at_1000_max": 0.385331, + "nauc_recall_at_1000_std": 0.452808, + "nauc_recall_at_1000_diff1": 0.075813, + "nauc_precision_at_1_max": 0.351465, + "nauc_precision_at_1_std": 0.226626, + "nauc_precision_at_1_diff1": 0.304025, + "nauc_precision_at_3_max": 0.292647, + "nauc_precision_at_3_std": 0.363288, + "nauc_precision_at_3_diff1": 0.031849, + "nauc_precision_at_5_max": 0.261826, + "nauc_precision_at_5_std": 0.399842, + "nauc_precision_at_5_diff1": -0.009491, + "nauc_precision_at_10_max": 0.226527, + "nauc_precision_at_10_std": 0.364585, + "nauc_precision_at_10_diff1": -0.053923, + "nauc_precision_at_20_max": 0.173519, + "nauc_precision_at_20_std": 0.318563, + "nauc_precision_at_20_diff1": -0.101655, + "nauc_precision_at_100_max": 0.073289, + "nauc_precision_at_100_std": 0.191603, + "nauc_precision_at_100_diff1": -0.114584, + "nauc_precision_at_1000_max": 0.073289, + "nauc_precision_at_1000_std": 0.191603, + "nauc_precision_at_1000_diff1": -0.114584, + "nauc_mrr_at_1_max": 0.351465, + "nauc_mrr_at_1_std": 0.226626, + "nauc_mrr_at_1_diff1": 0.304025, + "nauc_mrr_at_3_max": 0.378039, + "nauc_mrr_at_3_std": 0.304562, + "nauc_mrr_at_3_diff1": 0.261096, + "nauc_mrr_at_5_max": 0.382251, + "nauc_mrr_at_5_std": 0.314983, + "nauc_mrr_at_5_diff1": 0.261415, + "nauc_mrr_at_10_max": 0.386294, + "nauc_mrr_at_10_std": 0.317188, + "nauc_mrr_at_10_diff1": 0.262817, + "nauc_mrr_at_20_max": 0.385411, + "nauc_mrr_at_20_std": 0.315258, + "nauc_mrr_at_20_diff1": 0.261777, + "nauc_mrr_at_100_max": 0.384907, + "nauc_mrr_at_100_std": 0.31359, + "nauc_mrr_at_100_diff1": 0.262506, + "nauc_mrr_at_1000_max": 0.384907, + "nauc_mrr_at_1000_std": 0.31359, + "nauc_mrr_at_1000_diff1": 0.262506, + "main_score": 0.48223, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 146.75673055648804, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/CmedqaRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/CmedqaRetrieval.json new file mode 100644 index 0000000000..d7020612e6 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/CmedqaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301", + "task_name": "CmedqaRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.47437, + "ndcg_at_3": 0.46776, + "ndcg_at_5": 0.48836, + "ndcg_at_10": 0.51907, + "ndcg_at_20": 0.5456, + "ndcg_at_100": 0.58557, + "ndcg_at_1000": 0.58557, + "map_at_1": 0.31613, + "map_at_3": 0.40923, + "map_at_5": 0.43383, + "map_at_10": 0.45328, + "map_at_20": 0.46324, + "map_at_100": 0.47089, + "map_at_1000": 0.47089, + "recall_at_1": 0.31613, + "recall_at_3": 0.46008, + "recall_at_5": 0.52812, + "recall_at_10": 0.62017, + "recall_at_20": 0.71023, + "recall_at_100": 0.89401, + "recall_at_1000": 0.89401, + "precision_at_1": 0.47437, + "precision_at_3": 0.25915, + "precision_at_5": 0.18465, + "precision_at_10": 0.11113, + "precision_at_20": 0.06447, + "precision_at_100": 0.01654, + "precision_at_1000": 0.00165, + "mrr_at_1": 0.474369, + "mrr_at_3": 0.524298, + "mrr_at_5": 0.537476, + "mrr_at_10": 0.547637, + "mrr_at_20": 0.552796, + "mrr_at_100": 0.556312, + "mrr_at_1000": 0.556312, + "nauc_ndcg_at_1_max": 0.682111, + "nauc_ndcg_at_1_std": 0.423422, + "nauc_ndcg_at_1_diff1": 0.690742, + "nauc_ndcg_at_3_max": 0.611654, + "nauc_ndcg_at_3_std": 0.438705, + "nauc_ndcg_at_3_diff1": 0.578955, + "nauc_ndcg_at_5_max": 0.605926, + "nauc_ndcg_at_5_std": 0.455078, + "nauc_ndcg_at_5_diff1": 0.568711, + "nauc_ndcg_at_10_max": 0.608739, + "nauc_ndcg_at_10_std": 0.477907, + "nauc_ndcg_at_10_diff1": 0.56606, + "nauc_ndcg_at_20_max": 0.612945, + "nauc_ndcg_at_20_std": 0.491393, + "nauc_ndcg_at_20_diff1": 0.566109, + "nauc_ndcg_at_100_max": 0.622313, + "nauc_ndcg_at_100_std": 0.482217, + "nauc_ndcg_at_100_diff1": 0.575301, + "nauc_ndcg_at_1000_max": 0.622313, + "nauc_ndcg_at_1000_std": 0.482217, + "nauc_ndcg_at_1000_diff1": 0.575301, + "nauc_map_at_1_max": 0.437787, + "nauc_map_at_1_std": 0.24531, + "nauc_map_at_1_diff1": 0.577324, + "nauc_map_at_3_max": 0.545392, + "nauc_map_at_3_std": 0.37674, + "nauc_map_at_3_diff1": 0.566284, + "nauc_map_at_5_max": 0.569187, + "nauc_map_at_5_std": 0.413615, + "nauc_map_at_5_diff1": 0.563218, + "nauc_map_at_10_max": 0.583608, + "nauc_map_at_10_std": 0.437598, + "nauc_map_at_10_diff1": 0.563255, + "nauc_map_at_20_max": 0.588377, + "nauc_map_at_20_std": 0.445423, + "nauc_map_at_20_diff1": 0.56406, + "nauc_map_at_100_max": 0.591257, + "nauc_map_at_100_std": 0.445196, + "nauc_map_at_100_diff1": 0.56594, + "nauc_map_at_1000_max": 0.591257, + "nauc_map_at_1000_std": 0.445196, + "nauc_map_at_1000_diff1": 0.56594, + "nauc_recall_at_1_max": 0.437787, + "nauc_recall_at_1_std": 0.24531, + "nauc_recall_at_1_diff1": 0.577324, + "nauc_recall_at_3_max": 0.507833, + "nauc_recall_at_3_std": 0.388022, + "nauc_recall_at_3_diff1": 0.502758, + "nauc_recall_at_5_max": 0.518442, + "nauc_recall_at_5_std": 0.446319, + "nauc_recall_at_5_diff1": 0.468465, + "nauc_recall_at_10_max": 0.519493, + "nauc_recall_at_10_std": 0.514619, + "nauc_recall_at_10_diff1": 0.438916, + "nauc_recall_at_20_max": 0.510451, + "nauc_recall_at_20_std": 0.571843, + "nauc_recall_at_20_diff1": 0.410926, + "nauc_recall_at_100_max": 0.510475, + "nauc_recall_at_100_std": 0.599449, + "nauc_recall_at_100_diff1": 0.360487, + "nauc_recall_at_1000_max": 0.510475, + "nauc_recall_at_1000_std": 0.599449, + "nauc_recall_at_1000_diff1": 0.360487, + "nauc_precision_at_1_max": 0.682111, + "nauc_precision_at_1_std": 0.423422, + "nauc_precision_at_1_diff1": 0.690742, + "nauc_precision_at_3_max": 0.612724, + "nauc_precision_at_3_std": 0.494819, + "nauc_precision_at_3_diff1": 0.393453, + "nauc_precision_at_5_max": 0.579902, + "nauc_precision_at_5_std": 0.517022, + "nauc_precision_at_5_diff1": 0.312714, + "nauc_precision_at_10_max": 0.527519, + "nauc_precision_at_10_std": 0.515391, + "nauc_precision_at_10_diff1": 0.239159, + "nauc_precision_at_20_max": 0.463471, + "nauc_precision_at_20_std": 0.478831, + "nauc_precision_at_20_diff1": 0.175423, + "nauc_precision_at_100_max": 0.330998, + "nauc_precision_at_100_std": 0.307211, + "nauc_precision_at_100_diff1": 0.069768, + "nauc_precision_at_1000_max": 0.330998, + "nauc_precision_at_1000_std": 0.307211, + "nauc_precision_at_1000_diff1": 0.069768, + "nauc_mrr_at_1_max": 0.682111, + "nauc_mrr_at_1_std": 0.423422, + "nauc_mrr_at_1_diff1": 0.690742, + "nauc_mrr_at_3_max": 0.680732, + "nauc_mrr_at_3_std": 0.455051, + "nauc_mrr_at_3_diff1": 0.659469, + "nauc_mrr_at_5_max": 0.681261, + "nauc_mrr_at_5_std": 0.462265, + "nauc_mrr_at_5_diff1": 0.654002, + "nauc_mrr_at_10_max": 0.679242, + "nauc_mrr_at_10_std": 0.465905, + "nauc_mrr_at_10_diff1": 0.651968, + "nauc_mrr_at_20_max": 0.679112, + "nauc_mrr_at_20_std": 0.466404, + "nauc_mrr_at_20_diff1": 0.651714, + "nauc_mrr_at_100_max": 0.679313, + "nauc_mrr_at_100_std": 0.465102, + "nauc_mrr_at_100_diff1": 0.652477, + "nauc_mrr_at_1000_max": 0.679313, + "nauc_mrr_at_1000_std": 0.465102, + "nauc_mrr_at_1000_diff1": 0.652477, + "main_score": 0.51907, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 138.93327617645264, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/Cmnli.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/Cmnli.json new file mode 100644 index 0000000000..5b87ba1bbc --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/Cmnli.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "41bc36f332156f7adc9e38f53777c959b2ae9766", + "task_name": "Cmnli", + "mteb_version": "1.36.5", + "scores": { + "validation": [ + { + "similarity_accuracy": 0.866506, + "similarity_accuracy_threshold": 0.606543, + "similarity_f1": 0.870245, + "similarity_f1_threshold": 0.603646, + "similarity_precision": 0.868623, + "similarity_recall": 0.871873, + "similarity_ap": 0.93052, + "cosine_accuracy": 0.866506, + "cosine_accuracy_threshold": 0.606543, + "cosine_f1": 0.870245, + "cosine_f1_threshold": 0.603646, + "cosine_precision": 0.868623, + "cosine_recall": 0.871873, + "cosine_ap": 0.93052, + "manhattan_accuracy": 0.860613, + "manhattan_accuracy_threshold": 31.665583, + "manhattan_f1": 0.866091, + "manhattan_f1_threshold": 31.845669, + "manhattan_precision": 0.853221, + "manhattan_recall": 0.879355, + "manhattan_ap": 0.927735, + "euclidean_accuracy": 0.866627, + "euclidean_accuracy_threshold": 0.887011, + "euclidean_f1": 0.870184, + "euclidean_f1_threshold": 0.889923, + "euclidean_precision": 0.868967, + "euclidean_recall": 0.871405, + "euclidean_ap": 0.930516, + "dot_accuracy": 0.866747, + "dot_accuracy_threshold": 0.604472, + "dot_f1": 0.87047, + "dot_f1_threshold": 0.604472, + "dot_precision": 0.87047, + "dot_recall": 0.87047, + "dot_ap": 0.930569, + "max_accuracy": 0.866747, + "max_f1": 0.87047, + "max_precision": 0.87047, + "max_recall": 0.879355, + "max_ap": 0.930569, + "main_score": 0.930569, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 14.274982452392578, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/CovidRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/CovidRetrieval.json new file mode 100644 index 0000000000..a9ea1698cb --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/CovidRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "1271c7809071a13532e05f25fb53511ffce77117", + "task_name": "CovidRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.77555, + "ndcg_at_3": 0.84889, + "ndcg_at_5": 0.86292, + "ndcg_at_10": 0.87404, + "ndcg_at_20": 0.87842, + "ndcg_at_100": 0.88108, + "ndcg_at_1000": 0.88108, + "map_at_1": 0.77397, + "map_at_3": 0.83136, + "map_at_5": 0.83921, + "map_at_10": 0.84424, + "map_at_20": 0.8455, + "map_at_100": 0.84591, + "map_at_1000": 0.84591, + "recall_at_1": 0.77397, + "recall_at_3": 0.89858, + "recall_at_5": 0.93256, + "recall_at_10": 0.96523, + "recall_at_20": 0.98209, + "recall_at_100": 0.99579, + "recall_at_1000": 0.99579, + "precision_at_1": 0.77555, + "precision_at_3": 0.30172, + "precision_at_5": 0.18799, + "precision_at_10": 0.09758, + "precision_at_20": 0.04963, + "precision_at_100": 0.01006, + "precision_at_1000": 0.00101, + "mrr_at_1": 0.775553, + "mrr_at_3": 0.831753, + "mrr_at_5": 0.839498, + "mrr_at_10": 0.844071, + "mrr_at_20": 0.845336, + "mrr_at_100": 0.845745, + "mrr_at_1000": 0.845745, + "nauc_ndcg_at_1_max": 0.352049, + "nauc_ndcg_at_1_std": -0.138459, + "nauc_ndcg_at_1_diff1": 0.86209, + "nauc_ndcg_at_3_max": 0.409504, + "nauc_ndcg_at_3_std": -0.094972, + "nauc_ndcg_at_3_diff1": 0.835209, + "nauc_ndcg_at_5_max": 0.407739, + "nauc_ndcg_at_5_std": -0.113233, + "nauc_ndcg_at_5_diff1": 0.835843, + "nauc_ndcg_at_10_max": 0.411573, + "nauc_ndcg_at_10_std": -0.089296, + "nauc_ndcg_at_10_diff1": 0.843413, + "nauc_ndcg_at_20_max": 0.40674, + "nauc_ndcg_at_20_std": -0.08599, + "nauc_ndcg_at_20_diff1": 0.844189, + "nauc_ndcg_at_100_max": 0.400653, + "nauc_ndcg_at_100_std": -0.090879, + "nauc_ndcg_at_100_diff1": 0.845588, + "nauc_ndcg_at_1000_max": 0.400653, + "nauc_ndcg_at_1000_std": -0.090879, + "nauc_ndcg_at_1000_diff1": 0.845588, + "nauc_map_at_1_max": 0.352353, + "nauc_map_at_1_std": -0.141357, + "nauc_map_at_1_diff1": 0.864989, + "nauc_map_at_3_max": 0.393105, + "nauc_map_at_3_std": -0.107623, + "nauc_map_at_3_diff1": 0.84385, + "nauc_map_at_5_max": 0.391425, + "nauc_map_at_5_std": -0.116617, + "nauc_map_at_5_diff1": 0.844426, + "nauc_map_at_10_max": 0.392557, + "nauc_map_at_10_std": -0.105341, + "nauc_map_at_10_diff1": 0.847275, + "nauc_map_at_20_max": 0.391597, + "nauc_map_at_20_std": -0.104162, + "nauc_map_at_20_diff1": 0.847504, + "nauc_map_at_100_max": 0.39077, + "nauc_map_at_100_std": -0.104789, + "nauc_map_at_100_diff1": 0.847665, + "nauc_map_at_1000_max": 0.39077, + "nauc_map_at_1000_std": -0.104789, + "nauc_map_at_1000_diff1": 0.847665, + "nauc_recall_at_1_max": 0.352353, + "nauc_recall_at_1_std": -0.141357, + "nauc_recall_at_1_diff1": 0.864989, + "nauc_recall_at_3_max": 0.480932, + "nauc_recall_at_3_std": -0.052897, + "nauc_recall_at_3_diff1": 0.79728, + "nauc_recall_at_5_max": 0.508466, + "nauc_recall_at_5_std": -0.115316, + "nauc_recall_at_5_diff1": 0.781217, + "nauc_recall_at_10_max": 0.639664, + "nauc_recall_at_10_std": 0.087873, + "nauc_recall_at_10_diff1": 0.809463, + "nauc_recall_at_20_max": 0.717614, + "nauc_recall_at_20_std": 0.301828, + "nauc_recall_at_20_diff1": 0.795353, + "nauc_recall_at_100_max": 0.930542, + "nauc_recall_at_100_std": 0.901949, + "nauc_recall_at_100_diff1": 0.832491, + "nauc_recall_at_1000_max": 0.930542, + "nauc_recall_at_1000_std": 0.901949, + "nauc_recall_at_1000_diff1": 0.832491, + "nauc_precision_at_1_max": 0.352049, + "nauc_precision_at_1_std": -0.138459, + "nauc_precision_at_1_diff1": 0.86209, + "nauc_precision_at_3_max": 0.430334, + "nauc_precision_at_3_std": -0.025088, + "nauc_precision_at_3_diff1": 0.680117, + "nauc_precision_at_5_max": 0.420275, + "nauc_precision_at_5_std": -0.048965, + "nauc_precision_at_5_diff1": 0.589139, + "nauc_precision_at_10_max": 0.371302, + "nauc_precision_at_10_std": 0.236178, + "nauc_precision_at_10_diff1": 0.344698, + "nauc_precision_at_20_max": 0.284954, + "nauc_precision_at_20_std": 0.410991, + "nauc_precision_at_20_diff1": 0.121427, + "nauc_precision_at_100_max": 0.061716, + "nauc_precision_at_100_std": 0.589643, + "nauc_precision_at_100_diff1": -0.281399, + "nauc_precision_at_1000_max": 0.061716, + "nauc_precision_at_1000_std": 0.589643, + "nauc_precision_at_1000_diff1": -0.281399, + "nauc_mrr_at_1_max": 0.352049, + "nauc_mrr_at_1_std": -0.138459, + "nauc_mrr_at_1_diff1": 0.86209, + "nauc_mrr_at_3_max": 0.393563, + "nauc_mrr_at_3_std": -0.100953, + "nauc_mrr_at_3_diff1": 0.843793, + "nauc_mrr_at_5_max": 0.391789, + "nauc_mrr_at_5_std": -0.110787, + "nauc_mrr_at_5_diff1": 0.844548, + "nauc_mrr_at_10_max": 0.392475, + "nauc_mrr_at_10_std": -0.104458, + "nauc_mrr_at_10_diff1": 0.847703, + "nauc_mrr_at_20_max": 0.391515, + "nauc_mrr_at_20_std": -0.103271, + "nauc_mrr_at_20_diff1": 0.847936, + "nauc_mrr_at_100_max": 0.390689, + "nauc_mrr_at_100_std": -0.103896, + "nauc_mrr_at_100_diff1": 0.848099, + "nauc_mrr_at_1000_max": 0.390689, + "nauc_mrr_at_1000_std": -0.103896, + "nauc_mrr_at_1000_diff1": 0.848099, + "main_score": 0.87404, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 91.08627414703369, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/DuRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/DuRetrieval.json new file mode 100644 index 0000000000..910aa51faf --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/DuRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a1a333e290fe30b10f3f56498e3a0d911a693ced", + "task_name": "DuRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.9565, + "ndcg_at_3": 0.93666, + "ndcg_at_5": 0.92715, + "ndcg_at_10": 0.9386, + "ndcg_at_20": 0.94796, + "ndcg_at_100": 0.95428, + "ndcg_at_1000": 0.95428, + "map_at_1": 0.29142, + "map_at_3": 0.62966, + "map_at_5": 0.79172, + "map_at_10": 0.88967, + "map_at_20": 0.90868, + "map_at_100": 0.91331, + "map_at_1000": 0.91331, + "recall_at_1": 0.29142, + "recall_at_3": 0.64278, + "recall_at_5": 0.81832, + "recall_at_10": 0.93836, + "recall_at_20": 0.97374, + "recall_at_100": 0.99326, + "recall_at_1000": 0.99326, + "precision_at_1": 0.9565, + "precision_at_3": 0.83567, + "precision_at_5": 0.7045, + "precision_at_10": 0.43925, + "precision_at_20": 0.23603, + "precision_at_100": 0.04882, + "precision_at_1000": 0.00488, + "mrr_at_1": 0.9565, + "mrr_at_3": 0.970083, + "mrr_at_5": 0.970858, + "mrr_at_10": 0.971579, + "mrr_at_20": 0.971766, + "mrr_at_100": 0.971804, + "mrr_at_1000": 0.971804, + "nauc_ndcg_at_1_max": 0.811863, + "nauc_ndcg_at_1_std": 0.712697, + "nauc_ndcg_at_1_diff1": 0.512852, + "nauc_ndcg_at_3_max": 0.562563, + "nauc_ndcg_at_3_std": 0.618202, + "nauc_ndcg_at_3_diff1": 0.047698, + "nauc_ndcg_at_5_max": 0.482255, + "nauc_ndcg_at_5_std": 0.491469, + "nauc_ndcg_at_5_diff1": 0.116774, + "nauc_ndcg_at_10_max": 0.50263, + "nauc_ndcg_at_10_std": 0.487954, + "nauc_ndcg_at_10_diff1": 0.158395, + "nauc_ndcg_at_20_max": 0.556649, + "nauc_ndcg_at_20_std": 0.543252, + "nauc_ndcg_at_20_diff1": 0.167857, + "nauc_ndcg_at_100_max": 0.575991, + "nauc_ndcg_at_100_std": 0.582628, + "nauc_ndcg_at_100_diff1": 0.157206, + "nauc_ndcg_at_1000_max": 0.575991, + "nauc_ndcg_at_1000_std": 0.582628, + "nauc_ndcg_at_1000_diff1": 0.157206, + "nauc_map_at_1_max": -0.131303, + "nauc_map_at_1_std": -0.518528, + "nauc_map_at_1_diff1": 0.552347, + "nauc_map_at_3_max": -0.010923, + "nauc_map_at_3_std": -0.473903, + "nauc_map_at_3_diff1": 0.371743, + "nauc_map_at_5_max": 0.152687, + "nauc_map_at_5_std": -0.251076, + "nauc_map_at_5_diff1": 0.265082, + "nauc_map_at_10_max": 0.392691, + "nauc_map_at_10_std": 0.22832, + "nauc_map_at_10_diff1": 0.142784, + "nauc_map_at_20_max": 0.456656, + "nauc_map_at_20_std": 0.418458, + "nauc_map_at_20_diff1": 0.106822, + "nauc_map_at_100_max": 0.463384, + "nauc_map_at_100_std": 0.454374, + "nauc_map_at_100_diff1": 0.095142, + "nauc_map_at_1000_max": 0.463384, + "nauc_map_at_1000_std": 0.454374, + "nauc_map_at_1000_diff1": 0.095142, + "nauc_recall_at_1_max": -0.131303, + "nauc_recall_at_1_std": -0.518528, + "nauc_recall_at_1_diff1": 0.552347, + "nauc_recall_at_3_max": -0.050171, + "nauc_recall_at_3_std": -0.518258, + "nauc_recall_at_3_diff1": 0.367493, + "nauc_recall_at_5_max": 0.07596, + "nauc_recall_at_5_std": -0.379676, + "nauc_recall_at_5_diff1": 0.290235, + "nauc_recall_at_10_max": 0.338115, + "nauc_recall_at_10_std": 0.053535, + "nauc_recall_at_10_diff1": 0.224616, + "nauc_recall_at_20_max": 0.507538, + "nauc_recall_at_20_std": 0.400235, + "nauc_recall_at_20_diff1": 0.194146, + "nauc_recall_at_100_max": 0.674811, + "nauc_recall_at_100_std": 0.674069, + "nauc_recall_at_100_diff1": 0.075576, + "nauc_recall_at_1000_max": 0.674811, + "nauc_recall_at_1000_std": 0.674069, + "nauc_recall_at_1000_diff1": 0.075576, + "nauc_precision_at_1_max": 0.811863, + "nauc_precision_at_1_std": 0.712697, + "nauc_precision_at_1_diff1": 0.512852, + "nauc_precision_at_3_max": 0.430955, + "nauc_precision_at_3_std": 0.801939, + "nauc_precision_at_3_diff1": -0.545671, + "nauc_precision_at_5_max": 0.330959, + "nauc_precision_at_5_std": 0.791235, + "nauc_precision_at_5_diff1": -0.479381, + "nauc_precision_at_10_max": 0.230025, + "nauc_precision_at_10_std": 0.736162, + "nauc_precision_at_10_diff1": -0.397533, + "nauc_precision_at_20_max": 0.17563, + "nauc_precision_at_20_std": 0.69705, + "nauc_precision_at_20_diff1": -0.370606, + "nauc_precision_at_100_max": 0.142013, + "nauc_precision_at_100_std": 0.657399, + "nauc_precision_at_100_diff1": -0.365443, + "nauc_precision_at_1000_max": 0.142013, + "nauc_precision_at_1000_std": 0.657399, + "nauc_precision_at_1000_diff1": -0.365443, + "nauc_mrr_at_1_max": 0.811863, + "nauc_mrr_at_1_std": 0.712697, + "nauc_mrr_at_1_diff1": 0.512852, + "nauc_mrr_at_3_max": 0.857655, + "nauc_mrr_at_3_std": 0.772317, + "nauc_mrr_at_3_diff1": 0.548247, + "nauc_mrr_at_5_max": 0.854823, + "nauc_mrr_at_5_std": 0.769362, + "nauc_mrr_at_5_diff1": 0.545857, + "nauc_mrr_at_10_max": 0.851525, + "nauc_mrr_at_10_std": 0.764226, + "nauc_mrr_at_10_diff1": 0.546919, + "nauc_mrr_at_20_max": 0.85054, + "nauc_mrr_at_20_std": 0.762661, + "nauc_mrr_at_20_diff1": 0.545189, + "nauc_mrr_at_100_max": 0.850342, + "nauc_mrr_at_100_std": 0.762347, + "nauc_mrr_at_100_diff1": 0.544708, + "nauc_mrr_at_1000_max": 0.850342, + "nauc_mrr_at_1000_std": 0.762347, + "nauc_mrr_at_1000_diff1": 0.544708, + "main_score": 0.9386, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 101.3613133430481, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/EcomRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/EcomRetrieval.json new file mode 100644 index 0000000000..7d18daa450 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/EcomRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "687de13dc7294d6fd9be10c6945f9e8fec8166b9", + "task_name": "EcomRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.573, + "ndcg_at_3": 0.67756, + "ndcg_at_5": 0.70729, + "ndcg_at_10": 0.72687, + "ndcg_at_20": 0.73759, + "ndcg_at_100": 0.74481, + "ndcg_at_1000": 0.74481, + "map_at_1": 0.573, + "map_at_3": 0.65183, + "map_at_5": 0.66838, + "map_at_10": 0.67637, + "map_at_20": 0.67937, + "map_at_100": 0.68039, + "map_at_1000": 0.68039, + "recall_at_1": 0.573, + "recall_at_3": 0.752, + "recall_at_5": 0.824, + "recall_at_10": 0.885, + "recall_at_20": 0.927, + "recall_at_100": 0.966, + "recall_at_1000": 0.966, + "precision_at_1": 0.573, + "precision_at_3": 0.25067, + "precision_at_5": 0.1648, + "precision_at_10": 0.0885, + "precision_at_20": 0.04635, + "precision_at_100": 0.00966, + "precision_at_1000": 0.00097, + "mrr_at_1": 0.573, + "mrr_at_3": 0.651833, + "mrr_at_5": 0.668383, + "mrr_at_10": 0.676373, + "mrr_at_20": 0.67937, + "mrr_at_100": 0.680386, + "mrr_at_1000": 0.680386, + "nauc_ndcg_at_1_max": 0.353532, + "nauc_ndcg_at_1_std": 0.002037, + "nauc_ndcg_at_1_diff1": 0.795857, + "nauc_ndcg_at_3_max": 0.399097, + "nauc_ndcg_at_3_std": 0.084595, + "nauc_ndcg_at_3_diff1": 0.764243, + "nauc_ndcg_at_5_max": 0.404842, + "nauc_ndcg_at_5_std": 0.101574, + "nauc_ndcg_at_5_diff1": 0.758077, + "nauc_ndcg_at_10_max": 0.418343, + "nauc_ndcg_at_10_std": 0.12268, + "nauc_ndcg_at_10_diff1": 0.758866, + "nauc_ndcg_at_20_max": 0.4113, + "nauc_ndcg_at_20_std": 0.117088, + "nauc_ndcg_at_20_diff1": 0.762941, + "nauc_ndcg_at_100_max": 0.413461, + "nauc_ndcg_at_100_std": 0.110819, + "nauc_ndcg_at_100_diff1": 0.766884, + "nauc_ndcg_at_1000_max": 0.413461, + "nauc_ndcg_at_1000_std": 0.110819, + "nauc_ndcg_at_1000_diff1": 0.766884, + "nauc_map_at_1_max": 0.353532, + "nauc_map_at_1_std": 0.002037, + "nauc_map_at_1_diff1": 0.795857, + "nauc_map_at_3_max": 0.386723, + "nauc_map_at_3_std": 0.0596, + "nauc_map_at_3_diff1": 0.773212, + "nauc_map_at_5_max": 0.389243, + "nauc_map_at_5_std": 0.067225, + "nauc_map_at_5_diff1": 0.770284, + "nauc_map_at_10_max": 0.393451, + "nauc_map_at_10_std": 0.073272, + "nauc_map_at_10_diff1": 0.770816, + "nauc_map_at_20_max": 0.391388, + "nauc_map_at_20_std": 0.071285, + "nauc_map_at_20_diff1": 0.771751, + "nauc_map_at_100_max": 0.391632, + "nauc_map_at_100_std": 0.070787, + "nauc_map_at_100_diff1": 0.772267, + "nauc_map_at_1000_max": 0.391632, + "nauc_map_at_1000_std": 0.070787, + "nauc_map_at_1000_diff1": 0.772267, + "nauc_recall_at_1_max": 0.353532, + "nauc_recall_at_1_std": 0.002037, + "nauc_recall_at_1_diff1": 0.795857, + "nauc_recall_at_3_max": 0.444086, + "nauc_recall_at_3_std": 0.176852, + "nauc_recall_at_3_diff1": 0.731429, + "nauc_recall_at_5_max": 0.478041, + "nauc_recall_at_5_std": 0.266061, + "nauc_recall_at_5_diff1": 0.700288, + "nauc_recall_at_10_max": 0.597914, + "nauc_recall_at_10_std": 0.480308, + "nauc_recall_at_10_diff1": 0.679032, + "nauc_recall_at_20_max": 0.61906, + "nauc_recall_at_20_std": 0.615831, + "nauc_recall_at_20_diff1": 0.686044, + "nauc_recall_at_100_max": 0.917696, + "nauc_recall_at_100_std": 0.937538, + "nauc_recall_at_100_diff1": 0.724708, + "nauc_recall_at_1000_max": 0.917696, + "nauc_recall_at_1000_std": 0.937538, + "nauc_recall_at_1000_diff1": 0.724708, + "nauc_precision_at_1_max": 0.353532, + "nauc_precision_at_1_std": 0.002037, + "nauc_precision_at_1_diff1": 0.795857, + "nauc_precision_at_3_max": 0.444086, + "nauc_precision_at_3_std": 0.176852, + "nauc_precision_at_3_diff1": 0.731429, + "nauc_precision_at_5_max": 0.478041, + "nauc_precision_at_5_std": 0.266061, + "nauc_precision_at_5_diff1": 0.700288, + "nauc_precision_at_10_max": 0.597914, + "nauc_precision_at_10_std": 0.480308, + "nauc_precision_at_10_diff1": 0.679032, + "nauc_precision_at_20_max": 0.61906, + "nauc_precision_at_20_std": 0.615831, + "nauc_precision_at_20_diff1": 0.686044, + "nauc_precision_at_100_max": 0.917696, + "nauc_precision_at_100_std": 0.937538, + "nauc_precision_at_100_diff1": 0.724708, + "nauc_precision_at_1000_max": 0.917696, + "nauc_precision_at_1000_std": 0.937538, + "nauc_precision_at_1000_diff1": 0.724708, + "nauc_mrr_at_1_max": 0.353532, + "nauc_mrr_at_1_std": 0.002037, + "nauc_mrr_at_1_diff1": 0.795857, + "nauc_mrr_at_3_max": 0.386723, + "nauc_mrr_at_3_std": 0.0596, + "nauc_mrr_at_3_diff1": 0.773212, + "nauc_mrr_at_5_max": 0.389243, + "nauc_mrr_at_5_std": 0.067225, + "nauc_mrr_at_5_diff1": 0.770284, + "nauc_mrr_at_10_max": 0.393451, + "nauc_mrr_at_10_std": 0.073272, + "nauc_mrr_at_10_diff1": 0.770816, + "nauc_mrr_at_20_max": 0.391388, + "nauc_mrr_at_20_std": 0.071285, + "nauc_mrr_at_20_diff1": 0.771751, + "nauc_mrr_at_100_max": 0.391632, + "nauc_mrr_at_100_std": 0.070787, + "nauc_mrr_at_100_diff1": 0.772267, + "nauc_mrr_at_1000_max": 0.391632, + "nauc_mrr_at_1000_std": 0.070787, + "nauc_mrr_at_1000_diff1": 0.772267, + "main_score": 0.72687, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 56.825522899627686, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/FEVERHardNegatives.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/FEVERHardNegatives.json new file mode 100644 index 0000000000..054ea26aa6 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/FEVERHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "080c9ed6267b65029207906e815d44a9240bafca", + "task_name": "FEVERHardNegatives", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.918, + "ndcg_at_3": 0.93624, + "ndcg_at_5": 0.9401, + "ndcg_at_10": 0.94531, + "ndcg_at_20": 0.94768, + "ndcg_at_100": 0.94983, + "ndcg_at_1000": 0.94983, + "map_at_1": 0.86355, + "map_at_3": 0.91671, + "map_at_5": 0.92126, + "map_at_10": 0.92517, + "map_at_20": 0.92622, + "map_at_100": 0.92676, + "map_at_1000": 0.92676, + "recall_at_1": 0.86355, + "recall_at_3": 0.953, + "recall_at_5": 0.96501, + "recall_at_10": 0.97899, + "recall_at_20": 0.98671, + "recall_at_100": 0.99488, + "recall_at_1000": 0.99488, + "precision_at_1": 0.918, + "precision_at_3": 0.348, + "precision_at_5": 0.2136, + "precision_at_10": 0.11, + "precision_at_20": 0.05595, + "precision_at_100": 0.01147, + "precision_at_1000": 0.00115, + "mrr_at_1": 0.918, + "mrr_at_3": 0.9495, + "mrr_at_5": 0.95085, + "mrr_at_10": 0.951461, + "mrr_at_20": 0.951688, + "mrr_at_100": 0.95173, + "mrr_at_1000": 0.95173, + "nauc_ndcg_at_1_max": 0.235306, + "nauc_ndcg_at_1_std": -0.321793, + "nauc_ndcg_at_1_diff1": 0.795086, + "nauc_ndcg_at_3_max": 0.267866, + "nauc_ndcg_at_3_std": -0.048482, + "nauc_ndcg_at_3_diff1": 0.594016, + "nauc_ndcg_at_5_max": 0.272921, + "nauc_ndcg_at_5_std": -0.036112, + "nauc_ndcg_at_5_diff1": 0.581213, + "nauc_ndcg_at_10_max": 0.274524, + "nauc_ndcg_at_10_std": -0.039874, + "nauc_ndcg_at_10_diff1": 0.577328, + "nauc_ndcg_at_20_max": 0.276754, + "nauc_ndcg_at_20_std": -0.048239, + "nauc_ndcg_at_20_diff1": 0.585156, + "nauc_ndcg_at_100_max": 0.271324, + "nauc_ndcg_at_100_std": -0.084389, + "nauc_ndcg_at_100_diff1": 0.610773, + "nauc_ndcg_at_1000_max": 0.271324, + "nauc_ndcg_at_1000_std": -0.084389, + "nauc_ndcg_at_1000_diff1": 0.610773, + "nauc_map_at_1_max": 0.212411, + "nauc_map_at_1_std": -0.161522, + "nauc_map_at_1_diff1": 0.6407, + "nauc_map_at_3_max": 0.223767, + "nauc_map_at_3_std": -0.057588, + "nauc_map_at_3_diff1": 0.572802, + "nauc_map_at_5_max": 0.240677, + "nauc_map_at_5_std": -0.054325, + "nauc_map_at_5_diff1": 0.57291, + "nauc_map_at_10_max": 0.244323, + "nauc_map_at_10_std": -0.054531, + "nauc_map_at_10_diff1": 0.572627, + "nauc_map_at_20_max": 0.245742, + "nauc_map_at_20_std": -0.056381, + "nauc_map_at_20_diff1": 0.575926, + "nauc_map_at_100_max": 0.24537, + "nauc_map_at_100_std": -0.062601, + "nauc_map_at_100_diff1": 0.58021, + "nauc_map_at_1000_max": 0.24537, + "nauc_map_at_1000_std": -0.062601, + "nauc_map_at_1000_diff1": 0.58021, + "nauc_recall_at_1_max": 0.212411, + "nauc_recall_at_1_std": -0.161522, + "nauc_recall_at_1_diff1": 0.6407, + "nauc_recall_at_3_max": 0.304669, + "nauc_recall_at_3_std": 0.244024, + "nauc_recall_at_3_diff1": 0.395895, + "nauc_recall_at_5_max": 0.328373, + "nauc_recall_at_5_std": 0.294266, + "nauc_recall_at_5_diff1": 0.33202, + "nauc_recall_at_10_max": 0.362664, + "nauc_recall_at_10_std": 0.463916, + "nauc_recall_at_10_diff1": 0.175217, + "nauc_recall_at_20_max": 0.443509, + "nauc_recall_at_20_std": 0.680405, + "nauc_recall_at_20_diff1": 0.027413, + "nauc_recall_at_100_max": 0.452468, + "nauc_recall_at_100_std": 0.499286, + "nauc_recall_at_100_diff1": 0.095149, + "nauc_recall_at_1000_max": 0.452468, + "nauc_recall_at_1000_std": 0.499286, + "nauc_recall_at_1000_diff1": 0.095149, + "nauc_precision_at_1_max": 0.235306, + "nauc_precision_at_1_std": -0.321793, + "nauc_precision_at_1_diff1": 0.795086, + "nauc_precision_at_3_max": -0.034917, + "nauc_precision_at_3_std": 0.08372, + "nauc_precision_at_3_diff1": -0.189386, + "nauc_precision_at_5_max": -0.006695, + "nauc_precision_at_5_std": 0.085296, + "nauc_precision_at_5_diff1": -0.240655, + "nauc_precision_at_10_max": -0.039975, + "nauc_precision_at_10_std": 0.048105, + "nauc_precision_at_10_diff1": -0.269115, + "nauc_precision_at_20_max": -0.035404, + "nauc_precision_at_20_std": 0.030168, + "nauc_precision_at_20_diff1": -0.267292, + "nauc_precision_at_100_max": -0.064461, + "nauc_precision_at_100_std": -0.056505, + "nauc_precision_at_100_diff1": -0.219641, + "nauc_precision_at_1000_max": -0.064461, + "nauc_precision_at_1000_std": -0.056505, + "nauc_precision_at_1000_diff1": -0.219641, + "nauc_mrr_at_1_max": 0.235306, + "nauc_mrr_at_1_std": -0.321793, + "nauc_mrr_at_1_diff1": 0.795086, + "nauc_mrr_at_3_max": 0.329766, + "nauc_mrr_at_3_std": -0.239938, + "nauc_mrr_at_3_diff1": 0.793783, + "nauc_mrr_at_5_max": 0.316351, + "nauc_mrr_at_5_std": -0.258581, + "nauc_mrr_at_5_diff1": 0.796131, + "nauc_mrr_at_10_max": 0.309948, + "nauc_mrr_at_10_std": -0.262331, + "nauc_mrr_at_10_diff1": 0.795993, + "nauc_mrr_at_20_max": 0.306707, + "nauc_mrr_at_20_std": -0.263933, + "nauc_mrr_at_20_diff1": 0.795468, + "nauc_mrr_at_100_max": 0.306108, + "nauc_mrr_at_100_std": -0.26464, + "nauc_mrr_at_100_diff1": 0.795292, + "nauc_mrr_at_1000_max": 0.306108, + "nauc_mrr_at_1000_std": -0.26464, + "nauc_mrr_at_1000_diff1": 0.795292, + "main_score": 0.94531, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 177.0243697166443, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/FiQA2018.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/FiQA2018.json new file mode 100644 index 0000000000..d720e84408 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/FiQA2018.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "27a168819829fe9bcd655c2df245fb19452e8e06", + "task_name": "FiQA2018", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.65432, + "ndcg_at_3": 0.60692, + "ndcg_at_5": 0.61948, + "ndcg_at_10": 0.64754, + "ndcg_at_20": 0.67248, + "ndcg_at_100": 0.69502, + "ndcg_at_1000": 0.69502, + "map_at_1": 0.3515, + "map_at_3": 0.50009, + "map_at_5": 0.53831, + "map_at_10": 0.56688, + "map_at_20": 0.57899, + "map_at_100": 0.5849, + "map_at_1000": 0.5849, + "recall_at_1": 0.3515, + "recall_at_3": 0.54711, + "recall_at_5": 0.62091, + "recall_at_10": 0.71215, + "recall_at_20": 0.78728, + "recall_at_100": 0.87464, + "recall_at_1000": 0.87464, + "precision_at_1": 0.65432, + "precision_at_3": 0.40072, + "precision_at_5": 0.28981, + "precision_at_10": 0.17454, + "precision_at_20": 0.09907, + "precision_at_100": 0.02275, + "precision_at_1000": 0.00227, + "mrr_at_1": 0.654321, + "mrr_at_3": 0.708333, + "mrr_at_5": 0.717284, + "mrr_at_10": 0.725028, + "mrr_at_20": 0.727697, + "mrr_at_100": 0.728446, + "mrr_at_1000": 0.728446, + "nauc_ndcg_at_1_max": 0.577339, + "nauc_ndcg_at_1_std": 0.361425, + "nauc_ndcg_at_1_diff1": 0.585664, + "nauc_ndcg_at_3_max": 0.459646, + "nauc_ndcg_at_3_std": 0.320414, + "nauc_ndcg_at_3_diff1": 0.430696, + "nauc_ndcg_at_5_max": 0.454564, + "nauc_ndcg_at_5_std": 0.317496, + "nauc_ndcg_at_5_diff1": 0.439113, + "nauc_ndcg_at_10_max": 0.460599, + "nauc_ndcg_at_10_std": 0.335731, + "nauc_ndcg_at_10_diff1": 0.444234, + "nauc_ndcg_at_20_max": 0.47556, + "nauc_ndcg_at_20_std": 0.36205, + "nauc_ndcg_at_20_diff1": 0.451564, + "nauc_ndcg_at_100_max": 0.482958, + "nauc_ndcg_at_100_std": 0.367772, + "nauc_ndcg_at_100_diff1": 0.448508, + "nauc_ndcg_at_1000_max": 0.482958, + "nauc_ndcg_at_1000_std": 0.367772, + "nauc_ndcg_at_1000_diff1": 0.448508, + "nauc_map_at_1_max": 0.167912, + "nauc_map_at_1_std": 0.012873, + "nauc_map_at_1_diff1": 0.480988, + "nauc_map_at_3_max": 0.329472, + "nauc_map_at_3_std": 0.157596, + "nauc_map_at_3_diff1": 0.452379, + "nauc_map_at_5_max": 0.385329, + "nauc_map_at_5_std": 0.229572, + "nauc_map_at_5_diff1": 0.441844, + "nauc_map_at_10_max": 0.415981, + "nauc_map_at_10_std": 0.279078, + "nauc_map_at_10_diff1": 0.432434, + "nauc_map_at_20_max": 0.425016, + "nauc_map_at_20_std": 0.29569, + "nauc_map_at_20_diff1": 0.43348, + "nauc_map_at_100_max": 0.427241, + "nauc_map_at_100_std": 0.298876, + "nauc_map_at_100_diff1": 0.431773, + "nauc_map_at_1000_max": 0.427241, + "nauc_map_at_1000_std": 0.298876, + "nauc_map_at_1000_diff1": 0.431773, + "nauc_recall_at_1_max": 0.167912, + "nauc_recall_at_1_std": 0.012873, + "nauc_recall_at_1_diff1": 0.480988, + "nauc_recall_at_3_max": 0.294898, + "nauc_recall_at_3_std": 0.149899, + "nauc_recall_at_3_diff1": 0.40887, + "nauc_recall_at_5_max": 0.339007, + "nauc_recall_at_5_std": 0.215259, + "nauc_recall_at_5_diff1": 0.3825, + "nauc_recall_at_10_max": 0.372897, + "nauc_recall_at_10_std": 0.32205, + "nauc_recall_at_10_diff1": 0.345222, + "nauc_recall_at_20_max": 0.391469, + "nauc_recall_at_20_std": 0.409639, + "nauc_recall_at_20_diff1": 0.343609, + "nauc_recall_at_100_max": 0.401473, + "nauc_recall_at_100_std": 0.469923, + "nauc_recall_at_100_diff1": 0.290721, + "nauc_recall_at_1000_max": 0.401473, + "nauc_recall_at_1000_std": 0.469923, + "nauc_recall_at_1000_diff1": 0.290721, + "nauc_precision_at_1_max": 0.577339, + "nauc_precision_at_1_std": 0.361425, + "nauc_precision_at_1_diff1": 0.585664, + "nauc_precision_at_3_max": 0.462802, + "nauc_precision_at_3_std": 0.431748, + "nauc_precision_at_3_diff1": 0.114188, + "nauc_precision_at_5_max": 0.45384, + "nauc_precision_at_5_std": 0.485715, + "nauc_precision_at_5_diff1": 0.022995, + "nauc_precision_at_10_max": 0.408241, + "nauc_precision_at_10_std": 0.503674, + "nauc_precision_at_10_diff1": -0.061916, + "nauc_precision_at_20_max": 0.361543, + "nauc_precision_at_20_std": 0.491897, + "nauc_precision_at_20_diff1": -0.110808, + "nauc_precision_at_100_max": 0.283961, + "nauc_precision_at_100_std": 0.405025, + "nauc_precision_at_100_diff1": -0.167647, + "nauc_precision_at_1000_max": 0.283961, + "nauc_precision_at_1000_std": 0.405025, + "nauc_precision_at_1000_diff1": -0.167647, + "nauc_mrr_at_1_max": 0.577339, + "nauc_mrr_at_1_std": 0.361425, + "nauc_mrr_at_1_diff1": 0.585664, + "nauc_mrr_at_3_max": 0.587316, + "nauc_mrr_at_3_std": 0.393474, + "nauc_mrr_at_3_diff1": 0.568018, + "nauc_mrr_at_5_max": 0.587971, + "nauc_mrr_at_5_std": 0.390966, + "nauc_mrr_at_5_diff1": 0.57003, + "nauc_mrr_at_10_max": 0.589658, + "nauc_mrr_at_10_std": 0.397755, + "nauc_mrr_at_10_diff1": 0.569563, + "nauc_mrr_at_20_max": 0.589717, + "nauc_mrr_at_20_std": 0.398659, + "nauc_mrr_at_20_diff1": 0.568717, + "nauc_mrr_at_100_max": 0.590293, + "nauc_mrr_at_100_std": 0.398906, + "nauc_mrr_at_100_diff1": 0.569189, + "nauc_mrr_at_1000_max": 0.590293, + "nauc_mrr_at_1000_std": 0.398906, + "nauc_mrr_at_1000_diff1": 0.569189, + "main_score": 0.64754, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 60.506704330444336, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/HotpotQAHardNegatives.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/HotpotQAHardNegatives.json new file mode 100644 index 0000000000..0b3529edec --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/HotpotQAHardNegatives.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "617612fa63afcb60e3b134bed8b7216a99707c37", + "task_name": "HotpotQAHardNegatives", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.889, + "ndcg_at_3": 0.81631, + "ndcg_at_5": 0.84214, + "ndcg_at_10": 0.86074, + "ndcg_at_20": 0.86822, + "ndcg_at_100": 0.87904, + "ndcg_at_1000": 0.87904, + "map_at_1": 0.4445, + "map_at_3": 0.77058, + "map_at_5": 0.79286, + "map_at_10": 0.8047, + "map_at_20": 0.80787, + "map_at_100": 0.81036, + "map_at_1000": 0.81036, + "recall_at_1": 0.4445, + "recall_at_3": 0.8075, + "recall_at_5": 0.8585, + "recall_at_10": 0.905, + "recall_at_20": 0.929, + "recall_at_100": 0.9755, + "recall_at_1000": 0.9755, + "precision_at_1": 0.889, + "precision_at_3": 0.53833, + "precision_at_5": 0.3434, + "precision_at_10": 0.181, + "precision_at_20": 0.0929, + "precision_at_100": 0.01951, + "precision_at_1000": 0.00195, + "mrr_at_1": 0.889, + "mrr_at_3": 0.922833, + "mrr_at_5": 0.925183, + "mrr_at_10": 0.926821, + "mrr_at_20": 0.927257, + "mrr_at_100": 0.927444, + "mrr_at_1000": 0.927444, + "nauc_ndcg_at_1_max": 0.424685, + "nauc_ndcg_at_1_std": 0.291002, + "nauc_ndcg_at_1_diff1": 0.612757, + "nauc_ndcg_at_3_max": 0.345483, + "nauc_ndcg_at_3_std": 0.437667, + "nauc_ndcg_at_3_diff1": 0.052163, + "nauc_ndcg_at_5_max": 0.364938, + "nauc_ndcg_at_5_std": 0.476479, + "nauc_ndcg_at_5_diff1": 0.068611, + "nauc_ndcg_at_10_max": 0.37478, + "nauc_ndcg_at_10_std": 0.489518, + "nauc_ndcg_at_10_diff1": 0.076906, + "nauc_ndcg_at_20_max": 0.371186, + "nauc_ndcg_at_20_std": 0.474547, + "nauc_ndcg_at_20_diff1": 0.084856, + "nauc_ndcg_at_100_max": 0.370221, + "nauc_ndcg_at_100_std": 0.465461, + "nauc_ndcg_at_100_diff1": 0.102797, + "nauc_ndcg_at_1000_max": 0.370221, + "nauc_ndcg_at_1000_std": 0.465461, + "nauc_ndcg_at_1000_diff1": 0.102797, + "nauc_map_at_1_max": 0.424685, + "nauc_map_at_1_std": 0.291002, + "nauc_map_at_1_diff1": 0.612757, + "nauc_map_at_3_max": 0.32076, + "nauc_map_at_3_std": 0.424303, + "nauc_map_at_3_diff1": 0.005867, + "nauc_map_at_5_max": 0.335665, + "nauc_map_at_5_std": 0.452509, + "nauc_map_at_5_diff1": 0.017447, + "nauc_map_at_10_max": 0.340572, + "nauc_map_at_10_std": 0.459149, + "nauc_map_at_10_diff1": 0.021958, + "nauc_map_at_20_max": 0.339235, + "nauc_map_at_20_std": 0.454361, + "nauc_map_at_20_diff1": 0.023484, + "nauc_map_at_100_max": 0.338567, + "nauc_map_at_100_std": 0.453242, + "nauc_map_at_100_diff1": 0.025752, + "nauc_map_at_1000_max": 0.338567, + "nauc_map_at_1000_std": 0.453242, + "nauc_map_at_1000_diff1": 0.025752, + "nauc_recall_at_1_max": 0.424685, + "nauc_recall_at_1_std": 0.291002, + "nauc_recall_at_1_diff1": 0.612757, + "nauc_recall_at_3_max": 0.331175, + "nauc_recall_at_3_std": 0.47592, + "nauc_recall_at_3_diff1": -0.06212, + "nauc_recall_at_5_max": 0.363474, + "nauc_recall_at_5_std": 0.56028, + "nauc_recall_at_5_diff1": -0.058796, + "nauc_recall_at_10_max": 0.390403, + "nauc_recall_at_10_std": 0.630097, + "nauc_recall_at_10_diff1": -0.083833, + "nauc_recall_at_20_max": 0.383484, + "nauc_recall_at_20_std": 0.60026, + "nauc_recall_at_20_diff1": -0.085197, + "nauc_recall_at_100_max": 0.419863, + "nauc_recall_at_100_std": 0.667057, + "nauc_recall_at_100_diff1": -0.066951, + "nauc_recall_at_1000_max": 0.419863, + "nauc_recall_at_1000_std": 0.667057, + "nauc_recall_at_1000_diff1": -0.066951, + "nauc_precision_at_1_max": 0.424685, + "nauc_precision_at_1_std": 0.291002, + "nauc_precision_at_1_diff1": 0.612757, + "nauc_precision_at_3_max": 0.331175, + "nauc_precision_at_3_std": 0.47592, + "nauc_precision_at_3_diff1": -0.06212, + "nauc_precision_at_5_max": 0.363474, + "nauc_precision_at_5_std": 0.56028, + "nauc_precision_at_5_diff1": -0.058796, + "nauc_precision_at_10_max": 0.390403, + "nauc_precision_at_10_std": 0.630097, + "nauc_precision_at_10_diff1": -0.083833, + "nauc_precision_at_20_max": 0.383484, + "nauc_precision_at_20_std": 0.60026, + "nauc_precision_at_20_diff1": -0.085197, + "nauc_precision_at_100_max": 0.419863, + "nauc_precision_at_100_std": 0.667057, + "nauc_precision_at_100_diff1": -0.066951, + "nauc_precision_at_1000_max": 0.419863, + "nauc_precision_at_1000_std": 0.667057, + "nauc_precision_at_1000_diff1": -0.066951, + "nauc_mrr_at_1_max": 0.424685, + "nauc_mrr_at_1_std": 0.291002, + "nauc_mrr_at_1_diff1": 0.612757, + "nauc_mrr_at_3_max": 0.499793, + "nauc_mrr_at_3_std": 0.379719, + "nauc_mrr_at_3_diff1": 0.618091, + "nauc_mrr_at_5_max": 0.490611, + "nauc_mrr_at_5_std": 0.373692, + "nauc_mrr_at_5_diff1": 0.617753, + "nauc_mrr_at_10_max": 0.487557, + "nauc_mrr_at_10_std": 0.367836, + "nauc_mrr_at_10_diff1": 0.614938, + "nauc_mrr_at_20_max": 0.485675, + "nauc_mrr_at_20_std": 0.365314, + "nauc_mrr_at_20_diff1": 0.616267, + "nauc_mrr_at_100_max": 0.4848, + "nauc_mrr_at_100_std": 0.364168, + "nauc_mrr_at_100_diff1": 0.616535, + "nauc_mrr_at_1000_max": 0.4848, + "nauc_mrr_at_1000_std": 0.364168, + "nauc_mrr_at_1000_diff1": 0.616535, + "main_score": 0.86074, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 127.46883988380432, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/IFlyTek.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/IFlyTek.json new file mode 100644 index 0000000000..52ff16b473 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/IFlyTek.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "421605374b29664c5fc098418fe20ada9bd55f8a", + "task_name": "IFlyTek", + "mteb_version": "1.36.5", + "scores": { + "validation": [ + { + "accuracy": 0.576991, + "f1": 0.400848, + "f1_weighted": 0.570334, + "scores_per_experiment": [ + { + "accuracy": 0.582532, + "f1": 0.396199, + "f1_weighted": 0.575416 + }, + { + "accuracy": 0.571374, + "f1": 0.395993, + "f1_weighted": 0.562821 + }, + { + "accuracy": 0.575991, + "f1": 0.40841, + "f1_weighted": 0.572475 + }, + { + "accuracy": 0.574067, + "f1": 0.400297, + "f1_weighted": 0.566854 + }, + { + "accuracy": 0.580993, + "f1": 0.403342, + "f1_weighted": 0.574103 + } + ], + "main_score": 0.576991, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 50.83578848838806, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/ImdbClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/ImdbClassification.json new file mode 100644 index 0000000000..1e1ba20b73 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/ImdbClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "3d86128a09e091d6018b6d26cad27f2739fc2db7", + "task_name": "ImdbClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.9698, + "f1": 0.969789, + "f1_weighted": 0.969789, + "ap": 0.95064, + "ap_weighted": 0.95064, + "scores_per_experiment": [ + { + "accuracy": 0.97416, + "f1": 0.97416, + "f1_weighted": 0.97416, + "ap": 0.962304, + "ap_weighted": 0.962304 + }, + { + "accuracy": 0.97324, + "f1": 0.973239, + "f1_weighted": 0.973239, + "ap": 0.962619, + "ap_weighted": 0.962619 + }, + { + "accuracy": 0.96604, + "f1": 0.966021, + "f1_weighted": 0.966021, + "ap": 0.940345, + "ap_weighted": 0.940345 + }, + { + "accuracy": 0.97012, + "f1": 0.97011, + "f1_weighted": 0.97011, + "ap": 0.948376, + "ap_weighted": 0.948376 + }, + { + "accuracy": 0.97408, + "f1": 0.974079, + "f1_weighted": 0.974079, + "ap": 0.959162, + "ap_weighted": 0.959162 + }, + { + "accuracy": 0.97172, + "f1": 0.971716, + "f1_weighted": 0.971716, + "ap": 0.953453, + "ap_weighted": 0.953453 + }, + { + "accuracy": 0.9666, + "f1": 0.966584, + "f1_weighted": 0.966584, + "ap": 0.941952, + "ap_weighted": 0.941952 + }, + { + "accuracy": 0.9576, + "f1": 0.957546, + "f1_weighted": 0.957546, + "ap": 0.924309, + "ap_weighted": 0.924309 + }, + { + "accuracy": 0.97148, + "f1": 0.971475, + "f1_weighted": 0.971475, + "ap": 0.952231, + "ap_weighted": 0.952231 + }, + { + "accuracy": 0.97296, + "f1": 0.97296, + "f1_weighted": 0.97296, + "ap": 0.961648, + "ap_weighted": 0.961648 + } + ], + "main_score": 0.9698, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 49.78438591957092, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/JDReview.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/JDReview.json new file mode 100644 index 0000000000..e18417c943 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/JDReview.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "b7c64bd89eb87f8ded463478346f76731f07bf8b", + "task_name": "JDReview", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.899437, + "f1": 0.855714, + "f1_weighted": 0.905606, + "ap": 0.633345, + "ap_weighted": 0.633345, + "scores_per_experiment": [ + { + "accuracy": 0.915572, + "f1": 0.872413, + "f1_weighted": 0.919053, + "ap": 0.664004, + "ap_weighted": 0.664004 + }, + { + "accuracy": 0.894934, + "f1": 0.849435, + "f1_weighted": 0.901456, + "ap": 0.61948, + "ap_weighted": 0.61948 + }, + { + "accuracy": 0.894934, + "f1": 0.851183, + "f1_weighted": 0.901898, + "ap": 0.624916, + "ap_weighted": 0.624916 + }, + { + "accuracy": 0.896811, + "f1": 0.852559, + "f1_weighted": 0.903327, + "ap": 0.626494, + "ap_weighted": 0.626494 + }, + { + "accuracy": 0.893058, + "f1": 0.849817, + "f1_weighted": 0.900467, + "ap": 0.623462, + "ap_weighted": 0.623462 + }, + { + "accuracy": 0.88743, + "f1": 0.843235, + "f1_weighted": 0.89555, + "ap": 0.611224, + "ap_weighted": 0.611224 + }, + { + "accuracy": 0.913696, + "f1": 0.870006, + "f1_weighted": 0.917372, + "ap": 0.658843, + "ap_weighted": 0.658843 + }, + { + "accuracy": 0.913696, + "f1": 0.870849, + "f1_weighted": 0.917604, + "ap": 0.661123, + "ap_weighted": 0.661123 + }, + { + "accuracy": 0.900563, + "f1": 0.858747, + "f1_weighted": 0.907051, + "ap": 0.640571, + "ap_weighted": 0.640571 + }, + { + "accuracy": 0.883677, + "f1": 0.838901, + "f1_weighted": 0.892282, + "ap": 0.603332, + "ap_weighted": 0.603332 + } + ], + "main_score": 0.899437, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 17.543463468551636, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/LCQMC.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/LCQMC.json new file mode 100644 index 0000000000..65600c2cdb --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/LCQMC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "17f9b096f80380fce5ed12a9be8be7784b337daf", + "task_name": "LCQMC", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.784937, + "spearman": 0.807044, + "cosine_pearson": 0.784937, + "cosine_spearman": 0.807044, + "manhattan_pearson": 0.817418, + "manhattan_spearman": 0.806785, + "euclidean_pearson": 0.818083, + "euclidean_spearman": 0.807059, + "main_score": 0.807044, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 18.196677923202515, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/MMarcoReranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/MMarcoReranking.json new file mode 100644 index 0000000000..77a4c12ae3 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/MMarcoReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "8e0c766dbe9e16e1d221116a3f36795fbade07f6", + "task_name": "MMarcoReranking", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "map": 0.376452, + "mrr": 0.366865, + "nAUC_map_max": 0.071087, + "nAUC_map_std": -0.27992, + "nAUC_map_diff1": 0.218469, + "nAUC_mrr_max": 0.079017, + "nAUC_mrr_std": -0.277124, + "nAUC_mrr_diff1": 0.218843, + "main_score": 0.376452, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 57.06361532211304, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/MMarcoRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/MMarcoRetrieval.json new file mode 100644 index 0000000000..3d6447c044 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/MMarcoRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "539bbde593d947e2a124ba72651aafc09eb33fc2", + "task_name": "MMarcoRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.80688, + "ndcg_at_3": 0.86353, + "ndcg_at_5": 0.87832, + "ndcg_at_10": 0.88777, + "ndcg_at_20": 0.89099, + "ndcg_at_100": 0.89356, + "ndcg_at_1000": 0.89356, + "map_at_1": 0.78194, + "map_at_3": 0.84743, + "map_at_5": 0.85643, + "map_at_10": 0.86065, + "map_at_20": 0.86162, + "map_at_100": 0.86204, + "map_at_1000": 0.86204, + "recall_at_1": 0.78194, + "recall_at_3": 0.90347, + "recall_at_5": 0.93861, + "recall_at_10": 0.96667, + "recall_at_20": 0.97898, + "recall_at_100": 0.99191, + "recall_at_1000": 0.99191, + "precision_at_1": 0.80688, + "precision_at_3": 0.31891, + "precision_at_5": 0.19943, + "precision_at_10": 0.10284, + "precision_at_20": 0.05211, + "precision_at_100": 0.01057, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.806877, + "mrr_at_3": 0.853438, + "mrr_at_5": 0.860738, + "mrr_at_10": 0.864443, + "mrr_at_20": 0.865249, + "mrr_at_100": 0.865607, + "mrr_at_1000": 0.865607, + "nauc_ndcg_at_1_max": 0.405091, + "nauc_ndcg_at_1_std": -0.292537, + "nauc_ndcg_at_1_diff1": 0.856605, + "nauc_ndcg_at_3_max": 0.463335, + "nauc_ndcg_at_3_std": -0.213601, + "nauc_ndcg_at_3_diff1": 0.835379, + "nauc_ndcg_at_5_max": 0.480007, + "nauc_ndcg_at_5_std": -0.173928, + "nauc_ndcg_at_5_diff1": 0.839611, + "nauc_ndcg_at_10_max": 0.478442, + "nauc_ndcg_at_10_std": -0.163973, + "nauc_ndcg_at_10_diff1": 0.841442, + "nauc_ndcg_at_20_max": 0.472459, + "nauc_ndcg_at_20_std": -0.174161, + "nauc_ndcg_at_20_diff1": 0.843767, + "nauc_ndcg_at_100_max": 0.463826, + "nauc_ndcg_at_100_std": -0.192494, + "nauc_ndcg_at_100_diff1": 0.844564, + "nauc_ndcg_at_1000_max": 0.463826, + "nauc_ndcg_at_1000_std": -0.192494, + "nauc_ndcg_at_1000_diff1": 0.844564, + "nauc_map_at_1_max": 0.357556, + "nauc_map_at_1_std": -0.325911, + "nauc_map_at_1_diff1": 0.850503, + "nauc_map_at_3_max": 0.44142, + "nauc_map_at_3_std": -0.245511, + "nauc_map_at_3_diff1": 0.839589, + "nauc_map_at_5_max": 0.450599, + "nauc_map_at_5_std": -0.224735, + "nauc_map_at_5_diff1": 0.842191, + "nauc_map_at_10_max": 0.449736, + "nauc_map_at_10_std": -0.221847, + "nauc_map_at_10_diff1": 0.843194, + "nauc_map_at_20_max": 0.448335, + "nauc_map_at_20_std": -0.224349, + "nauc_map_at_20_diff1": 0.843897, + "nauc_map_at_100_max": 0.447198, + "nauc_map_at_100_std": -0.226719, + "nauc_map_at_100_diff1": 0.844004, + "nauc_map_at_1000_max": 0.447198, + "nauc_map_at_1000_std": -0.226719, + "nauc_map_at_1000_diff1": 0.844004, + "nauc_recall_at_1_max": 0.357556, + "nauc_recall_at_1_std": -0.325911, + "nauc_recall_at_1_diff1": 0.850503, + "nauc_recall_at_3_max": 0.538727, + "nauc_recall_at_3_std": -0.096101, + "nauc_recall_at_3_diff1": 0.805001, + "nauc_recall_at_5_max": 0.655705, + "nauc_recall_at_5_std": 0.143556, + "nauc_recall_at_5_diff1": 0.80955, + "nauc_recall_at_10_max": 0.783995, + "nauc_recall_at_10_std": 0.494091, + "nauc_recall_at_10_diff1": 0.804022, + "nauc_recall_at_20_max": 0.845806, + "nauc_recall_at_20_std": 0.677366, + "nauc_recall_at_20_diff1": 0.82563, + "nauc_recall_at_100_max": 0.894652, + "nauc_recall_at_100_std": 0.859613, + "nauc_recall_at_100_diff1": 0.85185, + "nauc_recall_at_1000_max": 0.894652, + "nauc_recall_at_1000_std": 0.859613, + "nauc_recall_at_1000_diff1": 0.85185, + "nauc_precision_at_1_max": 0.405091, + "nauc_precision_at_1_std": -0.292537, + "nauc_precision_at_1_diff1": 0.856605, + "nauc_precision_at_3_max": 0.328581, + "nauc_precision_at_3_std": 0.105189, + "nauc_precision_at_3_diff1": 0.261904, + "nauc_precision_at_5_max": 0.288247, + "nauc_precision_at_5_std": 0.260903, + "nauc_precision_at_5_diff1": 0.087796, + "nauc_precision_at_10_max": 0.213161, + "nauc_precision_at_10_std": 0.358782, + "nauc_precision_at_10_diff1": -0.085727, + "nauc_precision_at_20_max": 0.152614, + "nauc_precision_at_20_std": 0.359737, + "nauc_precision_at_20_diff1": -0.171729, + "nauc_precision_at_100_max": 0.064318, + "nauc_precision_at_100_std": 0.317168, + "nauc_precision_at_100_diff1": -0.280361, + "nauc_precision_at_1000_max": 0.064318, + "nauc_precision_at_1000_std": 0.317168, + "nauc_precision_at_1000_diff1": -0.280361, + "nauc_mrr_at_1_max": 0.405091, + "nauc_mrr_at_1_std": -0.292537, + "nauc_mrr_at_1_diff1": 0.856605, + "nauc_mrr_at_3_max": 0.451466, + "nauc_mrr_at_3_std": -0.23168, + "nauc_mrr_at_3_diff1": 0.845291, + "nauc_mrr_at_5_max": 0.455546, + "nauc_mrr_at_5_std": -0.21827, + "nauc_mrr_at_5_diff1": 0.847109, + "nauc_mrr_at_10_max": 0.454144, + "nauc_mrr_at_10_std": -0.216484, + "nauc_mrr_at_10_diff1": 0.847687, + "nauc_mrr_at_20_max": 0.452743, + "nauc_mrr_at_20_std": -0.218898, + "nauc_mrr_at_20_diff1": 0.848113, + "nauc_mrr_at_100_max": 0.451726, + "nauc_mrr_at_100_std": -0.221104, + "nauc_mrr_at_100_diff1": 0.848213, + "nauc_mrr_at_1000_max": 0.451726, + "nauc_mrr_at_1000_std": -0.221104, + "nauc_mrr_at_1000_diff1": 0.848213, + "main_score": 0.88777, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 140.79387640953064, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/MTOPDomainClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/MTOPDomainClassification.json new file mode 100644 index 0000000000..42231d7abc --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/MTOPDomainClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d80d48c1eb48d3562165c59d59d0034df9fff0bf", + "task_name": "MTOPDomainClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.992818, + "f1": 0.992228, + "f1_weighted": 0.992825, + "scores_per_experiment": [ + { + "accuracy": 0.992704, + "f1": 0.992124, + "f1_weighted": 0.992711 + }, + { + "accuracy": 0.992932, + "f1": 0.992232, + "f1_weighted": 0.992938 + }, + { + "accuracy": 0.992932, + "f1": 0.992513, + "f1_weighted": 0.992936 + }, + { + "accuracy": 0.992704, + "f1": 0.992081, + "f1_weighted": 0.992714 + }, + { + "accuracy": 0.99202, + "f1": 0.991272, + "f1_weighted": 0.992032 + }, + { + "accuracy": 0.992704, + "f1": 0.992304, + "f1_weighted": 0.992712 + }, + { + "accuracy": 0.992704, + "f1": 0.992276, + "f1_weighted": 0.992709 + }, + { + "accuracy": 0.992932, + "f1": 0.992165, + "f1_weighted": 0.992942 + }, + { + "accuracy": 0.993388, + "f1": 0.992868, + "f1_weighted": 0.993394 + }, + { + "accuracy": 0.99316, + "f1": 0.992448, + "f1_weighted": 0.993165 + } + ], + "main_score": 0.992818, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 17.87173819541931, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/MassiveIntentClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/MassiveIntentClassification.json new file mode 100644 index 0000000000..fb0961d070 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/MassiveIntentClassification.json @@ -0,0 +1,135 @@ +{ + "dataset_revision": "4672e20407010da34463acc759c162ca9734bca6", + "task_name": "MassiveIntentClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.878144, + "f1": 0.860115, + "f1_weighted": 0.873773, + "scores_per_experiment": [ + { + "accuracy": 0.885003, + "f1": 0.862566, + "f1_weighted": 0.882543 + }, + { + "accuracy": 0.8692, + "f1": 0.855192, + "f1_weighted": 0.862698 + }, + { + "accuracy": 0.863484, + "f1": 0.849603, + "f1_weighted": 0.85433 + }, + { + "accuracy": 0.879623, + "f1": 0.860842, + "f1_weighted": 0.875442 + }, + { + "accuracy": 0.866174, + "f1": 0.85071, + "f1_weighted": 0.85809 + }, + { + "accuracy": 0.886012, + "f1": 0.866088, + "f1_weighted": 0.885585 + }, + { + "accuracy": 0.871217, + "f1": 0.859225, + "f1_weighted": 0.862913 + }, + { + "accuracy": 0.87996, + "f1": 0.86281, + "f1_weighted": 0.876804 + }, + { + "accuracy": 0.889711, + "f1": 0.865689, + "f1_weighted": 0.888821 + }, + { + "accuracy": 0.891056, + "f1": 0.868422, + "f1_weighted": 0.890505 + } + ], + "main_score": 0.878144, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.858978, + "f1": 0.83208, + "f1_weighted": 0.85303, + "scores_per_experiment": [ + { + "accuracy": 0.869536, + "f1": 0.836219, + "f1_weighted": 0.866721 + }, + { + "accuracy": 0.85306, + "f1": 0.828667, + "f1_weighted": 0.843561 + }, + { + "accuracy": 0.84768, + "f1": 0.822704, + "f1_weighted": 0.836716 + }, + { + "accuracy": 0.860794, + "f1": 0.832139, + "f1_weighted": 0.857203 + }, + { + "accuracy": 0.846335, + "f1": 0.824397, + "f1_weighted": 0.836834 + }, + { + "accuracy": 0.86382, + "f1": 0.837979, + "f1_weighted": 0.861836 + }, + { + "accuracy": 0.849697, + "f1": 0.824108, + "f1_weighted": 0.83864 + }, + { + "accuracy": 0.862475, + "f1": 0.832993, + "f1_weighted": 0.858662 + }, + { + "accuracy": 0.872898, + "f1": 0.841394, + "f1_weighted": 0.871011 + }, + { + "accuracy": 0.863484, + "f1": 0.8402, + "f1_weighted": 0.859115 + } + ], + "main_score": 0.858978, + "hf_subset": "zh-CN", + "languages": [ + "cmo-Hans" + ] + } + ] + }, + "evaluation_time": 84.03193402290344, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/MassiveScenarioClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/MassiveScenarioClassification.json new file mode 100644 index 0000000000..ecc913df46 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/MassiveScenarioClassification.json @@ -0,0 +1,135 @@ +{ + "dataset_revision": "fad2c6e8459f9e1c45d9315f4953d921437d70f8", + "task_name": "MassiveScenarioClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.932582, + "f1": 0.926593, + "f1_weighted": 0.932165, + "scores_per_experiment": [ + { + "accuracy": 0.931406, + "f1": 0.926686, + "f1_weighted": 0.930981 + }, + { + "accuracy": 0.933759, + "f1": 0.928553, + "f1_weighted": 0.93369 + }, + { + "accuracy": 0.932078, + "f1": 0.924799, + "f1_weighted": 0.931226 + }, + { + "accuracy": 0.933087, + "f1": 0.926546, + "f1_weighted": 0.932328 + }, + { + "accuracy": 0.933087, + "f1": 0.926576, + "f1_weighted": 0.93264 + }, + { + "accuracy": 0.931406, + "f1": 0.925308, + "f1_weighted": 0.930923 + }, + { + "accuracy": 0.932414, + "f1": 0.926084, + "f1_weighted": 0.932214 + }, + { + "accuracy": 0.931406, + "f1": 0.925276, + "f1_weighted": 0.931181 + }, + { + "accuracy": 0.932078, + "f1": 0.925673, + "f1_weighted": 0.931427 + }, + { + "accuracy": 0.935104, + "f1": 0.930431, + "f1_weighted": 0.935041 + } + ], + "main_score": 0.932582, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "accuracy": 0.921352, + "f1": 0.916772, + "f1_weighted": 0.920551, + "scores_per_experiment": [ + { + "accuracy": 0.922663, + "f1": 0.918957, + "f1_weighted": 0.921888 + }, + { + "accuracy": 0.922327, + "f1": 0.918544, + "f1_weighted": 0.921696 + }, + { + "accuracy": 0.923336, + "f1": 0.918184, + "f1_weighted": 0.922351 + }, + { + "accuracy": 0.921991, + "f1": 0.917392, + "f1_weighted": 0.920941 + }, + { + "accuracy": 0.920309, + "f1": 0.91544, + "f1_weighted": 0.919544 + }, + { + "accuracy": 0.919637, + "f1": 0.915077, + "f1_weighted": 0.918713 + }, + { + "accuracy": 0.923336, + "f1": 0.91885, + "f1_weighted": 0.922242 + }, + { + "accuracy": 0.918292, + "f1": 0.911939, + "f1_weighted": 0.917649 + }, + { + "accuracy": 0.920646, + "f1": 0.915788, + "f1_weighted": 0.919737 + }, + { + "accuracy": 0.920982, + "f1": 0.917546, + "f1_weighted": 0.920748 + } + ], + "main_score": 0.921352, + "hf_subset": "zh-CN", + "languages": [ + "cmo-Hans" + ] + } + ] + }, + "evaluation_time": 76.1633517742157, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/MedicalRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/MedicalRetrieval.json new file mode 100644 index 0000000000..78fcd96218 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/MedicalRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "2039188fb5800a9803ba5048df7b76e6fb151fc6", + "task_name": "MedicalRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.622, + "ndcg_at_3": 0.67699, + "ndcg_at_5": 0.69102, + "ndcg_at_10": 0.7034, + "ndcg_at_20": 0.71343, + "ndcg_at_100": 0.72981, + "ndcg_at_1000": 0.72981, + "map_at_1": 0.622, + "map_at_3": 0.6635, + "map_at_5": 0.6713, + "map_at_10": 0.67647, + "map_at_20": 0.67931, + "map_at_100": 0.68154, + "map_at_1000": 0.68154, + "recall_at_1": 0.622, + "recall_at_3": 0.716, + "recall_at_5": 0.75, + "recall_at_10": 0.788, + "recall_at_20": 0.827, + "recall_at_100": 0.916, + "recall_at_1000": 0.916, + "precision_at_1": 0.622, + "precision_at_3": 0.23867, + "precision_at_5": 0.15, + "precision_at_10": 0.0788, + "precision_at_20": 0.04135, + "precision_at_100": 0.00916, + "precision_at_1000": 0.00092, + "mrr_at_1": 0.622, + "mrr_at_3": 0.6635, + "mrr_at_5": 0.6713, + "mrr_at_10": 0.676465, + "mrr_at_20": 0.67931, + "mrr_at_100": 0.68154, + "mrr_at_1000": 0.68154, + "nauc_ndcg_at_1_max": 0.81669, + "nauc_ndcg_at_1_std": 0.616397, + "nauc_ndcg_at_1_diff1": 0.857877, + "nauc_ndcg_at_3_max": 0.82475, + "nauc_ndcg_at_3_std": 0.674786, + "nauc_ndcg_at_3_diff1": 0.820478, + "nauc_ndcg_at_5_max": 0.819395, + "nauc_ndcg_at_5_std": 0.680703, + "nauc_ndcg_at_5_diff1": 0.809645, + "nauc_ndcg_at_10_max": 0.814587, + "nauc_ndcg_at_10_std": 0.688676, + "nauc_ndcg_at_10_diff1": 0.804688, + "nauc_ndcg_at_20_max": 0.81276, + "nauc_ndcg_at_20_std": 0.68551, + "nauc_ndcg_at_20_diff1": 0.806406, + "nauc_ndcg_at_100_max": 0.814773, + "nauc_ndcg_at_100_std": 0.682076, + "nauc_ndcg_at_100_diff1": 0.808566, + "nauc_ndcg_at_1000_max": 0.814773, + "nauc_ndcg_at_1000_std": 0.682076, + "nauc_ndcg_at_1000_diff1": 0.808566, + "nauc_map_at_1_max": 0.81669, + "nauc_map_at_1_std": 0.616397, + "nauc_map_at_1_diff1": 0.857877, + "nauc_map_at_3_max": 0.823735, + "nauc_map_at_3_std": 0.661449, + "nauc_map_at_3_diff1": 0.830201, + "nauc_map_at_5_max": 0.820809, + "nauc_map_at_5_std": 0.664437, + "nauc_map_at_5_diff1": 0.824579, + "nauc_map_at_10_max": 0.819071, + "nauc_map_at_10_std": 0.667689, + "nauc_map_at_10_diff1": 0.822801, + "nauc_map_at_20_max": 0.818607, + "nauc_map_at_20_std": 0.666711, + "nauc_map_at_20_diff1": 0.823343, + "nauc_map_at_100_max": 0.818807, + "nauc_map_at_100_std": 0.666183, + "nauc_map_at_100_diff1": 0.823647, + "nauc_map_at_1000_max": 0.818807, + "nauc_map_at_1000_std": 0.666183, + "nauc_map_at_1000_diff1": 0.823647, + "nauc_recall_at_1_max": 0.81669, + "nauc_recall_at_1_std": 0.616397, + "nauc_recall_at_1_diff1": 0.857877, + "nauc_recall_at_3_max": 0.827534, + "nauc_recall_at_3_std": 0.717169, + "nauc_recall_at_3_diff1": 0.788956, + "nauc_recall_at_5_max": 0.813009, + "nauc_recall_at_5_std": 0.737951, + "nauc_recall_at_5_diff1": 0.7552, + "nauc_recall_at_10_max": 0.792264, + "nauc_recall_at_10_std": 0.774689, + "nauc_recall_at_10_diff1": 0.728342, + "nauc_recall_at_20_max": 0.777591, + "nauc_recall_at_20_std": 0.772887, + "nauc_recall_at_20_diff1": 0.723908, + "nauc_recall_at_100_max": 0.778339, + "nauc_recall_at_100_std": 0.804155, + "nauc_recall_at_100_diff1": 0.682545, + "nauc_recall_at_1000_max": 0.778339, + "nauc_recall_at_1000_std": 0.804155, + "nauc_recall_at_1000_diff1": 0.682545, + "nauc_precision_at_1_max": 0.81669, + "nauc_precision_at_1_std": 0.616397, + "nauc_precision_at_1_diff1": 0.857877, + "nauc_precision_at_3_max": 0.827534, + "nauc_precision_at_3_std": 0.717169, + "nauc_precision_at_3_diff1": 0.788956, + "nauc_precision_at_5_max": 0.813009, + "nauc_precision_at_5_std": 0.737951, + "nauc_precision_at_5_diff1": 0.7552, + "nauc_precision_at_10_max": 0.792264, + "nauc_precision_at_10_std": 0.774689, + "nauc_precision_at_10_diff1": 0.728342, + "nauc_precision_at_20_max": 0.777591, + "nauc_precision_at_20_std": 0.772887, + "nauc_precision_at_20_diff1": 0.723908, + "nauc_precision_at_100_max": 0.778339, + "nauc_precision_at_100_std": 0.804155, + "nauc_precision_at_100_diff1": 0.682545, + "nauc_precision_at_1000_max": 0.778339, + "nauc_precision_at_1000_std": 0.804155, + "nauc_precision_at_1000_diff1": 0.682545, + "nauc_mrr_at_1_max": 0.81669, + "nauc_mrr_at_1_std": 0.616397, + "nauc_mrr_at_1_diff1": 0.857877, + "nauc_mrr_at_3_max": 0.823735, + "nauc_mrr_at_3_std": 0.661449, + "nauc_mrr_at_3_diff1": 0.830201, + "nauc_mrr_at_5_max": 0.820809, + "nauc_mrr_at_5_std": 0.664437, + "nauc_mrr_at_5_diff1": 0.824579, + "nauc_mrr_at_10_max": 0.819071, + "nauc_mrr_at_10_std": 0.667689, + "nauc_mrr_at_10_diff1": 0.822801, + "nauc_mrr_at_20_max": 0.818607, + "nauc_mrr_at_20_std": 0.666711, + "nauc_mrr_at_20_diff1": 0.823343, + "nauc_mrr_at_100_max": 0.818807, + "nauc_mrr_at_100_std": 0.666183, + "nauc_mrr_at_100_diff1": 0.823647, + "nauc_mrr_at_1000_max": 0.818807, + "nauc_mrr_at_1000_std": 0.666183, + "nauc_mrr_at_1000_diff1": 0.823647, + "main_score": 0.7034, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 68.22556781768799, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/MedrxivClusteringP2P.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/MedrxivClusteringP2P.v2.json new file mode 100644 index 0000000000..5b9aa58766 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/MedrxivClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "e7a26af6f3ae46b30dde8737f02c07b1505bcc73", + "task_name": "MedrxivClusteringP2P.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.535631, + 0.519434, + 0.522423, + 0.512896, + 0.524717, + 0.517477, + 0.50328, + 0.507135, + 0.520148, + 0.516036 + ] + }, + "v_measure": 0.517918, + "v_measure_std": 0.008643, + "main_score": 0.517918, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 19.331128120422363, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/MedrxivClusteringS2S.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/MedrxivClusteringS2S.v2.json new file mode 100644 index 0000000000..24e784f722 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/MedrxivClusteringS2S.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "35191c8c0dca72d8ff3efcd72aa802307d469663", + "task_name": "MedrxivClusteringS2S.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.516142, + 0.507475, + 0.519792, + 0.506592, + 0.504882, + 0.51366, + 0.501175, + 0.505081, + 0.518154, + 0.512744 + ] + }, + "v_measure": 0.51057, + "v_measure_std": 0.006034, + "main_score": 0.51057, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 16.850826501846313, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/MindSmallReranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/MindSmallReranking.json new file mode 100644 index 0000000000..cbb35e877f --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/MindSmallReranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "59042f120c80e8afa9cdbb224f67076cec0fc9a7", + "task_name": "MindSmallReranking", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "map": 0.32519, + "mrr": 0.336163, + "nAUC_map_max": -0.268356, + "nAUC_map_std": 0.013275, + "nAUC_map_diff1": 0.144098, + "nAUC_mrr_max": -0.21668, + "nAUC_mrr_std": 0.022134, + "nAUC_mrr_diff1": 0.129776, + "main_score": 0.32519, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 154.56287360191345, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/MultilingualSentiment.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/MultilingualSentiment.json new file mode 100644 index 0000000000..867a51bce7 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/MultilingualSentiment.json @@ -0,0 +1,137 @@ +{ + "dataset_revision": "46958b007a63fdbf239b7672c25d0bea67b5ea1a", + "task_name": "MultilingualSentiment", + "mteb_version": "1.36.5", + "scores": { + "validation": [ + { + "accuracy": 0.820767, + "f1": 0.821992, + "f1_weighted": 0.821992, + "scores_per_experiment": [ + { + "accuracy": 0.821667, + "f1": 0.822949, + "f1_weighted": 0.822949 + }, + { + "accuracy": 0.822, + "f1": 0.821402, + "f1_weighted": 0.821402 + }, + { + "accuracy": 0.830667, + "f1": 0.8315, + "f1_weighted": 0.8315 + }, + { + "accuracy": 0.82, + "f1": 0.822085, + "f1_weighted": 0.822085 + }, + { + "accuracy": 0.820667, + "f1": 0.822725, + "f1_weighted": 0.822725 + }, + { + "accuracy": 0.794333, + "f1": 0.79601, + "f1_weighted": 0.79601 + }, + { + "accuracy": 0.825333, + "f1": 0.826544, + "f1_weighted": 0.826544 + }, + { + "accuracy": 0.822, + "f1": 0.822362, + "f1_weighted": 0.822362 + }, + { + "accuracy": 0.827333, + "f1": 0.828522, + "f1_weighted": 0.828522 + }, + { + "accuracy": 0.823667, + "f1": 0.825818, + "f1_weighted": 0.825818 + } + ], + "main_score": 0.820767, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ], + "test": [ + { + "accuracy": 0.8319, + "f1": 0.833109, + "f1_weighted": 0.833109, + "scores_per_experiment": [ + { + "accuracy": 0.828333, + "f1": 0.829728, + "f1_weighted": 0.829728 + }, + { + "accuracy": 0.834667, + "f1": 0.834211, + "f1_weighted": 0.834211 + }, + { + "accuracy": 0.837667, + "f1": 0.838557, + "f1_weighted": 0.838557 + }, + { + "accuracy": 0.830667, + "f1": 0.832467, + "f1_weighted": 0.832467 + }, + { + "accuracy": 0.832, + "f1": 0.833846, + "f1_weighted": 0.833846 + }, + { + "accuracy": 0.810667, + "f1": 0.812262, + "f1_weighted": 0.812262 + }, + { + "accuracy": 0.835667, + "f1": 0.837025, + "f1_weighted": 0.837025 + }, + { + "accuracy": 0.835333, + "f1": 0.835858, + "f1_weighted": 0.835858 + }, + { + "accuracy": 0.838, + "f1": 0.839198, + "f1_weighted": 0.839198 + }, + { + "accuracy": 0.836, + "f1": 0.837938, + "f1_weighted": 0.837938 + } + ], + "main_score": 0.8319, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 33.283143043518066, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/Ocnli.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/Ocnli.json new file mode 100644 index 0000000000..b6867df441 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/Ocnli.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "66e76a618a34d6d565d5538088562851e6daa7ec", + "task_name": "Ocnli", + "mteb_version": "1.36.5", + "scores": { + "validation": [ + { + "similarity_accuracy": 0.82079, + "similarity_accuracy_threshold": 0.585115, + "similarity_f1": 0.836941, + "similarity_f1_threshold": 0.568239, + "similarity_precision": 0.768551, + "similarity_recall": 0.918691, + "similarity_ap": 0.860639, + "cosine_accuracy": 0.82079, + "cosine_accuracy_threshold": 0.585115, + "cosine_f1": 0.836941, + "cosine_f1_threshold": 0.568239, + "cosine_precision": 0.768551, + "cosine_recall": 0.918691, + "cosine_ap": 0.860639, + "manhattan_accuracy": 0.811045, + "manhattan_accuracy_threshold": 31.196552, + "manhattan_f1": 0.828263, + "manhattan_f1_threshold": 32.324425, + "manhattan_precision": 0.773602, + "manhattan_recall": 0.891235, + "manhattan_ap": 0.857137, + "euclidean_accuracy": 0.82079, + "euclidean_accuracy_threshold": 0.911905, + "euclidean_f1": 0.836538, + "euclidean_f1_threshold": 0.929243, + "euclidean_precision": 0.767873, + "euclidean_recall": 0.918691, + "euclidean_ap": 0.860517, + "dot_accuracy": 0.821873, + "dot_accuracy_threshold": 0.583232, + "dot_f1": 0.83591, + "dot_f1_threshold": 0.583232, + "dot_precision": 0.79206, + "dot_recall": 0.8849, + "dot_ap": 0.860762, + "max_accuracy": 0.821873, + "max_f1": 0.836941, + "max_precision": 0.79206, + "max_recall": 0.918691, + "max_ap": 0.860762, + "main_score": 0.860762, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 8.793944358825684, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/OnlineShopping.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/OnlineShopping.json new file mode 100644 index 0000000000..d1b5e9732c --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/OnlineShopping.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "e610f2ebd179a8fda30ae534c3878750a96db120", + "task_name": "OnlineShopping", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.9556, + "f1": 0.955546, + "f1_weighted": 0.955617, + "ap": 0.945279, + "ap_weighted": 0.945279, + "scores_per_experiment": [ + { + "accuracy": 0.959, + "f1": 0.958893, + "f1_weighted": 0.958994, + "ap": 0.942508, + "ap_weighted": 0.942508 + }, + { + "accuracy": 0.951, + "f1": 0.950964, + "f1_weighted": 0.951028, + "ap": 0.942232, + "ap_weighted": 0.942232 + }, + { + "accuracy": 0.96, + "f1": 0.959936, + "f1_weighted": 0.960013, + "ap": 0.948611, + "ap_weighted": 0.948611 + }, + { + "accuracy": 0.96, + "f1": 0.959915, + "f1_weighted": 0.960004, + "ap": 0.945984, + "ap_weighted": 0.945984 + }, + { + "accuracy": 0.958, + "f1": 0.957951, + "f1_weighted": 0.95802, + "ap": 0.948581, + "ap_weighted": 0.948581 + }, + { + "accuracy": 0.952, + "f1": 0.951972, + "f1_weighted": 0.952028, + "ap": 0.944947, + "ap_weighted": 0.944947 + }, + { + "accuracy": 0.951, + "f1": 0.950978, + "f1_weighted": 0.951028, + "ap": 0.944953, + "ap_weighted": 0.944953 + }, + { + "accuracy": 0.953, + "f1": 0.952971, + "f1_weighted": 0.953027, + "ap": 0.945854, + "ap_weighted": 0.945854 + }, + { + "accuracy": 0.962, + "f1": 0.961912, + "f1_weighted": 0.962, + "ap": 0.947796, + "ap_weighted": 0.947796 + }, + { + "accuracy": 0.95, + "f1": 0.949966, + "f1_weighted": 0.950029, + "ap": 0.941325, + "ap_weighted": 0.941325 + } + ], + "main_score": 0.9556, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 19.422606229782104, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/PAWSX.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/PAWSX.json new file mode 100644 index 0000000000..a3ba0ef786 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/PAWSX.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "9c6a90e430ac22b5779fb019a23e820b11a8b5e1", + "task_name": "PAWSX", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.620973, + "spearman": 0.664419, + "cosine_pearson": 0.620973, + "cosine_spearman": 0.664419, + "manhattan_pearson": 0.648982, + "manhattan_spearman": 0.663835, + "euclidean_pearson": 0.650067, + "euclidean_spearman": 0.664513, + "main_score": 0.664419, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 12.41031265258789, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/QBQTC.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/QBQTC.json new file mode 100644 index 0000000000..8a62364dca --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/QBQTC.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "790b0510dc52b1553e8c49f3d2afb48c0e5c48b7", + "task_name": "QBQTC", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.509597, + "spearman": 0.518826, + "cosine_pearson": 0.509597, + "cosine_spearman": 0.518826, + "manhattan_pearson": 0.516425, + "manhattan_spearman": 0.518413, + "euclidean_pearson": 0.516898, + "euclidean_spearman": 0.518851, + "main_score": 0.518826, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 14.569149017333984, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/SCIDOCS.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/SCIDOCS.json new file mode 100644 index 0000000000..3e7c026b95 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/SCIDOCS.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88", + "task_name": "SCIDOCS", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.271, + "ndcg_at_3": 0.2384, + "ndcg_at_5": 0.21357, + "ndcg_at_10": 0.26412, + "ndcg_at_20": 0.30509, + "ndcg_at_100": 0.37072, + "ndcg_at_1000": 0.37072, + "map_at_1": 0.05508, + "map_at_3": 0.10896, + "map_at_5": 0.13563, + "map_at_10": 0.16281, + "map_at_20": 0.17973, + "map_at_100": 0.19484, + "map_at_1000": 0.19484, + "recall_at_1": 0.05508, + "recall_at_3": 0.13902, + "recall_at_5": 0.19632, + "recall_at_10": 0.28973, + "recall_at_20": 0.38582, + "recall_at_100": 0.59815, + "recall_at_1000": 0.59815, + "precision_at_1": 0.271, + "precision_at_3": 0.228, + "precision_at_5": 0.1934, + "precision_at_10": 0.1428, + "precision_at_20": 0.09505, + "precision_at_100": 0.0295, + "precision_at_1000": 0.00295, + "mrr_at_1": 0.271, + "mrr_at_3": 0.368, + "mrr_at_5": 0.38955, + "mrr_at_10": 0.408071, + "mrr_at_20": 0.414735, + "mrr_at_100": 0.418359, + "mrr_at_1000": 0.418359, + "nauc_ndcg_at_1_max": 0.14996, + "nauc_ndcg_at_1_std": 0.197798, + "nauc_ndcg_at_1_diff1": 0.204993, + "nauc_ndcg_at_3_max": 0.198927, + "nauc_ndcg_at_3_std": 0.297631, + "nauc_ndcg_at_3_diff1": 0.141748, + "nauc_ndcg_at_5_max": 0.232526, + "nauc_ndcg_at_5_std": 0.341152, + "nauc_ndcg_at_5_diff1": 0.131198, + "nauc_ndcg_at_10_max": 0.248527, + "nauc_ndcg_at_10_std": 0.378658, + "nauc_ndcg_at_10_diff1": 0.127571, + "nauc_ndcg_at_20_max": 0.26111, + "nauc_ndcg_at_20_std": 0.412562, + "nauc_ndcg_at_20_diff1": 0.11714, + "nauc_ndcg_at_100_max": 0.260669, + "nauc_ndcg_at_100_std": 0.411874, + "nauc_ndcg_at_100_diff1": 0.104018, + "nauc_ndcg_at_1000_max": 0.260669, + "nauc_ndcg_at_1000_std": 0.411874, + "nauc_ndcg_at_1000_diff1": 0.104018, + "nauc_map_at_1_max": 0.147815, + "nauc_map_at_1_std": 0.19594, + "nauc_map_at_1_diff1": 0.205271, + "nauc_map_at_3_max": 0.203188, + "nauc_map_at_3_std": 0.305334, + "nauc_map_at_3_diff1": 0.157641, + "nauc_map_at_5_max": 0.239308, + "nauc_map_at_5_std": 0.348183, + "nauc_map_at_5_diff1": 0.137158, + "nauc_map_at_10_max": 0.254659, + "nauc_map_at_10_std": 0.379176, + "nauc_map_at_10_diff1": 0.130936, + "nauc_map_at_20_max": 0.262572, + "nauc_map_at_20_std": 0.404045, + "nauc_map_at_20_diff1": 0.123406, + "nauc_map_at_100_max": 0.263859, + "nauc_map_at_100_std": 0.408017, + "nauc_map_at_100_diff1": 0.11764, + "nauc_map_at_1000_max": 0.263859, + "nauc_map_at_1000_std": 0.408017, + "nauc_map_at_1000_diff1": 0.11764, + "nauc_recall_at_1_max": 0.147815, + "nauc_recall_at_1_std": 0.19594, + "nauc_recall_at_1_diff1": 0.205271, + "nauc_recall_at_3_max": 0.207119, + "nauc_recall_at_3_std": 0.331056, + "nauc_recall_at_3_diff1": 0.111219, + "nauc_recall_at_5_max": 0.250934, + "nauc_recall_at_5_std": 0.381021, + "nauc_recall_at_5_diff1": 0.094026, + "nauc_recall_at_10_max": 0.257964, + "nauc_recall_at_10_std": 0.415923, + "nauc_recall_at_10_diff1": 0.084609, + "nauc_recall_at_20_max": 0.266624, + "nauc_recall_at_20_std": 0.456532, + "nauc_recall_at_20_diff1": 0.05987, + "nauc_recall_at_100_max": 0.236842, + "nauc_recall_at_100_std": 0.407398, + "nauc_recall_at_100_diff1": 0.016355, + "nauc_recall_at_1000_max": 0.236842, + "nauc_recall_at_1000_std": 0.407398, + "nauc_recall_at_1000_diff1": 0.016355, + "nauc_precision_at_1_max": 0.14996, + "nauc_precision_at_1_std": 0.197798, + "nauc_precision_at_1_diff1": 0.204993, + "nauc_precision_at_3_max": 0.21205, + "nauc_precision_at_3_std": 0.334741, + "nauc_precision_at_3_diff1": 0.111612, + "nauc_precision_at_5_max": 0.255021, + "nauc_precision_at_5_std": 0.384157, + "nauc_precision_at_5_diff1": 0.094181, + "nauc_precision_at_10_max": 0.260366, + "nauc_precision_at_10_std": 0.416503, + "nauc_precision_at_10_diff1": 0.084041, + "nauc_precision_at_20_max": 0.267612, + "nauc_precision_at_20_std": 0.454421, + "nauc_precision_at_20_diff1": 0.058151, + "nauc_precision_at_100_max": 0.237595, + "nauc_precision_at_100_std": 0.400744, + "nauc_precision_at_100_diff1": 0.011572, + "nauc_precision_at_1000_max": 0.237595, + "nauc_precision_at_1000_std": 0.400744, + "nauc_precision_at_1000_diff1": 0.011572, + "nauc_mrr_at_1_max": 0.14996, + "nauc_mrr_at_1_std": 0.197798, + "nauc_mrr_at_1_diff1": 0.204993, + "nauc_mrr_at_3_max": 0.174106, + "nauc_mrr_at_3_std": 0.240202, + "nauc_mrr_at_3_diff1": 0.153949, + "nauc_mrr_at_5_max": 0.190423, + "nauc_mrr_at_5_std": 0.252781, + "nauc_mrr_at_5_diff1": 0.163123, + "nauc_mrr_at_10_max": 0.189537, + "nauc_mrr_at_10_std": 0.251729, + "nauc_mrr_at_10_diff1": 0.161967, + "nauc_mrr_at_20_max": 0.18912, + "nauc_mrr_at_20_std": 0.250609, + "nauc_mrr_at_20_diff1": 0.162209, + "nauc_mrr_at_100_max": 0.188183, + "nauc_mrr_at_100_std": 0.2497, + "nauc_mrr_at_100_diff1": 0.161785, + "nauc_mrr_at_1000_max": 0.188183, + "nauc_mrr_at_1000_std": 0.2497, + "nauc_mrr_at_1000_diff1": 0.161785, + "main_score": 0.26412, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 53.24364376068115, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/SICK-R.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/SICK-R.json new file mode 100644 index 0000000000..a96ee3b532 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/SICK-R.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "20a6d6f312dd54037fe07a32d58e5e168867909d", + "task_name": "SICK-R", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.877057, + "spearman": 0.843305, + "cosine_pearson": 0.877057, + "cosine_spearman": 0.843305, + "manhattan_pearson": 0.854134, + "manhattan_spearman": 0.844687, + "euclidean_pearson": 0.851871, + "euclidean_spearman": 0.843309, + "main_score": 0.843305, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 14.968255758285522, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/STS12.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/STS12.json new file mode 100644 index 0000000000..026a5bd0f6 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/STS12.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "a0d554a64d88156834ff5ae9920b964011b16384", + "task_name": "STS12", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.904787, + "spearman": 0.847519, + "cosine_pearson": 0.904787, + "cosine_spearman": 0.847519, + "manhattan_pearson": 0.895531, + "manhattan_spearman": 0.849693, + "euclidean_pearson": 0.894364, + "euclidean_spearman": 0.84746, + "main_score": 0.847519, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 13.369229793548584, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/STS13.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/STS13.json new file mode 100644 index 0000000000..04ede00e8a --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/STS13.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "7e90230a92c190f1bf69ae9002b8cea547a64cca", + "task_name": "STS13", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.920236, + "spearman": 0.925484, + "cosine_pearson": 0.920236, + "cosine_spearman": 0.925484, + "manhattan_pearson": 0.918664, + "manhattan_spearman": 0.923456, + "euclidean_pearson": 0.920997, + "euclidean_spearman": 0.9255, + "main_score": 0.925484, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.867910146713257, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/STS14.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/STS14.json new file mode 100644 index 0000000000..7bff9097e2 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/STS14.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "6031580fec1f6af667f0bd2da0a551cf4f0b2375", + "task_name": "STS14", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.911976, + "spearman": 0.903995, + "cosine_pearson": 0.911976, + "cosine_spearman": 0.903995, + "manhattan_pearson": 0.905292, + "manhattan_spearman": 0.903143, + "euclidean_pearson": 0.907242, + "euclidean_spearman": 0.903953, + "main_score": 0.903995, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 29.6851167678833, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/STS15.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/STS15.json new file mode 100644 index 0000000000..b4f67cb62f --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/STS15.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "ae752c7c21bf194d8b67fd573edf7ae58183cbe3", + "task_name": "STS15", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.914015, + "spearman": 0.924218, + "cosine_pearson": 0.914015, + "cosine_spearman": 0.924218, + "manhattan_pearson": 0.916789, + "manhattan_spearman": 0.92329, + "euclidean_pearson": 0.917924, + "euclidean_spearman": 0.924169, + "main_score": 0.924218, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 10.30591630935669, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/STS17.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/STS17.json new file mode 100644 index 0000000000..e378a1e324 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/STS17.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "faeb762787bd10488a50c8b5be4a3b82e411949c", + "task_name": "STS17", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.9347, + "spearman": 0.932298, + "cosine_pearson": 0.9347, + "cosine_spearman": 0.932298, + "manhattan_pearson": 0.928658, + "manhattan_spearman": 0.932801, + "euclidean_pearson": 0.928613, + "euclidean_spearman": 0.932377, + "main_score": 0.932298, + "hf_subset": "en-en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.880078792572021, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/STS22.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/STS22.v2.json new file mode 100644 index 0000000000..b4be25ea3a --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/STS22.v2.json @@ -0,0 +1,41 @@ +{ + "dataset_revision": "d31f33a128469b20e357535c39b82fb3c3f6f2bd", + "task_name": "STS22.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.709193, + "spearman": 0.723988, + "cosine_pearson": 0.709193, + "cosine_spearman": 0.723988, + "manhattan_pearson": 0.723035, + "manhattan_spearman": 0.722053, + "euclidean_pearson": 0.723753, + "euclidean_spearman": 0.723456, + "main_score": 0.723988, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "pearson": 0.735099, + "spearman": 0.729942, + "cosine_pearson": 0.735099, + "cosine_spearman": 0.729942, + "manhattan_pearson": 0.714998, + "manhattan_spearman": 0.730285, + "euclidean_pearson": 0.716012, + "euclidean_spearman": 0.730071, + "main_score": 0.729942, + "hf_subset": "zh", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 24.964032411575317, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/STSB.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/STSB.json new file mode 100644 index 0000000000..a89460eac3 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/STSB.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "0cde68302b3541bb8b3c340dc0644b0b745b3dc0", + "task_name": "STSB", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.849772, + "spearman": 0.86562, + "cosine_pearson": 0.849772, + "cosine_spearman": 0.86562, + "manhattan_pearson": 0.849177, + "manhattan_spearman": 0.86455, + "euclidean_pearson": 0.849884, + "euclidean_spearman": 0.86561, + "main_score": 0.86562, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 11.355995893478394, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/STSBenchmark.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/STSBenchmark.json new file mode 100644 index 0000000000..b5b127d2fd --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/STSBenchmark.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "b0fddb56ed78048fa8b90373c8a3cfc37b684831", + "task_name": "STSBenchmark", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.899861, + "spearman": 0.915074, + "cosine_pearson": 0.899861, + "cosine_spearman": 0.915074, + "manhattan_pearson": 0.908117, + "manhattan_spearman": 0.914484, + "euclidean_pearson": 0.908988, + "euclidean_spearman": 0.915041, + "main_score": 0.915074, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 9.160970211029053, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/SprintDuplicateQuestions.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/SprintDuplicateQuestions.json new file mode 100644 index 0000000000..f6f67b0e44 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/SprintDuplicateQuestions.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46", + "task_name": "SprintDuplicateQuestions", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "similarity_accuracy": 0.998842, + "similarity_accuracy_threshold": 0.880172, + "similarity_f1": 0.941117, + "similarity_f1_threshold": 0.880172, + "similarity_precision": 0.947315, + "similarity_recall": 0.935, + "similarity_ap": 0.973421, + "cosine_accuracy": 0.998842, + "cosine_accuracy_threshold": 0.880172, + "cosine_f1": 0.941117, + "cosine_f1_threshold": 0.880172, + "cosine_precision": 0.947315, + "cosine_recall": 0.935, + "cosine_ap": 0.973421, + "manhattan_accuracy": 0.998861, + "manhattan_accuracy_threshold": 17.594849, + "manhattan_f1": 0.942065, + "manhattan_f1_threshold": 17.594849, + "manhattan_precision": 0.949239, + "manhattan_recall": 0.935, + "manhattan_ap": 0.973555, + "euclidean_accuracy": 0.998851, + "euclidean_accuracy_threshold": 0.4886, + "euclidean_f1": 0.941591, + "euclidean_f1_threshold": 0.489309, + "euclidean_precision": 0.948276, + "euclidean_recall": 0.935, + "euclidean_ap": 0.973458, + "dot_accuracy": 0.998772, + "dot_accuracy_threshold": 0.878452, + "dot_f1": 0.937751, + "dot_f1_threshold": 0.878452, + "dot_precision": 0.941532, + "dot_recall": 0.934, + "dot_ap": 0.972762, + "max_accuracy": 0.998861, + "max_f1": 0.942065, + "max_precision": 0.949239, + "max_recall": 0.935, + "max_ap": 0.973555, + "main_score": 0.973555, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 16.548149585723877, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/StackExchangeClustering.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/StackExchangeClustering.v2.json new file mode 100644 index 0000000000..bcb8243310 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/StackExchangeClustering.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "6cbc1f7b2bc0622f2e39d2c77fa502909748c259", + "task_name": "StackExchangeClustering.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.807731, + 0.80963, + 0.808109, + 0.798824, + 0.80454, + 0.807816, + 0.806716, + 0.80609, + 0.80604, + 0.810271 + ] + }, + "v_measure": 0.806577, + "v_measure_std": 0.003049, + "main_score": 0.806577, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 30.491961240768433, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/StackExchangeClusteringP2P.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/StackExchangeClusteringP2P.v2.json new file mode 100644 index 0000000000..e8f823ff1e --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/StackExchangeClusteringP2P.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "815ca46b2622cec33ccafc3735d572c266efdb44", + "task_name": "StackExchangeClusteringP2P.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.518984, + 0.521905, + 0.524851, + 0.532553, + 0.524316, + 0.533981, + 0.530143, + 0.536067, + 0.527055, + 0.524497 + ] + }, + "v_measure": 0.527435, + "v_measure_std": 0.005273, + "main_score": 0.527435, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 34.84761643409729, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/SummEvalSummarization.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/SummEvalSummarization.v2.json new file mode 100644 index 0000000000..b32acd73c4 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/SummEvalSummarization.v2.json @@ -0,0 +1,24 @@ +{ + "dataset_revision": "cda12ad7615edc362dbf25a00fdd61d3b1eaf93c", + "task_name": "SummEvalSummarization.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "pearson": 0.407495, + "spearman": 0.364364, + "cosine_spearman": 0.364364, + "cosine_pearson": 0.407495, + "dot_spearman": 0.349571, + "dot_pearson": 0.395742, + "main_score": 0.364364, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 12.321335315704346, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/T2Reranking.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/T2Reranking.json new file mode 100644 index 0000000000..ce65af468b --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/T2Reranking.json @@ -0,0 +1,26 @@ +{ + "dataset_revision": "76631901a18387f85eaa53e5450019b87ad58ef9", + "task_name": "T2Reranking", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "map": 0.673278, + "mrr": 0.773587, + "nAUC_map_max": 0.304517, + "nAUC_map_std": 0.055887, + "nAUC_map_diff1": -0.086749, + "nAUC_mrr_max": 0.23006, + "nAUC_mrr_std": 0.00191, + "nAUC_mrr_diff1": -0.070448, + "main_score": 0.673278, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 145.66250228881836, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/T2Retrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/T2Retrieval.json new file mode 100644 index 0000000000..c66161ceac --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/T2Retrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "8731a845f1bf500a4f111cf1070785c793d10e64", + "task_name": "T2Retrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.9303, + "ndcg_at_3": 0.90207, + "ndcg_at_5": 0.89217, + "ndcg_at_10": 0.89261, + "ndcg_at_20": 0.90635, + "ndcg_at_100": 0.91915, + "ndcg_at_1000": 0.91915, + "map_at_1": 0.29133, + "map_at_3": 0.58219, + "map_at_5": 0.71781, + "map_at_10": 0.83035, + "map_at_20": 0.8567, + "map_at_100": 0.86335, + "map_at_1000": 0.86335, + "recall_at_1": 0.29133, + "recall_at_3": 0.59502, + "recall_at_5": 0.74473, + "recall_at_10": 0.88165, + "recall_at_20": 0.9316, + "recall_at_100": 0.97175, + "recall_at_1000": 0.97175, + "precision_at_1": 0.9303, + "precision_at_3": 0.78812, + "precision_at_5": 0.66369, + "precision_at_10": 0.442, + "precision_at_20": 0.24234, + "precision_at_100": 0.05096, + "precision_at_1000": 0.0051, + "mrr_at_1": 0.9303, + "mrr_at_3": 0.945394, + "mrr_at_5": 0.947452, + "mrr_at_10": 0.948581, + "mrr_at_20": 0.948947, + "mrr_at_100": 0.949132, + "mrr_at_1000": 0.949132, + "nauc_ndcg_at_1_max": 0.815149, + "nauc_ndcg_at_1_std": 0.678362, + "nauc_ndcg_at_1_diff1": 0.460492, + "nauc_ndcg_at_3_max": 0.682406, + "nauc_ndcg_at_3_std": 0.646888, + "nauc_ndcg_at_3_diff1": 0.07849, + "nauc_ndcg_at_5_max": 0.605134, + "nauc_ndcg_at_5_std": 0.585612, + "nauc_ndcg_at_5_diff1": 0.081857, + "nauc_ndcg_at_10_max": 0.526237, + "nauc_ndcg_at_10_std": 0.495387, + "nauc_ndcg_at_10_diff1": 0.124986, + "nauc_ndcg_at_20_max": 0.57129, + "nauc_ndcg_at_20_std": 0.553732, + "nauc_ndcg_at_20_diff1": 0.126863, + "nauc_ndcg_at_100_max": 0.623038, + "nauc_ndcg_at_100_std": 0.607141, + "nauc_ndcg_at_100_diff1": 0.127863, + "nauc_ndcg_at_1000_max": 0.623038, + "nauc_ndcg_at_1000_std": 0.607141, + "nauc_ndcg_at_1000_diff1": 0.127863, + "nauc_map_at_1_max": -0.273316, + "nauc_map_at_1_std": -0.491137, + "nauc_map_at_1_diff1": 0.551434, + "nauc_map_at_3_max": -0.169588, + "nauc_map_at_3_std": -0.440807, + "nauc_map_at_3_diff1": 0.396, + "nauc_map_at_5_max": -0.008057, + "nauc_map_at_5_std": -0.254188, + "nauc_map_at_5_diff1": 0.303993, + "nauc_map_at_10_max": 0.350391, + "nauc_map_at_10_std": 0.253874, + "nauc_map_at_10_diff1": 0.142842, + "nauc_map_at_20_max": 0.483797, + "nauc_map_at_20_std": 0.457077, + "nauc_map_at_20_diff1": 0.092004, + "nauc_map_at_100_max": 0.509217, + "nauc_map_at_100_std": 0.491036, + "nauc_map_at_100_diff1": 0.084917, + "nauc_map_at_1000_max": 0.509217, + "nauc_map_at_1000_std": 0.491036, + "nauc_map_at_1000_diff1": 0.084917, + "nauc_recall_at_1_max": -0.273316, + "nauc_recall_at_1_std": -0.491137, + "nauc_recall_at_1_diff1": 0.551434, + "nauc_recall_at_3_max": -0.209483, + "nauc_recall_at_3_std": -0.477309, + "nauc_recall_at_3_diff1": 0.384998, + "nauc_recall_at_5_max": -0.098683, + "nauc_recall_at_5_std": -0.350884, + "nauc_recall_at_5_diff1": 0.302096, + "nauc_recall_at_10_max": 0.226948, + "nauc_recall_at_10_std": 0.129579, + "nauc_recall_at_10_diff1": 0.141982, + "nauc_recall_at_20_max": 0.42491, + "nauc_recall_at_20_std": 0.455372, + "nauc_recall_at_20_diff1": 0.065091, + "nauc_recall_at_100_max": 0.561523, + "nauc_recall_at_100_std": 0.633837, + "nauc_recall_at_100_diff1": 0.033883, + "nauc_recall_at_1000_max": 0.561523, + "nauc_recall_at_1000_std": 0.633837, + "nauc_recall_at_1000_diff1": 0.033883, + "nauc_precision_at_1_max": 0.815149, + "nauc_precision_at_1_std": 0.678362, + "nauc_precision_at_1_diff1": 0.460492, + "nauc_precision_at_3_max": 0.645229, + "nauc_precision_at_3_std": 0.789457, + "nauc_precision_at_3_diff1": -0.448542, + "nauc_precision_at_5_max": 0.581417, + "nauc_precision_at_5_std": 0.803919, + "nauc_precision_at_5_diff1": -0.448517, + "nauc_precision_at_10_max": 0.515621, + "nauc_precision_at_10_std": 0.792641, + "nauc_precision_at_10_diff1": -0.401255, + "nauc_precision_at_20_max": 0.485892, + "nauc_precision_at_20_std": 0.777762, + "nauc_precision_at_20_diff1": -0.385233, + "nauc_precision_at_100_max": 0.460781, + "nauc_precision_at_100_std": 0.747251, + "nauc_precision_at_100_diff1": -0.379389, + "nauc_precision_at_1000_max": 0.460781, + "nauc_precision_at_1000_std": 0.747251, + "nauc_precision_at_1000_diff1": -0.379389, + "nauc_mrr_at_1_max": 0.815149, + "nauc_mrr_at_1_std": 0.678362, + "nauc_mrr_at_1_diff1": 0.460492, + "nauc_mrr_at_3_max": 0.852432, + "nauc_mrr_at_3_std": 0.735294, + "nauc_mrr_at_3_diff1": 0.458095, + "nauc_mrr_at_5_max": 0.852582, + "nauc_mrr_at_5_std": 0.735619, + "nauc_mrr_at_5_diff1": 0.459625, + "nauc_mrr_at_10_max": 0.851283, + "nauc_mrr_at_10_std": 0.734275, + "nauc_mrr_at_10_diff1": 0.459517, + "nauc_mrr_at_20_max": 0.850602, + "nauc_mrr_at_20_std": 0.733363, + "nauc_mrr_at_20_diff1": 0.45965, + "nauc_mrr_at_100_max": 0.850184, + "nauc_mrr_at_100_std": 0.732655, + "nauc_mrr_at_100_diff1": 0.459641, + "nauc_mrr_at_1000_max": 0.850184, + "nauc_mrr_at_1000_std": 0.732655, + "nauc_mrr_at_1000_diff1": 0.459641, + "main_score": 0.89261, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 893.2558205127716, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/TNews.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/TNews.json new file mode 100644 index 0000000000..1aefcbdd09 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/TNews.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "317f262bf1e6126357bbe89e875451e4b0938fe4", + "task_name": "TNews", + "mteb_version": "1.36.5", + "scores": { + "validation": [ + { + "accuracy": 0.59218, + "f1": 0.574401, + "f1_weighted": 0.590084, + "scores_per_experiment": [ + { + "accuracy": 0.591, + "f1": 0.571273, + "f1_weighted": 0.58781 + }, + { + "accuracy": 0.5936, + "f1": 0.574102, + "f1_weighted": 0.587801 + }, + { + "accuracy": 0.5953, + "f1": 0.577643, + "f1_weighted": 0.593579 + }, + { + "accuracy": 0.5942, + "f1": 0.578466, + "f1_weighted": 0.593584 + }, + { + "accuracy": 0.5956, + "f1": 0.580987, + "f1_weighted": 0.59665 + }, + { + "accuracy": 0.5966, + "f1": 0.580189, + "f1_weighted": 0.59379 + }, + { + "accuracy": 0.5909, + "f1": 0.573639, + "f1_weighted": 0.590623 + }, + { + "accuracy": 0.5893, + "f1": 0.570778, + "f1_weighted": 0.585762 + }, + { + "accuracy": 0.5841, + "f1": 0.562018, + "f1_weighted": 0.578613 + }, + { + "accuracy": 0.5912, + "f1": 0.574917, + "f1_weighted": 0.592623 + } + ], + "main_score": 0.59218, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 25.768748998641968, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/TRECCOVID.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/TRECCOVID.json new file mode 100644 index 0000000000..cd92bbe2e3 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/TRECCOVID.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bb9466bac8153a0349341eb1b22e06409e78ef4e", + "task_name": "TRECCOVID", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.92, + "ndcg_at_3": 0.92173, + "ndcg_at_5": 0.9087, + "ndcg_at_10": 0.87652, + "ndcg_at_20": 0.8453, + "ndcg_at_100": 0.68331, + "ndcg_at_1000": 0.27662, + "map_at_1": 0.00247, + "map_at_3": 0.00743, + "map_at_5": 0.01193, + "map_at_10": 0.02304, + "map_at_20": 0.04301, + "map_at_100": 0.14631, + "map_at_1000": 0.14631, + "recall_at_1": 0.00247, + "recall_at_3": 0.00765, + "recall_at_5": 0.01236, + "recall_at_10": 0.02417, + "recall_at_20": 0.04593, + "recall_at_100": 0.17282, + "recall_at_1000": 0.17282, + "precision_at_1": 0.96, + "precision_at_3": 0.96, + "precision_at_5": 0.944, + "precision_at_10": 0.92, + "precision_at_20": 0.889, + "precision_at_100": 0.7094, + "precision_at_1000": 0.07094, + "mrr_at_1": 0.96, + "mrr_at_3": 0.98, + "mrr_at_5": 0.98, + "mrr_at_10": 0.98, + "mrr_at_20": 0.98, + "mrr_at_100": 0.98, + "mrr_at_1000": 0.98, + "nauc_ndcg_at_1_max": 0.132772, + "nauc_ndcg_at_1_std": 0.02551, + "nauc_ndcg_at_1_diff1": 0.137636, + "nauc_ndcg_at_3_max": 0.188724, + "nauc_ndcg_at_3_std": -0.00744, + "nauc_ndcg_at_3_diff1": -0.091401, + "nauc_ndcg_at_5_max": 0.276997, + "nauc_ndcg_at_5_std": 0.123748, + "nauc_ndcg_at_5_diff1": -0.158786, + "nauc_ndcg_at_10_max": 0.097929, + "nauc_ndcg_at_10_std": 0.155521, + "nauc_ndcg_at_10_diff1": -0.151367, + "nauc_ndcg_at_20_max": 0.166072, + "nauc_ndcg_at_20_std": 0.231998, + "nauc_ndcg_at_20_diff1": -0.157196, + "nauc_ndcg_at_100_max": 0.154802, + "nauc_ndcg_at_100_std": -0.005534, + "nauc_ndcg_at_100_diff1": -0.153459, + "nauc_ndcg_at_1000_max": 0.05427, + "nauc_ndcg_at_1000_std": 0.49056, + "nauc_ndcg_at_1000_diff1": 0.122191, + "nauc_map_at_1_max": 0.05385, + "nauc_map_at_1_std": 0.390508, + "nauc_map_at_1_diff1": 0.082069, + "nauc_map_at_3_max": 0.033052, + "nauc_map_at_3_std": 0.351632, + "nauc_map_at_3_diff1": 0.018455, + "nauc_map_at_5_max": 0.056849, + "nauc_map_at_5_std": 0.455996, + "nauc_map_at_5_diff1": -0.019413, + "nauc_map_at_10_max": 0.034934, + "nauc_map_at_10_std": 0.506916, + "nauc_map_at_10_diff1": -0.031513, + "nauc_map_at_20_max": 0.050097, + "nauc_map_at_20_std": 0.554412, + "nauc_map_at_20_diff1": -0.010601, + "nauc_map_at_100_max": 0.093126, + "nauc_map_at_100_std": 0.463634, + "nauc_map_at_100_diff1": 0.052221, + "nauc_map_at_1000_max": 0.093126, + "nauc_map_at_1000_std": 0.463634, + "nauc_map_at_1000_diff1": 0.052221, + "nauc_recall_at_1_max": 0.05385, + "nauc_recall_at_1_std": 0.390508, + "nauc_recall_at_1_diff1": 0.082069, + "nauc_recall_at_3_max": -0.01207, + "nauc_recall_at_3_std": 0.347973, + "nauc_recall_at_3_diff1": 0.021781, + "nauc_recall_at_5_max": 0.003331, + "nauc_recall_at_5_std": 0.43425, + "nauc_recall_at_5_diff1": -0.009035, + "nauc_recall_at_10_max": -0.01964, + "nauc_recall_at_10_std": 0.463853, + "nauc_recall_at_10_diff1": -0.002674, + "nauc_recall_at_20_max": -0.005321, + "nauc_recall_at_20_std": 0.506515, + "nauc_recall_at_20_diff1": 0.023051, + "nauc_recall_at_100_max": 0.010316, + "nauc_recall_at_100_std": 0.415497, + "nauc_recall_at_100_diff1": 0.093266, + "nauc_recall_at_1000_max": 0.010316, + "nauc_recall_at_1000_std": 0.415497, + "nauc_recall_at_1000_diff1": 0.093266, + "nauc_precision_at_1_max": 0.861111, + "nauc_precision_at_1_std": 0.275444, + "nauc_precision_at_1_diff1": 0.24043, + "nauc_precision_at_3_max": 0.728836, + "nauc_precision_at_3_std": -0.107376, + "nauc_precision_at_3_diff1": -0.068472, + "nauc_precision_at_5_max": 0.673383, + "nauc_precision_at_5_std": 0.265316, + "nauc_precision_at_5_diff1": -0.304255, + "nauc_precision_at_10_max": 0.454903, + "nauc_precision_at_10_std": 0.428623, + "nauc_precision_at_10_diff1": -0.203597, + "nauc_precision_at_20_max": 0.360842, + "nauc_precision_at_20_std": 0.439205, + "nauc_precision_at_20_diff1": -0.120002, + "nauc_precision_at_100_max": 0.238637, + "nauc_precision_at_100_std": -0.017946, + "nauc_precision_at_100_diff1": -0.097051, + "nauc_precision_at_1000_max": 0.238637, + "nauc_precision_at_1000_std": -0.017946, + "nauc_precision_at_1000_diff1": -0.097051, + "nauc_mrr_at_1_max": 0.861111, + "nauc_mrr_at_1_std": 0.275444, + "nauc_mrr_at_1_diff1": 0.24043, + "nauc_mrr_at_3_max": 0.861111, + "nauc_mrr_at_3_std": 0.275444, + "nauc_mrr_at_3_diff1": 0.24043, + "nauc_mrr_at_5_max": 0.861111, + "nauc_mrr_at_5_std": 0.275444, + "nauc_mrr_at_5_diff1": 0.24043, + "nauc_mrr_at_10_max": 0.861111, + "nauc_mrr_at_10_std": 0.275444, + "nauc_mrr_at_10_diff1": 0.24043, + "nauc_mrr_at_20_max": 0.861111, + "nauc_mrr_at_20_std": 0.275444, + "nauc_mrr_at_20_diff1": 0.24043, + "nauc_mrr_at_100_max": 0.861111, + "nauc_mrr_at_100_std": 0.275444, + "nauc_mrr_at_100_diff1": 0.24043, + "nauc_mrr_at_1000_max": 0.861111, + "nauc_mrr_at_1000_std": 0.275444, + "nauc_mrr_at_1000_diff1": 0.24043, + "main_score": 0.87652, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 142.46919703483582, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/ThuNewsClusteringP2P.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/ThuNewsClusteringP2P.json new file mode 100644 index 0000000000..7df7d0e03e --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/ThuNewsClusteringP2P.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "5798586b105c0434e4f0fe5e767abe619442cf93", + "task_name": "ThuNewsClusteringP2P", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measure": 0.823367, + "v_measure_std": 0.013922, + "v_measures": [ + 0.854623, + 0.830183, + 0.815015, + 0.821029, + 0.827547, + 0.805233, + 0.80695, + 0.812427, + 0.828441, + 0.83222 + ], + "main_score": 0.823367, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 59.3122935295105, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/ThuNewsClusteringS2S.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/ThuNewsClusteringS2S.json new file mode 100644 index 0000000000..a97e3c5643 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/ThuNewsClusteringS2S.json @@ -0,0 +1,32 @@ +{ + "dataset_revision": "8a8b2caeda43f39e13c4bc5bea0f8a667896e10d", + "task_name": "ThuNewsClusteringS2S", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measure": 0.847408, + "v_measure_std": 0.009321, + "v_measures": [ + 0.841279, + 0.864502, + 0.837433, + 0.851879, + 0.852219, + 0.843707, + 0.856688, + 0.850649, + 0.845066, + 0.830655 + ], + "main_score": 0.847408, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 41.71358919143677, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/Touche2020Retrieval.v3.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/Touche2020Retrieval.v3.json new file mode 100644 index 0000000000..8874733834 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/Touche2020Retrieval.v3.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "431886eaecc48f067a3975b70d0949ea2862463c", + "task_name": "Touche2020Retrieval.v3", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "ndcg_at_1": 0.69388, + "ndcg_at_3": 0.74074, + "ndcg_at_5": 0.70226, + "ndcg_at_10": 0.63694, + "ndcg_at_20": 0.56106, + "ndcg_at_100": 0.6149, + "ndcg_at_1000": 0.6149, + "map_at_1": 0.02549, + "map_at_3": 0.07358, + "map_at_5": 0.11366, + "map_at_10": 0.19023, + "map_at_20": 0.26843, + "map_at_100": 0.36793, + "map_at_1000": 0.36793, + "recall_at_1": 0.02549, + "recall_at_3": 0.08175, + "recall_at_5": 0.129, + "recall_at_10": 0.2261, + "recall_at_20": 0.34769, + "recall_at_100": 0.6193, + "recall_at_1000": 0.6193, + "precision_at_1": 0.71429, + "precision_at_3": 0.79592, + "precision_at_5": 0.74286, + "precision_at_10": 0.64286, + "precision_at_20": 0.5051, + "precision_at_100": 0.19796, + "precision_at_1000": 0.0198, + "mrr_at_1": 0.714286, + "mrr_at_3": 0.85034, + "mrr_at_5": 0.85034, + "mrr_at_10": 0.85034, + "mrr_at_20": 0.85034, + "mrr_at_100": 0.85034, + "mrr_at_1000": 0.85034, + "nauc_ndcg_at_1_max": 0.254934, + "nauc_ndcg_at_1_std": 0.154215, + "nauc_ndcg_at_1_diff1": 0.544151, + "nauc_ndcg_at_3_max": 0.228601, + "nauc_ndcg_at_3_std": 0.139068, + "nauc_ndcg_at_3_diff1": 0.263899, + "nauc_ndcg_at_5_max": 0.250718, + "nauc_ndcg_at_5_std": 0.310617, + "nauc_ndcg_at_5_diff1": 0.239017, + "nauc_ndcg_at_10_max": 0.332281, + "nauc_ndcg_at_10_std": 0.586808, + "nauc_ndcg_at_10_diff1": 0.190839, + "nauc_ndcg_at_20_max": 0.315242, + "nauc_ndcg_at_20_std": 0.754736, + "nauc_ndcg_at_20_diff1": 0.203064, + "nauc_ndcg_at_100_max": 0.213169, + "nauc_ndcg_at_100_std": 0.841384, + "nauc_ndcg_at_100_diff1": 0.238884, + "nauc_ndcg_at_1000_max": 0.213169, + "nauc_ndcg_at_1000_std": 0.841384, + "nauc_ndcg_at_1000_diff1": 0.238884, + "nauc_map_at_1_max": 0.074357, + "nauc_map_at_1_std": 0.279047, + "nauc_map_at_1_diff1": 0.551532, + "nauc_map_at_3_max": 0.139275, + "nauc_map_at_3_std": 0.542554, + "nauc_map_at_3_diff1": 0.432262, + "nauc_map_at_5_max": 0.194502, + "nauc_map_at_5_std": 0.727509, + "nauc_map_at_5_diff1": 0.397474, + "nauc_map_at_10_max": 0.270561, + "nauc_map_at_10_std": 0.886303, + "nauc_map_at_10_diff1": 0.314566, + "nauc_map_at_20_max": 0.232804, + "nauc_map_at_20_std": 0.90874, + "nauc_map_at_20_diff1": 0.245431, + "nauc_map_at_100_max": 0.227389, + "nauc_map_at_100_std": 0.87572, + "nauc_map_at_100_diff1": 0.234627, + "nauc_map_at_1000_max": 0.227389, + "nauc_map_at_1000_std": 0.87572, + "nauc_map_at_1000_diff1": 0.234627, + "nauc_recall_at_1_max": 0.074357, + "nauc_recall_at_1_std": 0.279047, + "nauc_recall_at_1_diff1": 0.551532, + "nauc_recall_at_3_max": 0.064819, + "nauc_recall_at_3_std": 0.606198, + "nauc_recall_at_3_diff1": 0.387919, + "nauc_recall_at_5_max": 0.104348, + "nauc_recall_at_5_std": 0.758375, + "nauc_recall_at_5_diff1": 0.312634, + "nauc_recall_at_10_max": 0.167714, + "nauc_recall_at_10_std": 0.87368, + "nauc_recall_at_10_diff1": 0.197581, + "nauc_recall_at_20_max": 0.116231, + "nauc_recall_at_20_std": 0.874442, + "nauc_recall_at_20_diff1": 0.161968, + "nauc_recall_at_100_max": 0.007997, + "nauc_recall_at_100_std": 0.720268, + "nauc_recall_at_100_diff1": 0.147653, + "nauc_recall_at_1000_max": 0.007997, + "nauc_recall_at_1000_std": 0.720268, + "nauc_recall_at_1000_diff1": 0.147653, + "nauc_precision_at_1_max": 0.296951, + "nauc_precision_at_1_std": 0.139542, + "nauc_precision_at_1_diff1": 0.612619, + "nauc_precision_at_3_max": 0.192151, + "nauc_precision_at_3_std": 0.18183, + "nauc_precision_at_3_diff1": 0.346675, + "nauc_precision_at_5_max": 0.286721, + "nauc_precision_at_5_std": 0.503901, + "nauc_precision_at_5_diff1": 0.299412, + "nauc_precision_at_10_max": 0.370073, + "nauc_precision_at_10_std": 0.791943, + "nauc_precision_at_10_diff1": 0.079904, + "nauc_precision_at_20_max": 0.302529, + "nauc_precision_at_20_std": 0.655826, + "nauc_precision_at_20_diff1": -0.026783, + "nauc_precision_at_100_max": 0.122169, + "nauc_precision_at_100_std": -0.246818, + "nauc_precision_at_100_diff1": -0.11167, + "nauc_precision_at_1000_max": 0.122169, + "nauc_precision_at_1000_std": -0.246818, + "nauc_precision_at_1000_diff1": -0.11167, + "nauc_mrr_at_1_max": 0.296951, + "nauc_mrr_at_1_std": 0.139542, + "nauc_mrr_at_1_diff1": 0.612619, + "nauc_mrr_at_3_max": 0.307221, + "nauc_mrr_at_3_std": 0.088981, + "nauc_mrr_at_3_diff1": 0.61823, + "nauc_mrr_at_5_max": 0.307221, + "nauc_mrr_at_5_std": 0.088981, + "nauc_mrr_at_5_diff1": 0.61823, + "nauc_mrr_at_10_max": 0.307221, + "nauc_mrr_at_10_std": 0.088981, + "nauc_mrr_at_10_diff1": 0.61823, + "nauc_mrr_at_20_max": 0.307221, + "nauc_mrr_at_20_std": 0.088981, + "nauc_mrr_at_20_diff1": 0.61823, + "nauc_mrr_at_100_max": 0.307221, + "nauc_mrr_at_100_std": 0.088981, + "nauc_mrr_at_100_diff1": 0.61823, + "nauc_mrr_at_1000_max": 0.307221, + "nauc_mrr_at_1000_std": 0.088981, + "nauc_mrr_at_1000_diff1": 0.61823, + "main_score": 0.63694, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 274.2135167121887, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/ToxicConversationsClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/ToxicConversationsClassification.json new file mode 100644 index 0000000000..9c67dd550b --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/ToxicConversationsClassification.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "edfaf9da55d3dd50d43143d90c1ac476895ae6de", + "task_name": "ToxicConversationsClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.866455, + "f1": 0.725879, + "f1_weighted": 0.891036, + "ap": 0.352051, + "ap_weighted": 0.352051, + "scores_per_experiment": [ + { + "accuracy": 0.856445, + "f1": 0.713742, + "f1_weighted": 0.883881, + "ap": 0.336622, + "ap_weighted": 0.336622 + }, + { + "accuracy": 0.891113, + "f1": 0.753992, + "f1_weighted": 0.908601, + "ap": 0.385575, + "ap_weighted": 0.385575 + }, + { + "accuracy": 0.87207, + "f1": 0.733023, + "f1_weighted": 0.895214, + "ap": 0.362718, + "ap_weighted": 0.362718 + }, + { + "accuracy": 0.850098, + "f1": 0.70631, + "f1_weighted": 0.879297, + "ap": 0.327074, + "ap_weighted": 0.327074 + }, + { + "accuracy": 0.879395, + "f1": 0.740227, + "f1_weighted": 0.900284, + "ap": 0.369058, + "ap_weighted": 0.369058 + }, + { + "accuracy": 0.828613, + "f1": 0.683401, + "f1_weighted": 0.863895, + "ap": 0.300617, + "ap_weighted": 0.300617 + }, + { + "accuracy": 0.89502, + "f1": 0.759642, + "f1_weighted": 0.91149, + "ap": 0.394257, + "ap_weighted": 0.394257 + }, + { + "accuracy": 0.836914, + "f1": 0.691526, + "f1_weighted": 0.869797, + "ap": 0.308896, + "ap_weighted": 0.308896 + }, + { + "accuracy": 0.880371, + "f1": 0.741534, + "f1_weighted": 0.900998, + "ap": 0.370947, + "ap_weighted": 0.370947 + }, + { + "accuracy": 0.874512, + "f1": 0.735387, + "f1_weighted": 0.896903, + "ap": 0.364744, + "ap_weighted": 0.364744 + } + ], + "main_score": 0.866455, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 17.649029970169067, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/TweetSentimentExtractionClassification.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/TweetSentimentExtractionClassification.json new file mode 100644 index 0000000000..e8859b3fbf --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/TweetSentimentExtractionClassification.json @@ -0,0 +1,73 @@ +{ + "dataset_revision": "d604517c81ca91fe16a244d1248fc021f9ecee7a", + "task_name": "TweetSentimentExtractionClassification", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.719468, + "f1": 0.722865, + "f1_weighted": 0.715024, + "scores_per_experiment": [ + { + "accuracy": 0.716469, + "f1": 0.717728, + "f1_weighted": 0.70746 + }, + { + "accuracy": 0.708263, + "f1": 0.712486, + "f1_weighted": 0.704354 + }, + { + "accuracy": 0.734578, + "f1": 0.73807, + "f1_weighted": 0.731328 + }, + { + "accuracy": 0.735144, + "f1": 0.738871, + "f1_weighted": 0.731964 + }, + { + "accuracy": 0.733729, + "f1": 0.737057, + "f1_weighted": 0.730555 + }, + { + "accuracy": 0.729768, + "f1": 0.734082, + "f1_weighted": 0.726627 + }, + { + "accuracy": 0.689587, + "f1": 0.691967, + "f1_weighted": 0.683684 + }, + { + "accuracy": 0.73571, + "f1": 0.739286, + "f1_weighted": 0.732177 + }, + { + "accuracy": 0.696095, + "f1": 0.698242, + "f1_weighted": 0.687431 + }, + { + "accuracy": 0.715337, + "f1": 0.720866, + "f1_weighted": 0.714658 + } + ], + "main_score": 0.719468, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 19.44035315513611, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/TwentyNewsgroupsClustering.v2.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/TwentyNewsgroupsClustering.v2.json new file mode 100644 index 0000000000..2fb0a7d851 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/TwentyNewsgroupsClustering.v2.json @@ -0,0 +1,34 @@ +{ + "dataset_revision": "6125ec4e24fa026cec8a478383ee943acfbd5449", + "task_name": "TwentyNewsgroupsClustering.v2", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "v_measures": { + "Level 0": [ + 0.632413, + 0.628813, + 0.635846, + 0.653999, + 0.669713, + 0.631552, + 0.648535, + 0.66556, + 0.639301, + 0.645149 + ] + }, + "v_measure": 0.645088, + "v_measure_std": 0.013581, + "main_score": 0.645088, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 16.0293710231781, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/TwitterSemEval2015.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/TwitterSemEval2015.json new file mode 100644 index 0000000000..d9943349be --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/TwitterSemEval2015.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "70970daeab8776df92f5ea462b6173c0b46fd2d1", + "task_name": "TwitterSemEval2015", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "similarity_accuracy": 0.873875, + "similarity_accuracy_threshold": 0.921424, + "similarity_f1": 0.712448, + "similarity_f1_threshold": 0.905943, + "similarity_precision": 0.659479, + "similarity_recall": 0.77467, + "similarity_ap": 0.779442, + "cosine_accuracy": 0.873875, + "cosine_accuracy_threshold": 0.921424, + "cosine_f1": 0.712448, + "cosine_f1_threshold": 0.905943, + "cosine_precision": 0.659479, + "cosine_recall": 0.77467, + "cosine_ap": 0.779442, + "manhattan_accuracy": 0.873875, + "manhattan_accuracy_threshold": 14.343916, + "manhattan_f1": 0.714515, + "manhattan_f1_threshold": 15.245093, + "manhattan_precision": 0.671777, + "manhattan_recall": 0.763061, + "manhattan_ap": 0.780855, + "euclidean_accuracy": 0.873696, + "euclidean_accuracy_threshold": 0.396105, + "euclidean_f1": 0.713282, + "euclidean_f1_threshold": 0.430204, + "euclidean_precision": 0.669599, + "euclidean_recall": 0.763061, + "euclidean_ap": 0.779342, + "dot_accuracy": 0.873994, + "dot_accuracy_threshold": 0.919849, + "dot_f1": 0.710168, + "dot_f1_threshold": 0.907214, + "dot_precision": 0.663534, + "dot_recall": 0.763852, + "dot_ap": 0.779698, + "max_accuracy": 0.873994, + "max_f1": 0.714515, + "max_precision": 0.671777, + "max_recall": 0.77467, + "max_ap": 0.780855, + "main_score": 0.780855, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 14.061373949050903, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/TwitterURLCorpus.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/TwitterURLCorpus.json new file mode 100644 index 0000000000..5d78f86cc6 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/TwitterURLCorpus.json @@ -0,0 +1,58 @@ +{ + "dataset_revision": "8b6510b0b1fa4e4c4f879467980e9be563ec1cdf", + "task_name": "TwitterURLCorpus", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "similarity_accuracy": 0.89341, + "similarity_accuracy_threshold": 0.9152, + "similarity_f1": 0.792247, + "similarity_f1_threshold": 0.908065, + "similarity_precision": 0.758361, + "similarity_recall": 0.829304, + "similarity_ap": 0.866863, + "cosine_accuracy": 0.89341, + "cosine_accuracy_threshold": 0.9152, + "cosine_f1": 0.792247, + "cosine_f1_threshold": 0.908065, + "cosine_precision": 0.758361, + "cosine_recall": 0.829304, + "cosine_ap": 0.866863, + "manhattan_accuracy": 0.893488, + "manhattan_accuracy_threshold": 14.550603, + "manhattan_f1": 0.793054, + "manhattan_f1_threshold": 15.227173, + "manhattan_precision": 0.763892, + "manhattan_recall": 0.82453, + "manhattan_ap": 0.867263, + "euclidean_accuracy": 0.893274, + "euclidean_accuracy_threshold": 0.411596, + "euclidean_f1": 0.792507, + "euclidean_f1_threshold": 0.428022, + "euclidean_precision": 0.760456, + "euclidean_recall": 0.827379, + "euclidean_ap": 0.86683, + "dot_accuracy": 0.89341, + "dot_accuracy_threshold": 0.916363, + "dot_f1": 0.791893, + "dot_f1_threshold": 0.908096, + "dot_precision": 0.758099, + "dot_recall": 0.828842, + "dot_ap": 0.866322, + "max_accuracy": 0.893488, + "max_f1": 0.793054, + "max_precision": 0.763892, + "max_recall": 0.829304, + "max_ap": 0.867263, + "main_score": 0.867263, + "hf_subset": "default", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 25.858631134033203, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/VideoRetrieval.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/VideoRetrieval.json new file mode 100644 index 0000000000..ccf512775e --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/VideoRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "58c2597a5943a2ba48f4668c3b90d796283c5639", + "task_name": "VideoRetrieval", + "mteb_version": "1.36.5", + "scores": { + "dev": [ + { + "ndcg_at_1": 0.678, + "ndcg_at_3": 0.77419, + "ndcg_at_5": 0.78955, + "ndcg_at_10": 0.804, + "ndcg_at_20": 0.81049, + "ndcg_at_100": 0.81763, + "ndcg_at_1000": 0.81763, + "map_at_1": 0.678, + "map_at_3": 0.75133, + "map_at_5": 0.75993, + "map_at_10": 0.76602, + "map_at_20": 0.76789, + "map_at_100": 0.76897, + "map_at_1000": 0.76897, + "recall_at_1": 0.678, + "recall_at_3": 0.84, + "recall_at_5": 0.877, + "recall_at_10": 0.921, + "recall_at_20": 0.946, + "recall_at_100": 0.983, + "recall_at_1000": 0.983, + "precision_at_1": 0.678, + "precision_at_3": 0.28, + "precision_at_5": 0.1754, + "precision_at_10": 0.0921, + "precision_at_20": 0.0473, + "precision_at_100": 0.00983, + "precision_at_1000": 0.00098, + "mrr_at_1": 0.678, + "mrr_at_3": 0.751333, + "mrr_at_5": 0.759933, + "mrr_at_10": 0.766019, + "mrr_at_20": 0.767886, + "mrr_at_100": 0.768966, + "mrr_at_1000": 0.768966, + "nauc_ndcg_at_1_max": 0.053347, + "nauc_ndcg_at_1_std": -0.384263, + "nauc_ndcg_at_1_diff1": 0.789928, + "nauc_ndcg_at_3_max": 0.081979, + "nauc_ndcg_at_3_std": -0.348329, + "nauc_ndcg_at_3_diff1": 0.777947, + "nauc_ndcg_at_5_max": 0.08509, + "nauc_ndcg_at_5_std": -0.335048, + "nauc_ndcg_at_5_diff1": 0.768651, + "nauc_ndcg_at_10_max": 0.087166, + "nauc_ndcg_at_10_std": -0.305393, + "nauc_ndcg_at_10_diff1": 0.766671, + "nauc_ndcg_at_20_max": 0.087383, + "nauc_ndcg_at_20_std": -0.293024, + "nauc_ndcg_at_20_diff1": 0.76941, + "nauc_ndcg_at_100_max": 0.084284, + "nauc_ndcg_at_100_std": -0.31368, + "nauc_ndcg_at_100_diff1": 0.774095, + "nauc_ndcg_at_1000_max": 0.084284, + "nauc_ndcg_at_1000_std": -0.31368, + "nauc_ndcg_at_1000_diff1": 0.774095, + "nauc_map_at_1_max": 0.053347, + "nauc_map_at_1_std": -0.384263, + "nauc_map_at_1_diff1": 0.789928, + "nauc_map_at_3_max": 0.07061, + "nauc_map_at_3_std": -0.362574, + "nauc_map_at_3_diff1": 0.78141, + "nauc_map_at_5_max": 0.071578, + "nauc_map_at_5_std": -0.356536, + "nauc_map_at_5_diff1": 0.776986, + "nauc_map_at_10_max": 0.072179, + "nauc_map_at_10_std": -0.346387, + "nauc_map_at_10_diff1": 0.776389, + "nauc_map_at_20_max": 0.072016, + "nauc_map_at_20_std": -0.343898, + "nauc_map_at_20_diff1": 0.777098, + "nauc_map_at_100_max": 0.071582, + "nauc_map_at_100_std": -0.346662, + "nauc_map_at_100_diff1": 0.777773, + "nauc_map_at_1000_max": 0.071582, + "nauc_map_at_1000_std": -0.346662, + "nauc_map_at_1000_diff1": 0.777773, + "nauc_recall_at_1_max": 0.053347, + "nauc_recall_at_1_std": -0.384263, + "nauc_recall_at_1_diff1": 0.789928, + "nauc_recall_at_3_max": 0.130689, + "nauc_recall_at_3_std": -0.287304, + "nauc_recall_at_3_diff1": 0.763687, + "nauc_recall_at_5_max": 0.156801, + "nauc_recall_at_5_std": -0.220258, + "nauc_recall_at_5_diff1": 0.724149, + "nauc_recall_at_10_max": 0.20486, + "nauc_recall_at_10_std": 0.037141, + "nauc_recall_at_10_diff1": 0.689336, + "nauc_recall_at_20_max": 0.263798, + "nauc_recall_at_20_std": 0.345463, + "nauc_recall_at_20_diff1": 0.687182, + "nauc_recall_at_100_max": 0.486434, + "nauc_recall_at_100_std": 0.712665, + "nauc_recall_at_100_diff1": 0.725463, + "nauc_recall_at_1000_max": 0.486434, + "nauc_recall_at_1000_std": 0.712665, + "nauc_recall_at_1000_diff1": 0.725463, + "nauc_precision_at_1_max": 0.053347, + "nauc_precision_at_1_std": -0.384263, + "nauc_precision_at_1_diff1": 0.789928, + "nauc_precision_at_3_max": 0.130689, + "nauc_precision_at_3_std": -0.287304, + "nauc_precision_at_3_diff1": 0.763687, + "nauc_precision_at_5_max": 0.156801, + "nauc_precision_at_5_std": -0.220258, + "nauc_precision_at_5_diff1": 0.724149, + "nauc_precision_at_10_max": 0.20486, + "nauc_precision_at_10_std": 0.037141, + "nauc_precision_at_10_diff1": 0.689336, + "nauc_precision_at_20_max": 0.263798, + "nauc_precision_at_20_std": 0.345463, + "nauc_precision_at_20_diff1": 0.687182, + "nauc_precision_at_100_max": 0.486434, + "nauc_precision_at_100_std": 0.712665, + "nauc_precision_at_100_diff1": 0.725463, + "nauc_precision_at_1000_max": 0.486434, + "nauc_precision_at_1000_std": 0.712665, + "nauc_precision_at_1000_diff1": 0.725463, + "nauc_mrr_at_1_max": 0.053347, + "nauc_mrr_at_1_std": -0.384263, + "nauc_mrr_at_1_diff1": 0.789928, + "nauc_mrr_at_3_max": 0.07061, + "nauc_mrr_at_3_std": -0.362574, + "nauc_mrr_at_3_diff1": 0.78141, + "nauc_mrr_at_5_max": 0.071578, + "nauc_mrr_at_5_std": -0.356536, + "nauc_mrr_at_5_diff1": 0.776986, + "nauc_mrr_at_10_max": 0.072179, + "nauc_mrr_at_10_std": -0.346387, + "nauc_mrr_at_10_diff1": 0.776389, + "nauc_mrr_at_20_max": 0.072016, + "nauc_mrr_at_20_std": -0.343898, + "nauc_mrr_at_20_diff1": 0.777098, + "nauc_mrr_at_100_max": 0.071582, + "nauc_mrr_at_100_std": -0.346662, + "nauc_mrr_at_100_diff1": 0.777773, + "nauc_mrr_at_1000_max": 0.071582, + "nauc_mrr_at_1000_std": -0.346662, + "nauc_mrr_at_1000_diff1": 0.777773, + "main_score": 0.804, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 57.08095693588257, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/Waimai.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/Waimai.json new file mode 100644 index 0000000000..409bc7fe94 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/Waimai.json @@ -0,0 +1,95 @@ +{ + "dataset_revision": "339287def212450dcaa9df8c22bf93e9980c7023", + "task_name": "Waimai", + "mteb_version": "1.36.5", + "scores": { + "test": [ + { + "accuracy": 0.9174, + "f1": 0.905424, + "f1_weighted": 0.917195, + "ap": 0.804595, + "ap_weighted": 0.804595, + "scores_per_experiment": [ + { + "accuracy": 0.92, + "f1": 0.90839, + "f1_weighted": 0.919804, + "ap": 0.810051, + "ap_weighted": 0.810051 + }, + { + "accuracy": 0.922, + "f1": 0.909315, + "f1_weighted": 0.921186, + "ap": 0.818452, + "ap_weighted": 0.818452 + }, + { + "accuracy": 0.915, + "f1": 0.903516, + "f1_weighted": 0.915166, + "ap": 0.797343, + "ap_weighted": 0.797343 + }, + { + "accuracy": 0.919, + "f1": 0.90747, + "f1_weighted": 0.918902, + "ap": 0.807303, + "ap_weighted": 0.807303 + }, + { + "accuracy": 0.913, + "f1": 0.90244, + "f1_weighted": 0.913674, + "ap": 0.791239, + "ap_weighted": 0.791239 + }, + { + "accuracy": 0.918, + "f1": 0.90579, + "f1_weighted": 0.917661, + "ap": 0.806352, + "ap_weighted": 0.806352 + }, + { + "accuracy": 0.917, + "f1": 0.903586, + "f1_weighted": 0.916173, + "ap": 0.806785, + "ap_weighted": 0.806785 + }, + { + "accuracy": 0.915, + "f1": 0.902584, + "f1_weighted": 0.914757, + "ap": 0.799179, + "ap_weighted": 0.799179 + }, + { + "accuracy": 0.915, + "f1": 0.903364, + "f1_weighted": 0.915101, + "ap": 0.797623, + "ap_weighted": 0.797623 + }, + { + "accuracy": 0.92, + "f1": 0.907781, + "f1_weighted": 0.91953, + "ap": 0.81162, + "ap_weighted": 0.81162 + } + ], + "main_score": 0.9174, + "hf_subset": "default", + "languages": [ + "cmn-Hans" + ] + } + ] + }, + "evaluation_time": 18.563534021377563, + "kg_co2_emissions": null +} \ No newline at end of file diff --git a/results/ByteDance-Seed__Seed1.5-Embedding/3/model_meta.json b/results/ByteDance-Seed__Seed1.5-Embedding/3/model_meta.json new file mode 100644 index 0000000000..9b0f94bec3 --- /dev/null +++ b/results/ByteDance-Seed__Seed1.5-Embedding/3/model_meta.json @@ -0,0 +1 @@ +{"name": "ByteDance-Seed/Seed1.5-Embedding", "revision": "3", "release_date": "2025-04-25", "languages": ["eng-Latn","zho-Hans"], "n_parameters": null, "memory_usage_mb": null, "max_tokens": 32768, "embed_dim": 2048, "license": null, "open_weights": false, "public_training_code": null, "public_training_data": null, "framework": ["API"], "reference": null, "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": null, "adapted_from": null, "superseded_by": null, "is_cross_encoder": null, "modalities": ["text"], "loader": null} \ No newline at end of file