diff --git a/EXTERNAL_MODEL_RESULTS.json b/EXTERNAL_MODEL_RESULTS.json
index 10c57b4f..82ce19c8 100644
--- a/EXTERNAL_MODEL_RESULTS.json
+++ b/EXTERNAL_MODEL_RESULTS.json
@@ -781,123 +781,6 @@
"p-MRR": []
}
},
- "BAAI__bge-m3": {
- "BitextMining": {
- "f1": [
- {
- "Model": "bge-m3",
- "Tatoeba (rus-eng)": 93.42
- }
- ]
- },
- "Classification": {
- "accuracy": [
- {
- "Model": "bge-m3",
- "GeoreviewClassification": 48.27,
- "HeadlineClassification": 70.32,
- "InappropriatenessClassification": 59.87,
- "KinopoiskClassification": 58.23,
- "MassiveIntentClassification (ru)": 68.75,
- "MassiveScenarioClassification (ru)": 73.42,
- "RuReviewsClassification": 66.91,
- "RuSciBenchGRNTIClassification": 55.81,
- "RuSciBenchOECDClassification": 42.57
- }
- ]
- },
- "Clustering": {
- "v_measure": [
- {
- "Model": "bge-m3",
- "GeoreviewClusteringP2P": 63.09,
- "RuSciBenchGRNTIClusteringP2P": 50.83,
- "RuSciBenchOECDClusteringP2P": 43.21
- }
- ]
- },
- "PairClassification": {
- "max_ap": [
- {
- "Model": "bge-m3",
- "OpusparcusPC (ru)": 89.64,
- "TERRa": 60.6
- },
- {
- "Model": "bge-m3",
- "OpusparcusPC (ru)": 89.64,
- "TERRa": 60.6
- }
- ]
- },
- "Reranking": {
- "map": [
- {
- "Model": "bge-m3",
- "MIRACLReranking (ru)": 65.38
- },
- {
- "Model": "bge-m3",
- "RuBQReranking": 74.03
- }
- ]
- },
- "Retrieval": {
- "ndcg_at_10": [
- {
- "Model": "bge-m3",
- "ARCChallenge": 9.02,
- "AlphaNLI": 24.73,
- "HellaSwag": 25.67,
- "LEMBNarrativeQARetrieval": 45.76,
- "LEMBQMSumRetrieval": 35.54,
- "LEMBSummScreenFDRetrieval": 94.09,
- "LEMBWikimQARetrieval": 77.73,
- "MIRACLRetrieval (ru)": 70.16,
- "PIQA": 22.93,
- "Quail": 7.51,
- "RARbCode": 38.8,
- "RARbMath": 69.19,
- "RiaNewsRetrieval": 82.99,
- "RuBQRetrieval": 71.22,
- "SIQA": 4.89,
- "SpartQA": 7.49,
- "TempReasonL1": 0.99,
- "TempReasonL2Fact": 33.23,
- "TempReasonL2Pure": 0.68,
- "TempReasonL3Fact": 30.05,
- "TempReasonL3Pure": 5.28,
- "WinoGrande": 41.72
- }
- ]
- },
- "STS": {
- "cosine_spearman": [
- {
- "Model": "bge-m3",
- "RUParaPhraserSTS": 74.9,
- "RuSTSBenchmarkSTS": 79.87,
- "STS22 (ru)": 66.26,
- "STSBenchmarkMultilingualSTS (ru)": 79.27
- }
- ]
- },
- "Summarization": {
- "cosine_spearman": []
- },
- "MultilabelClassification": {
- "accuracy": [
- {
- "Model": "bge-m3",
- "CEDRClassification": 43.47,
- "SensitiveTopicsClassification": 26.25
- }
- ]
- },
- "InstructionRetrieval": {
- "p-MRR": []
- }
- },
"BAAI__bge-m3-instruct": {
"BitextMining": {
"f1": []
@@ -9150,1530 +9033,6 @@
"p-MRR": []
}
},
- "intfloat__multilingual-e5-base": {
- "BitextMining": {
- "f1": [
- {
- "Model": "multilingual-e5-base",
- "BornholmBitextMining": 46.4,
- "Tatoeba (csb-eng)": 24.29,
- "Tatoeba (spa-eng)": 96.97,
- "Tatoeba (kzj-eng)": 6.26,
- "Tatoeba (uzb-eng)": 62.63,
- "Tatoeba (mal-eng)": 96.72,
- "Tatoeba (dtp-eng)": 5.13,
- "Tatoeba (ces-eng)": 88.75,
- "Tatoeba (mon-eng)": 78.37,
- "Tatoeba (fry-eng)": 50.82,
- "Tatoeba (cbk-eng)": 60.66,
- "Tatoeba (awa-eng)": 68.39,
- "Tatoeba (cha-eng)": 16.95,
- "Tatoeba (nld-eng)": 93.2,
- "Tatoeba (mhr-eng)": 5.52,
- "Tatoeba (cmn-eng)": 93.35,
- "Tatoeba (arq-eng)": 26.61,
- "Tatoeba (kor-eng)": 83.37,
- "Tatoeba (pol-eng)": 94.57,
- "Tatoeba (uig-eng)": 62.97,
- "Tatoeba (zsm-eng)": 92.45,
- "Tatoeba (tuk-eng)": 19.67,
- "Tatoeba (ind-eng)": 90.26,
- "Tatoeba (aze-eng)": 84.71,
- "Tatoeba (xho-eng)": 73.24,
- "Tatoeba (war-eng)": 47.18,
- "Tatoeba (ast-eng)": 74.36,
- "Tatoeba (ido-eng)": 74.41,
- "Tatoeba (dsb-eng)": 34.36,
- "Tatoeba (est-eng)": 70.64,
- "Tatoeba (ina-eng)": 86.11,
- "Tatoeba (bel-eng)": 86.7,
- "Tatoeba (fin-eng)": 86.15,
- "Tatoeba (ang-eng)": 29.87,
- "Tatoeba (gle-eng)": 58.62,
- "Tatoeba (slk-eng)": 86.42,
- "Tatoeba (hye-eng)": 85.85,
- "Tatoeba (tgl-eng)": 83.78,
- "Tatoeba (pam-eng)": 6.92,
- "Tatoeba (eus-eng)": 56.26,
- "Tatoeba (slv-eng)": 81.93,
- "Tatoeba (nno-eng)": 82.67,
- "Tatoeba (wuu-eng)": 78.65,
- "Tatoeba (nds-eng)": 53.86,
- "Tatoeba (deu-eng)": 97.07,
- "Tatoeba (ita-eng)": 90.61,
- "Tatoeba (rus-eng)": 91.78,
- "Tatoeba (bos-eng)": 88.86,
- "Tatoeba (glg-eng)": 82.69,
- "Tatoeba (kaz-eng)": 75.56,
- "Tatoeba (nov-eng)": 66.96,
- "Tatoeba (tam-eng)": 85.12,
- "Tatoeba (dan-eng)": 91.23,
- "Tatoeba (oci-eng)": 35.79,
- "Tatoeba (arz-eng)": 66.79,
- "Tatoeba (gsw-eng)": 43.53,
- "Tatoeba (jav-eng)": 61.25,
- "Tatoeba (sqi-eng)": 90.06,
- "Tatoeba (vie-eng)": 94.55,
- "Tatoeba (lit-eng)": 75.53,
- "Tatoeba (isl-eng)": 76.9,
- "Tatoeba (hsb-eng)": 40.36,
- "Tatoeba (hun-eng)": 84.41,
- "Tatoeba (kab-eng)": 21.77,
- "Tatoeba (ceb-eng)": 45.46,
- "Tatoeba (ber-eng)": 23.59,
- "Tatoeba (jpn-eng)": 90.3,
- "Tatoeba (max-eng)": 52.4,
- "Tatoeba (ara-eng)": 82.86,
- "Tatoeba (nob-eng)": 95.9,
- "Tatoeba (cat-eng)": 84.09,
- "Tatoeba (orv-eng)": 16.0,
- "Tatoeba (cor-eng)": 4.38,
- "Tatoeba (tel-eng)": 88.49,
- "Tatoeba (ell-eng)": 89.96,
- "Tatoeba (bre-eng)": 5.44,
- "Tatoeba (swg-eng)": 42.33,
- "Tatoeba (pes-eng)": 87.18,
- "Tatoeba (cym-eng)": 65.69,
- "Tatoeba (heb-eng)": 74.26,
- "Tatoeba (urd-eng)": 86.2,
- "Tatoeba (amh-eng)": 74.93,
- "Tatoeba (lvs-eng)": 76.76,
- "Tatoeba (swe-eng)": 91.33,
- "Tatoeba (ukr-eng)": 88.29,
- "Tatoeba (gla-eng)": 43.08,
- "Tatoeba (mar-eng)": 86.62,
- "Tatoeba (khm-eng)": 47.27,
- "Tatoeba (tat-eng)": 66.92,
- "Tatoeba (lat-eng)": 39.62,
- "Tatoeba (pms-eng)": 44.61,
- "Tatoeba (hrv-eng)": 92.5,
- "Tatoeba (mkd-eng)": 73.76,
- "Tatoeba (bul-eng)": 88.95,
- "Tatoeba (kat-eng)": 77.83,
- "Tatoeba (hin-eng)": 93.13,
- "Tatoeba (fao-eng)": 64.72,
- "Tatoeba (ben-eng)": 81.05,
- "Tatoeba (epo-eng)": 92.07,
- "Tatoeba (por-eng)": 92.74,
- "Tatoeba (tur-eng)": 92.54,
- "Tatoeba (yue-eng)": 80.66,
- "Tatoeba (srp-eng)": 89.08,
- "Tatoeba (swh-eng)": 66.81,
- "Tatoeba (tha-eng)": 94.22,
- "Tatoeba (ron-eng)": 91.27,
- "Tatoeba (fra-eng)": 92.76,
- "Tatoeba (yid-eng)": 63.2,
- "Tatoeba (afr-eng)": 87.04,
- "Tatoeba (tzl-eng)": 34.44,
- "Tatoeba (lfn-eng)": 52.85,
- "Tatoeba (ile-eng)": 72.56,
- "Tatoeba (kur-eng)": 52.96
- }
- ]
- },
- "Classification": {
- "accuracy": [
- {
- "Model": "multilingual-e5-base",
- "AllegroReviews": 40.85,
- "AmazonCounterfactualClassification (en-ext)": 76.91,
- "AmazonCounterfactualClassification (en)": 77.36,
- "AmazonCounterfactualClassification (de)": 70.81,
- "AmazonCounterfactualClassification (ja)": 72.02,
- "AmazonPolarityClassification": 91.76,
- "AmazonReviewsClassification (en)": 47.54,
- "AmazonReviewsClassification (de)": 44.37,
- "AmazonReviewsClassification (es)": 43.38,
- "AmazonReviewsClassification (fr)": 40.94,
- "AmazonReviewsClassification (ja)": 39.57,
- "AmazonReviewsClassification (zh)": 38.34,
- "AngryTweetsClassification": 54.65,
- "Banking77Classification": 73.53,
- "CBD": 62.66,
- "DKHateClassification": 63.53,
- "DanishPoliticalCommentsClassification": 36.69,
- "EmotionClassification": 45.68,
- "GeoreviewClassification": 46.05,
- "HeadlineClassification": 75.64,
- "IFlyTek": 44.93,
- "ImdbClassification": 84.29,
- "InappropriatenessClassification": 58.78,
- "JDReview": 76.21,
- "KinopoiskClassification": 50.89,
- "LccSentimentClassification": 59.67,
- "MTOPDomainClassification (en)": 90.9,
- "MTOPDomainClassification (de)": 87.94,
- "MTOPDomainClassification (es)": 85.96,
- "MTOPDomainClassification (fr)": 84.79,
- "MTOPDomainClassification (hi)": 83.92,
- "MTOPDomainClassification (th)": 83.94,
- "MTOPIntentClassification (en)": 61.6,
- "MTOPIntentClassification (de)": 61.05,
- "MTOPIntentClassification (es)": 55.36,
- "MTOPIntentClassification (fr)": 55.51,
- "MTOPIntentClassification (hi)": 53.93,
- "MTOPIntentClassification (th)": 58.69,
- "MasakhaNEWSClassification (amh)": 83.8,
- "MasakhaNEWSClassification (eng)": 76.49,
- "MasakhaNEWSClassification (fra)": 79.69,
- "MasakhaNEWSClassification (hau)": 74.63,
- "MasakhaNEWSClassification (ibo)": 64.59,
- "MasakhaNEWSClassification (lin)": 70.57,
- "MasakhaNEWSClassification (lug)": 68.12,
- "MasakhaNEWSClassification (orm)": 71.75,
- "MasakhaNEWSClassification (pcm)": 91.05,
- "MasakhaNEWSClassification (run)": 73.35,
- "MasakhaNEWSClassification (sna)": 84.17,
- "MasakhaNEWSClassification (som)": 60.1,
- "MasakhaNEWSClassification (swa)": 70.74,
- "MasakhaNEWSClassification (tir)": 67.1,
- "MasakhaNEWSClassification (xho)": 76.03,
- "MasakhaNEWSClassification (yor)": 72.75,
- "MassiveIntentClassification (es)": 61.13,
- "MassiveIntentClassification (de)": 59.82,
- "MassiveIntentClassification (mn)": 46.8,
- "MassiveIntentClassification (sq)": 51.07,
- "MassiveIntentClassification (sv)": 61.78,
- "MassiveIntentClassification (tr)": 60.69,
- "MassiveIntentClassification (hy)": 48.77,
- "MassiveIntentClassification (az)": 51.36,
- "MassiveIntentClassification (da)": 60.16,
- "MassiveIntentClassification (my)": 46.67,
- "MassiveIntentClassification (th)": 59.63,
- "MassiveIntentClassification (sl)": 53.84,
- "MassiveIntentClassification (sw)": 45.24,
- "MassiveIntentClassification (pl)": 61.04,
- "MassiveIntentClassification (bn)": 51.69,
- "MassiveIntentClassification (nb)": 59.83,
- "MassiveIntentClassification (fi)": 58.91,
- "MassiveIntentClassification (jv)": 43.23,
- "MassiveIntentClassification (id)": 58.7,
- "MassiveIntentClassification (ka)": 37.56,
- "MassiveIntentClassification (af)": 49.82,
- "MassiveIntentClassification (km)": 32.14,
- "MassiveIntentClassification (lv)": 51.17,
- "MassiveIntentClassification (tl)": 48.99,
- "MassiveIntentClassification (ko)": 59.97,
- "MassiveIntentClassification (am)": 42.4,
- "MassiveIntentClassification (ar)": 50.2,
- "MassiveIntentClassification (el)": 58.07,
- "MassiveIntentClassification (fa)": 59.51,
- "MassiveIntentClassification (hi)": 56.75,
- "MassiveIntentClassification (fr)": 61.32,
- "MassiveIntentClassification (is)": 44.52,
- "MassiveIntentClassification (te)": 50.09,
- "MassiveIntentClassification (ja)": 62.3,
- "MassiveIntentClassification (vi)": 59.61,
- "MassiveIntentClassification (hu)": 57.69,
- "MassiveIntentClassification (en)": 65.71,
- "MassiveIntentClassification (ms)": 52.85,
- "MassiveIntentClassification (zh-CN)": 63.22,
- "MassiveIntentClassification (ro)": 56.83,
- "MassiveIntentClassification (he)": 55.3,
- "MassiveIntentClassification (ur)": 51.3,
- "MassiveIntentClassification (pt)": 62.12,
- "MassiveIntentClassification (it)": 61.29,
- "MassiveIntentClassification (ta)": 48.93,
- "MassiveIntentClassification (cy)": 37.05,
- "MassiveIntentClassification (kn)": 48.63,
- "MassiveIntentClassification (nl)": 61.23,
- "MassiveIntentClassification (ml)": 53.75,
- "MassiveIntentClassification (zh-TW)": 56.4,
- "MassiveIntentClassification (ru)": 62.78,
- "MassiveScenarioClassification (az)": 55.15,
- "MassiveScenarioClassification (fr)": 67.37,
- "MassiveScenarioClassification (ko)": 67.9,
- "MassiveScenarioClassification (da)": 67.46,
- "MassiveScenarioClassification (fi)": 64.94,
- "MassiveScenarioClassification (ro)": 63.5,
- "MassiveScenarioClassification (zh-TW)": 63.73,
- "MassiveScenarioClassification (ar)": 58.0,
- "MassiveScenarioClassification (kn)": 53.49,
- "MassiveScenarioClassification (nl)": 68.62,
- "MassiveScenarioClassification (sq)": 57.92,
- "MassiveScenarioClassification (ta)": 53.86,
- "MassiveScenarioClassification (am)": 50.33,
- "MassiveScenarioClassification (ml)": 59.89,
- "MassiveScenarioClassification (hu)": 65.75,
- "MassiveScenarioClassification (vi)": 66.35,
- "MassiveScenarioClassification (tl)": 54.36,
- "MassiveScenarioClassification (ka)": 43.38,
- "MassiveScenarioClassification (sl)": 58.3,
- "MassiveScenarioClassification (cy)": 43.84,
- "MassiveScenarioClassification (it)": 66.17,
- "MassiveScenarioClassification (is)": 53.28,
- "MassiveScenarioClassification (fa)": 63.92,
- "MassiveScenarioClassification (pt)": 65.49,
- "MassiveScenarioClassification (mn)": 51.87,
- "MassiveScenarioClassification (lv)": 56.42,
- "MassiveScenarioClassification (ru)": 68.21,
- "MassiveScenarioClassification (bn)": 57.0,
- "MassiveScenarioClassification (en)": 71.57,
- "MassiveScenarioClassification (hy)": 53.63,
- "MassiveScenarioClassification (tr)": 65.18,
- "MassiveScenarioClassification (ja)": 69.89,
- "MassiveScenarioClassification (nb)": 66.18,
- "MassiveScenarioClassification (sv)": 69.15,
- "MassiveScenarioClassification (id)": 63.6,
- "MassiveScenarioClassification (th)": 67.37,
- "MassiveScenarioClassification (de)": 68.4,
- "MassiveScenarioClassification (jv)": 51.94,
- "MassiveScenarioClassification (sw)": 52.64,
- "MassiveScenarioClassification (ms)": 58.35,
- "MassiveScenarioClassification (es)": 66.47,
- "MassiveScenarioClassification (te)": 54.24,
- "MassiveScenarioClassification (he)": 62.53,
- "MassiveScenarioClassification (my)": 50.77,
- "MassiveScenarioClassification (af)": 58.95,
- "MassiveScenarioClassification (el)": 65.38,
- "MassiveScenarioClassification (km)": 38.45,
- "MassiveScenarioClassification (pl)": 66.11,
- "MassiveScenarioClassification (ur)": 56.74,
- "MassiveScenarioClassification (hi)": 62.91,
- "MassiveScenarioClassification (zh-CN)": 70.24,
- "MultilingualSentiment": 65.28,
- "NoRecClassification": 57.58,
- "NordicLangClassification": 75.94,
- "NorwegianParliament": 59.94,
- "OnlineShopping": 88.4,
- "PAC": 70.87,
- "PolEmo2.0-IN": 67.66,
- "PolEmo2.0-OUT": 43.91,
- "RuReviewsClassification": 62.99,
- "RuSciBenchGRNTIClassification": 56.28,
- "RuSciBenchOECDClassification": 42.69,
- "ScalaDaClassification": 50.79,
- "ScalaNbClassification": 50.32,
- "TNews": 47.06,
- "ToxicConversationsClassification": 64.33,
- "TweetSentimentExtractionClassification": 62.8,
- "Waimai": 84.42
- }
- ]
- },
- "Clustering": {
- "v_measure": [
- {
- "Model": "multilingual-e5-base",
- "8TagsClustering": 24.97,
- "AlloProfClusteringP2P": 62.09,
- "AlloProfClusteringS2S": 32.98,
- "ArxivClusteringP2P": 43.35,
- "ArxivClusteringS2S": 36.0,
- "BiorxivClusteringP2P": 37.55,
- "BiorxivClusteringS2S": 30.33,
- "CLSClusteringP2P": 32.41,
- "CLSClusteringS2S": 36.99,
- "GeoreviewClusteringP2P": 54.46,
- "HALClusteringS2S": 22.48,
- "MLSUMClusteringP2P (ru)": 43.47,
- "MLSUMClusteringP2P": 43.48,
- "MLSUMClusteringS2S (ru)": 40.87,
- "MLSUMClusteringS2S": 38.53,
- "MasakhaNEWSClusteringP2P (amh)": 58.05,
- "MasakhaNEWSClusteringP2P (eng)": 43.8,
- "MasakhaNEWSClusteringP2P (fra)": 47.91,
- "MasakhaNEWSClusteringP2P (hau)": 44.78,
- "MasakhaNEWSClusteringP2P (ibo)": 44.97,
- "MasakhaNEWSClusteringP2P (lin)": 48.08,
- "MasakhaNEWSClusteringP2P (lug)": 50.15,
- "MasakhaNEWSClusteringP2P (orm)": 38.02,
- "MasakhaNEWSClusteringP2P (pcm)": 71.03,
- "MasakhaNEWSClusteringP2P (run)": 58.28,
- "MasakhaNEWSClusteringP2P (sna)": 59.25,
- "MasakhaNEWSClusteringP2P (som)": 37.27,
- "MasakhaNEWSClusteringP2P (swa)": 34.54,
- "MasakhaNEWSClusteringP2P (tir)": 53.44,
- "MasakhaNEWSClusteringP2P (xho)": 40.32,
- "MasakhaNEWSClusteringP2P (yor)": 37.97,
- "MasakhaNEWSClusteringS2S (amh)": 49.38,
- "MasakhaNEWSClusteringS2S (eng)": 45.76,
- "MasakhaNEWSClusteringS2S (fra)": 51.16,
- "MasakhaNEWSClusteringS2S (hau)": 16.11,
- "MasakhaNEWSClusteringS2S (ibo)": 24.38,
- "MasakhaNEWSClusteringS2S (lin)": 44.8,
- "MasakhaNEWSClusteringS2S (lug)": 45.67,
- "MasakhaNEWSClusteringS2S (orm)": 26.41,
- "MasakhaNEWSClusteringS2S (pcm)": 83.26,
- "MasakhaNEWSClusteringS2S (run)": 48.77,
- "MasakhaNEWSClusteringS2S (sna)": 43.9,
- "MasakhaNEWSClusteringS2S (som)": 25.43,
- "MasakhaNEWSClusteringS2S (swa)": 9.87,
- "MasakhaNEWSClusteringS2S (tir)": 51.66,
- "MasakhaNEWSClusteringS2S (xho)": 29.65,
- "MasakhaNEWSClusteringS2S (yor)": 30.12,
- "MedrxivClusteringP2P": 30.6,
- "MedrxivClusteringS2S": 28.73,
- "RedditClustering": 43.15,
- "RedditClusteringP2P": 61.69,
- "RuSciBenchGRNTIClusteringP2P": 51.56,
- "RuSciBenchOECDClusteringP2P": 44.79,
- "StackExchangeClustering": 55.31,
- "StackExchangeClusteringP2P": 33.51,
- "ThuNewsClusteringP2P": 40.98,
- "ThuNewsClusteringS2S": 52.36,
- "TwentyNewsgroupsClustering": 35.55
- }
- ]
- },
- "PairClassification": {
- "max_ap": [
- {
- "Model": "multilingual-e5-base",
- "CDSC-E": 72.7,
- "OpusparcusPC (de)": 95.83,
- "OpusparcusPC (en)": 98.71,
- "OpusparcusPC (fi)": 90.3,
- "OpusparcusPC (fr)": 92.12,
- "OpusparcusPC (ru)": 86.82,
- "OpusparcusPC (sv)": 93.05,
- "PSC": 99.14,
- "PawsXPairClassification (de)": 54.11,
- "PawsXPairClassification (en)": 55.79,
- "PawsXPairClassification (es)": 54.13,
- "PawsXPairClassification (fr)": 56.01,
- "PawsXPairClassification (ja)": 49.02,
- "PawsXPairClassification (ko)": 51.01,
- "PawsXPairClassification (zh)": 55.13,
- "SICK-E-PL": 68.76,
- "SprintDuplicateQuestions": 93.02,
- "TERRa": 54.96,
- "TwitterSemEval2015": 72.21,
- "TwitterURLCorpus": 85.48
- },
- {
- "Model": "multilingual-e5-base",
- "CDSC-E": 72.67,
- "Cmnli": 74.98,
- "Ocnli": 60.47,
- "OpusparcusPC (de)": 95.83,
- "OpusparcusPC (en)": 98.71,
- "OpusparcusPC (fi)": 90.3,
- "OpusparcusPC (fr)": 92.73,
- "OpusparcusPC (ru)": 86.82,
- "OpusparcusPC (sv)": 93.05,
- "PPC": 88.01,
- "PSC": 99.14,
- "PawsXPairClassification (de)": 54.26,
- "PawsXPairClassification (en)": 55.79,
- "PawsXPairClassification (es)": 54.13,
- "PawsXPairClassification (fr)": 56.93,
- "PawsXPairClassification (ja)": 49.15,
- "PawsXPairClassification (ko)": 51.01,
- "PawsXPairClassification (zh)": 55.13,
- "SICK-E-PL": 68.77,
- "SprintDuplicateQuestions": 93.02,
- "TERRa": 54.98,
- "TwitterSemEval2015": 72.21,
- "TwitterURLCorpus": 85.48
- },
- {
- "Model": "multilingual-e5-base",
- "CDSC-E": 72.67,
- "Cmnli": 74.51,
- "Ocnli": 59.63,
- "OpusparcusPC (fr)": 92.72,
- "PPC": 88.01,
- "PSC": 99.14,
- "PawsXPairClassification (fr)": 56.93,
- "SICK-E-PL": 68.77
- }
- ]
- },
- "Reranking": {
- "map": [
- {
- "Model": "multilingual-e5-base",
- "AlloprofReranking": 58.1,
- "AskUbuntuDupQuestions": 59.28,
- "CMedQAv1": 65.21,
- "CMedQAv2": 66.06,
- "MMarcoReranking": 21.76,
- "MindSmallReranking": 29.28,
- "RuBQReranking": 72.01,
- "SciDocsRR": 81.81,
- "StackOverflowDupQuestions": 49.75,
- "SyntecReranking": 85.43,
- "T2Reranking": 64.39
- },
- {
- "Model": "multilingual-e5-base",
- "MIRACLReranking (ru)": 60.47
- }
- ]
- },
- "Retrieval": {
- "ndcg_at_10": [
- {
- "Model": "multilingual-e5-base",
- "AILACasedocs": 26.05,
- "AILAStatutes": 20.37,
- "ARCChallenge": 9.61,
- "AlloprofRetrieval": 36.21,
- "AlphaNLI": 16.44,
- "AppsRetrieval": 20.94,
- "ArguAna": 44.21,
- "ArguAna-PL": 42.81,
- "BSARDRetrieval": 0.0,
- "CmedqaRetrieval": 27.2,
- "CodeFeedbackMT": 43.18,
- "CodeFeedbackST": 72.61,
- "CodeSearchNetCCRetrieval (python)": 85.37,
- "CodeSearchNetCCRetrieval (javascript)": 78.44,
- "CodeSearchNetCCRetrieval (go)": 72.29,
- "CodeSearchNetCCRetrieval (ruby)": 81.92,
- "CodeSearchNetCCRetrieval (java)": 78.85,
- "CodeSearchNetCCRetrieval (php)": 72.02,
- "CodeSearchNetRetrieval (python)": 85.68,
- "CodeSearchNetRetrieval (javascript)": 71.4,
- "CodeSearchNetRetrieval (go)": 89.47,
- "CodeSearchNetRetrieval (ruby)": 78.24,
- "CodeSearchNetRetrieval (java)": 78.39,
- "CodeSearchNetRetrieval (php)": 81.05,
- "CodeTransOceanContest": 51.89,
- "CodeTransOceanDL": 29.88,
- "CosQA": 31.12,
- "CovidRetrieval": 73.45,
- "DBPedia-PL": 30.23,
- "DuRetrieval": 81.64,
- "EcomRetrieval": 54.17,
- "FiQA-PL": 25.52,
- "FiQA2018": 38.15,
- "GerDaLIRSmall": 15.3,
- "HellaSwag": 24.79,
- "HotpotQA-PL": 63.52,
- "LEMBNarrativeQARetrieval": 23.6,
- "LEMBQMSumRetrieval": 25.16,
- "LEMBSummScreenFDRetrieval": 68.21,
- "LEMBWikimQARetrieval": 56.04,
- "LeCaRDv2": 59.0,
- "LegalBenchConsumerContractsQA": 69.02,
- "LegalBenchCorporateLobbying": 88.97,
- "LegalQuAD": 47.85,
- "LegalSummarization": 61.69,
- "MIRACLRetrieval (ru)": 61.6,
- "MMarcoRetrieval": 76.04,
- "MSMARCO-PL": 29.52,
- "MedicalRetrieval": 48.35,
- "MintakaRetrieval (ar)": 23.06,
- "MintakaRetrieval (de)": 29.8,
- "MintakaRetrieval (es)": 29.88,
- "MintakaRetrieval (fr)": 23.46,
- "MintakaRetrieval (hi)": 22.68,
- "MintakaRetrieval (it)": 29.77,
- "MintakaRetrieval (ja)": 22.98,
- "MintakaRetrieval (pt)": 30.62,
- "NFCorpus": 32.49,
- "NFCorpus-PL": 25.98,
- "NQ-PL": 44.8,
- "PIQA": 25.09,
- "Quail": 3.52,
- "Quora-PL": 81.22,
- "RARbCode": 52.16,
- "RARbMath": 65.35,
- "RiaNewsRetrieval": 70.24,
- "RuBQRetrieval": 69.58,
- "SCIDOCS": 17.17,
- "SCIDOCS-PL": 12.35,
- "SIQA": 3.72,
- "SciFact": 69.39,
- "SciFact-PL": 62.11,
- "SpartQA": 7.91,
- "StackOverflowQA": 85.11,
- "SyntecRetrieval": 80.49,
- "SyntheticText2SQL": 53.61,
- "T2Retrieval": 70.86,
- "TRECCOVID": 69.5,
- "TRECCOVID-PL": 66.06,
- "TempReasonL1": 0.72,
- "TempReasonL2Fact": 38.76,
- "TempReasonL2Pure": 1.63,
- "TempReasonL3Fact": 35.85,
- "TempReasonL3Pure": 7.11,
- "Touche2020": 21.5,
- "VideoRetrieval": 61.3,
- "WinoGrande": 56.18,
- "XPQARetrieval (ara-ara)": 39.97,
- "XPQARetrieval (eng-ara)": 17.23,
- "XPQARetrieval (ara-eng)": 34.35,
- "XPQARetrieval (deu-deu)": 72.11,
- "XPQARetrieval (eng-deu)": 28.91,
- "XPQARetrieval (deu-eng)": 61.46,
- "XPQARetrieval (spa-spa)": 58.35,
- "XPQARetrieval (eng-spa)": 25.27,
- "XPQARetrieval (spa-eng)": 51.07,
- "XPQARetrieval (fra-fra)": 59.56,
- "XPQARetrieval (eng-fra)": 23.69,
- "XPQARetrieval (fra-eng)": 53.9,
- "XPQARetrieval (hin-hin)": 70.59,
- "XPQARetrieval (eng-hin)": 27.57,
- "XPQARetrieval (hin-eng)": 63.69,
- "XPQARetrieval (ita-ita)": 70.38,
- "XPQARetrieval (eng-ita)": 26.06,
- "XPQARetrieval (ita-eng)": 56.2,
- "XPQARetrieval (jpn-jpn)": 71.97,
- "XPQARetrieval (eng-jpn)": 17.63,
- "XPQARetrieval (jpn-eng)": 61.03,
- "XPQARetrieval (kor-kor)": 36.12,
- "XPQARetrieval (eng-kor)": 20.15,
- "XPQARetrieval (kor-eng)": 29.27,
- "XPQARetrieval (pol-pol)": 48.05,
- "XPQARetrieval (eng-pol)": 19.48,
- "XPQARetrieval (pol-eng)": 40.18,
- "XPQARetrieval (por-por)": 44.78,
- "XPQARetrieval (eng-por)": 17.66,
- "XPQARetrieval (por-eng)": 40.58,
- "XPQARetrieval (tam-tam)": 35.21,
- "XPQARetrieval (eng-tam)": 12.64,
- "XPQARetrieval (tam-eng)": 26.73,
- "XPQARetrieval (cmn-cmn)": 67.06,
- "XPQARetrieval (eng-cmn)": 12.72,
- "XPQARetrieval (cmn-eng)": 53.53,
- "XPQARetrieval (fr)": 65.81
- }
- ]
- },
- "STS": {
- "cosine_spearman": [
- {
- "Model": "multilingual-e5-base",
- "AFQMC": 29.66,
- "ATEC": 37.01,
- "BIOSSES": 85.05,
- "BQ": 45.45,
- "CDSC-R": 90.09,
- "LCQMC": 74.15,
- "PAWSX": 12.13,
- "RUParaPhraserSTS": 70.17,
- "RuSTSBenchmarkSTS": 79.64,
- "SICK-R": 78.51,
- "SICK-R-PL": 71.23,
- "SICKFr": 75.76,
- "STS12": 76.7,
- "STS13": 78.02,
- "STS14": 76.6,
- "STS15": 88.16,
- "STS16": 84.28,
- "STS17 (en-de)": 82.08,
- "STS17 (es-en)": 76.56,
- "STS17 (fr-en)": 80.18,
- "STS17 (en-tr)": 63.3,
- "STS17 (ko-ko)": 79.95,
- "STS17 (en-ar)": 71.27,
- "STS17 (en-en)": 87.84,
- "STS17 (es-es)": 86.74,
- "STS17 (ar-ar)": 74.48,
- "STS17 (it-en)": 80.16,
- "STS17 (nl-en)": 79.29,
- "STS22 (fr)": 75.04,
- "STS22 (zh-en)": 69.8,
- "STS22 (ar)": 57.82,
- "STS22 (es-it)": 66.43,
- "STS22 (it)": 77.76,
- "STS22 (fr-pl)": 73.25,
- "STS22 (de-en)": 54.89,
- "STS22 (pl-en)": 70.37,
- "STS22 (zh)": 65.63,
- "STS22 (de)": 55.95,
- "STS22 (pl)": 34.08,
- "STS22 (de-fr)": 59.68,
- "STS22 (es)": 66.67,
- "STS22 (ru)": 60.67,
- "STS22 (es-en)": 74.0,
- "STS22 (en)": 62.26,
- "STS22 (tr)": 63.71,
- "STS22 (de-pl)": 39.35,
- "STSB": 79.04,
- "STSBenchmark": 85.64,
- "STSBenchmarkMultilingualSTS (zh)": 79.87,
- "STSBenchmarkMultilingualSTS (de)": 79.68,
- "STSBenchmarkMultilingualSTS (es)": 81.75,
- "STSBenchmarkMultilingualSTS (en)": 85.64,
- "STSBenchmarkMultilingualSTS (fr)": 80.85,
- "STSBenchmarkMultilingualSTS (pl)": 74.93,
- "STSBenchmarkMultilingualSTS (pt)": 67.16,
- "STSBenchmarkMultilingualSTS (nl)": 75.96,
- "STSBenchmarkMultilingualSTS (it)": 78.09,
- "STSBenchmarkMultilingualSTS (ru)": 79.33
- },
- {
- "Model": "multilingual-e5-base",
- "AFQMC": 29.66,
- "ATEC": 37.01,
- "BIOSSES": 85.05,
- "BQ": 45.45,
- "CDSC-R": 90.09,
- "LCQMC": 74.15,
- "PAWSX": 12.13,
- "RUParaPhraserSTS": 70.17,
- "RuSTSBenchmarkSTS": 79.64,
- "SICK-R": 78.51,
- "SICK-R-PL": 71.23,
- "SICKFr": 75.76,
- "STS12": 76.7,
- "STS13": 78.02,
- "STS14": 76.6,
- "STS15": 88.16,
- "STS16": 84.28,
- "STS17 (en-de)": 82.08,
- "STS17 (es-en)": 76.56,
- "STS17 (fr-en)": 80.18,
- "STS17 (en-tr)": 63.3,
- "STS17 (ko-ko)": 79.95,
- "STS17 (en-ar)": 71.27,
- "STS17 (en-en)": 87.84,
- "STS17 (es-es)": 86.74,
- "STS17 (ar-ar)": 74.48,
- "STS17 (it-en)": 80.16,
- "STS17 (nl-en)": 79.29,
- "STS22 (fr)": 75.04,
- "STS22 (zh-en)": 69.8,
- "STS22 (ar)": 57.82,
- "STS22 (es-it)": 66.43,
- "STS22 (it)": 77.76,
- "STS22 (fr-pl)": 73.25,
- "STS22 (de-en)": 54.89,
- "STS22 (pl-en)": 70.37,
- "STS22 (zh)": 65.63,
- "STS22 (de)": 55.95,
- "STS22 (pl)": 34.08,
- "STS22 (de-fr)": 59.68,
- "STS22 (es)": 66.67,
- "STS22 (ru)": 60.67,
- "STS22 (es-en)": 74.0,
- "STS22 (en)": 62.26,
- "STS22 (tr)": 63.71,
- "STS22 (de-pl)": 39.35,
- "STSB": 79.04,
- "STSBenchmark": 85.64,
- "STSBenchmarkMultilingualSTS (zh)": 79.87,
- "STSBenchmarkMultilingualSTS (de)": 79.68,
- "STSBenchmarkMultilingualSTS (es)": 81.75,
- "STSBenchmarkMultilingualSTS (en)": 85.64,
- "STSBenchmarkMultilingualSTS (fr)": 80.85,
- "STSBenchmarkMultilingualSTS (pl)": 74.93,
- "STSBenchmarkMultilingualSTS (pt)": 67.16,
- "STSBenchmarkMultilingualSTS (nl)": 75.96,
- "STSBenchmarkMultilingualSTS (it)": 78.09,
- "STSBenchmarkMultilingualSTS (ru)": 79.33
- },
- {
- "Model": "multilingual-e5-base",
- "AFQMC": 29.67,
- "ATEC": 37.01,
- "BQ": 45.45,
- "CDSC-R": 90.08,
- "LCQMC": 74.15,
- "PAWSX": 12.14,
- "QBQTC": 28.81,
- "SICK-R-PL": 71.23,
- "SICKFr": 76.23,
- "STS22 (zh)": 65.64,
- "STS22 (pl)": 34.07,
- "STSB": 79.05,
- "STSBenchmarkMultilingualSTS (fr)": 80.62
- }
- ]
- },
- "Summarization": {
- "cosine_spearman": [
- {
- "Model": "multilingual-e5-base",
- "SummEval": 30.23,
- "SummEvalFr": 32.96
- },
- {
- "Model": "multilingual-e5-base",
- "SummEval": 30.23,
- "SummEvalFr": 32.96
- },
- {
- "Model": "multilingual-e5-base",
- "SummEvalFr": 30.76
- }
- ]
- },
- "MultilabelClassification": {
- "accuracy": [
- {
- "Model": "multilingual-e5-base",
- "CEDRClassification": 42.32,
- "SensitiveTopicsClassification": 24.98
- }
- ]
- },
- "InstructionRetrieval": {
- "p-MRR": [
- {
- "Model": "multilingual-e5-base",
- "Core17InstructionRetrieval": 0.61,
- "News21InstructionRetrieval": -1.14,
- "Robust04InstructionRetrieval": -7.43
- }
- ]
- }
- },
- "intfloat__multilingual-e5-large": {
- "BitextMining": {
- "f1": [
- {
- "Model": "multilingual-e5-large",
- "BornholmBitextMining": 44.16,
- "Tatoeba (bre-eng)": 11.1,
- "Tatoeba (oci-eng)": 54.91,
- "Tatoeba (orv-eng)": 39.87,
- "Tatoeba (tur-eng)": 96.27,
- "Tatoeba (afr-eng)": 90.22,
- "Tatoeba (dtp-eng)": 7.03,
- "Tatoeba (glg-eng)": 93.34,
- "Tatoeba (sqi-eng)": 94.7,
- "Tatoeba (gla-eng)": 59.0,
- "Tatoeba (heb-eng)": 86.61,
- "Tatoeba (mal-eng)": 97.7,
- "Tatoeba (yid-eng)": 76.33,
- "Tatoeba (nob-eng)": 97.2,
- "Tatoeba (tha-eng)": 95.38,
- "Tatoeba (ind-eng)": 92.9,
- "Tatoeba (isl-eng)": 92.09,
- "Tatoeba (ces-eng)": 94.89,
- "Tatoeba (uig-eng)": 72.17,
- "Tatoeba (rus-eng)": 92.32,
- "Tatoeba (zsm-eng)": 94.53,
- "Tatoeba (war-eng)": 62.02,
- "Tatoeba (jpn-eng)": 95.28,
- "Tatoeba (hin-eng)": 94.48,
- "Tatoeba (mkd-eng)": 85.63,
- "Tatoeba (vie-eng)": 97.0,
- "Tatoeba (bos-eng)": 92.86,
- "Tatoeba (arq-eng)": 41.56,
- "Tatoeba (cha-eng)": 27.16,
- "Tatoeba (ell-eng)": 93.88,
- "Tatoeba (hye-eng)": 90.92,
- "Tatoeba (kaz-eng)": 79.67,
- "Tatoeba (xho-eng)": 80.87,
- "Tatoeba (arz-eng)": 74.73,
- "Tatoeba (fin-eng)": 95.44,
- "Tatoeba (gle-eng)": 71.48,
- "Tatoeba (ile-eng)": 79.16,
- "Tatoeba (ber-eng)": 38.9,
- "Tatoeba (mon-eng)": 87.53,
- "Tatoeba (aze-eng)": 87.61,
- "Tatoeba (srp-eng)": 93.1,
- "Tatoeba (tzl-eng)": 53.16,
- "Tatoeba (dsb-eng)": 48.44,
- "Tatoeba (pol-eng)": 96.6,
- "Tatoeba (eus-eng)": 77.82,
- "Tatoeba (nov-eng)": 71.62,
- "Tatoeba (tuk-eng)": 33.15,
- "Tatoeba (ukr-eng)": 93.32,
- "Tatoeba (est-eng)": 85.03,
- "Tatoeba (deu-eng)": 99.07,
- "Tatoeba (ido-eng)": 83.52,
- "Tatoeba (cym-eng)": 76.21,
- "Tatoeba (ara-eng)": 85.48,
- "Tatoeba (csb-eng)": 36.98,
- "Tatoeba (cbk-eng)": 69.26,
- "Tatoeba (ben-eng)": 83.02,
- "Tatoeba (slk-eng)": 93.13,
- "Tatoeba (fao-eng)": 72.62,
- "Tatoeba (pam-eng)": 9.32,
- "Tatoeba (hsb-eng)": 58.9,
- "Tatoeba (lat-eng)": 53.37,
- "Tatoeba (nno-eng)": 91.4,
- "Tatoeba (gsw-eng)": 51.65,
- "Tatoeba (cat-eng)": 91.03,
- "Tatoeba (urd-eng)": 89.21,
- "Tatoeba (kzj-eng)": 7.91,
- "Tatoeba (kor-eng)": 90.65,
- "Tatoeba (slv-eng)": 89.57,
- "Tatoeba (ast-eng)": 81.76,
- "Tatoeba (cmn-eng)": 95.28,
- "Tatoeba (cor-eng)": 6.28,
- "Tatoeba (tel-eng)": 91.34,
- "Tatoeba (kab-eng)": 36.54,
- "Tatoeba (yue-eng)": 88.71,
- "Tatoeba (swe-eng)": 95.3,
- "Tatoeba (pes-eng)": 92.14,
- "Tatoeba (hun-eng)": 94.01,
- "Tatoeba (tgl-eng)": 92.0,
- "Tatoeba (pms-eng)": 59.85,
- "Tatoeba (lvs-eng)": 90.06,
- "Tatoeba (swh-eng)": 71.61,
- "Tatoeba (uzb-eng)": 72.35,
- "Tatoeba (por-eng)": 93.63,
- "Tatoeba (ron-eng)": 94.87,
- "Tatoeba (nds-eng)": 69.28,
- "Tatoeba (fry-eng)": 63.43,
- "Tatoeba (khm-eng)": 59.96,
- "Tatoeba (nld-eng)": 96.63,
- "Tatoeba (lit-eng)": 88.48,
- "Tatoeba (awa-eng)": 72.27,
- "Tatoeba (amh-eng)": 80.69,
- "Tatoeba (jav-eng)": 75.46,
- "Tatoeba (mar-eng)": 88.58,
- "Tatoeba (spa-eng)": 97.1,
- "Tatoeba (lfn-eng)": 62.91,
- "Tatoeba (ceb-eng)": 55.31,
- "Tatoeba (bul-eng)": 92.93,
- "Tatoeba (tat-eng)": 73.51,
- "Tatoeba (kur-eng)": 66.83,
- "Tatoeba (mhr-eng)": 6.79,
- "Tatoeba (epo-eng)": 96.01,
- "Tatoeba (kat-eng)": 84.09,
- "Tatoeba (ina-eng)": 93.47,
- "Tatoeba (tam-eng)": 88.23,
- "Tatoeba (ita-eng)": 93.29,
- "Tatoeba (hrv-eng)": 96.15,
- "Tatoeba (fra-eng)": 93.42,
- "Tatoeba (wuu-eng)": 86.37,
- "Tatoeba (dan-eng)": 95.08,
- "Tatoeba (max-eng)": 63.41,
- "Tatoeba (ang-eng)": 40.18,
- "Tatoeba (bel-eng)": 91.08,
- "Tatoeba (swg-eng)": 55.64
- }
- ]
- },
- "Classification": {
- "accuracy": [
- {
- "Model": "multilingual-e5-large",
- "AllegroReviews": 41.14,
- "AmazonCounterfactualClassification (en-ext)": 78.73,
- "AmazonCounterfactualClassification (en)": 78.67,
- "AmazonCounterfactualClassification (de)": 68.66,
- "AmazonCounterfactualClassification (ja)": 78.8,
- "AmazonPolarityClassification": 93.26,
- "AmazonReviewsClassification (en)": 49.2,
- "AmazonReviewsClassification (de)": 46.5,
- "AmazonReviewsClassification (es)": 44.35,
- "AmazonReviewsClassification (fr)": 41.91,
- "AmazonReviewsClassification (ja)": 41.71,
- "AmazonReviewsClassification (zh)": 38.87,
- "AngryTweetsClassification": 54.95,
- "Banking77Classification": 75.88,
- "CBD": 69.9,
- "DKHateClassification": 66.02,
- "DanishPoliticalCommentsClassification": 38.27,
- "EmotionClassification": 47.58,
- "GeoreviewClassification": 49.69,
- "HeadlineClassification": 77.19,
- "IFlyTek": 45.47,
- "ImdbClassification": 90.23,
- "InappropriatenessClassification": 61.59,
- "JDReview": 80.99,
- "KinopoiskClassification": 56.59,
- "LccSentimentClassification": 59.6,
- "MTOPDomainClassification (en)": 91.81,
- "MTOPDomainClassification (de)": 90.44,
- "MTOPDomainClassification (es)": 88.34,
- "MTOPDomainClassification (fr)": 86.41,
- "MTOPDomainClassification (hi)": 86.84,
- "MTOPDomainClassification (th)": 86.88,
- "MTOPIntentClassification (en)": 64.29,
- "MTOPIntentClassification (de)": 65.97,
- "MTOPIntentClassification (es)": 61.9,
- "MTOPIntentClassification (fr)": 59.43,
- "MTOPIntentClassification (hi)": 59.17,
- "MTOPIntentClassification (th)": 62.59,
- "MasakhaNEWSClassification (amh)": 83.7,
- "MasakhaNEWSClassification (eng)": 78.26,
- "MasakhaNEWSClassification (fra)": 79.38,
- "MasakhaNEWSClassification (hau)": 76.17,
- "MasakhaNEWSClassification (ibo)": 70.05,
- "MasakhaNEWSClassification (lin)": 75.89,
- "MasakhaNEWSClassification (lug)": 73.63,
- "MasakhaNEWSClassification (orm)": 80.31,
- "MasakhaNEWSClassification (pcm)": 89.15,
- "MasakhaNEWSClassification (run)": 76.55,
- "MasakhaNEWSClassification (sna)": 86.99,
- "MasakhaNEWSClassification (som)": 64.63,
- "MasakhaNEWSClassification (swa)": 73.42,
- "MasakhaNEWSClassification (tir)": 72.06,
- "MasakhaNEWSClassification (xho)": 82.56,
- "MasakhaNEWSClassification (yor)": 81.09,
- "MassiveIntentClassification (he)": 62.44,
- "MassiveIntentClassification (id)": 63.51,
- "MassiveIntentClassification (fi)": 64.28,
- "MassiveIntentClassification (hu)": 64.0,
- "MassiveIntentClassification (nb)": 59.83,
- "MassiveIntentClassification (vi)": 63.39,
- "MassiveIntentClassification (ko)": 63.92,
- "MassiveIntentClassification (ta)": 53.41,
- "MassiveIntentClassification (te)": 53.96,
- "MassiveIntentClassification (da)": 60.16,
- "MassiveIntentClassification (ar)": 54.1,
- "MassiveIntentClassification (en)": 68.51,
- "MassiveIntentClassification (hi)": 60.93,
- "MassiveIntentClassification (bn)": 55.6,
- "MassiveIntentClassification (tr)": 64.61,
- "MassiveIntentClassification (am)": 45.48,
- "MassiveIntentClassification (es)": 64.01,
- "MassiveIntentClassification (lv)": 58.31,
- "MassiveIntentClassification (my)": 49.73,
- "MassiveIntentClassification (sq)": 57.3,
- "MassiveIntentClassification (th)": 62.75,
- "MassiveIntentClassification (sl)": 59.38,
- "MassiveIntentClassification (ml)": 57.58,
- "MassiveIntentClassification (is)": 53.3,
- "MassiveIntentClassification (ms)": 58.49,
- "MassiveIntentClassification (nl)": 65.0,
- "MassiveIntentClassification (az)": 54.68,
- "MassiveIntentClassification (ru)": 65.76,
- "MassiveIntentClassification (sv)": 61.78,
- "MassiveIntentClassification (ro)": 59.76,
- "MassiveIntentClassification (zh-TW)": 58.78,
- "MassiveIntentClassification (jv)": 48.96,
- "MassiveIntentClassification (fa)": 63.74,
- "MassiveIntentClassification (pl)": 65.07,
- "MassiveIntentClassification (km)": 34.88,
- "MassiveIntentClassification (ja)": 67.11,
- "MassiveIntentClassification (kn)": 53.45,
- "MassiveIntentClassification (fr)": 63.37,
- "MassiveIntentClassification (ka)": 41.45,
- "MassiveIntentClassification (sw)": 47.69,
- "MassiveIntentClassification (zh-CN)": 66.23,
- "MassiveIntentClassification (ur)": 54.6,
- "MassiveIntentClassification (tl)": 54.77,
- "MassiveIntentClassification (cy)": 44.22,
- "MassiveIntentClassification (de)": 63.82,
- "MassiveIntentClassification (af)": 53.69,
- "MassiveIntentClassification (it)": 63.89,
- "MassiveIntentClassification (el)": 64.34,
- "MassiveIntentClassification (mn)": 49.6,
- "MassiveIntentClassification (hy)": 50.89,
- "MassiveIntentClassification (pt)": 65.6,
- "MassiveScenarioClassification (en)": 73.04,
- "MassiveScenarioClassification (ta)": 58.76,
- "MassiveScenarioClassification (ml)": 63.17,
- "MassiveScenarioClassification (pt)": 68.33,
- "MassiveScenarioClassification (he)": 67.72,
- "MassiveScenarioClassification (ar)": 61.0,
- "MassiveScenarioClassification (pl)": 69.82,
- "MassiveScenarioClassification (vi)": 68.91,
- "MassiveScenarioClassification (ms)": 63.55,
- "MassiveScenarioClassification (sl)": 65.33,
- "MassiveScenarioClassification (hu)": 70.53,
- "MassiveScenarioClassification (my)": 54.03,
- "MassiveScenarioClassification (sq)": 63.79,
- "MassiveScenarioClassification (fi)": 68.62,
- "MassiveScenarioClassification (te)": 59.49,
- "MassiveScenarioClassification (ru)": 70.85,
- "MassiveScenarioClassification (am)": 52.69,
- "MassiveScenarioClassification (hi)": 66.85,
- "MassiveScenarioClassification (fr)": 68.74,
- "MassiveScenarioClassification (cy)": 51.25,
- "MassiveScenarioClassification (tr)": 68.12,
- "MassiveScenarioClassification (ro)": 66.06,
- "MassiveScenarioClassification (zh-CN)": 72.25,
- "MassiveScenarioClassification (kn)": 59.36,
- "MassiveScenarioClassification (nb)": 66.18,
- "MassiveScenarioClassification (de)": 71.25,
- "MassiveScenarioClassification (el)": 69.74,
- "MassiveScenarioClassification (es)": 69.07,
- "MassiveScenarioClassification (da)": 67.46,
- "MassiveScenarioClassification (ur)": 60.89,
- "MassiveScenarioClassification (af)": 62.35,
- "MassiveScenarioClassification (ko)": 70.54,
- "MassiveScenarioClassification (bn)": 61.85,
- "MassiveScenarioClassification (ja)": 73.16,
- "MassiveScenarioClassification (az)": 58.49,
- "MassiveScenarioClassification (fa)": 67.55,
- "MassiveScenarioClassification (sv)": 69.15,
- "MassiveScenarioClassification (zh-TW)": 64.35,
- "MassiveScenarioClassification (ka)": 47.82,
- "MassiveScenarioClassification (nl)": 71.11,
- "MassiveScenarioClassification (sw)": 56.27,
- "MassiveScenarioClassification (hy)": 55.76,
- "MassiveScenarioClassification (is)": 60.74,
- "MassiveScenarioClassification (mn)": 55.37,
- "MassiveScenarioClassification (it)": 69.45,
- "MassiveScenarioClassification (tl)": 60.71,
- "MassiveScenarioClassification (km)": 41.14,
- "MassiveScenarioClassification (th)": 69.06,
- "MassiveScenarioClassification (lv)": 64.28,
- "MassiveScenarioClassification (jv)": 56.24,
- "MassiveScenarioClassification (id)": 69.43,
- "MultilingualSentiment": 68.58,
- "NoRecClassification": 62.76,
- "NordicLangClassification": 82.29,
- "NorwegianParliament": 60.36,
- "OnlineShopping": 90.81,
- "PAC": 70.37,
- "PolEmo2.0-IN": 77.06,
- "PolEmo2.0-OUT": 53.38,
- "RuReviewsClassification": 65.28,
- "RuSciBenchGRNTIClassification": 58.2,
- "RuSciBenchOECDClassification": 43.91,
- "ScalaDaClassification": 50.77,
- "ScalaNbClassification": 50.44,
- "TNews": 48.38,
- "ToxicConversationsClassification": 66.01,
- "TweetSentimentExtractionClassification": 62.8,
- "Waimai": 85.02
- }
- ]
- },
- "Clustering": {
- "v_measure": [
- {
- "Model": "multilingual-e5-large",
- "8TagsClustering": 33.88,
- "AlloProfClusteringP2P": 62.99,
- "AlloProfClusteringS2S": 32.26,
- "BiorxivClusteringP2P": 35.5,
- "BiorxivClusteringS2S": 33.3,
- "CLSClusteringP2P": 40.68,
- "CLSClusteringS2S": 38.59,
- "GeoreviewClusteringP2P": 59.59,
- "HALClusteringS2S": 22.44,
- "MLSUMClusteringP2P (ru)": 42.79,
- "MLSUMClusteringP2P": 44.04,
- "MLSUMClusteringS2S (ru)": 44.32,
- "MLSUMClusteringS2S": 37.65,
- "MasakhaNEWSClusteringP2P (amh)": 67.16,
- "MasakhaNEWSClusteringP2P (eng)": 61.1,
- "MasakhaNEWSClusteringP2P (fra)": 40.94,
- "MasakhaNEWSClusteringP2P (hau)": 60.7,
- "MasakhaNEWSClusteringP2P (ibo)": 48.41,
- "MasakhaNEWSClusteringP2P (lin)": 57.69,
- "MasakhaNEWSClusteringP2P (lug)": 71.95,
- "MasakhaNEWSClusteringP2P (orm)": 60.14,
- "MasakhaNEWSClusteringP2P (pcm)": 80.84,
- "MasakhaNEWSClusteringP2P (run)": 59.91,
- "MasakhaNEWSClusteringP2P (sna)": 53.3,
- "MasakhaNEWSClusteringP2P (som)": 34.38,
- "MasakhaNEWSClusteringP2P (swa)": 33.25,
- "MasakhaNEWSClusteringP2P (tir)": 54.21,
- "MasakhaNEWSClusteringP2P (xho)": 41.12,
- "MasakhaNEWSClusteringP2P (yor)": 36.22,
- "MasakhaNEWSClusteringS2S (amh)": 47.24,
- "MasakhaNEWSClusteringS2S (eng)": 53.93,
- "MasakhaNEWSClusteringS2S (fra)": 30.56,
- "MasakhaNEWSClusteringS2S (hau)": 19.24,
- "MasakhaNEWSClusteringS2S (ibo)": 28.88,
- "MasakhaNEWSClusteringS2S (lin)": 42.22,
- "MasakhaNEWSClusteringS2S (lug)": 43.63,
- "MasakhaNEWSClusteringS2S (orm)": 26.29,
- "MasakhaNEWSClusteringS2S (pcm)": 59.77,
- "MasakhaNEWSClusteringS2S (run)": 51.46,
- "MasakhaNEWSClusteringS2S (sna)": 48.14,
- "MasakhaNEWSClusteringS2S (som)": 25.14,
- "MasakhaNEWSClusteringS2S (swa)": 7.28,
- "MasakhaNEWSClusteringS2S (tir)": 50.51,
- "MasakhaNEWSClusteringS2S (xho)": 30.98,
- "MasakhaNEWSClusteringS2S (yor)": 34.09,
- "MedrxivClusteringP2P": 31.7,
- "MedrxivClusteringS2S": 29.76,
- "RedditClustering": 46.91,
- "RedditClusteringP2P": 63.0,
- "RuSciBenchGRNTIClusteringP2P": 51.98,
- "RuSciBenchOECDClusteringP2P": 45.12,
- "StackExchangeClustering": 58.37,
- "StackExchangeClusteringP2P": 32.9,
- "ThuNewsClusteringP2P": 58.05,
- "ThuNewsClusteringS2S": 55.59,
- "TwentyNewsgroupsClustering": 39.4
- }
- ]
- },
- "PairClassification": {
- "max_ap": [
- {
- "Model": "multilingual-e5-large",
- "CDSC-E": 74.47,
- "OpusparcusPC (de)": 97.27,
- "OpusparcusPC (en)": 98.74,
- "OpusparcusPC (fi)": 94.26,
- "OpusparcusPC (fr)": 93.68,
- "OpusparcusPC (ru)": 89.64,
- "OpusparcusPC (sv)": 94.98,
- "PSC": 99.4,
- "PawsXPairClassification (de)": 56.81,
- "PawsXPairClassification (en)": 62.97,
- "PawsXPairClassification (es)": 56.85,
- "PawsXPairClassification (fr)": 58.68,
- "PawsXPairClassification (ja)": 50.7,
- "PawsXPairClassification (ko)": 52.08,
- "PawsXPairClassification (zh)": 56.82,
- "SICK-E-PL": 75.95,
- "SprintDuplicateQuestions": 93.14,
- "TERRa": 58.4,
- "TwitterSemEval2015": 75.28,
- "TwitterURLCorpus": 85.83
- },
- {
- "Model": "multilingual-e5-large",
- "CDSC-E": 74.47,
- "Cmnli": 78.18,
- "Ocnli": 61.6,
- "OpusparcusPC (de)": 97.27,
- "OpusparcusPC (en)": 98.74,
- "OpusparcusPC (fi)": 94.26,
- "OpusparcusPC (fr)": 93.89,
- "OpusparcusPC (ru)": 89.64,
- "OpusparcusPC (sv)": 94.98,
- "PPC": 92.18,
- "PSC": 99.39,
- "PawsXPairClassification (de)": 57.14,
- "PawsXPairClassification (en)": 62.97,
- "PawsXPairClassification (es)": 56.87,
- "PawsXPairClassification (fr)": 58.61,
- "PawsXPairClassification (ja)": 50.84,
- "PawsXPairClassification (ko)": 52.22,
- "PawsXPairClassification (zh)": 56.95,
- "SICK-E-PL": 75.96,
- "SprintDuplicateQuestions": 93.14,
- "TERRa": 58.42,
- "TwitterSemEval2015": 75.28,
- "TwitterURLCorpus": 85.83
- },
- {
- "Model": "multilingual-e5-large",
- "CDSC-E": 74.47,
- "Cmnli": 78.18,
- "Ocnli": 61.6,
- "OpusparcusPC (fr)": 93.89,
- "PPC": 92.18,
- "PSC": 99.39,
- "PawsXPairClassification (fr)": 58.5,
- "SICK-E-PL": 75.96
- }
- ]
- },
- "Reranking": {
- "map": [
- {
- "Model": "multilingual-e5-large",
- "AlloprofReranking": 57.37,
- "AskUbuntuDupQuestions": 59.24,
- "CMedQAv1": 68.25,
- "CMedQAv2": 68.56,
- "MMarcoReranking": 21.34,
- "MindSmallReranking": 30.24,
- "RuBQReranking": 75.6,
- "SciDocsRR": 84.22,
- "StackOverflowDupQuestions": 50.14,
- "SyntecReranking": 86.9,
- "T2Reranking": 65.83
- },
- {
- "Model": "multilingual-e5-large",
- "MIRACLReranking (ru)": 63.71
- }
- ]
- },
- "Retrieval": {
- "ndcg_at_10": [
- {
- "Model": "multilingual-e5-large",
- "AILACasedocs": 26.43,
- "AILAStatutes": 20.84,
- "ARCChallenge": 10.83,
- "AlloprofRetrieval": 38.15,
- "AlphaNLI": 13.59,
- "AppsRetrieval": 32.55,
- "ArguAna": 54.36,
- "ArguAna-PL": 53.02,
- "BSARDRetrieval": 0.27,
- "CmedqaRetrieval": 28.67,
- "CodeFeedbackMT": 42.78,
- "CodeFeedbackST": 74.26,
- "CodeSearchNetCCRetrieval (python)": 84.45,
- "CodeSearchNetCCRetrieval (javascript)": 77.67,
- "CodeSearchNetCCRetrieval (go)": 72.08,
- "CodeSearchNetCCRetrieval (ruby)": 81.94,
- "CodeSearchNetCCRetrieval (java)": 78.65,
- "CodeSearchNetCCRetrieval (php)": 72.21,
- "CodeSearchNetRetrieval (python)": 89.42,
- "CodeSearchNetRetrieval (javascript)": 75.54,
- "CodeSearchNetRetrieval (go)": 91.8,
- "CodeSearchNetRetrieval (ruby)": 81.43,
- "CodeSearchNetRetrieval (java)": 82.05,
- "CodeSearchNetRetrieval (php)": 84.5,
- "CodeTransOceanContest": 74.03,
- "CodeTransOceanDL": 31.28,
- "CosQA": 34.8,
- "CovidRetrieval": 75.51,
- "DBPedia-PL": 35.82,
- "DuRetrieval": 85.32,
- "EcomRetrieval": 54.75,
- "FiQA-PL": 33.0,
- "FiQA2018": 43.81,
- "GerDaLIRSmall": 15.72,
- "HellaSwag": 27.35,
- "HotpotQA-PL": 67.41,
- "LEMBNarrativeQARetrieval": 24.22,
- "LEMBQMSumRetrieval": 24.26,
- "LEMBSummScreenFDRetrieval": 71.12,
- "LEMBWikimQARetrieval": 56.8,
- "LeCaRDv2": 55.83,
- "LegalBenchConsumerContractsQA": 73.3,
- "LegalBenchCorporateLobbying": 89.72,
- "LegalQuAD": 43.17,
- "LegalSummarization": 62.1,
- "MIRACLRetrieval (ru)": 67.33,
- "MMarcoRetrieval": 79.2,
- "MSMARCO-PL": 33.38,
- "MedicalRetrieval": 51.44,
- "MintakaRetrieval (ar)": 26.5,
- "MintakaRetrieval (de)": 32.77,
- "MintakaRetrieval (es)": 34.23,
- "MintakaRetrieval (fr)": 25.2,
- "MintakaRetrieval (hi)": 27.45,
- "MintakaRetrieval (it)": 33.84,
- "MintakaRetrieval (ja)": 26.45,
- "MintakaRetrieval (pt)": 35.9,
- "NFCorpus": 33.95,
- "NFCorpus-PL": 30.24,
- "NQ-PL": 52.79,
- "PIQA": 28.82,
- "Quail": 4.85,
- "Quora-PL": 83.65,
- "RARbCode": 58.92,
- "RARbMath": 67.32,
- "RiaNewsRetrieval": 80.67,
- "RuBQRetrieval": 74.13,
- "SCIDOCS": 17.45,
- "SCIDOCS-PL": 13.81,
- "SIQA": 5.36,
- "SciFact": 70.42,
- "SciFact-PL": 65.66,
- "SpartQA": 5.64,
- "StackOverflowQA": 88.89,
- "SyntecRetrieval": 81.07,
- "SyntheticText2SQL": 53.07,
- "T2Retrieval": 76.11,
- "TRECCOVID": 71.21,
- "TRECCOVID-PL": 70.03,
- "TempReasonL1": 1.14,
- "TempReasonL2Fact": 42.96,
- "TempReasonL2Pure": 2.05,
- "TempReasonL3Fact": 38.22,
- "TempReasonL3Pure": 8.31,
- "Touche2020": 23.13,
- "VideoRetrieval": 58.25,
- "WinoGrande": 54.99,
- "XPQARetrieval (ara-ara)": 43.69,
- "XPQARetrieval (eng-ara)": 30.86,
- "XPQARetrieval (ara-eng)": 39.11,
- "XPQARetrieval (deu-deu)": 76.83,
- "XPQARetrieval (eng-deu)": 42.87,
- "XPQARetrieval (deu-eng)": 68.25,
- "XPQARetrieval (spa-spa)": 61.77,
- "XPQARetrieval (eng-spa)": 37.55,
- "XPQARetrieval (spa-eng)": 52.86,
- "XPQARetrieval (fra-fra)": 61.38,
- "XPQARetrieval (eng-fra)": 39.12,
- "XPQARetrieval (fra-eng)": 57.93,
- "XPQARetrieval (hin-hin)": 71.07,
- "XPQARetrieval (eng-hin)": 32.39,
- "XPQARetrieval (hin-eng)": 68.31,
- "XPQARetrieval (ita-ita)": 74.32,
- "XPQARetrieval (eng-ita)": 37.95,
- "XPQARetrieval (ita-eng)": 64.54,
- "XPQARetrieval (jpn-jpn)": 74.11,
- "XPQARetrieval (eng-jpn)": 38.31,
- "XPQARetrieval (jpn-eng)": 65.42,
- "XPQARetrieval (kor-kor)": 35.71,
- "XPQARetrieval (eng-kor)": 31.09,
- "XPQARetrieval (kor-eng)": 34.02,
- "XPQARetrieval (pol-pol)": 51.01,
- "XPQARetrieval (eng-pol)": 30.49,
- "XPQARetrieval (pol-eng)": 44.66,
- "XPQARetrieval (por-por)": 41.1,
- "XPQARetrieval (eng-por)": 22.03,
- "XPQARetrieval (por-eng)": 35.15,
- "XPQARetrieval (tam-tam)": 39.47,
- "XPQARetrieval (eng-tam)": 17.33,
- "XPQARetrieval (tam-eng)": 33.67,
- "XPQARetrieval (cmn-cmn)": 66.27,
- "XPQARetrieval (eng-cmn)": 26.24,
- "XPQARetrieval (cmn-eng)": 55.15,
- "XPQARetrieval (fr)": 66.15
- }
- ]
- },
- "STS": {
- "cosine_spearman": [
- {
- "Model": "multilingual-e5-large",
- "AFQMC": 33.01,
- "ATEC": 39.8,
- "BIOSSES": 82.49,
- "BQ": 46.44,
- "CDSC-R": 91.0,
- "LCQMC": 75.95,
- "PAWSX": 14.63,
- "RUParaPhraserSTS": 71.82,
- "RuSTSBenchmarkSTS": 83.15,
- "SICK-R": 80.23,
- "SICK-R-PL": 75.08,
- "SICKFr": 78.81,
- "STS12": 80.02,
- "STS13": 81.55,
- "STS14": 77.72,
- "STS15": 89.31,
- "STS16": 85.79,
- "STS17 (fr-en)": 85.62,
- "STS17 (nl-en)": 85.29,
- "STS17 (es-es)": 86.71,
- "STS17 (ar-ar)": 77.83,
- "STS17 (es-en)": 80.74,
- "STS17 (en-en)": 88.12,
- "STS17 (ko-ko)": 82.27,
- "STS17 (en-ar)": 75.03,
- "STS17 (en-tr)": 71.22,
- "STS17 (en-de)": 86.15,
- "STS17 (it-en)": 84.52,
- "STS22 (de)": 56.58,
- "STS22 (de-fr)": 67.96,
- "STS22 (ar)": 56.95,
- "STS22 (ru)": 59.89,
- "STS22 (de-en)": 56.59,
- "STS22 (en)": 63.66,
- "STS22 (tr)": 63.56,
- "STS22 (pl-en)": 65.54,
- "STS22 (zh)": 66.82,
- "STS22 (it)": 76.99,
- "STS22 (pl)": 34.65,
- "STS22 (es-it)": 68.92,
- "STS22 (fr)": 76.77,
- "STS22 (zh-en)": 65.95,
- "STS22 (fr-pl)": 50.71,
- "STS22 (es-en)": 72.51,
- "STS22 (es)": 64.6,
- "STS22 (de-pl)": 49.58,
- "STSB": 81.08,
- "STSBenchmark": 87.29,
- "STSBenchmarkMultilingualSTS (pl)": 81.06,
- "STSBenchmarkMultilingualSTS (ru)": 83.05,
- "STSBenchmarkMultilingualSTS (de)": 84.27,
- "STSBenchmarkMultilingualSTS (pt)": 73.31,
- "STSBenchmarkMultilingualSTS (en)": 87.29,
- "STSBenchmarkMultilingualSTS (es)": 83.81,
- "STSBenchmarkMultilingualSTS (zh)": 81.22,
- "STSBenchmarkMultilingualSTS (it)": 81.75,
- "STSBenchmarkMultilingualSTS (nl)": 81.63,
- "STSBenchmarkMultilingualSTS (fr)": 83.28
- },
- {
- "Model": "multilingual-e5-large",
- "AFQMC": 33.01,
- "ATEC": 39.8,
- "BIOSSES": 82.49,
- "BQ": 46.44,
- "CDSC-R": 91.0,
- "LCQMC": 75.95,
- "PAWSX": 14.63,
- "RUParaPhraserSTS": 71.82,
- "RuSTSBenchmarkSTS": 83.15,
- "SICK-R": 80.23,
- "SICK-R-PL": 75.08,
- "SICKFr": 78.81,
- "STS12": 80.02,
- "STS13": 81.55,
- "STS14": 77.72,
- "STS15": 89.31,
- "STS16": 85.79,
- "STS17 (fr-en)": 85.62,
- "STS17 (nl-en)": 85.29,
- "STS17 (es-es)": 86.71,
- "STS17 (ar-ar)": 77.83,
- "STS17 (es-en)": 80.74,
- "STS17 (en-en)": 88.12,
- "STS17 (ko-ko)": 82.27,
- "STS17 (en-ar)": 75.03,
- "STS17 (en-tr)": 71.22,
- "STS17 (en-de)": 86.15,
- "STS17 (it-en)": 84.52,
- "STS22 (de)": 56.58,
- "STS22 (de-fr)": 67.96,
- "STS22 (ar)": 56.95,
- "STS22 (ru)": 59.89,
- "STS22 (de-en)": 56.59,
- "STS22 (en)": 63.66,
- "STS22 (tr)": 63.56,
- "STS22 (pl-en)": 65.54,
- "STS22 (zh)": 66.82,
- "STS22 (it)": 76.99,
- "STS22 (pl)": 34.65,
- "STS22 (es-it)": 68.92,
- "STS22 (fr)": 76.77,
- "STS22 (zh-en)": 65.95,
- "STS22 (fr-pl)": 50.71,
- "STS22 (es-en)": 72.51,
- "STS22 (es)": 64.6,
- "STS22 (de-pl)": 49.58,
- "STSB": 81.08,
- "STSBenchmark": 87.29,
- "STSBenchmarkMultilingualSTS (pl)": 81.06,
- "STSBenchmarkMultilingualSTS (ru)": 83.05,
- "STSBenchmarkMultilingualSTS (de)": 84.27,
- "STSBenchmarkMultilingualSTS (pt)": 73.31,
- "STSBenchmarkMultilingualSTS (en)": 87.29,
- "STSBenchmarkMultilingualSTS (es)": 83.81,
- "STSBenchmarkMultilingualSTS (zh)": 81.22,
- "STSBenchmarkMultilingualSTS (it)": 81.75,
- "STSBenchmarkMultilingualSTS (nl)": 81.63,
- "STSBenchmarkMultilingualSTS (fr)": 83.28
- },
- {
- "Model": "multilingual-e5-large",
- "AFQMC": 33.02,
- "ATEC": 39.81,
- "BQ": 46.44,
- "CDSC-R": 91.0,
- "LCQMC": 75.95,
- "PAWSX": 14.63,
- "QBQTC": 29.77,
- "SICK-R-PL": 75.08,
- "SICKFr": 78.78,
- "STS22 (zh)": 65.64,
- "STS22 (pl)": 34.66,
- "STSB": 81.08,
- "STSBenchmarkMultilingualSTS (fr)": 82.53
- }
- ]
- },
- "Summarization": {
- "cosine_spearman": [
- {
- "Model": "multilingual-e5-large",
- "SummEval": 29.64,
- "SummEvalFr": 30.92
- },
- {
- "Model": "multilingual-e5-large",
- "SummEval": 29.64,
- "SummEvalFr": 30.92
- },
- {
- "Model": "multilingual-e5-large",
- "SummEvalFr": 30.92
- }
- ]
- },
- "MultilabelClassification": {
- "accuracy": [
- {
- "Model": "multilingual-e5-large",
- "CEDRClassification": 44.84,
- "SensitiveTopicsClassification": 27.17
- }
- ]
- },
- "InstructionRetrieval": {
- "p-MRR": [
- {
- "Model": "multilingual-e5-large",
- "Core17InstructionRetrieval": -1.62,
- "News21InstructionRetrieval": -0.06,
- "Robust04InstructionRetrieval": -7.48
- }
- ]
- }
- },
"intfloat__multilingual-e5-large-instruct": {
"BitextMining": {
"f1": [
@@ -15535,1417 +13894,6 @@
"p-MRR": []
}
},
- "sentence-transformers__LaBSE": {
- "BitextMining": {
- "f1": [
- {
- "Model": "LaBSE",
- "BUCC (de-en)": 99.35,
- "BUCC (fr-en)": 98.72,
- "BUCC (ru-en)": 97.78,
- "BUCC (zh-en)": 99.16,
- "BornholmBitextMining": 45.63,
- "Tatoeba (ber-eng)": 8.4,
- "Tatoeba (hye-eng)": 94.09,
- "Tatoeba (ces-eng)": 96.68,
- "Tatoeba (slk-eng)": 96.5,
- "Tatoeba (cat-eng)": 95.38,
- "Tatoeba (awa-eng)": 71.7,
- "Tatoeba (lat-eng)": 80.07,
- "Tatoeba (hsb-eng)": 67.11,
- "Tatoeba (swh-eng)": 84.5,
- "Tatoeba (ind-eng)": 93.66,
- "Tatoeba (xho-eng)": 91.55,
- "Tatoeba (nno-eng)": 94.48,
- "Tatoeba (csb-eng)": 52.57,
- "Tatoeba (kzj-eng)": 11.33,
- "Tatoeba (isl-eng)": 94.75,
- "Tatoeba (nld-eng)": 96.07,
- "Tatoeba (ell-eng)": 95.35,
- "Tatoeba (por-eng)": 94.14,
- "Tatoeba (ita-eng)": 92.72,
- "Tatoeba (vie-eng)": 97.2,
- "Tatoeba (uzb-eng)": 84.23,
- "Tatoeba (bul-eng)": 94.58,
- "Tatoeba (tgl-eng)": 96.02,
- "Tatoeba (gla-eng)": 85.66,
- "Tatoeba (mkd-eng)": 93.6,
- "Tatoeba (tel-eng)": 97.86,
- "Tatoeba (kaz-eng)": 87.49,
- "Tatoeba (zsm-eng)": 95.62,
- "Tatoeba (hin-eng)": 96.87,
- "Tatoeba (ile-eng)": 85.58,
- "Tatoeba (jpn-eng)": 95.38,
- "Tatoeba (orv-eng)": 38.93,
- "Tatoeba (kab-eng)": 4.31,
- "Tatoeba (yue-eng)": 89.58,
- "Tatoeba (bel-eng)": 95.0,
- "Tatoeba (dtp-eng)": 10.85,
- "Tatoeba (ron-eng)": 96.92,
- "Tatoeba (arz-eng)": 76.0,
- "Tatoeba (tzl-eng)": 58.88,
- "Tatoeba (slv-eng)": 96.03,
- "Tatoeba (jav-eng)": 79.77,
- "Tatoeba (pol-eng)": 97.22,
- "Tatoeba (heb-eng)": 91.53,
- "Tatoeba (ang-eng)": 59.28,
- "Tatoeba (ara-eng)": 88.8,
- "Tatoeba (tuk-eng)": 75.27,
- "Tatoeba (afr-eng)": 96.18,
- "Tatoeba (kor-eng)": 90.95,
- "Tatoeba (mhr-eng)": 15.74,
- "Tatoeba (fry-eng)": 89.31,
- "Tatoeba (urd-eng)": 93.22,
- "Tatoeba (srp-eng)": 94.43,
- "Tatoeba (cbk-eng)": 79.44,
- "Tatoeba (tat-eng)": 85.92,
- "Tatoeba (wuu-eng)": 90.18,
- "Tatoeba (war-eng)": 60.29,
- "Tatoeba (kat-eng)": 95.02,
- "Tatoeba (nds-eng)": 79.42,
- "Tatoeba (gle-eng)": 93.8,
- "Tatoeba (mal-eng)": 98.45,
- "Tatoeba (tha-eng)": 96.14,
- "Tatoeba (fin-eng)": 96.37,
- "Tatoeba (cor-eng)": 10.11,
- "Tatoeba (tur-eng)": 98.0,
- "Tatoeba (yid-eng)": 88.79,
- "Tatoeba (eus-eng)": 95.01,
- "Tatoeba (cmn-eng)": 95.1,
- "Tatoeba (fao-eng)": 87.4,
- "Tatoeba (lfn-eng)": 67.54,
- "Tatoeba (bos-eng)": 94.92,
- "Tatoeba (arq-eng)": 42.69,
- "Tatoeba (sqi-eng)": 96.76,
- "Tatoeba (ben-eng)": 88.55,
- "Tatoeba (pes-eng)": 94.7,
- "Tatoeba (fra-eng)": 94.86,
- "Tatoeba (deu-eng)": 99.2,
- "Tatoeba (spa-eng)": 98.4,
- "Tatoeba (oci-eng)": 65.81,
- "Tatoeba (ina-eng)": 95.37,
- "Tatoeba (hrv-eng)": 96.95,
- "Tatoeba (gsw-eng)": 46.5,
- "Tatoeba (swe-eng)": 95.63,
- "Tatoeba (bre-eng)": 15.07,
- "Tatoeba (hun-eng)": 96.55,
- "Tatoeba (uig-eng)": 92.4,
- "Tatoeba (mar-eng)": 92.65,
- "Tatoeba (nob-eng)": 98.4,
- "Tatoeba (rus-eng)": 93.75,
- "Tatoeba (ceb-eng)": 64.42,
- "Tatoeba (aze-eng)": 94.93,
- "Tatoeba (dsb-eng)": 64.81,
- "Tatoeba (tam-eng)": 89.0,
- "Tatoeba (est-eng)": 96.55,
- "Tatoeba (cym-eng)": 92.0,
- "Tatoeba (amh-eng)": 91.47,
- "Tatoeba (dan-eng)": 95.71,
- "Tatoeba (epo-eng)": 98.2,
- "Tatoeba (ast-eng)": 90.68,
- "Tatoeba (swg-eng)": 59.36,
- "Tatoeba (pms-eng)": 64.57,
- "Tatoeba (kur-eng)": 83.59,
- "Tatoeba (khm-eng)": 78.37,
- "Tatoeba (ido-eng)": 89.42,
- "Tatoeba (ukr-eng)": 93.97,
- "Tatoeba (mon-eng)": 95.91,
- "Tatoeba (nov-eng)": 74.38,
- "Tatoeba (cha-eng)": 31.77,
- "Tatoeba (pam-eng)": 10.73,
- "Tatoeba (lvs-eng)": 95.88,
- "Tatoeba (max-eng)": 63.26,
- "Tatoeba (lit-eng)": 96.47,
- "Tatoeba (glg-eng)": 96.82
- }
- ]
- },
- "Classification": {
- "accuracy": [
- {
- "Model": "LaBSE",
- "AllegroReviews": 34.89,
- "AmazonCounterfactualClassification (en-ext)": 76.09,
- "AmazonCounterfactualClassification (en)": 75.93,
- "AmazonCounterfactualClassification (de)": 73.17,
- "AmazonCounterfactualClassification (ja)": 76.42,
- "AmazonPolarityClassification": 68.95,
- "AmazonReviewsClassification (en)": 35.8,
- "AmazonReviewsClassification (de)": 39.92,
- "AmazonReviewsClassification (es)": 39.39,
- "AmazonReviewsClassification (fr)": 38.52,
- "AmazonReviewsClassification (ja)": 36.44,
- "AmazonReviewsClassification (zh)": 36.45,
- "AngryTweetsClassification": 51.11,
- "Banking77Classification": 69.85,
- "CBD": 65.71,
- "DanishPoliticalCommentsClassification": 38.34,
- "EmotionClassification": 37.22,
- "GeoreviewClassification": 40.86,
- "HeadlineClassification": 68.75,
- "IFlyTek": 43.19,
- "ImdbClassification": 62.04,
- "InappropriatenessClassification": 58.52,
- "JDReview": 79.14,
- "KinopoiskClassification": 46.77,
- "LccSentimentClassification": 50.07,
- "MTOPDomainClassification (en)": 86.06,
- "MTOPDomainClassification (de)": 86.95,
- "MTOPDomainClassification (es)": 84.07,
- "MTOPDomainClassification (fr)": 84.14,
- "MTOPDomainClassification (hi)": 85.11,
- "MTOPDomainClassification (th)": 81.24,
- "MTOPIntentClassification (en)": 63.03,
- "MTOPIntentClassification (de)": 63.42,
- "MTOPIntentClassification (es)": 64.44,
- "MTOPIntentClassification (fr)": 62.01,
- "MTOPIntentClassification (hi)": 62.58,
- "MTOPIntentClassification (th)": 64.61,
- "MasakhaNEWSClassification (amh)": 81.78,
- "MasakhaNEWSClassification (eng)": 77.77,
- "MasakhaNEWSClassification (fra)": 77.39,
- "MasakhaNEWSClassification (hau)": 73.12,
- "MasakhaNEWSClassification (ibo)": 69.1,
- "MasakhaNEWSClassification (lin)": 74.63,
- "MasakhaNEWSClassification (lug)": 57.44,
- "MasakhaNEWSClassification (orm)": 51.6,
- "MasakhaNEWSClassification (pcm)": 91.44,
- "MasakhaNEWSClassification (run)": 73.76,
- "MasakhaNEWSClassification (sna)": 87.18,
- "MasakhaNEWSClassification (som)": 60.03,
- "MasakhaNEWSClassification (swa)": 69.33,
- "MasakhaNEWSClassification (tir)": 61.73,
- "MasakhaNEWSClassification (xho)": 77.34,
- "MasakhaNEWSClassification (yor)": 77.13,
- "MassiveIntentClassification (ms)": 58.6,
- "MassiveIntentClassification (sl)": 59.37,
- "MassiveIntentClassification (hu)": 59.52,
- "MassiveIntentClassification (sv)": 59.66,
- "MassiveIntentClassification (nl)": 59.37,
- "MassiveIntentClassification (id)": 61.12,
- "MassiveIntentClassification (vi)": 56.67,
- "MassiveIntentClassification (zh-CN)": 63.86,
- "MassiveIntentClassification (am)": 55.71,
- "MassiveIntentClassification (ko)": 60.99,
- "MassiveIntentClassification (el)": 57.03,
- "MassiveIntentClassification (nb)": 57.91,
- "MassiveIntentClassification (jv)": 50.98,
- "MassiveIntentClassification (fa)": 62.33,
- "MassiveIntentClassification (ja)": 63.11,
- "MassiveIntentClassification (it)": 59.83,
- "MassiveIntentClassification (es)": 58.32,
- "MassiveIntentClassification (my)": 57.35,
- "MassiveIntentClassification (fi)": 60.12,
- "MassiveIntentClassification (he)": 56.55,
- "MassiveIntentClassification (is)": 54.9,
- "MassiveIntentClassification (tl)": 55.28,
- "MassiveIntentClassification (kn)": 56.24,
- "MassiveIntentClassification (ur)": 56.7,
- "MassiveIntentClassification (ml)": 57.91,
- "MassiveIntentClassification (tr)": 60.91,
- "MassiveIntentClassification (en)": 61.46,
- "MassiveIntentClassification (sw)": 51.62,
- "MassiveIntentClassification (km)": 48.55,
- "MassiveIntentClassification (ar)": 50.86,
- "MassiveIntentClassification (ru)": 60.67,
- "MassiveIntentClassification (te)": 58.32,
- "MassiveIntentClassification (ta)": 55.04,
- "MassiveIntentClassification (pt)": 60.16,
- "MassiveIntentClassification (zh-TW)": 59.51,
- "MassiveIntentClassification (pl)": 59.71,
- "MassiveIntentClassification (az)": 58.97,
- "MassiveIntentClassification (ro)": 57.92,
- "MassiveIntentClassification (ka)": 48.35,
- "MassiveIntentClassification (sq)": 58.03,
- "MassiveIntentClassification (bn)": 58.22,
- "MassiveIntentClassification (th)": 56.58,
- "MassiveIntentClassification (da)": 58.25,
- "MassiveIntentClassification (de)": 56.21,
- "MassiveIntentClassification (hy)": 56.2,
- "MassiveIntentClassification (mn)": 58.5,
- "MassiveIntentClassification (cy)": 50.16,
- "MassiveIntentClassification (af)": 56.12,
- "MassiveIntentClassification (fr)": 60.47,
- "MassiveIntentClassification (hi)": 59.4,
- "MassiveIntentClassification (lv)": 57.1,
- "MassiveScenarioClassification (te)": 64.13,
- "MassiveScenarioClassification (de)": 62.39,
- "MassiveScenarioClassification (el)": 64.58,
- "MassiveScenarioClassification (ml)": 62.26,
- "MassiveScenarioClassification (nl)": 65.16,
- "MassiveScenarioClassification (jv)": 58.29,
- "MassiveScenarioClassification (ro)": 62.41,
- "MassiveScenarioClassification (pt)": 63.28,
- "MassiveScenarioClassification (pl)": 64.58,
- "MassiveScenarioClassification (sl)": 64.25,
- "MassiveScenarioClassification (ta)": 59.08,
- "MassiveScenarioClassification (sw)": 58.36,
- "MassiveScenarioClassification (tl)": 60.23,
- "MassiveScenarioClassification (id)": 65.84,
- "MassiveScenarioClassification (ru)": 65.25,
- "MassiveScenarioClassification (sv)": 66.01,
- "MassiveScenarioClassification (ar)": 57.72,
- "MassiveScenarioClassification (fr)": 65.1,
- "MassiveScenarioClassification (th)": 64.34,
- "MassiveScenarioClassification (hy)": 61.25,
- "MassiveScenarioClassification (nb)": 64.29,
- "MassiveScenarioClassification (cy)": 56.13,
- "MassiveScenarioClassification (my)": 62.94,
- "MassiveScenarioClassification (bn)": 61.84,
- "MassiveScenarioClassification (it)": 64.09,
- "MassiveScenarioClassification (ko)": 67.26,
- "MassiveScenarioClassification (kn)": 61.74,
- "MassiveScenarioClassification (he)": 63.53,
- "MassiveScenarioClassification (am)": 62.02,
- "MassiveScenarioClassification (ka)": 53.38,
- "MassiveScenarioClassification (vi)": 61.05,
- "MassiveScenarioClassification (ur)": 61.52,
- "MassiveScenarioClassification (zh-TW)": 67.08,
- "MassiveScenarioClassification (en)": 66.41,
- "MassiveScenarioClassification (ms)": 65.63,
- "MassiveScenarioClassification (is)": 61.94,
- "MassiveScenarioClassification (km)": 56.18,
- "MassiveScenarioClassification (hu)": 65.82,
- "MassiveScenarioClassification (fi)": 64.58,
- "MassiveScenarioClassification (sq)": 64.54,
- "MassiveScenarioClassification (fa)": 67.46,
- "MassiveScenarioClassification (af)": 63.39,
- "MassiveScenarioClassification (mn)": 62.6,
- "MassiveScenarioClassification (da)": 65.24,
- "MassiveScenarioClassification (az)": 63.48,
- "MassiveScenarioClassification (es)": 63.61,
- "MassiveScenarioClassification (ja)": 67.72,
- "MassiveScenarioClassification (tr)": 65.43,
- "MassiveScenarioClassification (lv)": 61.87,
- "MassiveScenarioClassification (zh-CN)": 70.85,
- "MassiveScenarioClassification (hi)": 64.4,
- "MultilingualSentiment": 64.6,
- "NoRecClassification": 45.45,
- "NordicLangClassification": 35.39,
- "OnlineShopping": 85.63,
- "PAC": 68.11,
- "PolEmo2.0-IN": 64.0,
- "PolEmo2.0-OUT": 44.72,
- "RuReviewsClassification": 58.01,
- "RuSciBenchGRNTIClassification": 53.04,
- "RuSciBenchOECDClassification": 40.48,
- "TNews": 46.02,
- "ToxicConversationsClassification": 66.9,
- "TweetSentimentExtractionClassification": 58.82,
- "Waimai": 82.85
- }
- ]
- },
- "Clustering": {
- "v_measure": [
- {
- "Model": "LaBSE",
- "8TagsClustering": 12.96,
- "AlloProfClusteringP2P": 54.78,
- "AlloProfClusteringS2S": 31.6,
- "ArxivClusteringP2P": 32.13,
- "ArxivClusteringS2S": 22.05,
- "BiorxivClusteringP2P": 29.84,
- "BiorxivClusteringS2S": 20.57,
- "GeoreviewClusteringP2P": 52.19,
- "HALClusteringS2S": 20.62,
- "MLSUMClusteringP2P (ru)": 39.45,
- "MLSUMClusteringP2P": 42.09,
- "MLSUMClusteringS2S (ru)": 35.77,
- "MLSUMClusteringS2S": 34.84,
- "MasakhaNEWSClusteringP2P (amh)": 67.78,
- "MasakhaNEWSClusteringP2P (eng)": 48.16,
- "MasakhaNEWSClusteringP2P (fra)": 46.16,
- "MasakhaNEWSClusteringP2P (hau)": 39.77,
- "MasakhaNEWSClusteringP2P (ibo)": 62.67,
- "MasakhaNEWSClusteringP2P (lin)": 62.98,
- "MasakhaNEWSClusteringP2P (lug)": 47.76,
- "MasakhaNEWSClusteringP2P (orm)": 28.76,
- "MasakhaNEWSClusteringP2P (pcm)": 77.16,
- "MasakhaNEWSClusteringP2P (run)": 60.36,
- "MasakhaNEWSClusteringP2P (sna)": 63.57,
- "MasakhaNEWSClusteringP2P (som)": 34.94,
- "MasakhaNEWSClusteringP2P (swa)": 27.26,
- "MasakhaNEWSClusteringP2P (tir)": 51.59,
- "MasakhaNEWSClusteringP2P (xho)": 45.32,
- "MasakhaNEWSClusteringP2P (yor)": 48.73,
- "MasakhaNEWSClusteringS2S (amh)": 52.73,
- "MasakhaNEWSClusteringS2S (eng)": 32.6,
- "MasakhaNEWSClusteringS2S (fra)": 38.13,
- "MasakhaNEWSClusteringS2S (hau)": 31.62,
- "MasakhaNEWSClusteringS2S (ibo)": 32.27,
- "MasakhaNEWSClusteringS2S (lin)": 49.38,
- "MasakhaNEWSClusteringS2S (lug)": 47.63,
- "MasakhaNEWSClusteringS2S (orm)": 25.05,
- "MasakhaNEWSClusteringS2S (pcm)": 68.18,
- "MasakhaNEWSClusteringS2S (run)": 52.39,
- "MasakhaNEWSClusteringS2S (sna)": 46.9,
- "MasakhaNEWSClusteringS2S (som)": 24.08,
- "MasakhaNEWSClusteringS2S (swa)": 15.83,
- "MasakhaNEWSClusteringS2S (tir)": 49.07,
- "MasakhaNEWSClusteringS2S (xho)": 28.52,
- "MasakhaNEWSClusteringS2S (yor)": 32.26,
- "MedrxivClusteringP2P": 30.13,
- "MedrxivClusteringS2S": 24.82,
- "RedditClustering": 28.79,
- "RedditClusteringP2P": 49.14,
- "RuSciBenchGRNTIClusteringP2P": 49.07,
- "RuSciBenchOECDClusteringP2P": 41.97,
- "StackExchangeClustering": 35.43,
- "StackExchangeClusteringP2P": 28.83,
- "TwentyNewsgroupsClustering": 23.28
- }
- ]
- },
- "PairClassification": {
- "max_ap": [
- {
- "Model": "LaBSE",
- "CDSC-E": 68.92,
- "OpusparcusPC (de)": 96.58,
- "OpusparcusPC (en)": 98.12,
- "OpusparcusPC (fi)": 94.44,
- "OpusparcusPC (fr)": 93.96,
- "OpusparcusPC (ru)": 87.3,
- "OpusparcusPC (sv)": 93.69,
- "PSC": 97.42,
- "PawsXPairClassification (de)": 51.07,
- "PawsXPairClassification (en)": 54.07,
- "PawsXPairClassification (es)": 52.19,
- "PawsXPairClassification (fr)": 54.63,
- "PawsXPairClassification (ja)": 47.56,
- "PawsXPairClassification (ko)": 49.39,
- "PawsXPairClassification (zh)": 54.26,
- "SICK-E-PL": 63.77,
- "SprintDuplicateQuestions": 89.26,
- "TERRa": 55.71,
- "TwitterSemEval2015": 62.78,
- "TwitterURLCorpus": 84.58
- },
- {
- "Model": "LaBSE",
- "CDSC-E": 68.92,
- "OpusparcusPC (de)": 96.58,
- "OpusparcusPC (en)": 98.12,
- "OpusparcusPC (fi)": 94.44,
- "OpusparcusPC (fr)": 93.96,
- "OpusparcusPC (ru)": 87.3,
- "OpusparcusPC (sv)": 93.69,
- "PPC": 86.97,
- "PSC": 97.42,
- "PawsXPairClassification (de)": 51.45,
- "PawsXPairClassification (en)": 54.07,
- "PawsXPairClassification (es)": 52.19,
- "PawsXPairClassification (fr)": 54.69,
- "PawsXPairClassification (ja)": 47.74,
- "PawsXPairClassification (ko)": 49.42,
- "PawsXPairClassification (zh)": 54.55,
- "SICK-E-PL": 63.77,
- "SprintDuplicateQuestions": 89.26,
- "TERRa": 55.71,
- "TwitterSemEval2015": 62.78,
- "TwitterURLCorpus": 84.58
- },
- {
- "Model": "LaBSE",
- "CDSC-E": 68.91,
- "OpusparcusPC (fr)": 93.96,
- "PPC": 86.97,
- "PSC": 97.42,
- "PawsXPairClassification (fr)": 54.63,
- "SICK-E-PL": 63.77,
- "SprintDuplicateQuestions": 89.26,
- "TwitterSemEval2015": 62.78,
- "TwitterURLCorpus": 84.58
- }
- ]
- },
- "Reranking": {
- "map": [
- {
- "Model": "LaBSE",
- "AlloprofReranking": 49.51,
- "AskUbuntuDupQuestions": 52.75,
- "MMarcoReranking": 14.83,
- "MindSmallReranking": 29.81,
- "RuBQReranking": 55.13,
- "SciDocsRR": 68.72,
- "StackOverflowDupQuestions": 42.42,
- "SyntecReranking": 73.28,
- "T2Reranking": 63.29
- }
- ]
- },
- "Retrieval": {
- "ndcg_at_10": [
- {
- "Model": "LaBSE",
- "AILACasedocs": 17.67,
- "AILAStatutes": 16.72,
- "ARCChallenge": 3.78,
- "AlloprofRetrieval": 19.77,
- "AlphaNLI": 13.11,
- "AppsRetrieval": 2.39,
- "ArguAna": 34.18,
- "ArguAna-PL": 38.52,
- "BSARDRetrieval": 0.0,
- "CQADupstackRetrieval": 18.75,
- "ClimateFEVER": 3.83,
- "CmedqaRetrieval": 5.49,
- "CodeFeedbackMT": 17.98,
- "CodeFeedbackST": 35.98,
- "CodeSearchNetCCRetrieval (python)": 40.41,
- "CodeSearchNetCCRetrieval (javascript)": 52.0,
- "CodeSearchNetCCRetrieval (go)": 41.73,
- "CodeSearchNetCCRetrieval (ruby)": 41.46,
- "CodeSearchNetCCRetrieval (java)": 45.83,
- "CodeSearchNetCCRetrieval (php)": 31.96,
- "CodeSearchNetRetrieval (python)": 60.52,
- "CodeSearchNetRetrieval (javascript)": 46.24,
- "CodeSearchNetRetrieval (go)": 48.56,
- "CodeSearchNetRetrieval (ruby)": 51.34,
- "CodeSearchNetRetrieval (java)": 35.05,
- "CodeSearchNetRetrieval (php)": 46.03,
- "CodeTransOceanContest": 28.1,
- "CodeTransOceanDL": 26.45,
- "CosQA": 8.77,
- "CovidRetrieval": 28.6,
- "DBPedia": 15.57,
- "DBPedia-PL": 16.1,
- "DuRetrieval": 26.34,
- "EcomRetrieval": 25.42,
- "FEVER": 12.18,
- "FiQA-PL": 7.63,
- "FiQA2018": 7.0,
- "GerDaLIRSmall": 4.59,
- "HellaSwag": 5.59,
- "HotpotQA": 18.75,
- "HotpotQA-PL": 19.72,
- "LEMBNarrativeQARetrieval": 11.45,
- "LEMBQMSumRetrieval": 14.07,
- "LEMBSummScreenFDRetrieval": 40.52,
- "LEMBWikimQARetrieval": 28.1,
- "LeCaRDv2": 24.68,
- "LegalBenchConsumerContractsQA": 54.66,
- "LegalBenchCorporateLobbying": 69.39,
- "LegalQuAD": 16.64,
- "LegalSummarization": 53.89,
- "MMarcoRetrieval": 34.78,
- "MSMARCO": 7.6,
- "MSMARCO-PL": 7.22,
- "MedicalRetrieval": 6.68,
- "MintakaRetrieval (ar)": 14.06,
- "MintakaRetrieval (de)": 15.26,
- "MintakaRetrieval (es)": 15.65,
- "MintakaRetrieval (fr)": 15.53,
- "MintakaRetrieval (hi)": 13.67,
- "MintakaRetrieval (it)": 15.94,
- "MintakaRetrieval (ja)": 12.8,
- "MintakaRetrieval (pt)": 15.03,
- "NFCorpus": 16.54,
- "NFCorpus-PL": 17.45,
- "NQ": 8.42,
- "NQ-PL": 9.65,
- "PIQA": 6.53,
- "Quail": 1.91,
- "Quora-PL": 74.96,
- "QuoraRetrieval": 77.03,
- "RARbCode": 2.31,
- "RARbMath": 27.19,
- "RiaNewsRetrieval": 42.75,
- "RuBQRetrieval": 30.02,
- "SCIDOCS": 5.63,
- "SCIDOCS-PL": 7.48,
- "SIQA": 1.07,
- "SciFact": 38.2,
- "SciFact-PL": 39.79,
- "SpartQA": 1.56,
- "StackOverflowQA": 38.23,
- "SyntecRetrieval": 55.31,
- "SyntheticText2SQL": 43.28,
- "T2Retrieval": 25.32,
- "TRECCOVID": 16.34,
- "TRECCOVID-PL": 18.45,
- "TempReasonL1": 1.56,
- "TempReasonL2Fact": 7.06,
- "TempReasonL2Pure": 0.14,
- "TempReasonL3Fact": 8.74,
- "TempReasonL3Pure": 4.73,
- "Touche2020": 4.88,
- "VideoRetrieval": 22.04,
- "WinoGrande": 54.3,
- "XPQARetrieval (ara-ara)": 35.19,
- "XPQARetrieval (eng-ara)": 20.64,
- "XPQARetrieval (ara-eng)": 32.47,
- "XPQARetrieval (deu-deu)": 53.56,
- "XPQARetrieval (eng-deu)": 24.31,
- "XPQARetrieval (deu-eng)": 54.87,
- "XPQARetrieval (spa-spa)": 44.49,
- "XPQARetrieval (eng-spa)": 25.31,
- "XPQARetrieval (spa-eng)": 43.4,
- "XPQARetrieval (fra-fra)": 51.74,
- "XPQARetrieval (eng-fra)": 21.29,
- "XPQARetrieval (fra-eng)": 49.4,
- "XPQARetrieval (hin-hin)": 66.64,
- "XPQARetrieval (eng-hin)": 23.25,
- "XPQARetrieval (hin-eng)": 64.54,
- "XPQARetrieval (ita-ita)": 56.27,
- "XPQARetrieval (eng-ita)": 25.8,
- "XPQARetrieval (ita-eng)": 52.69,
- "XPQARetrieval (jpn-jpn)": 58.6,
- "XPQARetrieval (eng-jpn)": 21.49,
- "XPQARetrieval (jpn-eng)": 52.41,
- "XPQARetrieval (kor-kor)": 27.63,
- "XPQARetrieval (eng-kor)": 23.33,
- "XPQARetrieval (kor-eng)": 23.97,
- "XPQARetrieval (pol-pol)": 37.33,
- "XPQARetrieval (eng-pol)": 16.19,
- "XPQARetrieval (pol-eng)": 37.7,
- "XPQARetrieval (por-por)": 38.48,
- "XPQARetrieval (eng-por)": 19.41,
- "XPQARetrieval (por-eng)": 37.31,
- "XPQARetrieval (tam-tam)": 37.33,
- "XPQARetrieval (eng-tam)": 20.53,
- "XPQARetrieval (tam-eng)": 30.14,
- "XPQARetrieval (cmn-cmn)": 50.7,
- "XPQARetrieval (eng-cmn)": 20.59,
- "XPQARetrieval (cmn-eng)": 48.23,
- "XPQARetrieval (fr)": 51.74
- }
- ]
- },
- "STS": {
- "cosine_spearman": [
- {
- "Model": "LaBSE",
- "AFQMC": 21.02,
- "ATEC": 26.61,
- "BIOSSES": 78.7,
- "BQ": 42.6,
- "CDSC-R": 85.53,
- "LCQMC": 52.19,
- "PAWSX": 10.23,
- "RUParaPhraserSTS": 65.74,
- "RuSTSBenchmarkSTS": 73.34,
- "SICK-R": 69.99,
- "SICK-R-PL": 65.9,
- "SICKFr": 69.94,
- "STS12": 65.08,
- "STS13": 67.98,
- "STS14": 64.03,
- "STS15": 76.59,
- "STS16": 72.98,
- "STS17 (es-en)": 65.71,
- "STS17 (en-de)": 73.85,
- "STS17 (fr-en)": 76.98,
- "STS17 (en-tr)": 72.07,
- "STS17 (ko-ko)": 71.32,
- "STS17 (es-es)": 80.83,
- "STS17 (it-en)": 76.99,
- "STS17 (ar-ar)": 69.07,
- "STS17 (en-en)": 79.45,
- "STS17 (nl-en)": 75.22,
- "STS17 (en-ar)": 74.51,
- "STS22 (pl-en)": 69.41,
- "STS22 (de-en)": 50.14,
- "STS22 (es)": 63.18,
- "STS22 (de-pl)": 58.69,
- "STS22 (fr)": 77.95,
- "STS22 (fr-pl)": 61.98,
- "STS22 (de)": 48.58,
- "STS22 (pl)": 39.3,
- "STS22 (en)": 61.63,
- "STS22 (es-it)": 69.69,
- "STS22 (zh-en)": 64.02,
- "STS22 (de-fr)": 53.28,
- "STS22 (tr)": 58.15,
- "STS22 (it)": 72.22,
- "STS22 (ru)": 57.49,
- "STS22 (es-en)": 71.86,
- "STS22 (ar)": 57.67,
- "STS22 (zh)": 63.02,
- "STSB": 68.38,
- "STSBenchmark": 72.25,
- "STSBenchmarkMultilingualSTS (nl)": 70.22,
- "STSBenchmarkMultilingualSTS (de)": 72.43,
- "STSBenchmarkMultilingualSTS (pt)": 71.65,
- "STSBenchmarkMultilingualSTS (fr)": 75.1,
- "STSBenchmarkMultilingualSTS (en)": 72.25,
- "STSBenchmarkMultilingualSTS (pl)": 72.58,
- "STSBenchmarkMultilingualSTS (es)": 72.92,
- "STSBenchmarkMultilingualSTS (zh)": 69.5,
- "STSBenchmarkMultilingualSTS (ru)": 73.06,
- "STSBenchmarkMultilingualSTS (it)": 72.97
- },
- {
- "Model": "LaBSE",
- "BIOSSES": 78.7,
- "CDSC-R": 85.53,
- "SICK-R": 69.99,
- "SICK-R-PL": 65.9,
- "SICKFr": 69.94,
- "STS12": 65.08,
- "STS13": 67.98,
- "STS14": 64.03,
- "STS15": 76.59,
- "STS16": 72.98,
- "STS17 (ar-ar)": 69.07,
- "STS17 (en-ar)": 74.51,
- "STS17 (en-de)": 73.85,
- "STS17 (en-en)": 79.45,
- "STS17 (en-tr)": 72.07,
- "STS17 (es-en)": 65.71,
- "STS17 (es-es)": 80.83,
- "STS17 (fr-en)": 76.98,
- "STS17 (it-en)": 76.99,
- "STS17 (ko-ko)": 71.32,
- "STS17 (nl-en)": 75.22,
- "STS22 (ar)": 57.67,
- "STS22 (de)": 48.58,
- "STS22 (de-en)": 50.14,
- "STS22 (de-fr)": 53.28,
- "STS22 (de-pl)": 58.69,
- "STS22 (en)": 60.97,
- "STS22 (es)": 63.18,
- "STS22 (es-en)": 71.86,
- "STS22 (es-it)": 69.69,
- "STS22 (fr)": 77.95,
- "STS22 (fr-pl)": 61.98,
- "STS22 (it)": 72.22,
- "STS22 (pl)": 39.28,
- "STS22 (pl-en)": 69.41,
- "STS22 (ru)": 57.49,
- "STS22 (tr)": 58.15,
- "STS22 (zh)": 63.02,
- "STS22 (zh-en)": 64.02,
- "STSBenchmark": 72.25,
- "STSBenchmarkMultilingualSTS (fr)": 75.1
- }
- ]
- },
- "Summarization": {
- "cosine_spearman": [
- {
- "Model": "LaBSE",
- "SummEval": 31.05,
- "SummEvalFr": 30.16
- },
- {
- "Model": "LaBSE",
- "SummEval": 31.05,
- "SummEvalFr": 30.16
- },
- {
- "Model": "LaBSE",
- "SummEval": 31.05,
- "SummEvalFr": 30.16
- }
- ]
- },
- "MultilabelClassification": {
- "accuracy": [
- {
- "Model": "LaBSE",
- "CEDRClassification": 40.61,
- "SensitiveTopicsClassification": 22.23
- }
- ]
- },
- "InstructionRetrieval": {
- "p-MRR": [
- {
- "Model": "LaBSE",
- "Core17InstructionRetrieval": 1.49,
- "News21InstructionRetrieval": -1.11,
- "Robust04InstructionRetrieval": -9.37
- }
- ]
- }
- },
- "sentence-transformers__all-MiniLM-L12-v2": {
- "BitextMining": {
- "f1": [
- {
- "Model": "all-MiniLM-L12-v2",
- "BornholmBitextMining": 35.25,
- "Tatoeba (tat-eng)": 0.75,
- "Tatoeba (yid-eng)": 0.19,
- "Tatoeba (tzl-eng)": 6.87,
- "Tatoeba (ben-eng)": 0.02,
- "Tatoeba (sqi-eng)": 5.86,
- "Tatoeba (war-eng)": 6.18,
- "Tatoeba (nld-eng)": 12.56,
- "Tatoeba (ast-eng)": 9.99,
- "Tatoeba (awa-eng)": 0.44,
- "Tatoeba (jpn-eng)": 2.18,
- "Tatoeba (kat-eng)": 0.45,
- "Tatoeba (pes-eng)": 0.3,
- "Tatoeba (fra-eng)": 17.53,
- "Tatoeba (nds-eng)": 11.35,
- "Tatoeba (gle-eng)": 3.08,
- "Tatoeba (arz-eng)": 0.0,
- "Tatoeba (srp-eng)": 2.22,
- "Tatoeba (mhr-eng)": 0.0,
- "Tatoeba (bos-eng)": 7.05,
- "Tatoeba (heb-eng)": 0.3,
- "Tatoeba (orv-eng)": 0.15,
- "Tatoeba (kaz-eng)": 0.82,
- "Tatoeba (eus-eng)": 6.58,
- "Tatoeba (hsb-eng)": 2.89,
- "Tatoeba (max-eng)": 8.4,
- "Tatoeba (kab-eng)": 0.91,
- "Tatoeba (hrv-eng)": 5.68,
- "Tatoeba (deu-eng)": 13.89,
- "Tatoeba (kor-eng)": 0.9,
- "Tatoeba (slk-eng)": 4.2,
- "Tatoeba (tur-eng)": 3.69,
- "Tatoeba (ron-eng)": 8.77,
- "Tatoeba (nno-eng)": 7.45,
- "Tatoeba (ido-eng)": 11.08,
- "Tatoeba (est-eng)": 2.6,
- "Tatoeba (ceb-eng)": 3.95,
- "Tatoeba (bre-eng)": 3.68,
- "Tatoeba (lfn-eng)": 7.52,
- "Tatoeba (ina-eng)": 25.36,
- "Tatoeba (nov-eng)": 19.45,
- "Tatoeba (slv-eng)": 4.52,
- "Tatoeba (fry-eng)": 14.53,
- "Tatoeba (cbk-eng)": 9.76,
- "Tatoeba (lvs-eng)": 3.45,
- "Tatoeba (yue-eng)": 1.89,
- "Tatoeba (ces-eng)": 4.2,
- "Tatoeba (afr-eng)": 7.59,
- "Tatoeba (rus-eng)": 0.07,
- "Tatoeba (amh-eng)": 0.01,
- "Tatoeba (ang-eng)": 14.63,
- "Tatoeba (cat-eng)": 11.79,
- "Tatoeba (khm-eng)": 0.42,
- "Tatoeba (pam-eng)": 4.73,
- "Tatoeba (pms-eng)": 8.94,
- "Tatoeba (gsw-eng)": 9.9,
- "Tatoeba (swg-eng)": 11.9,
- "Tatoeba (tel-eng)": 0.67,
- "Tatoeba (nob-eng)": 8.02,
- "Tatoeba (uig-eng)": 0.4,
- "Tatoeba (bel-eng)": 0.85,
- "Tatoeba (lit-eng)": 1.56,
- "Tatoeba (isl-eng)": 3.44,
- "Tatoeba (swh-eng)": 5.82,
- "Tatoeba (tha-eng)": 0.67,
- "Tatoeba (mon-eng)": 0.06,
- "Tatoeba (hin-eng)": 0.0,
- "Tatoeba (swe-eng)": 7.31,
- "Tatoeba (epo-eng)": 8.5,
- "Tatoeba (ind-eng)": 5.3,
- "Tatoeba (tgl-eng)": 3.34,
- "Tatoeba (arq-eng)": 0.28,
- "Tatoeba (aze-eng)": 1.47,
- "Tatoeba (por-eng)": 11.36,
- "Tatoeba (hun-eng)": 3.93,
- "Tatoeba (kur-eng)": 7.3,
- "Tatoeba (urd-eng)": 0.0,
- "Tatoeba (csb-eng)": 5.21,
- "Tatoeba (glg-eng)": 12.6,
- "Tatoeba (wuu-eng)": 1.89,
- "Tatoeba (uzb-eng)": 2.2,
- "Tatoeba (fao-eng)": 5.92,
- "Tatoeba (mal-eng)": 0.24,
- "Tatoeba (dsb-eng)": 3.06,
- "Tatoeba (jav-eng)": 3.5,
- "Tatoeba (xho-eng)": 3.66,
- "Tatoeba (dan-eng)": 10.21,
- "Tatoeba (gla-eng)": 2.58,
- "Tatoeba (spa-eng)": 11.26,
- "Tatoeba (ber-eng)": 4.72,
- "Tatoeba (ukr-eng)": 0.57,
- "Tatoeba (cym-eng)": 5.13,
- "Tatoeba (cha-eng)": 13.07,
- "Tatoeba (pol-eng)": 4.29,
- "Tatoeba (fin-eng)": 3.65,
- "Tatoeba (bul-eng)": 0.23,
- "Tatoeba (tuk-eng)": 2.66,
- "Tatoeba (tam-eng)": 0.33,
- "Tatoeba (mar-eng)": 0.04,
- "Tatoeba (vie-eng)": 5.06,
- "Tatoeba (ell-eng)": 0.2,
- "Tatoeba (lat-eng)": 7.14,
- "Tatoeba (zsm-eng)": 5.99,
- "Tatoeba (ita-eng)": 12.57,
- "Tatoeba (ara-eng)": 0.43,
- "Tatoeba (cor-eng)": 2.47,
- "Tatoeba (oci-eng)": 8.72,
- "Tatoeba (mkd-eng)": 0.01,
- "Tatoeba (ile-eng)": 17.43,
- "Tatoeba (kzj-eng)": 3.64,
- "Tatoeba (cmn-eng)": 2.45,
- "Tatoeba (dtp-eng)": 3.31,
- "Tatoeba (hye-eng)": 0.5
- }
- ]
- },
- "Classification": {
- "accuracy": [
- {
- "Model": "all-MiniLM-L12-v2",
- "AllegroReviews": 23.85,
- "AmazonCounterfactualClassification (en-ext)": 67.24,
- "AmazonCounterfactualClassification (en)": 65.28,
- "AmazonCounterfactualClassification (de)": 57.1,
- "AmazonCounterfactualClassification (ja)": 59.91,
- "AmazonPolarityClassification": 62.98,
- "AmazonReviewsClassification (en)": 30.79,
- "AmazonReviewsClassification (de)": 25.91,
- "AmazonReviewsClassification (es)": 27.63,
- "AmazonReviewsClassification (fr)": 27.54,
- "AmazonReviewsClassification (ja)": 23.57,
- "AmazonReviewsClassification (zh)": 22.99,
- "AngryTweetsClassification": 42.87,
- "Banking77Classification": 80.4,
- "CBD": 48.46,
- "DanishPoliticalCommentsClassification": 27.07,
- "EmotionClassification": 41.17,
- "GeoreviewClassification": 23.49,
- "HeadlineClassification": 28.49,
- "IFlyTek": 15.31,
- "ImdbClassification": 59.76,
- "InappropriatenessClassification": 50.85,
- "JDReview": 59.57,
- "KinopoiskClassification": 34.17,
- "LccSentimentClassification": 41.93,
- "MTOPDomainClassification (en)": 91.9,
- "MTOPDomainClassification (de)": 72.04,
- "MTOPDomainClassification (es)": 72.99,
- "MTOPDomainClassification (fr)": 75.59,
- "MTOPDomainClassification (hi)": 40.36,
- "MTOPDomainClassification (th)": 17.1,
- "MTOPIntentClassification (en)": 62.84,
- "MTOPIntentClassification (de)": 43.41,
- "MTOPIntentClassification (es)": 41.88,
- "MTOPIntentClassification (fr)": 38.94,
- "MTOPIntentClassification (hi)": 17.75,
- "MTOPIntentClassification (th)": 5.63,
- "MasakhaNEWSClassification (amh)": 30.64,
- "MasakhaNEWSClassification (eng)": 76.62,
- "MasakhaNEWSClassification (fra)": 72.2,
- "MasakhaNEWSClassification (hau)": 52.59,
- "MasakhaNEWSClassification (ibo)": 54.26,
- "MasakhaNEWSClassification (lin)": 62.23,
- "MasakhaNEWSClassification (lug)": 47.62,
- "MasakhaNEWSClassification (orm)": 47.17,
- "MasakhaNEWSClassification (pcm)": 91.77,
- "MasakhaNEWSClassification (run)": 54.47,
- "MasakhaNEWSClassification (sna)": 66.53,
- "MasakhaNEWSClassification (som)": 40.27,
- "MasakhaNEWSClassification (swa)": 47.77,
- "MasakhaNEWSClassification (tir)": 21.18,
- "MasakhaNEWSClassification (xho)": 54.34,
- "MasakhaNEWSClassification (yor)": 58.61,
- "MassiveIntentClassification (ja)": 30.94,
- "MassiveIntentClassification (zh-CN)": 23.74,
- "MassiveIntentClassification (en)": 67.15,
- "MassiveIntentClassification (nb)": 41.91,
- "MassiveIntentClassification (ur)": 16.18,
- "MassiveIntentClassification (kn)": 3.08,
- "MassiveIntentClassification (ru)": 26.33,
- "MassiveIntentClassification (de)": 44.17,
- "MassiveIntentClassification (az)": 34.25,
- "MassiveIntentClassification (hu)": 38.0,
- "MassiveIntentClassification (ml)": 2.85,
- "MassiveIntentClassification (tr)": 35.9,
- "MassiveIntentClassification (ko)": 19.97,
- "MassiveIntentClassification (mn)": 23.25,
- "MassiveIntentClassification (ta)": 13.1,
- "MassiveIntentClassification (fr)": 44.82,
- "MassiveIntentClassification (el)": 28.7,
- "MassiveIntentClassification (sw)": 35.28,
- "MassiveIntentClassification (sv)": 40.42,
- "MassiveIntentClassification (vi)": 37.38,
- "MassiveIntentClassification (ms)": 36.21,
- "MassiveIntentClassification (hi)": 17.98,
- "MassiveIntentClassification (hy)": 8.69,
- "MassiveIntentClassification (pl)": 37.63,
- "MassiveIntentClassification (pt)": 45.12,
- "MassiveIntentClassification (fi)": 39.27,
- "MassiveIntentClassification (ar)": 20.94,
- "MassiveIntentClassification (da)": 44.43,
- "MassiveIntentClassification (af)": 38.94,
- "MassiveIntentClassification (sq)": 41.62,
- "MassiveIntentClassification (am)": 2.45,
- "MassiveIntentClassification (nl)": 41.85,
- "MassiveIntentClassification (es)": 40.91,
- "MassiveIntentClassification (it)": 43.17,
- "MassiveIntentClassification (km)": 4.99,
- "MassiveIntentClassification (id)": 39.66,
- "MassiveIntentClassification (bn)": 13.67,
- "MassiveIntentClassification (is)": 35.14,
- "MassiveIntentClassification (jv)": 36.69,
- "MassiveIntentClassification (ka)": 9.17,
- "MassiveIntentClassification (ro)": 41.71,
- "MassiveIntentClassification (sl)": 38.52,
- "MassiveIntentClassification (zh-TW)": 22.39,
- "MassiveIntentClassification (tl)": 38.56,
- "MassiveIntentClassification (fa)": 23.52,
- "MassiveIntentClassification (te)": 2.56,
- "MassiveIntentClassification (lv)": 38.61,
- "MassiveIntentClassification (my)": 4.38,
- "MassiveIntentClassification (th)": 10.54,
- "MassiveIntentClassification (he)": 23.65,
- "MassiveIntentClassification (cy)": 35.71,
- "MassiveScenarioClassification (pt)": 53.0,
- "MassiveScenarioClassification (zh-TW)": 31.16,
- "MassiveScenarioClassification (sv)": 46.83,
- "MassiveScenarioClassification (fa)": 29.0,
- "MassiveScenarioClassification (lv)": 42.75,
- "MassiveScenarioClassification (fr)": 53.76,
- "MassiveScenarioClassification (ml)": 7.25,
- "MassiveScenarioClassification (is)": 43.08,
- "MassiveScenarioClassification (sq)": 49.14,
- "MassiveScenarioClassification (am)": 7.41,
- "MassiveScenarioClassification (cy)": 41.4,
- "MassiveScenarioClassification (ro)": 49.97,
- "MassiveScenarioClassification (th)": 18.32,
- "MassiveScenarioClassification (kn)": 8.32,
- "MassiveScenarioClassification (ur)": 24.46,
- "MassiveScenarioClassification (id)": 44.35,
- "MassiveScenarioClassification (el)": 35.51,
- "MassiveScenarioClassification (he)": 25.68,
- "MassiveScenarioClassification (my)": 10.07,
- "MassiveScenarioClassification (de)": 52.07,
- "MassiveScenarioClassification (hi)": 23.02,
- "MassiveScenarioClassification (hu)": 44.09,
- "MassiveScenarioClassification (tl)": 48.31,
- "MassiveScenarioClassification (en)": 74.58,
- "MassiveScenarioClassification (vi)": 40.94,
- "MassiveScenarioClassification (bn)": 18.98,
- "MassiveScenarioClassification (ka)": 14.84,
- "MassiveScenarioClassification (hy)": 14.83,
- "MassiveScenarioClassification (pl)": 44.72,
- "MassiveScenarioClassification (sw)": 43.18,
- "MassiveScenarioClassification (it)": 51.71,
- "MassiveScenarioClassification (tr)": 41.79,
- "MassiveScenarioClassification (nl)": 49.15,
- "MassiveScenarioClassification (az)": 39.58,
- "MassiveScenarioClassification (da)": 49.47,
- "MassiveScenarioClassification (es)": 50.74,
- "MassiveScenarioClassification (ar)": 27.62,
- "MassiveScenarioClassification (sl)": 42.26,
- "MassiveScenarioClassification (jv)": 44.57,
- "MassiveScenarioClassification (ms)": 44.65,
- "MassiveScenarioClassification (fi)": 45.8,
- "MassiveScenarioClassification (km)": 9.75,
- "MassiveScenarioClassification (ru)": 28.75,
- "MassiveScenarioClassification (mn)": 29.03,
- "MassiveScenarioClassification (ta)": 19.38,
- "MassiveScenarioClassification (te)": 7.74,
- "MassiveScenarioClassification (ja)": 36.75,
- "MassiveScenarioClassification (ko)": 25.72,
- "MassiveScenarioClassification (zh-CN)": 33.18,
- "MassiveScenarioClassification (nb)": 47.36,
- "MassiveScenarioClassification (af)": 45.71,
- "MultilingualSentiment": 40.52,
- "NoRecClassification": 37.73,
- "NordicLangClassification": 54.17,
- "OnlineShopping": 58.65,
- "PAC": 59.53,
- "PolEmo2.0-IN": 38.32,
- "PolEmo2.0-OUT": 22.98,
- "RuReviewsClassification": 42.49,
- "RuSciBenchGRNTIClassification": 10.49,
- "RuSciBenchOECDClassification": 8.31,
- "TNews": 20.37,
- "ToxicConversationsClassification": 67.47,
- "TweetSentimentExtractionClassification": 54.25,
- "Waimai": 63.48
- }
- ]
- },
- "Clustering": {
- "v_measure": [
- {
- "Model": "all-MiniLM-L12-v2",
- "AlloProfClusteringP2P": 46.03,
- "AlloProfClusteringS2S": 31.83,
- "ArxivClusteringP2P": 46.07,
- "ArxivClusteringS2S": 37.5,
- "BiorxivClusteringP2P": 36.99,
- "BiorxivClusteringS2S": 33.21,
- "GeoreviewClusteringP2P": 20.76,
- "HALClusteringS2S": 19.58,
- "MLSUMClusteringP2P": 34.35,
- "MLSUMClusteringS2S": 29.3,
- "MasakhaNEWSClusteringP2P (amh)": 40.5,
- "MasakhaNEWSClusteringP2P (eng)": 55.86,
- "MasakhaNEWSClusteringP2P (fra)": 42.72,
- "MasakhaNEWSClusteringP2P (hau)": 26.61,
- "MasakhaNEWSClusteringP2P (ibo)": 44.26,
- "MasakhaNEWSClusteringP2P (lin)": 54.52,
- "MasakhaNEWSClusteringP2P (lug)": 43.87,
- "MasakhaNEWSClusteringP2P (orm)": 24.87,
- "MasakhaNEWSClusteringP2P (pcm)": 74.42,
- "MasakhaNEWSClusteringP2P (run)": 51.73,
- "MasakhaNEWSClusteringP2P (sna)": 46.89,
- "MasakhaNEWSClusteringP2P (som)": 31.17,
- "MasakhaNEWSClusteringP2P (swa)": 23.72,
- "MasakhaNEWSClusteringP2P (tir)": 44.08,
- "MasakhaNEWSClusteringP2P (xho)": 26.97,
- "MasakhaNEWSClusteringP2P (yor)": 32.51,
- "MasakhaNEWSClusteringS2S (amh)": 44.11,
- "MasakhaNEWSClusteringS2S (eng)": 40.71,
- "MasakhaNEWSClusteringS2S (fra)": 32.47,
- "MasakhaNEWSClusteringS2S (hau)": 20.63,
- "MasakhaNEWSClusteringS2S (ibo)": 35.33,
- "MasakhaNEWSClusteringS2S (lin)": 54.52,
- "MasakhaNEWSClusteringS2S (lug)": 51.42,
- "MasakhaNEWSClusteringS2S (orm)": 24.84,
- "MasakhaNEWSClusteringS2S (pcm)": 70.72,
- "MasakhaNEWSClusteringS2S (run)": 50.88,
- "MasakhaNEWSClusteringS2S (sna)": 46.6,
- "MasakhaNEWSClusteringS2S (som)": 29.87,
- "MasakhaNEWSClusteringS2S (swa)": 10.82,
- "MasakhaNEWSClusteringS2S (tir)": 43.63,
- "MasakhaNEWSClusteringS2S (xho)": 24.55,
- "MasakhaNEWSClusteringS2S (yor)": 32.85,
- "MedrxivClusteringP2P": 34.25,
- "MedrxivClusteringS2S": 32.24,
- "RedditClustering": 51.18,
- "RedditClusteringP2P": 54.8,
- "RuSciBenchGRNTIClusteringP2P": 10.65,
- "RuSciBenchOECDClusteringP2P": 10.19,
- "StackExchangeClustering": 53.05,
- "StackExchangeClusteringP2P": 33.13,
- "TwentyNewsgroupsClustering": 47.47
- }
- ]
- },
- "PairClassification": {
- "max_ap": [
- {
- "Model": "all-MiniLM-L12-v2",
- "CDSC-E": 49.04,
- "OpusparcusPC (de)": 91.2,
- "OpusparcusPC (en)": 97.41,
- "OpusparcusPC (fi)": 85.99,
- "OpusparcusPC (fr)": 87.35,
- "OpusparcusPC (ru)": 79.23,
- "OpusparcusPC (sv)": 84.87,
- "PSC": 87.92,
- "PawsXPairClassification (de)": 50.83,
- "PawsXPairClassification (en)": 58.62,
- "PawsXPairClassification (es)": 52.08,
- "PawsXPairClassification (fr)": 55.54,
- "PawsXPairClassification (ja)": 47.75,
- "PawsXPairClassification (ko)": 49.59,
- "PawsXPairClassification (zh)": 52.8,
- "SICK-E-PL": 49.63,
- "SprintDuplicateQuestions": 92.45,
- "TERRa": 46.4,
- "TwitterSemEval2015": 70.02,
- "TwitterURLCorpus": 84.77
- },
- {
- "Model": "all-MiniLM-L12-v2",
- "CDSC-E": 49.04,
- "OpusparcusPC (de)": 91.2,
- "OpusparcusPC (en)": 97.41,
- "OpusparcusPC (fi)": 85.99,
- "OpusparcusPC (fr)": 87.35,
- "OpusparcusPC (ru)": 79.23,
- "OpusparcusPC (sv)": 84.87,
- "PSC": 87.92,
- "PawsXPairClassification (de)": 51.07,
- "PawsXPairClassification (en)": 58.7,
- "PawsXPairClassification (es)": 52.08,
- "PawsXPairClassification (fr)": 55.59,
- "PawsXPairClassification (ja)": 47.88,
- "PawsXPairClassification (ko)": 49.65,
- "PawsXPairClassification (zh)": 53.01,
- "SICK-E-PL": 49.63,
- "SprintDuplicateQuestions": 92.58,
- "TERRa": 46.4,
- "TwitterSemEval2015": 70.02,
- "TwitterURLCorpus": 84.77
- }
- ]
- },
- "Reranking": {
- "map": [
- {
- "Model": "all-MiniLM-L12-v2",
- "AlloprofReranking": 67.01,
- "AskUbuntuDupQuestions": 64.06,
- "MMarcoReranking": 5.27,
- "MindSmallReranking": 31.02,
- "RuBQReranking": 38.51,
- "SciDocsRR": 87.2,
- "StackOverflowDupQuestions": 51.47,
- "SyntecReranking": 69.17,
- "T2Reranking": 60.32
- }
- ]
- },
- "Retrieval": {
- "ndcg_at_10": [
- {
- "Model": "all-MiniLM-L12-v2",
- "AILACasedocs": 16.8,
- "AILAStatutes": 20.71,
- "ARCChallenge": 10.23,
- "AlloprofRetrieval": 33.2,
- "AlphaNLI": 25.35,
- "AppsRetrieval": 5.97,
- "ArguAna": 47.13,
- "ArguAna-PL": 13.4,
- "BSARDRetrieval": 6.24,
- "CQADupstackRetrieval": 42.53,
- "ClimateFEVER": 21.57,
- "CmedqaRetrieval": 2.58,
- "CodeFeedbackMT": 27.07,
- "CodeFeedbackST": 61.01,
- "CodeSearchNetCCRetrieval (python)": 71.1,
- "CodeSearchNetCCRetrieval (javascript)": 70.13,
- "CodeSearchNetCCRetrieval (go)": 63.52,
- "CodeSearchNetCCRetrieval (ruby)": 72.28,
- "CodeSearchNetCCRetrieval (java)": 67.51,
- "CodeSearchNetCCRetrieval (php)": 60.81,
- "CodeSearchNetRetrieval (python)": 82.09,
- "CodeSearchNetRetrieval (javascript)": 67.52,
- "CodeSearchNetRetrieval (go)": 89.06,
- "CodeSearchNetRetrieval (ruby)": 74.77,
- "CodeSearchNetRetrieval (java)": 53.2,
- "CodeSearchNetRetrieval (php)": 77.36,
- "CodeTransOceanContest": 59.86,
- "CodeTransOceanDL": 19.94,
- "CosQA": 32.09,
- "CovidRetrieval": 10.79,
- "DBPedia": 33.35,
- "DuRetrieval": 6.62,
- "EcomRetrieval": 4.01,
- "FEVER": 55.9,
- "FiQA-PL": 5.82,
- "FiQA2018": 37.27,
- "GerDaLIRSmall": 1.35,
- "HellaSwag": 24.08,
- "HotpotQA": 44.59,
- "LEMBNarrativeQARetrieval": 19.64,
- "LEMBQMSumRetrieval": 13.08,
- "LEMBSummScreenFDRetrieval": 46.98,
- "LEMBWikimQARetrieval": 44.88,
- "LeCaRDv2": 18.77,
- "LegalBenchConsumerContractsQA": 60.21,
- "LegalBenchCorporateLobbying": 88.69,
- "LegalQuAD": 7.44,
- "LegalSummarization": 57.43,
- "MMarcoRetrieval": 7.46,
- "MSMARCO": 39.03,
- "MedicalRetrieval": 2.3,
- "MintakaRetrieval (ar)": 2.74,
- "MintakaRetrieval (de)": 20.04,
- "MintakaRetrieval (es)": 11.76,
- "MintakaRetrieval (fr)": 16.08,
- "MintakaRetrieval (hi)": 3.04,
- "MintakaRetrieval (it)": 11.83,
- "MintakaRetrieval (ja)": 7.31,
- "MintakaRetrieval (pt)": 13.66,
- "NFCorpus": 32.25,
- "NFCorpus-PL": 15.43,
- "NQ": 46.47,
- "PIQA": 26.44,
- "Quail": 3.08,
- "QuoraRetrieval": 87.75,
- "RARbCode": 42.44,
- "RARbMath": 66.36,
- "RuBQRetrieval": 8.84,
- "SCIDOCS": 21.82,
- "SCIDOCS-PL": 5.34,
- "SIQA": 2.09,
- "SciFact": 62.64,
- "SciFact-PL": 22.48,
- "SpartQA": 2.67,
- "StackOverflowQA": 80.63,
- "SyntecRetrieval": 60.8,
- "SyntheticText2SQL": 43.93,
- "T2Retrieval": 4.82,
- "TRECCOVID": 50.82,
- "TRECCOVID-PL": 16.52,
- "TempReasonL1": 1.66,
- "TempReasonL2Fact": 10.31,
- "TempReasonL2Pure": 0.63,
- "TempReasonL3Fact": 11.11,
- "TempReasonL3Pure": 6.63,
- "Touche2020": 17.22,
- "VideoRetrieval": 9.38,
- "WinoGrande": 27.2,
- "XPQARetrieval (ara-ara)": 7.83,
- "XPQARetrieval (eng-ara)": 2.55,
- "XPQARetrieval (ara-eng)": 8.88,
- "XPQARetrieval (deu-deu)": 56.77,
- "XPQARetrieval (eng-deu)": 18.2,
- "XPQARetrieval (deu-eng)": 30.06,
- "XPQARetrieval (spa-spa)": 42.22,
- "XPQARetrieval (eng-spa)": 7.53,
- "XPQARetrieval (spa-eng)": 26.27,
- "XPQARetrieval (fra-fra)": 55.9,
- "XPQARetrieval (eng-fra)": 14.89,
- "XPQARetrieval (fra-eng)": 34.15,
- "XPQARetrieval (hin-hin)": 33.26,
- "XPQARetrieval (eng-hin)": 6.44,
- "XPQARetrieval (hin-eng)": 6.98,
- "XPQARetrieval (ita-ita)": 58.68,
- "XPQARetrieval (eng-ita)": 8.56,
- "XPQARetrieval (ita-eng)": 28.71,
- "XPQARetrieval (jpn-jpn)": 39.53,
- "XPQARetrieval (eng-jpn)": 5.7,
- "XPQARetrieval (jpn-eng)": 13.75,
- "XPQARetrieval (kor-kor)": 13.44,
- "XPQARetrieval (eng-kor)": 7.43,
- "XPQARetrieval (kor-eng)": 7.4,
- "XPQARetrieval (pol-pol)": 28.07,
- "XPQARetrieval (eng-pol)": 10.03,
- "XPQARetrieval (pol-eng)": 16.58,
- "XPQARetrieval (por-por)": 34.09,
- "XPQARetrieval (eng-por)": 7.38,
- "XPQARetrieval (por-eng)": 22.59,
- "XPQARetrieval (tam-tam)": 9.07,
- "XPQARetrieval (eng-tam)": 4.15,
- "XPQARetrieval (tam-eng)": 3.76,
- "XPQARetrieval (cmn-cmn)": 21.07,
- "XPQARetrieval (eng-cmn)": 6.58,
- "XPQARetrieval (cmn-eng)": 9.39,
- "XPQARetrieval (fr)": 55.9
- }
- ]
- },
- "STS": {
- "cosine_spearman": [
- {
- "Model": "all-MiniLM-L12-v2",
- "AFQMC": 7.94,
- "ATEC": 12.97,
- "BIOSSES": 83.57,
- "BQ": 23.31,
- "CDSC-R": 82.5,
- "LCQMC": 21.04,
- "PAWSX": 7.31,
- "RUParaPhraserSTS": 45.47,
- "RuSTSBenchmarkSTS": 56.33,
- "SICK-R": 79.32,
- "SICK-R-PL": 54.26,
- "SICKFr": 63.16,
- "STS12": 73.08,
- "STS13": 82.13,
- "STS14": 76.73,
- "STS15": 85.58,
- "STS16": 80.23,
- "STS17 (ar-ar)": 58.71,
- "STS17 (es-es)": 78.37,
- "STS17 (en-tr)": 0.43,
- "STS17 (es-en)": 22.01,
- "STS17 (it-en)": 24.28,
- "STS17 (en-de)": 27.54,
- "STS17 (en-en)": 88.63,
- "STS17 (ko-ko)": 43.37,
- "STS17 (nl-en)": 24.51,
- "STS17 (en-ar)": 0.54,
- "STS17 (fr-en)": 30.7,
- "STS22 (de-fr)": 43.52,
- "STS22 (tr)": 21.6,
- "STS22 (en)": 66.0,
- "STS22 (ar)": 17.54,
- "STS22 (pl-en)": 42.67,
- "STS22 (es-it)": 40.71,
- "STS22 (pl)": 19.22,
- "STS22 (fr-pl)": 16.9,
- "STS22 (es)": 43.98,
- "STS22 (ru)": 11.19,
- "STS22 (it)": 47.48,
- "STS22 (de-en)": 42.86,
- "STS22 (de)": 22.53,
- "STS22 (zh-en)": 44.39,
- "STS22 (de-pl)": 1.63,
- "STS22 (fr)": 69.51,
- "STS22 (zh)": 33.15,
- "STS22 (es-en)": 53.99,
- "STSB": 36.66,
- "STSBenchmark": 83.09,
- "STSBenchmarkMultilingualSTS (es)": 65.33,
- "STSBenchmarkMultilingualSTS (zh)": 38.93,
- "STSBenchmarkMultilingualSTS (en)": 83.09,
- "STSBenchmarkMultilingualSTS (fr)": 66.68,
- "STSBenchmarkMultilingualSTS (it)": 60.71,
- "STSBenchmarkMultilingualSTS (nl)": 60.03,
- "STSBenchmarkMultilingualSTS (pt)": 63.85,
- "STSBenchmarkMultilingualSTS (ru)": 56.09,
- "STSBenchmarkMultilingualSTS (de)": 63.28,
- "STSBenchmarkMultilingualSTS (pl)": 60.2
- },
- {
- "Model": "all-MiniLM-L12-v2",
- "STS17 (ar-ar)": 58.71,
- "STS17 (en-ar)": 0.54,
- "STS17 (en-de)": 27.54,
- "STS17 (en-en)": 88.63,
- "STS17 (en-tr)": 0.43,
- "STS17 (es-en)": 22.01,
- "STS17 (es-es)": 78.37,
- "STS17 (fr-en)": 30.7,
- "STS17 (it-en)": 24.28,
- "STS17 (ko-ko)": 43.37,
- "STS17 (nl-en)": 24.51,
- "STS22 (ar)": 17.54,
- "STS22 (de)": 22.53,
- "STS22 (de-en)": 42.86,
- "STS22 (de-fr)": 43.52,
- "STS22 (de-pl)": 1.63,
- "STS22 (en)": 65.67,
- "STS22 (es)": 43.98,
- "STS22 (es-en)": 53.99,
- "STS22 (es-it)": 40.71,
- "STS22 (fr)": 69.51,
- "STS22 (fr-pl)": 16.9,
- "STS22 (it)": 47.48,
- "STS22 (pl)": 19.22,
- "STS22 (pl-en)": 42.67,
- "STS22 (ru)": 11.19,
- "STS22 (tr)": 21.6,
- "STS22 (zh)": 33.15,
- "STS22 (zh-en)": 44.39
- }
- ]
- },
- "Summarization": {
- "cosine_spearman": [
- {
- "Model": "all-MiniLM-L12-v2",
- "SummEval": 27.9,
- "SummEvalFr": 26.63
- },
- {
- "Model": "all-MiniLM-L12-v2",
- "SummEval": 27.9,
- "SummEvalFr": 26.63
- }
- ]
- },
- "MultilabelClassification": {
- "accuracy": [
- {
- "Model": "all-MiniLM-L12-v2",
- "CEDRClassification": 33.86,
- "SensitiveTopicsClassification": 18.05
- }
- ]
- },
- "InstructionRetrieval": {
- "p-MRR": [
- {
- "Model": "all-MiniLM-L12-v2",
- "Core17InstructionRetrieval": 1.69,
- "News21InstructionRetrieval": -0.35,
- "Robust04InstructionRetrieval": -3.59
- }
- ]
- }
- },
"sentence-transformers__all-MiniLM-L6-v2": {
"BitextMining": {
"f1": [
@@ -20510,721 +17458,6 @@
"p-MRR": []
}
},
- "sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2": {
- "BitextMining": {
- "f1": [
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "BUCC (de-en)": 97.11,
- "BUCC (fr-en)": 94.99,
- "BUCC (ru-en)": 95.06,
- "BUCC (zh-en)": 95.63,
- "BornholmBitextMining": 19.67,
- "Tatoeba (est-eng)": 97.33,
- "Tatoeba (aze-eng)": 62.1,
- "Tatoeba (oci-eng)": 38.57,
- "Tatoeba (eus-eng)": 23.18,
- "Tatoeba (sqi-eng)": 98.17,
- "Tatoeba (yue-eng)": 71.45,
- "Tatoeba (ara-eng)": 87.93,
- "Tatoeba (wuu-eng)": 76.0,
- "Tatoeba (lvs-eng)": 97.87,
- "Tatoeba (ron-eng)": 95.3,
- "Tatoeba (uzb-eng)": 17.14,
- "Tatoeba (ell-eng)": 95.43,
- "Tatoeba (lit-eng)": 93.16,
- "Tatoeba (isl-eng)": 24.07,
- "Tatoeba (awa-eng)": 33.43,
- "Tatoeba (tuk-eng)": 15.16,
- "Tatoeba (nld-eng)": 94.58,
- "Tatoeba (ido-eng)": 40.25,
- "Tatoeba (afr-eng)": 58.22,
- "Tatoeba (cmn-eng)": 94.93,
- "Tatoeba (max-eng)": 45.25,
- "Tatoeba (tgl-eng)": 13.09,
- "Tatoeba (ind-eng)": 92.74,
- "Tatoeba (mkd-eng)": 91.0,
- "Tatoeba (bel-eng)": 67.73,
- "Tatoeba (hrv-eng)": 95.98,
- "Tatoeba (bre-eng)": 5.56,
- "Tatoeba (bul-eng)": 92.65,
- "Tatoeba (hun-eng)": 91.58,
- "Tatoeba (srp-eng)": 92.24,
- "Tatoeba (pol-eng)": 94.28,
- "Tatoeba (mhr-eng)": 6.89,
- "Tatoeba (fra-eng)": 91.72,
- "Tatoeba (lfn-eng)": 47.02,
- "Tatoeba (cha-eng)": 15.98,
- "Tatoeba (hsb-eng)": 36.1,
- "Tatoeba (rus-eng)": 91.87,
- "Tatoeba (khm-eng)": 32.11,
- "Tatoeba (arq-eng)": 18.6,
- "Tatoeba (gla-eng)": 3.61,
- "Tatoeba (ces-eng)": 95.12,
- "Tatoeba (ita-eng)": 93.05,
- "Tatoeba (csb-eng)": 21.56,
- "Tatoeba (kor-eng)": 92.52,
- "Tatoeba (ile-eng)": 57.71,
- "Tatoeba (xho-eng)": 4.52,
- "Tatoeba (deu-eng)": 97.02,
- "Tatoeba (heb-eng)": 86.88,
- "Tatoeba (yid-eng)": 14.38,
- "Tatoeba (cym-eng)": 13.25,
- "Tatoeba (jpn-eng)": 90.41,
- "Tatoeba (tzl-eng)": 25.46,
- "Tatoeba (ast-eng)": 62.17,
- "Tatoeba (war-eng)": 7.25,
- "Tatoeba (tha-eng)": 96.72,
- "Tatoeba (fry-eng)": 31.13,
- "Tatoeba (dtp-eng)": 5.69,
- "Tatoeba (hye-eng)": 93.28,
- "Tatoeba (mar-eng)": 92.38,
- "Tatoeba (cbk-eng)": 55.37,
- "Tatoeba (uig-eng)": 24.39,
- "Tatoeba (cor-eng)": 3.42,
- "Tatoeba (nov-eng)": 47.99,
- "Tatoeba (slv-eng)": 96.92,
- "Tatoeba (kzj-eng)": 6.24,
- "Tatoeba (dan-eng)": 94.8,
- "Tatoeba (hin-eng)": 97.62,
- "Tatoeba (ukr-eng)": 92.82,
- "Tatoeba (jav-eng)": 17.04,
- "Tatoeba (swe-eng)": 94.42,
- "Tatoeba (swg-eng)": 26.31,
- "Tatoeba (cat-eng)": 94.42,
- "Tatoeba (ina-eng)": 79.13,
- "Tatoeba (mal-eng)": 32.2,
- "Tatoeba (gle-eng)": 11.62,
- "Tatoeba (tel-eng)": 36.4,
- "Tatoeba (zsm-eng)": 95.31,
- "Tatoeba (swh-eng)": 14.48,
- "Tatoeba (tam-eng)": 24.64,
- "Tatoeba (epo-eng)": 41.73,
- "Tatoeba (pms-eng)": 30.7,
- "Tatoeba (mon-eng)": 95.04,
- "Tatoeba (kat-eng)": 95.44,
- "Tatoeba (urd-eng)": 94.57,
- "Tatoeba (vie-eng)": 95.12,
- "Tatoeba (fin-eng)": 93.1,
- "Tatoeba (tur-eng)": 95.08,
- "Tatoeba (arz-eng)": 51.26,
- "Tatoeba (fao-eng)": 27.51,
- "Tatoeba (pes-eng)": 92.59,
- "Tatoeba (tat-eng)": 10.25,
- "Tatoeba (nds-eng)": 32.16,
- "Tatoeba (nno-eng)": 76.34,
- "Tatoeba (ber-eng)": 4.43,
- "Tatoeba (bos-eng)": 93.27,
- "Tatoeba (slk-eng)": 95.15,
- "Tatoeba (spa-eng)": 95.42,
- "Tatoeba (pam-eng)": 5.41,
- "Tatoeba (ben-eng)": 36.48,
- "Tatoeba (ang-eng)": 10.24,
- "Tatoeba (kur-eng)": 46.94,
- "Tatoeba (por-eng)": 92.13,
- "Tatoeba (orv-eng)": 15.1,
- "Tatoeba (dsb-eng)": 33.43,
- "Tatoeba (amh-eng)": 36.21,
- "Tatoeba (kab-eng)": 1.16,
- "Tatoeba (kaz-eng)": 34.89,
- "Tatoeba (nob-eng)": 97.73,
- "Tatoeba (ceb-eng)": 8.05,
- "Tatoeba (lat-eng)": 19.47,
- "Tatoeba (glg-eng)": 94.0,
- "Tatoeba (gsw-eng)": 25.74
- }
- ]
- },
- "Classification": {
- "accuracy": [
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "AllegroReviews": 30.88,
- "AmazonCounterfactualClassification (en-ext)": 69.99,
- "AmazonCounterfactualClassification (en)": 71.57,
- "AmazonCounterfactualClassification (de)": 68.35,
- "AmazonCounterfactualClassification (ja)": 63.45,
- "AmazonPolarityClassification": 69.21,
- "AmazonReviewsClassification (en)": 35.11,
- "AmazonReviewsClassification (de)": 35.91,
- "AmazonReviewsClassification (es)": 37.49,
- "AmazonReviewsClassification (fr)": 35.3,
- "AmazonReviewsClassification (ja)": 33.24,
- "AmazonReviewsClassification (zh)": 35.26,
- "AngryTweetsClassification": 50.9,
- "Banking77Classification": 79.77,
- "CBD": 57.68,
- "DanishPoliticalCommentsClassification": 37.58,
- "EmotionClassification": 42.37,
- "GeoreviewClassification": 38.24,
- "HeadlineClassification": 68.3,
- "IFlyTek": 39.88,
- "ImdbClassification": 60.46,
- "InappropriatenessClassification": 58.18,
- "JDReview": 70.26,
- "KinopoiskClassification": 41.45,
- "LccSentimentClassification": 54.53,
- "MTOPDomainClassification (en)": 87.06,
- "MTOPDomainClassification (de)": 79.2,
- "MTOPDomainClassification (es)": 83.04,
- "MTOPDomainClassification (fr)": 78.63,
- "MTOPDomainClassification (hi)": 81.36,
- "MTOPDomainClassification (th)": 79.99,
- "MTOPIntentClassification (en)": 65.52,
- "MTOPIntentClassification (de)": 54.23,
- "MTOPIntentClassification (es)": 60.28,
- "MTOPIntentClassification (fr)": 54.05,
- "MTOPIntentClassification (hi)": 59.9,
- "MTOPIntentClassification (th)": 61.96,
- "MasakhaNEWSClassification (amh)": 64.28,
- "MasakhaNEWSClassification (eng)": 74.7,
- "MasakhaNEWSClassification (fra)": 76.09,
- "MasakhaNEWSClassification (hau)": 47.96,
- "MasakhaNEWSClassification (ibo)": 42.46,
- "MasakhaNEWSClassification (lin)": 59.26,
- "MasakhaNEWSClassification (lug)": 42.29,
- "MasakhaNEWSClassification (orm)": 34.98,
- "MasakhaNEWSClassification (pcm)": 89.54,
- "MasakhaNEWSClassification (run)": 47.2,
- "MasakhaNEWSClassification (sna)": 57.56,
- "MasakhaNEWSClassification (som)": 34.8,
- "MasakhaNEWSClassification (swa)": 46.05,
- "MasakhaNEWSClassification (tir)": 27.94,
- "MasakhaNEWSClassification (xho)": 44.81,
- "MasakhaNEWSClassification (yor)": 52.92,
- "MassiveIntentClassification (de)": 50.71,
- "MassiveIntentClassification (he)": 52.55,
- "MassiveIntentClassification (th)": 58.92,
- "MassiveIntentClassification (sw)": 29.56,
- "MassiveIntentClassification (zh-TW)": 58.74,
- "MassiveIntentClassification (vi)": 56.62,
- "MassiveIntentClassification (ko)": 50.36,
- "MassiveIntentClassification (en)": 66.89,
- "MassiveIntentClassification (hy)": 51.6,
- "MassiveIntentClassification (tl)": 33.67,
- "MassiveIntentClassification (nl)": 59.52,
- "MassiveIntentClassification (ur)": 52.79,
- "MassiveIntentClassification (fi)": 57.56,
- "MassiveIntentClassification (am)": 36.77,
- "MassiveIntentClassification (ka)": 43.03,
- "MassiveIntentClassification (it)": 59.66,
- "MassiveIntentClassification (kn)": 41.0,
- "MassiveIntentClassification (cy)": 26.13,
- "MassiveIntentClassification (lv)": 54.72,
- "MassiveIntentClassification (sq)": 56.6,
- "MassiveIntentClassification (ms)": 54.81,
- "MassiveIntentClassification (bn)": 35.38,
- "MassiveIntentClassification (hi)": 58.37,
- "MassiveIntentClassification (hu)": 60.44,
- "MassiveIntentClassification (nb)": 55.52,
- "MassiveIntentClassification (mn)": 51.77,
- "MassiveIntentClassification (sl)": 57.35,
- "MassiveIntentClassification (tr)": 59.91,
- "MassiveIntentClassification (fr)": 57.52,
- "MassiveIntentClassification (id)": 59.9,
- "MassiveIntentClassification (my)": 52.03,
- "MassiveIntentClassification (zh-CN)": 62.0,
- "MassiveIntentClassification (pl)": 59.43,
- "MassiveIntentClassification (es)": 59.7,
- "MassiveIntentClassification (ja)": 60.9,
- "MassiveIntentClassification (ta)": 36.82,
- "MassiveIntentClassification (el)": 58.7,
- "MassiveIntentClassification (ar)": 45.15,
- "MassiveIntentClassification (pt)": 61.29,
- "MassiveIntentClassification (jv)": 32.37,
- "MassiveIntentClassification (af)": 45.87,
- "MassiveIntentClassification (ru)": 59.06,
- "MassiveIntentClassification (te)": 40.77,
- "MassiveIntentClassification (az)": 47.43,
- "MassiveIntentClassification (is)": 30.87,
- "MassiveIntentClassification (ml)": 42.44,
- "MassiveIntentClassification (ro)": 58.44,
- "MassiveIntentClassification (km)": 40.04,
- "MassiveIntentClassification (fa)": 61.03,
- "MassiveIntentClassification (da)": 57.75,
- "MassiveIntentClassification (sv)": 59.43,
- "MassiveScenarioClassification (zh-CN)": 67.45,
- "MassiveScenarioClassification (bn)": 41.17,
- "MassiveScenarioClassification (sw)": 34.86,
- "MassiveScenarioClassification (es)": 65.07,
- "MassiveScenarioClassification (tr)": 66.53,
- "MassiveScenarioClassification (ar)": 51.71,
- "MassiveScenarioClassification (fi)": 63.74,
- "MassiveScenarioClassification (az)": 52.09,
- "MassiveScenarioClassification (mn)": 57.07,
- "MassiveScenarioClassification (km)": 46.95,
- "MassiveScenarioClassification (ro)": 64.2,
- "MassiveScenarioClassification (he)": 59.22,
- "MassiveScenarioClassification (en)": 71.54,
- "MassiveScenarioClassification (am)": 41.89,
- "MassiveScenarioClassification (zh-TW)": 65.72,
- "MassiveScenarioClassification (ko)": 55.71,
- "MassiveScenarioClassification (sl)": 64.01,
- "MassiveScenarioClassification (kn)": 45.72,
- "MassiveScenarioClassification (da)": 66.87,
- "MassiveScenarioClassification (tl)": 37.39,
- "MassiveScenarioClassification (de)": 57.4,
- "MassiveScenarioClassification (is)": 37.55,
- "MassiveScenarioClassification (sq)": 64.34,
- "MassiveScenarioClassification (id)": 66.17,
- "MassiveScenarioClassification (cy)": 31.71,
- "MassiveScenarioClassification (hu)": 66.57,
- "MassiveScenarioClassification (nl)": 65.53,
- "MassiveScenarioClassification (jv)": 38.62,
- "MassiveScenarioClassification (hy)": 56.11,
- "MassiveScenarioClassification (th)": 67.05,
- "MassiveScenarioClassification (af)": 53.63,
- "MassiveScenarioClassification (my)": 59.09,
- "MassiveScenarioClassification (fa)": 65.89,
- "MassiveScenarioClassification (ja)": 66.49,
- "MassiveScenarioClassification (ka)": 50.66,
- "MassiveScenarioClassification (vi)": 60.73,
- "MassiveScenarioClassification (ta)": 42.63,
- "MassiveScenarioClassification (el)": 66.14,
- "MassiveScenarioClassification (sv)": 67.14,
- "MassiveScenarioClassification (lv)": 59.82,
- "MassiveScenarioClassification (ru)": 65.25,
- "MassiveScenarioClassification (it)": 65.01,
- "MassiveScenarioClassification (ms)": 61.73,
- "MassiveScenarioClassification (pt)": 65.83,
- "MassiveScenarioClassification (ur)": 60.41,
- "MassiveScenarioClassification (pl)": 65.04,
- "MassiveScenarioClassification (nb)": 64.25,
- "MassiveScenarioClassification (hi)": 65.23,
- "MassiveScenarioClassification (te)": 46.49,
- "MassiveScenarioClassification (fr)": 64.52,
- "MassiveScenarioClassification (ml)": 47.73,
- "MultilingualSentiment": 61.9,
- "NoRecClassification": 46.7,
- "NordicLangClassification": 42.52,
- "OnlineShopping": 84.89,
- "PAC": 65.76,
- "PolEmo2.0-IN": 57.76,
- "PolEmo2.0-OUT": 28.7,
- "RuReviewsClassification": 58.88,
- "RuSciBenchGRNTIClassification": 53.19,
- "RuSciBenchOECDClassification": 41.41,
- "TNews": 39.19,
- "ToxicConversationsClassification": 66.07,
- "TweetSentimentExtractionClassification": 56.12,
- "Waimai": 82.27
- }
- ]
- },
- "Clustering": {
- "v_measure": [
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "8TagsClustering": 23.24,
- "AlloProfClusteringP2P": 56.06,
- "AlloProfClusteringS2S": 42.16,
- "ArxivClusteringP2P": 38.33,
- "ArxivClusteringS2S": 31.55,
- "BiorxivClusteringP2P": 33.49,
- "BiorxivClusteringS2S": 29.44,
- "BlurbsClusteringP2P": 32.46,
- "BlurbsClusteringS2S": 14.33,
- "GeoreviewClusteringP2P": 53.37,
- "HALClusteringS2S": 23.21,
- "MLSUMClusteringP2P (ru)": 37.0,
- "MLSUMClusteringP2P": 39.97,
- "MLSUMClusteringS2S (ru)": 38.16,
- "MLSUMClusteringS2S": 36.55,
- "MasakhaNEWSClusteringP2P (amh)": 40.36,
- "MasakhaNEWSClusteringP2P (eng)": 49.96,
- "MasakhaNEWSClusteringP2P (fra)": 36.58,
- "MasakhaNEWSClusteringP2P (hau)": 19.39,
- "MasakhaNEWSClusteringP2P (ibo)": 33.81,
- "MasakhaNEWSClusteringP2P (lin)": 51.98,
- "MasakhaNEWSClusteringP2P (lug)": 41.88,
- "MasakhaNEWSClusteringP2P (orm)": 22.23,
- "MasakhaNEWSClusteringP2P (pcm)": 64.64,
- "MasakhaNEWSClusteringP2P (run)": 48.03,
- "MasakhaNEWSClusteringP2P (sna)": 44.62,
- "MasakhaNEWSClusteringP2P (som)": 27.54,
- "MasakhaNEWSClusteringP2P (swa)": 22.69,
- "MasakhaNEWSClusteringP2P (tir)": 42.02,
- "MasakhaNEWSClusteringP2P (xho)": 27.68,
- "MasakhaNEWSClusteringP2P (yor)": 27.29,
- "MasakhaNEWSClusteringS2S (amh)": 42.28,
- "MasakhaNEWSClusteringS2S (eng)": 25.74,
- "MasakhaNEWSClusteringS2S (fra)": 33.9,
- "MasakhaNEWSClusteringS2S (hau)": 9.2,
- "MasakhaNEWSClusteringS2S (ibo)": 33.37,
- "MasakhaNEWSClusteringS2S (lin)": 47.76,
- "MasakhaNEWSClusteringS2S (lug)": 45.15,
- "MasakhaNEWSClusteringS2S (orm)": 22.08,
- "MasakhaNEWSClusteringS2S (pcm)": 58.42,
- "MasakhaNEWSClusteringS2S (run)": 47.41,
- "MasakhaNEWSClusteringS2S (sna)": 43.0,
- "MasakhaNEWSClusteringS2S (som)": 26.22,
- "MasakhaNEWSClusteringS2S (swa)": 13.53,
- "MasakhaNEWSClusteringS2S (tir)": 42.4,
- "MasakhaNEWSClusteringS2S (xho)": 21.03,
- "MasakhaNEWSClusteringS2S (yor)": 27.04,
- "MedrxivClusteringP2P": 31.52,
- "MedrxivClusteringS2S": 30.87,
- "RedditClustering": 42.02,
- "RedditClusteringP2P": 50.73,
- "RuSciBenchGRNTIClusteringP2P": 48.22,
- "RuSciBenchOECDClusteringP2P": 41.68,
- "StackExchangeClustering": 49.6,
- "StackExchangeClusteringP2P": 31.69,
- "TenKGnadClusteringP2P": 36.13,
- "TenKGnadClusteringS2S": 22.26,
- "TwentyNewsgroupsClustering": 39.28
- }
- ]
- },
- "PairClassification": {
- "max_ap": [
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "CDSC-E": 72.22,
- "OpusparcusPC (de)": 96.63,
- "OpusparcusPC (en)": 98.59,
- "OpusparcusPC (fi)": 93.2,
- "OpusparcusPC (fr)": 92.01,
- "OpusparcusPC (ru)": 88.25,
- "OpusparcusPC (sv)": 93.99,
- "PSC": 97.14,
- "PawsXPairClassification (de)": 53.26,
- "PawsXPairClassification (en)": 55.94,
- "PawsXPairClassification (es)": 54.61,
- "PawsXPairClassification (fr)": 56.94,
- "PawsXPairClassification (ja)": 48.66,
- "PawsXPairClassification (ko)": 49.69,
- "PawsXPairClassification (zh)": 54.3,
- "SICK-E-PL": 71.94,
- "SprintDuplicateQuestions": 89.46,
- "TERRa": 58.56,
- "TwitterSemEval2015": 62.06,
- "TwitterURLCorpus": 83.83
- },
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "CDSC-E": 72.33,
- "OpusparcusPC (de)": 96.63,
- "OpusparcusPC (en)": 98.59,
- "OpusparcusPC (fi)": 93.2,
- "OpusparcusPC (fr)": 92.01,
- "OpusparcusPC (ru)": 88.25,
- "OpusparcusPC (sv)": 93.99,
- "PPC": 92.36,
- "PSC": 97.14,
- "PawsXPairClassification (de)": 53.34,
- "PawsXPairClassification (en)": 55.94,
- "PawsXPairClassification (es)": 54.61,
- "PawsXPairClassification (fr)": 57.03,
- "PawsXPairClassification (ja)": 48.84,
- "PawsXPairClassification (ko)": 49.86,
- "PawsXPairClassification (zh)": 54.59,
- "SICK-E-PL": 71.94,
- "SprintDuplicateQuestions": 91.15,
- "TERRa": 58.56,
- "TwitterSemEval2015": 65.06,
- "TwitterURLCorpus": 83.83
- },
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "CDSC-E": 72.22,
- "OpusparcusPC (fr)": 92.01,
- "PPC": 91.8,
- "PSC": 97.14,
- "PawsXPairClassification (fr)": 56.94,
- "SICK-E-PL": 71.94,
- "SprintDuplicateQuestions": 89.46,
- "TwitterSemEval2015": 62.06,
- "TwitterURLCorpus": 83.83
- }
- ]
- },
- "Reranking": {
- "map": [
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "AlloprofReranking": 49.01,
- "AskUbuntuDupQuestions": 60.49,
- "MMarcoReranking": 16.14,
- "MindSmallReranking": 30.37,
- "RuBQReranking": 52.8,
- "SciDocsRR": 77.78,
- "StackOverflowDupQuestions": 45.85,
- "SyntecReranking": 75.03,
- "T2Reranking": 65.28
- }
- ]
- },
- "Retrieval": {
- "ndcg_at_10": [
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "AILACasedocs": 13.66,
- "AILAStatutes": 20.52,
- "ARCChallenge": 6.19,
- "AlloprofRetrieval": 26.63,
- "AlphaNLI": 20.89,
- "AppsRetrieval": 2.0,
- "ArguAna": 44.88,
- "ArguAna-PL": 37.83,
- "BSARDRetrieval": 0.0,
- "CQADupstackRetrieval": 30.7,
- "ClimateFEVER": 18.49,
- "CmedqaRetrieval": 10.78,
- "CodeFeedbackMT": 12.53,
- "CodeFeedbackST": 32.53,
- "CodeSearchNetCCRetrieval (python)": 44.49,
- "CodeSearchNetCCRetrieval (javascript)": 43.56,
- "CodeSearchNetCCRetrieval (go)": 28.54,
- "CodeSearchNetCCRetrieval (ruby)": 43.84,
- "CodeSearchNetCCRetrieval (java)": 26.23,
- "CodeSearchNetCCRetrieval (php)": 16.3,
- "CodeSearchNetRetrieval (python)": 58.15,
- "CodeSearchNetRetrieval (javascript)": 46.41,
- "CodeSearchNetRetrieval (go)": 61.49,
- "CodeSearchNetRetrieval (ruby)": 56.44,
- "CodeSearchNetRetrieval (java)": 32.49,
- "CodeSearchNetRetrieval (php)": 45.01,
- "CodeTransOceanContest": 25.67,
- "CodeTransOceanDL": 11.65,
- "CosQA": 14.24,
- "CovidRetrieval": 30.11,
- "DBPedia": 22.63,
- "DBPedia-PL": 18.0,
- "DuRetrieval": 34.72,
- "EcomRetrieval": 13.32,
- "FEVER": 52.66,
- "FiQA-PL": 12.49,
- "FiQA2018": 20.33,
- "GerDaLIRSmall": 2.62,
- "HellaSwag": 16.98,
- "HotpotQA": 30.01,
- "HotpotQA-PL": 22.76,
- "LEMBNarrativeQARetrieval": 13.82,
- "LEMBQMSumRetrieval": 11.02,
- "LEMBSummScreenFDRetrieval": 38.12,
- "LEMBWikimQARetrieval": 40.84,
- "LeCaRDv2": 32.03,
- "LegalBenchConsumerContractsQA": 49.81,
- "LegalBenchCorporateLobbying": 88.51,
- "LegalQuAD": 13.31,
- "LegalSummarization": 54.97,
- "MMarcoRetrieval": 46.62,
- "MSMARCO": 23.72,
- "MSMARCO-PL": 10.39,
- "MedicalRetrieval": 15.46,
- "MintakaRetrieval (ar)": 12.61,
- "MintakaRetrieval (de)": 21.77,
- "MintakaRetrieval (es)": 21.59,
- "MintakaRetrieval (fr)": 21.53,
- "MintakaRetrieval (hi)": 16.76,
- "MintakaRetrieval (it)": 22.23,
- "MintakaRetrieval (ja)": 14.33,
- "MintakaRetrieval (pt)": 22.52,
- "NFCorpus": 23.45,
- "NFCorpus-PL": 17.16,
- "NQ": 29.8,
- "NQ-PL": 12.56,
- "PIQA": 15.79,
- "Quail": 2.96,
- "Quora-PL": 77.18,
- "QuoraRetrieval": 86.55,
- "RARbCode": 8.48,
- "RARbMath": 30.02,
- "RiaNewsRetrieval": 44.82,
- "RuBQRetrieval": 29.7,
- "SCIDOCS": 0.03,
- "SCIDOCS-PL": 10.26,
- "SIQA": 0.88,
- "SciFact": 48.37,
- "SciFact-PL": 40.24,
- "SpartQA": 4.94,
- "StackOverflowQA": 37.94,
- "SyntecRetrieval": 65.54,
- "SyntheticText2SQL": 30.57,
- "T2Retrieval": 30.31,
- "TRECCOVID": 39.12,
- "TRECCOVID-PL": 34.38,
- "TempReasonL1": 1.43,
- "TempReasonL2Fact": 6.21,
- "TempReasonL2Pure": 0.22,
- "TempReasonL3Fact": 6.77,
- "TempReasonL3Pure": 4.9,
- "Touche2020": 16.06,
- "VideoRetrieval": 14.71,
- "WinoGrande": 46.52,
- "XPQARetrieval (ara-ara)": 22.97,
- "XPQARetrieval (eng-ara)": 17.17,
- "XPQARetrieval (ara-eng)": 25.5,
- "XPQARetrieval (deu-deu)": 42.62,
- "XPQARetrieval (eng-deu)": 26.52,
- "XPQARetrieval (deu-eng)": 48.73,
- "XPQARetrieval (spa-spa)": 38.24,
- "XPQARetrieval (eng-spa)": 26.09,
- "XPQARetrieval (spa-eng)": 41.51,
- "XPQARetrieval (fra-fra)": 42.51,
- "XPQARetrieval (eng-fra)": 26.09,
- "XPQARetrieval (fra-eng)": 43.08,
- "XPQARetrieval (hin-hin)": 52.09,
- "XPQARetrieval (eng-hin)": 24.08,
- "XPQARetrieval (hin-eng)": 49.11,
- "XPQARetrieval (ita-ita)": 51.63,
- "XPQARetrieval (eng-ita)": 29.34,
- "XPQARetrieval (ita-eng)": 46.53,
- "XPQARetrieval (jpn-jpn)": 51.57,
- "XPQARetrieval (eng-jpn)": 23.87,
- "XPQARetrieval (jpn-eng)": 44.93,
- "XPQARetrieval (kor-kor)": 21.36,
- "XPQARetrieval (eng-kor)": 21.51,
- "XPQARetrieval (kor-eng)": 22.59,
- "XPQARetrieval (pol-pol)": 28.5,
- "XPQARetrieval (eng-pol)": 17.08,
- "XPQARetrieval (pol-eng)": 26.54,
- "XPQARetrieval (por-por)": 32.33,
- "XPQARetrieval (eng-por)": 19.76,
- "XPQARetrieval (por-eng)": 34.21,
- "XPQARetrieval (tam-tam)": 6.37,
- "XPQARetrieval (eng-tam)": 5.36,
- "XPQARetrieval (tam-eng)": 9.03,
- "XPQARetrieval (cmn-cmn)": 44.16,
- "XPQARetrieval (eng-cmn)": 19.03,
- "XPQARetrieval (cmn-eng)": 40.08,
- "XPQARetrieval (fr)": 42.51
- }
- ]
- },
- "STS": {
- "cosine_spearman": [
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "AFQMC": 14.3,
- "ATEC": 18.42,
- "BIOSSES": 74.18,
- "BQ": 38.53,
- "CDSC-R": 88.98,
- "LCQMC": 63.96,
- "PAWSX": 10.13,
- "RUParaPhraserSTS": 61.87,
- "RuSTSBenchmarkSTS": 79.55,
- "SICK-R": 79.61,
- "SICK-R-PL": 68.77,
- "SICKFr": 75.1,
- "STS12": 76.02,
- "STS13": 80.7,
- "STS14": 78.85,
- "STS15": 85.84,
- "STS16": 81.05,
- "STS17 (es-es)": 85.56,
- "STS17 (nl-en)": 81.71,
- "STS17 (ko-ko)": 77.03,
- "STS17 (en-ar)": 81.22,
- "STS17 (it-en)": 82.35,
- "STS17 (en-tr)": 76.74,
- "STS17 (en-de)": 84.22,
- "STS17 (fr-en)": 76.59,
- "STS17 (en-en)": 86.87,
- "STS17 (ar-ar)": 79.16,
- "STS17 (es-en)": 84.44,
- "STS22 (pl)": 33.74,
- "STS22 (es)": 56.56,
- "STS22 (en)": 62.07,
- "STS22 (fr)": 70.55,
- "STS22 (es-en)": 67.33,
- "STS22 (fr-pl)": 50.71,
- "STS22 (pl-en)": 69.02,
- "STS22 (de-fr)": 51.73,
- "STS22 (de)": 44.64,
- "STS22 (ar)": 46.2,
- "STS22 (de-pl)": 44.22,
- "STS22 (zh-en)": 65.71,
- "STS22 (it)": 55.22,
- "STS22 (tr)": 53.39,
- "STS22 (ru)": 57.08,
- "STS22 (zh)": 58.75,
- "STS22 (es-it)": 47.67,
- "STS22 (de-en)": 52.65,
- "STSB": 78.91,
- "STSBenchmark": 84.42,
- "STSBenchmarkMultilingualSTS (es)": 81.1,
- "STSBenchmarkMultilingualSTS (zh)": 80.47,
- "STSBenchmarkMultilingualSTS (en)": 84.42,
- "STSBenchmarkMultilingualSTS (de)": 78.87,
- "STSBenchmarkMultilingualSTS (nl)": 79.54,
- "STSBenchmarkMultilingualSTS (pl)": 78.29,
- "STSBenchmarkMultilingualSTS (fr)": 79.9,
- "STSBenchmarkMultilingualSTS (pt)": 80.16,
- "STSBenchmarkMultilingualSTS (it)": 80.39,
- "STSBenchmarkMultilingualSTS (ru)": 79.32
- },
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "BIOSSES": 74.18,
- "CDSC-R": 88.98,
- "SICK-R": 79.61,
- "SICK-R-PL": 68.77,
- "SICKFr": 75.1,
- "STS12": 76.02,
- "STS13": 80.7,
- "STS14": 78.85,
- "STS15": 85.84,
- "STS16": 81.05,
- "STS17 (ar-ar)": 79.16,
- "STS17 (en-ar)": 81.22,
- "STS17 (en-de)": 84.22,
- "STS17 (en-en)": 86.87,
- "STS17 (en-tr)": 76.74,
- "STS17 (es-en)": 84.44,
- "STS17 (es-es)": 85.56,
- "STS17 (fr-en)": 76.59,
- "STS17 (it-en)": 82.35,
- "STS17 (ko-ko)": 77.03,
- "STS17 (nl-en)": 81.71,
- "STS22 (pl)": 33.73,
- "STS22 (fr)": 70.55,
- "STSBenchmark": 84.42,
- "STSBenchmarkMultilingualSTS (fr)": 79.9
- }
- ]
- },
- "Summarization": {
- "cosine_spearman": [
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "SummEval": 30.67,
- "SummEvalFr": 29.2
- },
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "SummEval": 30.67,
- "SummEvalFr": 29.2
- },
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "SummEval": 30.67,
- "SummEvalFr": 29.2
- }
- ]
- },
- "MultilabelClassification": {
- "accuracy": [
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "CEDRClassification": 37.76,
- "SensitiveTopicsClassification": 24.84
- }
- ]
- },
- "InstructionRetrieval": {
- "p-MRR": [
- {
- "Model": "paraphrase-multilingual-MiniLM-L12-v2",
- "Core17InstructionRetrieval": -0.62,
- "News21InstructionRetrieval": -0.37,
- "Robust04InstructionRetrieval": -2.94
- }
- ]
- }
- },
"sentence-transformers__paraphrase-multilingual-mpnet-base-v2": {
"BitextMining": {
"f1": [
diff --git a/config.yaml b/config.yaml
index be1e9b4d..ca3d4f15 100644
--- a/config.yaml
+++ b/config.yaml
@@ -53,6 +53,11 @@ tasks:
metric: "p-MRR"
metric_description: "paired mean reciprocal rank (p-MRR)"
task_description: "Retrieval w/Instructions is the task of finding relevant documents for a query that has detailed instructions."
+ SummaryRetrieval:
+ icon: "📜🔎"
+ metric: "f1"
+ metric_description: "[F1](https://huggingface.co/spaces/evaluate-metric/f1)"
+ task_description: "SummaryRetrieval is the task of finding true summary for a text."
boards:
en:
title: English
@@ -374,6 +379,79 @@ boards:
MultilabelClassification:
- CEDRClassification
- SensitiveTopicsClassification
+ fa:
+ title: Persian
+ language_long: "Persian (Farsi)"
+ has_overall: true
+ acronym: null
+ icon: "🇮🇷"
+ special_icons: null
+ credits: "[MCINect: ](https://huggingface.co/MCINext)[Erfan Zeinivand](https://github.com/Erfun76), [Morteza Alikhani](https://github.com/mortezaalikhani), [Mehran Sarmadi](https://github.com/mehran-sarmadi)"
+ tasks:
+ Classification:
+ - PersianFoodSentimentClassification
+ - SynPerChatbotConvSAClassification
+ - SynPerChatbotConvSAToneClassification
+ - SynPerChatbotSatisfactionLevelClassification
+ - SynPerChatbotRAGToneClassification
+ - SynPerChatbotToneClassification
+ - PersianTextTone
+ - SIDClassification
+ - DeepSentiPers
+ - PersianTextEmotion
+ - SentimentDKSF
+ - NLPTwitterAnalysisClassification
+ - DigikalamagClassification
+ - MassiveIntentClassification (fa)
+ - MassiveScenarioClassification (fa)
+ Clustering:
+ - BeytooteClustering
+ - DigikalamagClustering
+ - HamshahriClustring
+ - NLPTwitterAnalysisClustering
+ - SIDClustring
+ PairClassification:
+ - FarsTail
+ - CExaPPC
+ - SynPerChatbotRAGFAQPC
+ - FarsiParaphraseDetection
+ - SynPerTextKeywordsPC
+ - SynPerQAFaPC
+ - ParsinluEntail
+ - ParsinluQueryParaphPC
+ Reranking:
+ - MIRACLReranking (fa)
+ - WikipediaRerankingMultilingual (fa)
+ Retrieval:
+ - SynPerQARetrieval
+ - SynPerChatbotTopicsRetrieval
+ - SynPerChatbotRAGTopicsRetrieval
+ - SynPerChatbotRAGFAQRetrieval
+ - PersianWebDocumentRetrieval
+ - WikipediaRetrievalMultilingual (fa)
+ - MIRACLRetrieval (fa)
+ - ClimateFEVER-Fa
+ - DBPedia-Fa
+ - HotpotQA-Fa
+ - MSMARCO-Fa
+ - NQ-Fa
+ - ArguAna-Fa
+ - CQADupstackRetrieval-Fa
+ - FiQA2018-Fa
+ - NFCorpus-Fa
+ - QuoraRetrieval-Fa
+ - SCIDOCS-Fa
+ - SciFact-Fa
+ - TRECCOVID-Fa
+ - Touche2020-Fa
+ STS:
+ - Farsick
+ - SynPerSTS
+ - Query2Query
+ SummaryRetrieval:
+ - SAMSumFa
+ - SynPerChatbotSumSRetrieval
+ - SynPerChatbotRAGSumSRetrieval
se:
title: Swedish
language_long: Swedish
diff --git a/model_meta.yaml b/model_meta.yaml
index 223126ce..e770cd9d 100644
--- a/model_meta.yaml
+++ b/model_meta.yaml
@@ -1810,6 +1810,96 @@ model_meta:
is_proprietary: false
is_sentence_transformers_compatible: false
uses_instruct: true
+ BAAI__bge-m3-unsupervised:
+ link: https://huggingface.co/BAAI/bge-m3-unsupervised
+ seq_len: 8194
+ size: 568
+ dim: 1024
+ is_external: true
+ is_proprietary: false
+ is_sentence_transformers_compatible: true
+ uses_instruct: false
+ jinaai__jina-embeddings-v3:
+ link: https://huggingface.co/jinaai/jina-embeddings-v3
+ seq_len: 8194
+ size: 572
+ dim: 1024
+ is_external: true
+ is_proprietary: false
+ is_sentence_transformers_compatible: true
+ uses_instruct: false
+ Alibaba-NLP__gte-multilingual-base:
+ link: https://huggingface.co/Alibaba-NLP/gte-multilingual-base
+ seq_len: 8192
+ size: 305
+ dim: 768
+ is_external: true
+ is_proprietary: false
+ is_sentence_transformers_compatible: true
+ uses_instruct: false
+ HooshvareLab__bert-base-parsbert-uncased:
+ link: https://huggingface.co/HooshvareLab/bert-base-parsbert-uncased
+ seq_len: 512
+ size: 110
+ dim: 768
+ is_external: true
+ is_proprietary: false
+ is_sentence_transformers_compatible: true
+ uses_instruct: false
+ m3hrdadfi__bert-zwnj-wnli-mean-tokens:
+ link: https://huggingface.co/m3hrdadfi/bert-zwnj-wnli-mean-tokens
+ seq_len: 512
+ size: 110
+ dim: 768
+ is_external: true
+ is_proprietary: false
+ is_sentence_transformers_compatible: true
+ uses_instruct: false
+ m3hrdadfi__roberta-zwnj-wnli-mean-tokens:
+ link: https://huggingface.co/HooshvareLab/roberta-fa-zwnj-base
+ seq_len: 514
+ size: 110
+ dim: 768
+ is_external: true
+ is_proprietary: false
+ is_sentence_transformers_compatible: true
+ uses_instruct: false
+ myrkur__sentence-transformer-parsbert-fa:
+ link: https://huggingface.co/myrkur/sentence-transformer-parsbert-fa
+ seq_len: 512
+ size: 163
+ dim: 768
+ is_external: true
+ is_proprietary: false
+ is_sentence_transformers_compatible: true
+ uses_instruct: false
+ PartAI__TookaBERT-Base:
+ link: https://huggingface.co/PartAI/TookaBERT-Base
+ seq_len: 512
+ size: 123
+ dim: 768
+ is_external: true
+ is_proprietary: false
+ is_sentence_transformers_compatible: true
+ uses_instruct: false
+ PartAI__Tooka-SBERT:
+ link: https://huggingface.co/PartAI/Tooka-SBERT
+ seq_len: 512
+ size: 353
+ dim: 1024
+ is_external: true
+ is_proprietary: false
+ is_sentence_transformers_compatible: true
+ uses_instruct: false
+ sbunlp__fabert:
+ link: https://huggingface.co/sbunlp/fabert
+ seq_len: 512
+ size: 124
+ dim: 768
+ is_external: true
+ is_proprietary: false
+ is_sentence_transformers_compatible: true
+ uses_instruct: false
models_to_skip:
- michaelfeil/ct2fast-e5-large-v2
- McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-unsup-simcse
diff --git a/refresh.py b/refresh.py
index 51e8be4b..36b93cc7 100644
--- a/refresh.py
+++ b/refresh.py
@@ -25,6 +25,8 @@
"InstructionRetrieval": "Retrieval w/Instructions",
"PairClassification": "Pair Classification",
"BitextMining": "Bitext Mining",
+ "SummaryRetrieval": "Summary Retrieval",
+
}
TASK_TO_METRIC = {k: [v["metric"]] for k, v in TASKS_CONFIG.items()}