Add FaMTEB (Farsi/Persian Text Embedding Benchmark)

mteb/abstasks/TaskMetadata.py

@@ -96,6 +96,7 @@
     "Summarization",
     "InstructionRetrieval",
     "Speed",
+    "SummaryRetrieval",
 ]
 
 

mteb/benchmarks/benchmarks.py

@@ -1222,6 +1222,86 @@ def load_results(
 }""",
 )
 
+FA_MTEB = Benchmark(
+    name="FaMTEB(fas, beta)",
+    tasks=get_tasks(
+        languages=["fas"],
+        tasks=[
+            # Classification
+            "PersianFoodSentimentClassification",
+            "SynPerChatbotConvSAClassification",
+            "SynPerChatbotConvSAToneChatbotClassification",
+            "SynPerChatbotConvSAToneUserClassification",
+            "SynPerChatbotSatisfactionLevelClassification",
+            "SynPerChatbotRAGToneChatbotClassification",
+            "SynPerChatbotRAGToneUserClassification",
+            "SynPerChatbotToneChatbotClassification",
+            "SynPerChatbotToneUserClassification",
+            "PersianTextTone",
+            "SIDClassification",
+            "DeepSentiPers",
+            "PersianTextEmotion",
+            "SentimentDKSF",
+            "NLPTwitterAnalysisClassification",
+            "DigikalamagClassification",
+            "MassiveIntentClassification",
+            "MassiveScenarioClassification",
+            # Clustering
+            "BeytooteClustering",
+            "DigikalamagClustering",
+            "HamshahriClustring",
+            "NLPTwitterAnalysisClustering",
+            "SIDClustring",
+            # PairClassification
+            "FarsTail",
+            "CExaPPC",
+            "SynPerChatbotRAGFAQPC",
+            "FarsiParaphraseDetection",
+            "SynPerTextKeywordsPC",
+            "SynPerQAPC",
+            "ParsinluEntail",
+            "ParsinluQueryParaphPC",
+            # Reranking
+            "MIRACLReranking",
+            "WikipediaRerankingMultilingual",
+            # Retrieval
+            "SynPerQARetrieval",
+            "SynPerChatbotTopicsRetrieval",
+            "SynPerChatbotRAGTopicsRetrieval",
+            "SynPerChatbotRAGFAQRetrieval",
+            "PersianWebDocumentRetrieval",
+            "WikipediaRetrievalMultilingual",
+            "MIRACLRetrieval",
+            "ClimateFEVER-Fa",
+            "DBPedia-Fa",
+            "HotpotQA-Fa",
+            "MSMARCO-Fa",
+            "NQ-Fa",
+            "ArguAna-Fa",
+            "CQADupstackRetrieval-Fa",
+            "FiQA2018-Fa",
+            "NFCorpus-Fa",
+            "QuoraRetrieval-Fa",
+            "SCIDOCS-Fa",
+            "SciFact-Fa",
+            "TRECCOVID-Fa",
+            "Touche2020-Fa",
+            # STS
+            "Farsick",
+            "SynPerSTS",
+            "Query2Query",
+            # SummaryRetrieval
+            "SAMSumFa",
+            "SynPerChatbotSumSRetrieval",
+            "SynPerChatbotRAGSumSRetrieval",
+        ],
+    ),
+    description="Main Persian (Farsi) benchmarks from MTEB",
+    reference=None,
+    citation=None,
+    contacts=["mehran-sarmadi", "ERfun", "morteza20"],
+)
+
 CHEMTEB = Benchmark(
     name="ChemTEB",
     tasks=get_tasks(

mteb/tasks/Classification/__init__.py

@@ -57,6 +57,7 @@
 from .eng.YahooAnswersTopicsClassification import *
 from .eng.YelpReviewFullClassification import *
 from .est.estonian_valence import *
+from .fas.FaMTEBClassification import *
 from .fas.PersianFoodSentimentClassification import *
 from .fil.FilipinoHateSpeechClassification import *
 from .fil.FilipinoShopeeReviewsClassification import *