Skip to content

fix: Add multilingual Benchmark #1252

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
KennethEnevoldsen merged 3 commits into from
Sep 28, 2024
Merged

fix: Add multilingual Benchmark #1252

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
78bc9ac
fix: Add multilingual bench
KennethEnevoldsen Sep 28, 2024
7e44d48
Update mteb/benchmarks/benchmarks.py
KennethEnevoldsen Sep 28, 2024
fc5f6c6
format
KennethEnevoldsen Sep 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 143 additions & 0 deletions mteb/benchmarks/benchmarks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,3 +544,146 @@ def __getitem__(self, index):
reference=None,
citation=None,
)


MTEB_multilingual = Benchmark(
name="MTEB(Multilingual)",
tasks=get_tasks(
tasks=[
"BornholmBitextMining",
"BibleNLPBitextMining",
"BUCC.v2",
"DiaBlaBitextMining",
"FloresBitextMining",
"IN22GenBitextMining",
"IndicGenBenchFloresBitextMining",
"NollySentiBitextMining",
"NorwegianCourtsBitextMining",
"NTREXBitextMining",
"NusaTranslationBitextMining",
"NusaXBitextMining",
"Tatoeba",
"BulgarianStoreReviewSentimentClassfication",
"CzechProductReviewSentimentClassification",
"GreekLegalCodeClassification",
"DBpediaClassification",
"FinancialPhrasebankClassification",
"PoemSentimentClassification",
"ToxicConversationsClassification",
"TweetTopicSingleClassification",
"EstonianValenceClassification",
"FilipinoShopeeReviewsClassification",
"GujaratiNewsClassification",
"SentimentAnalysisHindi",
"IndonesianIdClickbaitClassification",
"ItaCaseholdClassification",
"KorSarcasmClassification",
"KurdishSentimentClassification",
"MacedonianTweetSentimentClassification",
"AfriSentiClassification",
"AmazonCounterfactualClassification",
"CataloniaTweetClassification",
"CyrillicTurkicLangClassification",
"IndicLangClassification",
"MasakhaNEWSClassification",
"MassiveIntentClassification",
"MultiHateClassification",
"NordicLangClassification",
"NusaParagraphEmotionClassification",
"NusaX-senti",
"ScalaClassification",
"SwissJudgementClassification",
"NepaliNewsClassification",
"OdiaNewsClassification",
"PunjabiNewsClassification",
"PolEmo2.0-OUT",
"PAC",
"SinhalaNewsClassification",
"CSFDSKMovieReviewSentimentClassification",
"SiswatiNewsClassification",
"SlovakMovieReviewSentimentClassification",
"SwahiliNewsClassification",
"DalajClassification",
"TswanaNewsClassification",
"IsiZuluNewsClassification",
"WikiCitiesClustering",
"MasakhaNEWSClusteringS2S",
"RomaniBibleClustering",
"ArXivHierarchicalClusteringP2P",
"ArXivHierarchicalClusteringS2S",
"BigPatentClustering.v2",
"BiorxivClusteringP2P.v2",
"MedrxivClusteringP2P.v2",
"StackExchangeClustering.v2",
"AlloProfClusteringS2S.v2",
"HALClusteringS2S.v2",
"SIB200ClusteringS2S",
"WikiClusteringP2P.v2",
"SNLHierarchicalClusteringP2P",
"PlscClusteringP2P.v2",
"SwednClusteringP2P",
"CLSClusteringP2P.v2",
"StackOverflowQA",
"TwitterHjerneRetrieval",
"AILAStatutes",
"ArguAna",
"HagridRetrieval",
"LegalBenchCorporateLobbying",
"LEMBPasskeyRetrieval",
"SCIDOCS",
"SpartQA",
"TempReasonL1",
"TRECCOVID",
"WinoGrande",
"BelebeleRetrieval",
"MLQARetrieval",
"StatcanDialogueDatasetRetrieval",
"WikipediaRetrievalMultilingual",
"CovidRetrieval",
"Core17InstructionRetrieval",
"News21InstructionRetrieval",
"Robust04InstructionRetrieval",
"KorHateSpeechMLClassification",
"MalteseNewsClassification",
"MultiEURLEXMultilabelClassification",
"BrazilianToxicTweetsClassification",
"CEDRClassification",
"CTKFactsNLI",
"SprintDuplicateQuestions",
"TwitterURLCorpus",
"ArmenianParaphrasePC",
"indonli",
"OpusparcusPC",
"PawsXPairClassification",
"RTE3",
"XNLI",
"PpcPC",
"TERRa",
"WebLINXCandidatesReranking",
"AlloprofReranking",
"VoyageMMarcoReranking",
"WikipediaRerankingMultilingual",
"RuBQReranking",
"T2Reranking",
"GermanSTSBenchmark",
"SICK-R",
"STS12",
"STS13",
"STS14",
"STS15",
"STSBenchmark",
"FaroeseSTS",
"FinParaSTS",
"JSICK",
"IndicCrosslingualSTS",
"SemRel24STS",
"STS17",
"STS22.v2",
"STSES",
"STSB",
],
),
description="The Multilingual benchmarks from MMTEB. Currently under development.",
reference=None,
citation=None,
)
6 changes: 4 additions & 2 deletions scripts/mmteb_create_author_list.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -907,7 +907,7 @@
" affiations[aff] = aff_id\n",
" aff_id += 1\n",
" aff_string += f\"{affiations[aff]},\"\n",
" \n",
"\n",
" # remove last comma\n",
" aff_string = aff_string[:-1]\n",
"\n",
Expand Down Expand Up @@ -936,7 +936,9 @@
"last_author2 = \"Siva\"\n",
"last_author__ = [a for a in author_list if last_author2 in a][0]\n",
"# remove from author list\n",
"author_list = [a for a in author_list if last_author1 not in a and last_author2 not in a]\n",
"author_list = [\n",
" a for a in author_list if last_author1 not in a and last_author2 not in a\n",
"]\n",
"\n",
"author_list.append(last_author__)\n",
"author_list.append(last_author_)"
Expand Down