Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
431 changes: 239 additions & 192 deletions mteb/benchmarks/benchmarks.py

Large diffs are not rendered by default.

22 changes: 11 additions & 11 deletions mteb/tasks/BitextMining/dan/BornholmskBitextMining.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,18 @@ class BornholmBitextMining(AbsTaskBitextMining):
annotations_creators="expert-annotated",
dialect=["da-dan-bornholm"],
sample_creation="created",
bibtex_citation="""
bibtex_citation=r"""
@inproceedings{derczynskiBornholmskNaturalLanguage2019,
title = {Bornholmsk natural language processing: Resources and tools},
url = {https://pure.itu.dk/ws/files/84551091/W19_6138.pdf},
shorttitle = {Bornholmsk natural language processing},
pages = {338--344},
booktitle = {Proceedings of the Nordic Conference of Computational Linguistics (2019)},
publisher = {Linköping University Electronic Press},
author = {Derczynski, Leon and Kjeldsen, Alex Speed},
urldate = {2024-04-24},
date = {2019},
file = {Available Version (via Google Scholar):/Users/au554730/Zotero/storage/FBQ73ZYN/Derczynski and Kjeldsen - 2019 - Bornholmsk natural language processing Resources .pdf:application/pdf},
author = {Derczynski, Leon and Kjeldsen, Alex Speed},
booktitle = {Proceedings of the Nordic Conference of Computational Linguistics (2019)},
date = {2019},
file = {Available Version (via Google Scholar):/Users/au554730/Zotero/storage/FBQ73ZYN/Derczynski and Kjeldsen - 2019 - Bornholmsk natural language processing Resources .pdf:application/pdf},
pages = {338--344},
publisher = {Linköping University Electronic Press},
shorttitle = {Bornholmsk natural language processing},
title = {Bornholmsk natural language processing: Resources and tools},
url = {https://pure.itu.dk/ws/files/84551091/W19_6138.pdf},
urldate = {2024-04-24},
}
""",
prompt="Retrieve parallel sentences.",
Expand Down
37 changes: 19 additions & 18 deletions mteb/tasks/BitextMining/eng/PubChemSMILESBitextMining.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,24 +41,25 @@ class PubChemSMILESBitextMining(MultilingualTask, AbsTaskBitextMining):
annotations_creators="derived",
dialect=[],
sample_creation="created",
bibtex_citation="""
@article{kasmaee2024chemteb,
title={ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain},
author={Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila},
journal={arXiv preprint arXiv:2412.00532},
year={2024}
}
@article{kim2023pubchem,
title={PubChem 2023 update},
author={Kim, Sunghwan and Chen, Jie and Cheng, Tiejun and Gindulyte, Asta and He, Jia and He, Siqian and Li, Qingliang and Shoemaker, Benjamin A and Thiessen, Paul A and Yu, Bo and others},
journal={Nucleic acids research},
volume={51},
number={D1},
pages={D1373--D1380},
year={2023},
publisher={Oxford University Press}
}
""",
bibtex_citation=r"""
@article{kasmaee2024chemteb,
author = {Kasmaee, Ali Shiraee and Khodadad, Mohammad and Saloot, Mohammad Arshi and Sherck, Nick and Dokas, Stephen and Mahyar, Hamidreza and Samiee, Soheila},
journal = {arXiv preprint arXiv:2412.00532},
title = {ChemTEB: Chemical Text Embedding Benchmark, an Overview of Embedding Models Performance \& Efficiency on a Specific Domain},
year = {2024},
}

@article{kim2023pubchem,
author = {Kim, Sunghwan and Chen, Jie and Cheng, Tiejun and Gindulyte, Asta and He, Jia and He, Siqian and Li, Qingliang and Shoemaker, Benjamin A and Thiessen, Paul A and Yu, Bo and others},
journal = {Nucleic acids research},
number = {D1},
pages = {D1373--D1380},
publisher = {Oxford University Press},
title = {PubChem 2023 update},
volume = {51},
year = {2023},
}
""",
)

def dataset_transform(self):
Expand Down
38 changes: 20 additions & 18 deletions mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,24 +43,26 @@ class BUCCBitextMining(AbsTaskBitextMining, MultilingualTask):
annotations_creators="human-annotated",
dialect=[],
sample_creation="human-translated",
bibtex_citation="""@inproceedings{zweigenbaum-etal-2017-overview,
title = "Overview of the Second {BUCC} Shared Task: Spotting Parallel Sentences in Comparable Corpora",
author = "Zweigenbaum, Pierre and
Sharoff, Serge and
Rapp, Reinhard",
editor = "Sharoff, Serge and
Zweigenbaum, Pierre and
Rapp, Reinhard",
booktitle = "Proceedings of the 10th Workshop on Building and Using Comparable Corpora",
month = aug,
year = "2017",
address = "Vancouver, Canada",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/W17-2512",
doi = "10.18653/v1/W17-2512",
pages = "60--67",
abstract = "This paper presents the BUCC 2017 shared task on parallel sentence extraction from comparable corpora. It recalls the design of the datasets, presents their final construction and statistics and the methods used to evaluate system results. 13 runs were submitted to the shared task by 4 teams, covering three of the four proposed language pairs: French-English (7 runs), German-English (3 runs), and Chinese-English (3 runs). The best F-scores as measured against the gold standard were 0.84 (German-English), 0.80 (French-English), and 0.43 (Chinese-English). Because of the design of the dataset, in which not all gold parallel sentence pairs are known, these are only minimum values. We examined manually a small sample of the false negative sentence pairs for the most precise French-English runs and estimated the number of parallel sentence pairs not yet in the provided gold standard. Adding them to the gold standard leads to revised estimates for the French-English F-scores of at most +1.5pt. This suggests that the BUCC 2017 datasets provide a reasonable approximate evaluation of the parallel sentence spotting task.",
}""",
bibtex_citation=r"""
@inproceedings{zweigenbaum-etal-2017-overview,
abstract = {This paper presents the BUCC 2017 shared task on parallel sentence extraction from comparable corpora. It recalls the design of the datasets, presents their final construction and statistics and the methods used to evaluate system results. 13 runs were submitted to the shared task by 4 teams, covering three of the four proposed language pairs: French-English (7 runs), German-English (3 runs), and Chinese-English (3 runs). The best F-scores as measured against the gold standard were 0.84 (German-English), 0.80 (French-English), and 0.43 (Chinese-English). Because of the design of the dataset, in which not all gold parallel sentence pairs are known, these are only minimum values. We examined manually a small sample of the false negative sentence pairs for the most precise French-English runs and estimated the number of parallel sentence pairs not yet in the provided gold standard. Adding them to the gold standard leads to revised estimates for the French-English F-scores of at most +1.5pt. This suggests that the BUCC 2017 datasets provide a reasonable approximate evaluation of the parallel sentence spotting task.},
address = {Vancouver, Canada},
author = {Zweigenbaum, Pierre and
Sharoff, Serge and
Rapp, Reinhard},
booktitle = {Proceedings of the 10th Workshop on Building and Using Comparable Corpora},
doi = {10.18653/v1/W17-2512},
editor = {Sharoff, Serge and
Zweigenbaum, Pierre and
Rapp, Reinhard},
month = aug,
pages = {60--67},
publisher = {Association for Computational Linguistics},
title = {Overview of the Second {BUCC} Shared Task: Spotting Parallel Sentences in Comparable Corpora},
url = {https://aclanthology.org/W17-2512},
year = {2017},
}
""",
)

def dataset_transform(self):
Expand Down
38 changes: 20 additions & 18 deletions mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,23 +38,25 @@ class BUCCBitextMiningFast(AbsTaskBitextMining, MultilingualTask):
annotations_creators="human-annotated",
dialect=[],
sample_creation="human-translated",
bibtex_citation="""@inproceedings{zweigenbaum-etal-2017-overview,
title = "Overview of the Second {BUCC} Shared Task: Spotting Parallel Sentences in Comparable Corpora",
author = "Zweigenbaum, Pierre and
Sharoff, Serge and
Rapp, Reinhard",
editor = "Sharoff, Serge and
Zweigenbaum, Pierre and
Rapp, Reinhard",
booktitle = "Proceedings of the 10th Workshop on Building and Using Comparable Corpora",
month = aug,
year = "2017",
address = "Vancouver, Canada",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/W17-2512",
doi = "10.18653/v1/W17-2512",
pages = "60--67",
abstract = "This paper presents the BUCC 2017 shared task on parallel sentence extraction from comparable corpora. It recalls the design of the datasets, presents their final construction and statistics and the methods used to evaluate system results. 13 runs were submitted to the shared task by 4 teams, covering three of the four proposed language pairs: French-English (7 runs), German-English (3 runs), and Chinese-English (3 runs). The best F-scores as measured against the gold standard were 0.84 (German-English), 0.80 (French-English), and 0.43 (Chinese-English). Because of the design of the dataset, in which not all gold parallel sentence pairs are known, these are only minimum values. We examined manually a small sample of the false negative sentence pairs for the most precise French-English runs and estimated the number of parallel sentence pairs not yet in the provided gold standard. Adding them to the gold standard leads to revised estimates for the French-English F-scores of at most +1.5pt. This suggests that the BUCC 2017 datasets provide a reasonable approximate evaluation of the parallel sentence spotting task.",
}""",
bibtex_citation=r"""
@inproceedings{zweigenbaum-etal-2017-overview,
abstract = {This paper presents the BUCC 2017 shared task on parallel sentence extraction from comparable corpora. It recalls the design of the datasets, presents their final construction and statistics and the methods used to evaluate system results. 13 runs were submitted to the shared task by 4 teams, covering three of the four proposed language pairs: French-English (7 runs), German-English (3 runs), and Chinese-English (3 runs). The best F-scores as measured against the gold standard were 0.84 (German-English), 0.80 (French-English), and 0.43 (Chinese-English). Because of the design of the dataset, in which not all gold parallel sentence pairs are known, these are only minimum values. We examined manually a small sample of the false negative sentence pairs for the most precise French-English runs and estimated the number of parallel sentence pairs not yet in the provided gold standard. Adding them to the gold standard leads to revised estimates for the French-English F-scores of at most +1.5pt. This suggests that the BUCC 2017 datasets provide a reasonable approximate evaluation of the parallel sentence spotting task.},
address = {Vancouver, Canada},
author = {Zweigenbaum, Pierre and
Sharoff, Serge and
Rapp, Reinhard},
booktitle = {Proceedings of the 10th Workshop on Building and Using Comparable Corpora},
doi = {10.18653/v1/W17-2512},
editor = {Sharoff, Serge and
Zweigenbaum, Pierre and
Rapp, Reinhard},
month = aug,
pages = {60--67},
publisher = {Association for Computational Linguistics},
title = {Overview of the Second {BUCC} Shared Task: Spotting Parallel Sentences in Comparable Corpora},
url = {https://aclanthology.org/W17-2512},
year = {2017},
}
""",
adapted_from=["BUCC"],
)
14 changes: 8 additions & 6 deletions mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py
Original file line number Diff line number Diff line change
Expand Up @@ -884,12 +884,14 @@ class BibleNLPBitextMining(AbsTaskBitextMining, MultilingualTask):
annotations_creators="expert-annotated",
dialect=[],
sample_creation="created",
bibtex_citation="""@article{akerman2023ebible,
title={The eBible Corpus: Data and Model Benchmarks for Bible Translation for Low-Resource Languages},
author={Akerman, Vesa and Baines, David and Daspit, Damien and Hermjakob, Ulf and Jang, Taeho and Leong, Colin and Martin, Michael and Mathew, Joel and Robie, Jonathan and Schwarting, Marcus},
journal={arXiv preprint arXiv:2304.09919},
year={2023}
}""",
bibtex_citation=r"""
@article{akerman2023ebible,
author = {Akerman, Vesa and Baines, David and Daspit, Damien and Hermjakob, Ulf and Jang, Taeho and Leong, Colin and Martin, Michael and Mathew, Joel and Robie, Jonathan and Schwarting, Marcus},
journal = {arXiv preprint arXiv:2304.09919},
title = {The eBible Corpus: Data and Model Benchmarks for Bible Translation for Low-Resource Languages},
year = {2023},
}
""",
)

def load_data(self, **kwargs: Any) -> None:
Expand Down
18 changes: 9 additions & 9 deletions mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,15 @@ class DiaBLaBitextMining(AbsTaskBitextMining, MultilingualTask):
annotations_creators="human-annotated",
dialect=[],
sample_creation="created",
bibtex_citation="""
@inproceedings{gonzalez2019diabla,
title={DiaBLa: A Corpus of Bilingual Spontaneous Written Dialogues for Machine Translation},
author={González, Matilde and García, Clara and Sánchez, Lucía},
booktitle={Proceedings of the 12th Language Resources and Evaluation Conference},
pages={4192--4198},
year={2019}
}
""",
bibtex_citation=r"""
@inproceedings{gonzalez2019diabla,
author = {González, Matilde and García, Clara and Sánchez, Lucía},
booktitle = {Proceedings of the 12th Language Resources and Evaluation Conference},
pages = {4192--4198},
title = {DiaBLa: A Corpus of Bilingual Spontaneous Written Dialogues for Machine Translation},
year = {2019},
}
""",
)

def load_data(self, **kwargs):
Expand Down
18 changes: 9 additions & 9 deletions mteb/tasks/BitextMining/multilingual/FloresBitextMining.py
Original file line number Diff line number Diff line change
Expand Up @@ -259,15 +259,15 @@ class FloresBitextMining(AbsTaskBitextMining, MultilingualTask):
annotations_creators="human-annotated",
dialect=[],
sample_creation="created",
bibtex_citation="""
@inproceedings{goyal2022flores,
title={The FLORES-101 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation},
author={Goyal, Naman and Gao, Cynthia and Chaudhary, Vishrav and Chen, Peng-Jen and Wenzek, Guillaume and Ju, Da and Krishnan, Sanjana and Ranzato, Marc'Aurelio and Guzm{\'a}n, Francisco},
booktitle={Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
pages={19--35},
year={2022}
}
""",
bibtex_citation=r"""
@inproceedings{goyal2022flores,
author = {Goyal, Naman and Gao, Cynthia and Chaudhary, Vishrav and Chen, Peng-Jen and Wenzek, Guillaume and Ju, Da and Krishnan, Sanjana and Ranzato, Marc'Aurelio and Guzm{\'a}n, Francisco},
booktitle = {Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
pages = {19--35},
title = {The FLORES-101 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation},
year = {2022},
}
""",
)

def load_data(self, **kwargs: Any) -> None:
Expand Down
20 changes: 11 additions & 9 deletions mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,15 +91,17 @@ class IN22ConvBitextMining(AbsTaskBitextMining, MultilingualTask):
annotations_creators="expert-annotated",
dialect=[],
sample_creation="created",
bibtex_citation="""@article{gala2023indictrans,
title={IndicTrans2: Towards High-Quality and Accessible Machine Translation Models for all 22 Scheduled Indian Languages},
author={Jay Gala and Pranjal A Chitale and A K Raghavan and Varun Gumma and Sumanth Doddapaneni and Aswanth Kumar M and Janki Atul Nawale and Anupama Sujatha and Ratish Puduppully and Vivek Raghavan and Pratyush Kumar and Mitesh M Khapra and Raj Dabre and Anoop Kunchukuttan},
journal={Transactions on Machine Learning Research},
issn={2835-8856},
year={2023},
url={https://openreview.net/forum?id=vfT4YuzAYA},
note={}
}""",
bibtex_citation=r"""
@article{gala2023indictrans,
author = {Jay Gala and Pranjal A Chitale and A K Raghavan and Varun Gumma and Sumanth Doddapaneni and Aswanth Kumar M and Janki Atul Nawale and Anupama Sujatha and Ratish Puduppully and Vivek Raghavan and Pratyush Kumar and Mitesh M Khapra and Raj Dabre and Anoop Kunchukuttan},
issn = {2835-8856},
journal = {Transactions on Machine Learning Research},
note = {},
title = {IndicTrans2: Towards High-Quality and Accessible Machine Translation Models for all 22 Scheduled Indian Languages},
url = {https://openreview.net/forum?id=vfT4YuzAYA},
year = {2023},
}
""",
)

def load_data(self, **kwargs: Any) -> None:
Expand Down
20 changes: 11 additions & 9 deletions mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,15 +85,17 @@ class IN22GenBitextMining(AbsTaskBitextMining, MultilingualTask):
annotations_creators="expert-annotated",
dialect=[],
sample_creation="created",
bibtex_citation="""@article{gala2023indictrans,
title={IndicTrans2: Towards High-Quality and Accessible Machine Translation Models for all 22 Scheduled Indian Languages},
author={Jay Gala and Pranjal A Chitale and A K Raghavan and Varun Gumma and Sumanth Doddapaneni and Aswanth Kumar M and Janki Atul Nawale and Anupama Sujatha and Ratish Puduppully and Vivek Raghavan and Pratyush Kumar and Mitesh M Khapra and Raj Dabre and Anoop Kunchukuttan},
journal={Transactions on Machine Learning Research},
issn={2835-8856},
year={2023},
url={https://openreview.net/forum?id=vfT4YuzAYA},
note={}
}""",
bibtex_citation=r"""
@article{gala2023indictrans,
author = {Jay Gala and Pranjal A Chitale and A K Raghavan and Varun Gumma and Sumanth Doddapaneni and Aswanth Kumar M and Janki Atul Nawale and Anupama Sujatha and Ratish Puduppully and Vivek Raghavan and Pratyush Kumar and Mitesh M Khapra and Raj Dabre and Anoop Kunchukuttan},
issn = {2835-8856},
journal = {Transactions on Machine Learning Research},
note = {},
title = {IndicTrans2: Towards High-Quality and Accessible Machine Translation Models for all 22 Scheduled Indian Languages},
url = {https://openreview.net/forum?id=vfT4YuzAYA},
year = {2023},
}
""",
)

def load_data(self, **kwargs: Any) -> None:
Expand Down
38 changes: 19 additions & 19 deletions mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,26 +59,26 @@ class IWSLT2017BitextMining(AbsTaskBitextMining, MultilingualTask):
annotations_creators="expert-annotated",
dialect=[],
sample_creation="found",
bibtex_citation="""
bibtex_citation=r"""
@inproceedings{cettolo-etal-2017-overview,
title = "Overview of the {IWSLT} 2017 Evaluation Campaign",
author = {Cettolo, Mauro and
Federico, Marcello and
Bentivogli, Luisa and
Niehues, Jan and
St{\"u}ker, Sebastian and
Sudoh, Katsuhito and
Yoshino, Koichiro and
Federmann, Christian},
editor = "Sakti, Sakriani and
Utiyama, Masao",
booktitle = "Proceedings of the 14th International Conference on Spoken Language Translation",
month = dec # " 14-15",
year = "2017",
address = "Tokyo, Japan",
publisher = "International Workshop on Spoken Language Translation",
url = "https://aclanthology.org/2017.iwslt-1.1",
pages = "2--14",
address = {Tokyo, Japan},
author = {Cettolo, Mauro and
Federico, Marcello and
Bentivogli, Luisa and
Niehues, Jan and
St{\"u}ker, Sebastian and
Sudoh, Katsuhito and
Yoshino, Koichiro and
Federmann, Christian},
booktitle = {Proceedings of the 14th International Conference on Spoken Language Translation},
editor = {Sakti, Sakriani and
Utiyama, Masao},
month = dec # { 14-15},
pages = {2--14},
publisher = {International Workshop on Spoken Language Translation},
title = {Overview of the {IWSLT} 2017 Evaluation Campaign},
url = {https://aclanthology.org/2017.iwslt-1.1},
year = {2017},
}
""",
)
Expand Down
Loading