diff --git a/mteb/benchmarks/benchmarks.py b/mteb/benchmarks/benchmarks.py index d5efbc092f..e872143ee5 100644 --- a/mteb/benchmarks/benchmarks.py +++ b/mteb/benchmarks/benchmarks.py @@ -978,3 +978,27 @@ def load_results( year={2024} }""", ) + +NANOBEIR = Benchmark( + name="NanoBEIR", + tasks=get_tasks( + tasks=[ + "NanoArguAnaRetrieval", + "NanoClimateFeverRetrieval", + "NanoDBPediaRetrieval", + "NanoFEVERRetrieval", + "NanoFiQA2018Retrieval", + "NanoHotpotQARetrieval", + "NanoMSMARCORetrieval", + "NanoNFCorpusRetrieval", + "NanoNQRetrieval", + "NanoQuoraRetrieval", + "NanoSCIDOCSRetrieval", + "NanoSciFactRetrieval", + "NanoTouche2020Retrieval", + ], + ), + description="A benchmark to evaluate with subsets of BEIR datasets to use less computational power", + reference="https://huggingface.co/collections/zeta-alpha-ai/nanobeir-66e1a0af21dfd93e620cd9f6", + citation=None, +) diff --git a/mteb/models/misc_models.py b/mteb/models/misc_models.py index 2429cce39b..61dc549b15 100644 --- a/mteb/models/misc_models.py +++ b/mteb/models/misc_models.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from mteb.model_meta import ModelMeta Haon_Chen__speed_embedding_7b_instruct = ModelMeta( diff --git a/mteb/tasks/Retrieval/__init__.py b/mteb/tasks/Retrieval/__init__.py index ca41d4354f..d83df7ec5e 100644 --- a/mteb/tasks/Retrieval/__init__.py +++ b/mteb/tasks/Retrieval/__init__.py @@ -64,6 +64,19 @@ from .eng.MLQuestions import * from .eng.MSMARCORetrieval import * from .eng.MSMARCOv2Retrieval import * +from .eng.NanoArguAnaRetrieval import * +from .eng.NanoClimateFeverRetrieval import * +from .eng.NanoDBPediaRetrieval import * +from .eng.NanoFEVERRetrieval import * +from .eng.NanoFiQA2018Retrieval import * +from .eng.NanoHotpotQARetrieval import * +from .eng.NanoMSMARCORetrieval import * +from .eng.NanoNFCorpusRetrieval import * +from .eng.NanoNQRetrieval import * +from .eng.NanoQuoraRetrieval import * +from .eng.NanoSCIDOCSRetrieval import * +from .eng.NanoSciFactRetrieval import * +from .eng.NanoTouche2020Retrieval import * from .eng.NarrativeQARetrieval import * from .eng.NFCorpusRetrieval import * from .eng.NQRetrieval import * diff --git a/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py new file mode 100644 index 0000000000..2230368b94 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoArguAnaRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoArguAnaRetrieval", + description="NanoArguAna is a smaller subset of ArguAna, a dataset for argument retrieval in debate contexts.", + reference="http://argumentation.bplaced.net/arguana/data", + dataset={ + "path": "zeta-alpha-ai/NanoArguAna", + "revision": "8f4a982d470a32c45817738b9d29042ca55d75ad", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2020-01-01", "2020-12-31"], + domains=["Medical", "Written"], + task_subtypes=["Discourse coherence"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{boteva2016, + author = {Boteva, Vera and Gholipour, Demian and Sokolov, Artem and Riezler, Stefan}, + title = {A Full-Text Learning to Rank Dataset for Medical Information Retrieval}, + journal = {Proceedings of the 38th European Conference on Information Retrieval}, + journal-abbrev = {ECIR}, + year = {2016}, + city = {Padova}, + country = {Italy}, + url = {http://www.cl.uni-heidelberg.de/~riezler/publications/papers/ECIR2016.pdf} +}""", + prompt={"query": "Given a claim, find documents that refute the claim"}, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoArguAna", + "corpus", + revision="8f4a982d470a32c45817738b9d29042ca55d75ad", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoArguAna", + "queries", + revision="8f4a982d470a32c45817738b9d29042ca55d75ad", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoArguAna", + "qrels", + revision="8f4a982d470a32c45817738b9d29042ca55d75ad", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py new file mode 100644 index 0000000000..0185a454d3 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoClimateFeverRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoClimateFeverRetrieval", + description="NanoClimateFever is a small version of the BEIR dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change.", + reference="https://arxiv.org/abs/2012.00614", + dataset={ + "path": "zeta-alpha-ai/NanoClimateFEVER", + "revision": "96741bfa30b9f56db8c9eb7d08e775ed6474f206", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2020-01-01", "2020-12-31"], + domains=["Non-fiction", "Academic", "News"], + task_subtypes=["Claim verification"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@misc{diggelmann2021climatefever, + title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, + author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold}, + year={2021}, + eprint={2012.00614}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +}""", + prompt={ + "query": "Given a claim about climate change, retrieve documents that support or refute the claim" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoClimateFEVER", + "corpus", + revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoClimateFEVER", + "queries", + revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoClimateFEVER", + "qrels", + revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py new file mode 100644 index 0000000000..caa638743c --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoDBPediaRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoDBPediaRetrieval", + description="NanoDBPediaRetrieval is a small version of the standard test collection for entity search over the DBpedia knowledge base.", + reference="https://huggingface.co/datasets/zeta-alpha-ai/NanoDBPedia", + dataset={ + "path": "zeta-alpha-ai/NanoDBPedia", + "revision": "438f1c25129f05db6238699b5afdc9c6b58d2096", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2015-01-01", "2015-12-31"], + domains=["Encyclopaedic"], + task_subtypes=["Topic classification"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{lehmann2015dbpedia, title={DBpedia: A large-scale, multilingual knowledge base extracted from Wikipedia}, author={Lehmann, Jens and et al.}, journal={Semantic Web}, year={2015}}""", + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoDBPedia", + "corpus", + revision="438f1c25129f05db6238699b5afdc9c6b58d2096", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoDBPedia", + "queries", + revision="438f1c25129f05db6238699b5afdc9c6b58d2096", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoDBPedia", + "qrels", + revision="438f1c25129f05db6238699b5afdc9c6b58d2096", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py new file mode 100644 index 0000000000..6bdd0ab4cf --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoFEVERRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoFEVERRetrieval", + description="NanoFEVER is a smaller version of " + + "FEVER (Fact Extraction and VERification), which consists of 185,445 claims generated by altering sentences" + + " extracted from Wikipedia and subsequently verified without knowledge of the sentence they were" + + " derived from.", + reference="https://fever.ai/", + dataset={ + "path": "zeta-alpha-ai/NanoFEVER", + "revision": "a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2018-01-01", "2018-12-31"], + domains=["Academic", "Encyclopaedic"], + task_subtypes=["Claim verification"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{thorne-etal-2018-fever, + title = "{FEVER}: a Large-scale Dataset for Fact Extraction and {VER}ification", + author = "Thorne, James and + Vlachos, Andreas and + Christodoulopoulos, Christos and + Mittal, Arpit", + editor = "Walker, Marilyn and + Ji, Heng and + Stent, Amanda", + booktitle = "Proceedings of the 2018 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)", + month = jun, + year = "2018", + address = "New Orleans, Louisiana", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/N18-1074", + doi = "10.18653/v1/N18-1074", + pages = "809--819", + abstract = "In this paper we introduce a new publicly available dataset for verification against textual sources, FEVER: Fact Extraction and VERification. It consists of 185,445 claims generated by altering sentences extracted from Wikipedia and subsequently verified without knowledge of the sentence they were derived from. The claims are classified as Supported, Refuted or NotEnoughInfo by annotators achieving 0.6841 in Fleiss kappa. For the first two classes, the annotators also recorded the sentence(s) forming the necessary evidence for their judgment. To characterize the challenge of the dataset presented, we develop a pipeline approach and compare it to suitably designed oracles. The best accuracy we achieve on labeling a claim accompanied by the correct evidence is 31.87{\%}, while if we ignore the evidence we achieve 50.91{\%}. Thus we believe that FEVER is a challenging testbed that will help stimulate progress on claim verification against textual sources.", +}""", + prompt={ + "query": "Given a claim, retrieve documents that support or refute the claim" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoFEVER", + "corpus", + revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoFEVER", + "queries", + revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoFEVER", + "qrels", + revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py new file mode 100644 index 0000000000..1a3467c1d7 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoFiQA2018Retrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoFiQA2018Retrieval", + description="NanoFiQA2018 is a smaller subset of the Financial Opinion Mining and Question Answering dataset.", + reference="https://sites.google.com/view/fiqa/", + dataset={ + "path": "zeta-alpha-ai/NanoFiQA2018", + "revision": "4163ba032953d5044a7a6244261413f609c14342", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2018-01-01", "2018-12-31"], + domains=["Academic", "Social"], + task_subtypes=["Sentiment/Hate speech"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{ +thakur2021beir, +title={{BEIR}: A Heterogeneous Benchmark for Zero-shot Evaluation of Information Retrieval Models}, +author={Nandan Thakur and Nils Reimers and Andreas R{\"u}ckl{\'e} and Abhishek Srivastava and Iryna Gurevych}, +booktitle={Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)}, +year={2021}, +url={https://openreview.net/forum?id=wCu6T5xFjeJ} +}""", + prompt={ + "query": "Given a financial question, retrieve user replies that best answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoFiQA2018", + "corpus", + revision="4163ba032953d5044a7a6244261413f609c14342", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoFiQA2018", + "queries", + revision="4163ba032953d5044a7a6244261413f609c14342", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoFiQA2018", + "qrels", + revision="4163ba032953d5044a7a6244261413f609c14342", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py new file mode 100644 index 0000000000..4389aeafa8 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoHotpotQARetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoHotpotQARetrieval", + description="NanoHotpotQARetrieval is a smaller subset of the " + + "HotpotQA dataset, which is a question answering dataset featuring natural, multi-hop questions, with strong" + + " supervision for supporting facts to enable more explainable question answering systems.", + reference="https://hotpotqa.github.io/", + dataset={ + "path": "zeta-alpha-ai/NanoHotpotQA", + "revision": "d79c0cdda980aba54842756770928035e1b61a51", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2018-01-01", "2018-12-31"], + domains=["Web", "Written"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{yang-etal-2018-hotpotqa, + title = "{H}otpot{QA}: A Dataset for Diverse, Explainable Multi-hop Question Answering", + author = "Yang, Zhilin and + Qi, Peng and + Zhang, Saizheng and + Bengio, Yoshua and + Cohen, William and + Salakhutdinov, Ruslan and + Manning, Christopher D.", + editor = "Riloff, Ellen and + Chiang, David and + Hockenmaier, Julia and + Tsujii, Jun{'}ichi", + booktitle = "Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing", + month = oct # "-" # nov, + year = "2018", + address = "Brussels, Belgium", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/D18-1259", + doi = "10.18653/v1/D18-1259", + pages = "2369--2380", + abstract = "Existing question answering (QA) datasets fail to train QA systems to perform complex reasoning and provide explanations for answers. We introduce HotpotQA, a new dataset with 113k Wikipedia-based question-answer pairs with four key features: (1) the questions require finding and reasoning over multiple supporting documents to answer; (2) the questions are diverse and not constrained to any pre-existing knowledge bases or knowledge schemas; (3) we provide sentence-level supporting facts required for reasoning, allowing QA systems to reason with strong supervision and explain the predictions; (4) we offer a new type of factoid comparison questions to test QA systems{'} ability to extract relevant facts and perform necessary comparison. We show that HotpotQA is challenging for the latest QA systems, and the supporting facts enable models to improve performance and make explainable predictions.", +}""", + prompt={ + "query": "Given a multi-hop question, retrieve documents that can help answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoHotpotQA", + "corpus", + revision="d79c0cdda980aba54842756770928035e1b61a51", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoHotpotQA", + "queries", + revision="d79c0cdda980aba54842756770928035e1b61a51", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoHotpotQA", + "qrels", + revision="d79c0cdda980aba54842756770928035e1b61a51", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py new file mode 100644 index 0000000000..8a2f51e7fd --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoMSMARCORetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoMSMARCORetrieval", + description="NanoMSMARCORetrieval is a smaller subset of MS MARCO, a collection of datasets focused on deep learning in search.", + reference="https://microsoft.github.io/msmarco/", + dataset={ + "path": "zeta-alpha-ai/NanoMSMARCO", + "revision": "7b8ff22f2771dc65ac5b439f222eb19a1f56abda", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2016-01-01", "2016-12-31"], + domains=["Web"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{DBLP:journals/corr/NguyenRSGTMD16, + author = {Tri Nguyen and + Mir Rosenberg and + Xia Song and + Jianfeng Gao and + Saurabh Tiwary and + Rangan Majumder and + Li Deng}, + title = {{MS} {MARCO:} {A} Human Generated MAchine Reading COmprehension Dataset}, + journal = {CoRR}, + volume = {abs/1611.09268}, + year = {2016}, + url = {http://arxiv.org/abs/1611.09268}, + archivePrefix = {arXiv}, + eprint = {1611.09268}, + timestamp = {Mon, 13 Aug 2018 16:49:03 +0200}, + biburl = {https://dblp.org/rec/journals/corr/NguyenRSGTMD16.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} +}""", + prompt={ + "query": "Given a web search query, retrieve relevant passages that answer the query" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoMSMARCO", + "corpus", + revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoMSMARCO", + "queries", + revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoMSMARCO", + "qrels", + revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py new file mode 100644 index 0000000000..0f6ac8533a --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoNFCorpusRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoNFCorpusRetrieval", + description="NanoNFCorpus is a smaller subset of NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval.", + reference="https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/", + dataset={ + "path": "zeta-alpha-ai/NanoNFCorpus", + "revision": "dd542a7efb9ad2136b9e00768b60fca9038f8156", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2016-01-01", "2016-12-31"], + domains=["Medical", "Academic", "Written"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{boteva2016, + author = {Boteva, Vera and Gholipour, Demian and Sokolov, Artem and Riezler, Stefan}, + title = {A Full-Text Learning to Rank Dataset for Medical Information Retrieval}, + journal = {Proceedings of the 38th European Conference on Information Retrieval}, + journal-abbrev = {ECIR}, + year = {2016}, + city = {Padova}, + country = {Italy}, + url = {http://www.cl.uni-heidelberg.de/~riezler/publications/papers/ECIR2016.pdf} +}""", + prompt={ + "query": "Given a question, retrieve relevant documents that best answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoNFCorpus", + "corpus", + revision="dd542a7efb9ad2136b9e00768b60fca9038f8156", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoNFCorpus", + "queries", + revision="dd542a7efb9ad2136b9e00768b60fca9038f8156", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoNFCorpus", + "qrels", + revision="dd542a7efb9ad2136b9e00768b60fca9038f8156", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py new file mode 100644 index 0000000000..5aa831f799 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoNQRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoNQRetrieval", + description="NanoNQ is a smaller subset of a dataset which contains questions from real users, and it requires QA systems to read and comprehend an entire Wikipedia article that may or may not contain the answer to the question.", + reference="https://ai.google.com/research/NaturalQuestions", + dataset={ + "path": "zeta-alpha-ai/NanoNQ", + "revision": "77540146379abf95df8326a3c5bb9eb21c7146c3", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2019-01-01", "2019-12-31"], + domains=["Academic", "Web"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{47761,title = {Natural Questions: a Benchmark for Question Answering Research}, + author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh + and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee + and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le + and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational + Linguistics}}""", + prompt={ + "query": "Given a question, retrieve Wikipedia passages that answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoNQ", + "corpus", + revision="77540146379abf95df8326a3c5bb9eb21c7146c3", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoNQ", + "queries", + revision="77540146379abf95df8326a3c5bb9eb21c7146c3", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoNQ", + "qrels", + revision="77540146379abf95df8326a3c5bb9eb21c7146c3", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py new file mode 100644 index 0000000000..1391d12b93 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoQuoraRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoQuoraRetrieval", + description="NanoQuoraRetrieval is a smaller subset of the " + + "QuoraRetrieval dataset, which is based on questions that are marked as duplicates on the Quora platform. Given a" + + " question, find other (duplicate) questions.", + reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs", + dataset={ + "path": "zeta-alpha-ai/NanoQuoraRetrieval", + "revision": "2ab2d73e6c862026282808b913a34f4136928545", + }, + type="Retrieval", + category="s2s", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2017-01-01", "2017-12-31"], + domains=["Social"], + task_subtypes=["Duplicate Detection"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@misc{quora-question-pairs, + author = {DataCanary, hilfialkaff, Lili Jiang, Meg Risdal, Nikhil Dandekar, tomtung}, + title = {Quora Question Pairs}, + publisher = {Kaggle}, + year = {2017}, + url = {https://kaggle.com/competitions/quora-question-pairs} +}""", + prompt={ + "query": "Given a question, retrieve questions that are semantically equivalent to the given question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoQuoraRetrieval", + "corpus", + revision="2ab2d73e6c862026282808b913a34f4136928545", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoQuoraRetrieval", + "queries", + revision="2ab2d73e6c862026282808b913a34f4136928545", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoQuoraRetrieval", + "qrels", + revision="2ab2d73e6c862026282808b913a34f4136928545", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py new file mode 100644 index 0000000000..2d27e1a2dc --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoSCIDOCSRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoSCIDOCSRetrieval", + description="NanoFiQA2018 is a smaller subset of " + + "SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation" + + " prediction, to document classification and recommendation.", + reference="https://allenai.org/data/scidocs", + dataset={ + "path": "zeta-alpha-ai/NanoSCIDOCS", + "revision": "484eb90549fc3f0b9c42b3551e80ceb999515537", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2020-01-01", "2020-12-31"], + domains=["Academic", "Written", "Non-fiction"], + task_subtypes=[], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{specter2020cohan, + title={SPECTER: Document-level Representation Learning using Citation-informed Transformers}, + author={Arman Cohan and Sergey Feldman and Iz Beltagy and Doug Downey and Daniel S. Weld}, + booktitle={ACL}, + year={2020} +}""", + prompt={ + "query": "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoSCIDOCS", + "corpus", + revision="484eb90549fc3f0b9c42b3551e80ceb999515537", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoSCIDOCS", + "queries", + revision="484eb90549fc3f0b9c42b3551e80ceb999515537", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoSCIDOCS", + "qrels", + revision="484eb90549fc3f0b9c42b3551e80ceb999515537", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py new file mode 100644 index 0000000000..aff949d319 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoSciFactRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoSciFactRetrieval", + description="NanoSciFact is a smaller subset of SciFact, which verifies scientific claims using evidence from the research literature containing scientific paper abstracts.", + reference="https://github.com/allenai/scifact", + dataset={ + "path": "zeta-alpha-ai/NanoSciFact", + "revision": "309f1d1ae3ae2e092444a8a0c25bed59b82318bc", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2018-01-01", "2018-12-31"], + domains=["Academic", "Medical", "Written"], + task_subtypes=["Claim verification"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{specter2020cohan, + title={SPECTER: Document-level Representation Learning using Citation-informed Transformers}, + author={Arman Cohan and Sergey Feldman and Iz Beltagy and Doug Downey and Daniel S. Weld}, + booktitle={ACL}, + year={2020} +}""", + prompt={ + "query": "Given a scientific claim, retrieve documents that support or refute the claim" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoSciFact", + "corpus", + revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoSciFact", + "queries", + revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoSciFact", + "qrels", + revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py new file mode 100644 index 0000000000..656b5494a0 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoTouche2020Retrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoTouche2020Retrieval", + description="NanoTouche2020 is a smaller subset of Touché Task 1: Argument Retrieval for Controversial Questions.", + reference="https://webis.de/events/touche-20/shared-task-1.html", + dataset={ + "path": "zeta-alpha-ai/NanoTouche2020", + "revision": "0d2f26ed8c5ad309f95c7f9499c70a40e140fccd", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2020-09-23", "2020-09-23"), + domains=["Academic"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@dataset{potthast_2022_6862281, + author = {Potthast, Martin and + Gienapp, Lukas and + Wachsmuth, Henning and + Hagen, Matthias and + Fröbe, Maik and + Bondarenko, Alexander and + Ajjour, Yamen and + Stein, Benno}, + title = {{Touché20-Argument-Retrieval-for-Controversial- + Questions}}, + month = jul, + year = 2022, + publisher = {Zenodo}, + doi = {10.5281/zenodo.6862281}, + url = {https://doi.org/10.5281/zenodo.6862281} +}""", + prompt={ + "query": "Given a question, retrieve detailed and persuasive arguments that answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoTouche2020", + "corpus", + revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoTouche2020", + "queries", + revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoTouche2020", + "qrels", + revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd", + ) + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: {sample["corpus-id"]: 1} + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + self.data_loaded = True