From 540a8fdfd4e3281b468b01ea13748ac4116e29cd Mon Sep 17 00:00:00 2001 From: Kavya <92774828+KGupta10@users.noreply.github.com> Date: Wed, 11 Dec 2024 08:19:58 -0800 Subject: [PATCH 1/9] add NanoClimateFeverRetrieval task, still requires some debugging --- mteb/tasks/Retrieval/__init__.py | 1 + .../eng/NanoClimateFeverRetrieval.py | 95 +++++++++++++++++++ .../tasks/Retrieval/eng/tempCodeRunnerFile.py | 6 ++ 3 files changed, 102 insertions(+) create mode 100644 mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py diff --git a/mteb/tasks/Retrieval/__init__.py b/mteb/tasks/Retrieval/__init__.py index ca41d4354f..58ccf39af5 100644 --- a/mteb/tasks/Retrieval/__init__.py +++ b/mteb/tasks/Retrieval/__init__.py @@ -145,3 +145,4 @@ from .vie.VieQuADRetrieval import * from .zho.CMTEBRetrieval import * from .zho.LeCaRDv2Retrieval import * +from .eng.NanoClimateFeverRetrieval import * \ No newline at end of file diff --git a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py new file mode 100644 index 0000000000..7c01a86de5 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval + +from datasets import load_dataset + +from sentence_transformers import SentenceTransformer +from mteb import MTEB + + +class NanoClimateFeverRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoClimateFeverRetrieval", + description="NanoClimateFever is a small version of the BEIR dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change.", + reference="https://huggingface.co/collections/zeta-alpha-ai/nanobeir-66e1a0af21dfd93e620cd9f6", + dataset={ + "path": "zeta-alpha-ai/NanoClimateFEVER", + "revision": "7bda449ec7e1965490bb862bae3d8c0f419b5611de561c7fd4ce7d7274b843ac", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=None, + domains=None, + task_subtypes=None, + license=None, + annotations_creators=None, + dialect=None, + sample_creation=None, + bibtex_citation="""@misc{diggelmann2021climatefever, + title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, + author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold}, + year={2021}, + eprint={2012.00614}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +}""", + prompt={ + "query": "Given a claim about climate change, retrieve documents that support or refute the claim" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoClimateFEVER", 'corpus') + self.queries = load_dataset("zeta-alpha-ai/NanoClimateFEVER", 'queries') + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoClimateFEVER", 'qrels') + + self.corpus = { + split: { + sample['_id']: {'_id': sample['_id'], 'text': sample['text']} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: { + sample['_id']: sample['text'] + for sample in self.queries[split] + } + for split in self.queries + } + + self.relevant_docs = { + split: { + sample['query-id']: { + sample['corpus-id']: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True + +model_name = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" + + +model = SentenceTransformer(model_name) +evaluation = MTEB(tasks=[NanoClimateFeverRetrieval()]) +evaluation.run(model) diff --git a/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py b/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py new file mode 100644 index 0000000000..4f0bc4e942 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py @@ -0,0 +1,6 @@ + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) \ No newline at end of file From 80e1139b6d6c13af1be05807fc85fd7032c2657f Mon Sep 17 00:00:00 2001 From: Kavya <92774828+KGupta10@users.noreply.github.com> Date: Thu, 12 Dec 2024 00:22:03 -0800 Subject: [PATCH 2/9] move task to correct place in init file --- mteb/tasks/Retrieval/__init__.py | 4 ++-- mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/mteb/tasks/Retrieval/__init__.py b/mteb/tasks/Retrieval/__init__.py index 58ccf39af5..e65a0750f8 100644 --- a/mteb/tasks/Retrieval/__init__.py +++ b/mteb/tasks/Retrieval/__init__.py @@ -64,6 +64,7 @@ from .eng.MLQuestions import * from .eng.MSMARCORetrieval import * from .eng.MSMARCOv2Retrieval import * +from .eng.NanoClimateFeverRetrieval import * from .eng.NarrativeQARetrieval import * from .eng.NFCorpusRetrieval import * from .eng.NQRetrieval import * @@ -144,5 +145,4 @@ from .tur.TurHistQuad import * from .vie.VieQuADRetrieval import * from .zho.CMTEBRetrieval import * -from .zho.LeCaRDv2Retrieval import * -from .eng.NanoClimateFeverRetrieval import * \ No newline at end of file +from .zho.LeCaRDv2Retrieval import * \ No newline at end of file diff --git a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py index 7c01a86de5..ae13d1c395 100644 --- a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py @@ -85,11 +85,4 @@ def load_data(self, **kwargs): # print(self.queries) # print("relevant_docs") # print(self.relevant_docs) - self.data_loaded = True - -model_name = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" - - -model = SentenceTransformer(model_name) -evaluation = MTEB(tasks=[NanoClimateFeverRetrieval()]) -evaluation.run(model) + self.data_loaded = True \ No newline at end of file From 775676d1dfb9f6de423d855b75df06412f8ad64b Mon Sep 17 00:00:00 2001 From: Kavya <92774828+KGupta10@users.noreply.github.com> Date: Thu, 12 Dec 2024 22:34:16 -0800 Subject: [PATCH 3/9] add all Nano datasets and results --- mteb/tasks/Retrieval/__init__.py | 13 ++- .../Retrieval/eng/NanoArguAnaRetrieval.py | 83 ++++++++++++++++ .../eng/NanoClimateFeverRetrieval.py | 46 ++++----- .../Retrieval/eng/NanoDBPediaRetrieval.py | 72 ++++++++++++++ .../tasks/Retrieval/eng/NanoFEVERRetrieval.py | 95 ++++++++++++++++++ .../Retrieval/eng/NanoFiQA2018Retrieval.py | 83 ++++++++++++++++ .../Retrieval/eng/NanoHotpotQARetrieval.py | 98 +++++++++++++++++++ .../Retrieval/eng/NanoMSMARCORetrieval.py | 94 ++++++++++++++++++ .../Retrieval/eng/NanoNFCorpusRetrieval.py | 85 ++++++++++++++++ mteb/tasks/Retrieval/eng/NanoNQRetrieval.py | 80 +++++++++++++++ .../tasks/Retrieval/eng/NanoQuoraRetrieval.py | 83 ++++++++++++++++ .../Retrieval/eng/NanoSCIDOCSRetrieval.py | 81 +++++++++++++++ .../Retrieval/eng/NanoSciFactRetrieval.py | 80 +++++++++++++++ .../Retrieval/eng/NanoTouche2020Retrieval.py | 91 +++++++++++++++++ 14 files changed, 1057 insertions(+), 27 deletions(-) create mode 100644 mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoNQRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py create mode 100644 mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py diff --git a/mteb/tasks/Retrieval/__init__.py b/mteb/tasks/Retrieval/__init__.py index e65a0750f8..a588c94aff 100644 --- a/mteb/tasks/Retrieval/__init__.py +++ b/mteb/tasks/Retrieval/__init__.py @@ -64,7 +64,18 @@ from .eng.MLQuestions import * from .eng.MSMARCORetrieval import * from .eng.MSMARCOv2Retrieval import * +from .eng.NanoArguAnaRetrieval import * from .eng.NanoClimateFeverRetrieval import * +from .eng.NanoDBPediaRetrieval import * +from .eng.NanoFEVERRetrieval import * +from .eng.NanoFiQA2018Retrieval import * +from .eng.NanoHotpotQARetrieval import * +from .eng.NanoMSMARCORetrieval import * +from .eng.NanoNFCorpusRetrieval import * +from .eng.NanoQuoraRetrieval import * +from .eng.NanoSCIDOCSRetrieval import * +from .eng.NanoSciFactRetrieval import * +from .eng.NanoTouche2020Retrieval import * from .eng.NarrativeQARetrieval import * from .eng.NFCorpusRetrieval import * from .eng.NQRetrieval import * @@ -145,4 +156,4 @@ from .tur.TurHistQuad import * from .vie.VieQuADRetrieval import * from .zho.CMTEBRetrieval import * -from .zho.LeCaRDv2Retrieval import * \ No newline at end of file +from .zho.LeCaRDv2Retrieval import * diff --git a/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py new file mode 100644 index 0000000000..bd20e00475 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoArguAnaRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoArguAnaRetrieval", + description="NanoArguAna is a smaller subset of ArguAna, a dataset for argument retrieval in debate contexts.", + reference="http://argumentation.bplaced.net/arguana/data", + dataset={ + "path": "zeta-alpha-ai/NanoArguAna", + "revision": "main", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2020-01-01", "2020-12-31"], + domains=["Medical", "Written"], + task_subtypes=["Discourse coherence"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{boteva2016, + author = {Boteva, Vera and Gholipour, Demian and Sokolov, Artem and Riezler, Stefan}, + title = {A Full-Text Learning to Rank Dataset for Medical Information Retrieval}, + journal = {Proceedings of the 38th European Conference on Information Retrieval}, + journal-abbrev = {ECIR}, + year = {2016}, + city = {Padova}, + country = {Italy}, + url = {http://www.cl.uni-heidelberg.de/~riezler/publications/papers/ECIR2016.pdf} +}""", + prompt={"query": "Given a claim, find documents that refute the claim"}, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoArguAna", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoArguAna", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoArguAna", "qrels") + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True \ No newline at end of file diff --git a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py index ae13d1c395..499a474513 100644 --- a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py @@ -1,23 +1,18 @@ from __future__ import annotations -from mteb.abstasks.TaskMetadata import TaskMetadata - -from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval - from datasets import load_dataset -from sentence_transformers import SentenceTransformer -from mteb import MTEB - +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata class NanoClimateFeverRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NanoClimateFeverRetrieval", description="NanoClimateFever is a small version of the BEIR dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change.", - reference="https://huggingface.co/collections/zeta-alpha-ai/nanobeir-66e1a0af21dfd93e620cd9f6", + reference="https://arxiv.org/abs/2012.00614", dataset={ "path": "zeta-alpha-ai/NanoClimateFEVER", - "revision": "7bda449ec7e1965490bb862bae3d8c0f419b5611de561c7fd4ce7d7274b843ac", + "revision": "main", }, type="Retrieval", category="s2p", @@ -25,13 +20,13 @@ class NanoClimateFeverRetrieval(AbsTaskRetrieval): eval_splits=["train"], eval_langs=["eng-Latn"], main_score="ndcg_at_10", - date=None, - domains=None, - task_subtypes=None, - license=None, - annotations_creators=None, - dialect=None, - sample_creation=None, + date=["2020-01-01", "2020-12-31"], + domains=["Non-fiction", "Academic", "News"], + task_subtypes=["Claim verification"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", bibtex_citation="""@misc{diggelmann2021climatefever, title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold}, @@ -49,30 +44,29 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoClimateFEVER", 'corpus') - self.queries = load_dataset("zeta-alpha-ai/NanoClimateFEVER", 'queries') - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoClimateFEVER", 'qrels') + self.corpus = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "qrels") self.corpus = { split: { - sample['_id']: {'_id': sample['_id'], 'text': sample['text']} + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} for sample in self.corpus[split] } for split in self.corpus } self.queries = { - split: { - sample['_id']: sample['text'] - for sample in self.queries[split] - } + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} for split in self.queries } self.relevant_docs = { split: { - sample['query-id']: { - sample['corpus-id']: 1 # Assuming a score of 1 for relevant documents + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents } for sample in self.relevant_docs[split] } diff --git a/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py new file mode 100644 index 0000000000..5be349b458 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + +class NanoDBPediaRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoDBPediaRetrieval", + description="NanoDBPediaRetrieval is a small version of the standard test collection for entity search over the DBpedia knowledge base.", + reference="https://huggingface.co/datasets/zeta-alpha-ai/NanoDBPedia", + dataset={ + "path": "zeta-alpha-ai/NanoDBPedia", + "revision": "main", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2015-01-01", "2015-12-31"], + domains=["Encyclopaedic"], + task_subtypes=["Topic classification"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{lehmann2015dbpedia, title={DBpedia: A large-scale, multilingual knowledge base extracted from Wikipedia}, author={Lehmann, Jens and et al.}, journal={Semantic Web}, year={2015}}""", + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoDBPedia", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoDBPedia", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoDBPedia", "qrels") + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py new file mode 100644 index 0000000000..1dfa3d1309 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + +class NanoFEVERRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoFEVERRetrieval", + description="NanoFEVER is a smaller version of " + "FEVER (Fact Extraction and VERification), which consists of 185,445 claims generated by altering sentences" + + " extracted from Wikipedia and subsequently verified without knowledge of the sentence they were" + + " derived from.", + reference="https://fever.ai/", + dataset={ + "path": "zeta-alpha-ai/NanoFEVER", + "revision": "main", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2018-01-01", "2018-12-31"], + domains=["Academic", "Encyclopaedic"], + task_subtypes=["Claim verification"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{thorne-etal-2018-fever, + title = "{FEVER}: a Large-scale Dataset for Fact Extraction and {VER}ification", + author = "Thorne, James and + Vlachos, Andreas and + Christodoulopoulos, Christos and + Mittal, Arpit", + editor = "Walker, Marilyn and + Ji, Heng and + Stent, Amanda", + booktitle = "Proceedings of the 2018 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)", + month = jun, + year = "2018", + address = "New Orleans, Louisiana", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/N18-1074", + doi = "10.18653/v1/N18-1074", + pages = "809--819", + abstract = "In this paper we introduce a new publicly available dataset for verification against textual sources, FEVER: Fact Extraction and VERification. It consists of 185,445 claims generated by altering sentences extracted from Wikipedia and subsequently verified without knowledge of the sentence they were derived from. The claims are classified as Supported, Refuted or NotEnoughInfo by annotators achieving 0.6841 in Fleiss kappa. For the first two classes, the annotators also recorded the sentence(s) forming the necessary evidence for their judgment. To characterize the challenge of the dataset presented, we develop a pipeline approach and compare it to suitably designed oracles. The best accuracy we achieve on labeling a claim accompanied by the correct evidence is 31.87{\%}, while if we ignore the evidence we achieve 50.91{\%}. Thus we believe that FEVER is a challenging testbed that will help stimulate progress on claim verification against textual sources.", +}""", + prompt={ + "query": "Given a claim, retrieve documents that support or refute the claim" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoFEVER", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoFEVER", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoFEVER", "qrels") + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True \ No newline at end of file diff --git a/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py new file mode 100644 index 0000000000..30438eb951 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + +class NanoFiQA2018Retrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoFiQA2018Retrieval", + description="NanoFiQA2018 is a smaller subset of the Financial Opinion Mining and Question Answering dataset.", + reference="https://sites.google.com/view/fiqa/", + dataset={ + "path": "zeta-alpha-ai/NanoFiQA2018", + "revision": "main", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2018-01-01", "2018-12-31"], + domains=["Academic", "Social"], + task_subtypes=["Sentiment/Hate speech"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{ +thakur2021beir, +title={{BEIR}: A Heterogeneous Benchmark for Zero-shot Evaluation of Information Retrieval Models}, +author={Nandan Thakur and Nils Reimers and Andreas R{\"u}ckl{\'e} and Abhishek Srivastava and Iryna Gurevych}, +booktitle={Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)}, +year={2021}, +url={https://openreview.net/forum?id=wCu6T5xFjeJ} +}""", + prompt={ + "query": "Given a financial question, retrieve user replies that best answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoFiQA2018", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoFiQA2018", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoFiQA2018", "qrels") + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True + diff --git a/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py new file mode 100644 index 0000000000..eb0cc5255d --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + +class NanoHotpotQARetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoHotpotQARetrieval", + description="NanoHotpotQARetrieval is a smaller subset of the " + "HotpotQA dataset, which is a question answering dataset featuring natural, multi-hop questions, with strong" + + " supervision for supporting facts to enable more explainable question answering systems.", + reference="https://hotpotqa.github.io/", + dataset={ + "path": "zeta-alpha-ai/NanoHotpotQA", + "revision": "main", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2018-01-01", "2018-12-31"], + domains=["Web", "Written"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{yang-etal-2018-hotpotqa, + title = "{H}otpot{QA}: A Dataset for Diverse, Explainable Multi-hop Question Answering", + author = "Yang, Zhilin and + Qi, Peng and + Zhang, Saizheng and + Bengio, Yoshua and + Cohen, William and + Salakhutdinov, Ruslan and + Manning, Christopher D.", + editor = "Riloff, Ellen and + Chiang, David and + Hockenmaier, Julia and + Tsujii, Jun{'}ichi", + booktitle = "Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing", + month = oct # "-" # nov, + year = "2018", + address = "Brussels, Belgium", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/D18-1259", + doi = "10.18653/v1/D18-1259", + pages = "2369--2380", + abstract = "Existing question answering (QA) datasets fail to train QA systems to perform complex reasoning and provide explanations for answers. We introduce HotpotQA, a new dataset with 113k Wikipedia-based question-answer pairs with four key features: (1) the questions require finding and reasoning over multiple supporting documents to answer; (2) the questions are diverse and not constrained to any pre-existing knowledge bases or knowledge schemas; (3) we provide sentence-level supporting facts required for reasoning, allowing QA systems to reason with strong supervision and explain the predictions; (4) we offer a new type of factoid comparison questions to test QA systems{'} ability to extract relevant facts and perform necessary comparison. We show that HotpotQA is challenging for the latest QA systems, and the supporting facts enable models to improve performance and make explainable predictions.", +}""", + prompt={ + "query": "Given a multi-hop question, retrieve documents that can help answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoHotpotQA", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoHotpotQA", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoHotpotQA", "qrels") + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True \ No newline at end of file diff --git a/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py new file mode 100644 index 0000000000..d77c238f4d --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + +class NanoMSMARCORetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoMSMARCORetrieval", + description="NanoMSMARCORetrieval is a smaller subset of MS MARCO, a collection of datasets focused on deep learning in search.", + reference="https://microsoft.github.io/msmarco/", + dataset={ + "path": "zeta-alpha-ai/NanoMSMARCO", + "revision": "main", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2016-01-01", "2016-12-31"], + domains=["Web"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{DBLP:journals/corr/NguyenRSGTMD16, + author = {Tri Nguyen and + Mir Rosenberg and + Xia Song and + Jianfeng Gao and + Saurabh Tiwary and + Rangan Majumder and + Li Deng}, + title = {{MS} {MARCO:} {A} Human Generated MAchine Reading COmprehension Dataset}, + journal = {CoRR}, + volume = {abs/1611.09268}, + year = {2016}, + url = {http://arxiv.org/abs/1611.09268}, + archivePrefix = {arXiv}, + eprint = {1611.09268}, + timestamp = {Mon, 13 Aug 2018 16:49:03 +0200}, + biburl = {https://dblp.org/rec/journals/corr/NguyenRSGTMD16.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} +}""", + prompt={ + "query": "Given a web search query, retrieve relevant passages that answer the query" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoMSMARCO", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoMSMARCO", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoMSMARCO", "qrels") + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True \ No newline at end of file diff --git a/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py new file mode 100644 index 0000000000..d54c3bc443 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoNFCorpusRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoNFCorpusRetrieval", + description="NanoNFCorpus is a smaller subset of NFCorpus: A Full-Text Learning to Rank Dataset for Medical Information Retrieval.", + reference="https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/", + dataset={ + "path": "zeta-alpha-ai/NanoNFCorpus", + "revision": "main", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2016-01-01", "2016-12-31"], + domains=["Medical", "Academic", "Written"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{boteva2016, + author = {Boteva, Vera and Gholipour, Demian and Sokolov, Artem and Riezler, Stefan}, + title = {A Full-Text Learning to Rank Dataset for Medical Information Retrieval}, + journal = {Proceedings of the 38th European Conference on Information Retrieval}, + journal-abbrev = {ECIR}, + year = {2016}, + city = {Padova}, + country = {Italy}, + url = {http://www.cl.uni-heidelberg.de/~riezler/publications/papers/ECIR2016.pdf} +}""", + prompt={ + "query": "Given a question, retrieve relevant documents that best answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoNFCorpus", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoNFCorpus", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoNFCorpus", "qrels") + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True \ No newline at end of file diff --git a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py new file mode 100644 index 0000000000..6c67e2fb96 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + +class NanoNQRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoNQRetrieval", + description="NanoNQ is a smaller subset of a dataset which contains questions from real users, and it requires QA systems to read and comprehend an entire Wikipedia article that may or may not contain the answer to the question.", + reference="https://ai.google.com/research/NaturalQuestions", + dataset={ + "path": "zeta-alpha-ai/NanoNQ", + "revision": "main", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2019-01-01", "2019-12-31"], + domains=["Academic", "Web"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@article{47761,title = {Natural Questions: a Benchmark for Question Answering Research}, + author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh + and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee + and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le + and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational + Linguistics}}""", + prompt={ + "query": "Given a question, retrieve Wikipedia passages that answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoNQ", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoNQ", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoNQ", "qrels") + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py new file mode 100644 index 0000000000..35c34b9c81 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class NanoQuoraRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoQuoraRetrieval", + description="NanoQuoraRetrieval is a smaller subset of the " + "QuoraRetrieval dataset, which is based on questions that are marked as duplicates on the Quora platform. Given a" + + " question, find other (duplicate) questions.", + reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs", + dataset={ + "path": "zeta-alpha-ai/NanoQuoraRetrieval", + "revision": "main", + }, + type="Retrieval", + category="s2s", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2017-01-01", "2017-12-31"], + domains=["Social"], + task_subtypes=["Duplicate Detection"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@misc{quora-question-pairs, + author = {DataCanary, hilfialkaff, Lili Jiang, Meg Risdal, Nikhil Dandekar, tomtung}, + title = {Quora Question Pairs}, + publisher = {Kaggle}, + year = {2017}, + url = {https://kaggle.com/competitions/quora-question-pairs} +}""", + prompt={ + "query": "Given a question, retrieve questions that are semantically equivalent to the given question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoQuoraRetrieval", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoQuoraRetrieval", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoQuoraRetrieval", "qrels") + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py new file mode 100644 index 0000000000..9c8db32871 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + +class NanoSCIDOCSRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoSCIDOCSRetrieval", + description="NanoFiQA2018 is a smaller subset of " + "SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation" + + " prediction, to document classification and recommendation.", + reference="https://allenai.org/data/scidocs", + dataset={ + "path": "zeta-alpha-ai/NanoSCIDOCS", + "revision": "main", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2020-01-01", "2020-12-31"], + domains=["Academic", "Written", "Non-fiction"], + task_subtypes=[], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{specter2020cohan, + title={SPECTER: Document-level Representation Learning using Citation-informed Transformers}, + author={Arman Cohan and Sergey Feldman and Iz Beltagy and Doug Downey and Daniel S. Weld}, + booktitle={ACL}, + year={2020} +}""", + prompt={ + "query": "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoSCIDOCS", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoSCIDOCS", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoSCIDOCS", "qrels") + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True \ No newline at end of file diff --git a/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py new file mode 100644 index 0000000000..3f637a7134 --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + +class NanoSciFactRetrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoSciFactRetrieval", + description="NanoSciFact is a smaller subset of SciFact, which verifies scientific claims using evidence from the research literature containing scientific paper abstracts.", + reference="https://github.com/allenai/scifact", + dataset={ + "path": "zeta-alpha-ai/NanoSciFact", + "revision": "main", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=["2018-01-01", "2018-12-31"], + domains=["Academic", "Medical", "Written"], + task_subtypes=["Claim verification"], + license="cc-by-4.0", + annotations_creators="expert-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@inproceedings{specter2020cohan, + title={SPECTER: Document-level Representation Learning using Citation-informed Transformers}, + author={Arman Cohan and Sergey Feldman and Iz Beltagy and Doug Downey and Daniel S. Weld}, + booktitle={ACL}, + year={2020} +}""", + prompt={ + "query": "Given a scientific claim, retrieve documents that support or refute the claim" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoSciFact", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoSciFact", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoSciFact", "qrels") + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True \ No newline at end of file diff --git a/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py new file mode 100644 index 0000000000..46f543385c --- /dev/null +++ b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from datasets import load_dataset + +from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval +from mteb.abstasks.TaskMetadata import TaskMetadata + +class NanoTouche2020Retrieval(AbsTaskRetrieval): + metadata = TaskMetadata( + name="NanoTouche2020Retrieval", + description="NanoTouche2020 is a smaller subset of Touché Task 1: Argument Retrieval for Controversial Questions.", + reference="https://webis.de/events/touche-20/shared-task-1.html", + dataset={ + "path": "zeta-alpha-ai/NanoTouche2020", + "revision": "main", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2020-09-23", "2020-09-23"), + domains=["Academic"], + task_subtypes=["Question answering"], + license="cc-by-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@dataset{potthast_2022_6862281, + author = {Potthast, Martin and + Gienapp, Lukas and + Wachsmuth, Henning and + Hagen, Matthias and + Fröbe, Maik and + Bondarenko, Alexander and + Ajjour, Yamen and + Stein, Benno}, + title = {{Touché20-Argument-Retrieval-for-Controversial- + Questions}}, + month = jul, + year = 2022, + publisher = {Zenodo}, + doi = {10.5281/zenodo.6862281}, + url = {https://doi.org/10.5281/zenodo.6862281} +}""", + prompt={ + "query": "Given a question, retrieve detailed and persuasive arguments that answer the question" + }, + ) + + def load_data(self, **kwargs): + if self.data_loaded: + return + + self.corpus = load_dataset("zeta-alpha-ai/NanoTouche2020", "corpus") + self.queries = load_dataset("zeta-alpha-ai/NanoTouche2020", "queries") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoTouche2020", "qrels") + + self.corpus = { + split: { + sample["_id"]: {"_id": sample["_id"], "text": sample["text"]} + for sample in self.corpus[split] + } + for split in self.corpus + } + + self.queries = { + split: {sample["_id"]: sample["text"] for sample in self.queries[split]} + for split in self.queries + } + + self.relevant_docs = { + split: { + sample["query-id"]: { + sample[ + "corpus-id" + ]: 1 # Assuming a score of 1 for relevant documents + } + for sample in self.relevant_docs[split] + } + for split in self.relevant_docs + } + + # print("corpus") + # print(self.corpus) + # print("queries") + # print(self.queries) + # print("relevant_docs") + # print(self.relevant_docs) + self.data_loaded = True \ No newline at end of file From f471ec68a3d6c0d0d145ec4fa8d7e3604187af17 Mon Sep 17 00:00:00 2001 From: Kavya <92774828+KGupta10@users.noreply.github.com> Date: Thu, 12 Dec 2024 22:41:48 -0800 Subject: [PATCH 4/9] format code --- mteb/models/misc_models.py | 2 ++ .../Retrieval/eng/NanoArguAnaRetrieval.py | 14 ++---------- .../eng/NanoClimateFeverRetrieval.py | 15 +++---------- .../Retrieval/eng/NanoDBPediaRetrieval.py | 13 ++--------- .../tasks/Retrieval/eng/NanoFEVERRetrieval.py | 22 ++++++------------- .../Retrieval/eng/NanoFiQA2018Retrieval.py | 14 ++---------- .../Retrieval/eng/NanoHotpotQARetrieval.py | 20 +++++------------ .../Retrieval/eng/NanoMSMARCORetrieval.py | 15 +++---------- .../Retrieval/eng/NanoNFCorpusRetrieval.py | 14 ++---------- mteb/tasks/Retrieval/eng/NanoNQRetrieval.py | 13 ++--------- .../tasks/Retrieval/eng/NanoQuoraRetrieval.py | 19 +++++----------- .../Retrieval/eng/NanoSCIDOCSRetrieval.py | 20 +++++------------ .../Retrieval/eng/NanoSciFactRetrieval.py | 15 +++---------- .../Retrieval/eng/NanoTouche2020Retrieval.py | 15 +++---------- .../tasks/Retrieval/eng/tempCodeRunnerFile.py | 12 +++++----- 15 files changed, 54 insertions(+), 169 deletions(-) diff --git a/mteb/models/misc_models.py b/mteb/models/misc_models.py index 2429cce39b..61dc549b15 100644 --- a/mteb/models/misc_models.py +++ b/mteb/models/misc_models.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from mteb.model_meta import ModelMeta Haon_Chen__speed_embedding_7b_instruct = ModelMeta( diff --git a/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py index bd20e00475..6012574fb9 100644 --- a/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py @@ -64,20 +64,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) - self.data_loaded = True \ No newline at end of file + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py index 499a474513..de326982a4 100644 --- a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py @@ -5,6 +5,7 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata + class NanoClimateFeverRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NanoClimateFeverRetrieval", @@ -63,20 +64,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) - self.data_loaded = True \ No newline at end of file + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py index 5be349b458..00adb31db5 100644 --- a/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py @@ -5,6 +5,7 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata + class NanoDBPediaRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NanoDBPediaRetrieval", @@ -53,20 +54,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py index 1dfa3d1309..c2c0df3d4f 100644 --- a/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py @@ -5,12 +5,14 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata + class NanoFEVERRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NanoFEVERRetrieval", - description="NanoFEVER is a smaller version of " + "FEVER (Fact Extraction and VERification), which consists of 185,445 claims generated by altering sentences" - + " extracted from Wikipedia and subsequently verified without knowledge of the sentence they were" - + " derived from.", + description="NanoFEVER is a smaller version of " + + "FEVER (Fact Extraction and VERification), which consists of 185,445 claims generated by altering sentences" + + " extracted from Wikipedia and subsequently verified without knowledge of the sentence they were" + + " derived from.", reference="https://fever.ai/", dataset={ "path": "zeta-alpha-ai/NanoFEVER", @@ -76,20 +78,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) - self.data_loaded = True \ No newline at end of file + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py index 30438eb951..71908c9714 100644 --- a/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py @@ -5,6 +5,7 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata + class NanoFiQA2018Retrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NanoFiQA2018Retrieval", @@ -63,21 +64,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) self.data_loaded = True - diff --git a/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py index eb0cc5255d..16c5806a74 100644 --- a/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py @@ -5,11 +5,13 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata + class NanoHotpotQARetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NanoHotpotQARetrieval", - description="NanoHotpotQARetrieval is a smaller subset of the " + "HotpotQA dataset, which is a question answering dataset featuring natural, multi-hop questions, with strong" - + " supervision for supporting facts to enable more explainable question answering systems.", + description="NanoHotpotQARetrieval is a smaller subset of the " + + "HotpotQA dataset, which is a question answering dataset featuring natural, multi-hop questions, with strong" + + " supervision for supporting facts to enable more explainable question answering systems.", reference="https://hotpotqa.github.io/", dataset={ "path": "zeta-alpha-ai/NanoHotpotQA", @@ -79,20 +81,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) - self.data_loaded = True \ No newline at end of file + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py index d77c238f4d..7124cec484 100644 --- a/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py @@ -5,6 +5,7 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata + class NanoMSMARCORetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NanoMSMARCORetrieval", @@ -75,20 +76,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) - self.data_loaded = True \ No newline at end of file + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py index d54c3bc443..2173007864 100644 --- a/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py @@ -66,20 +66,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) - self.data_loaded = True \ No newline at end of file + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py index 6c67e2fb96..a8185667ff 100644 --- a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py @@ -5,6 +5,7 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata + class NanoNQRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NanoNQRetrieval", @@ -61,20 +62,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py index 35c34b9c81..6a1a6c44b1 100644 --- a/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py @@ -9,8 +9,9 @@ class NanoQuoraRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NanoQuoraRetrieval", - description="NanoQuoraRetrieval is a smaller subset of the " + "QuoraRetrieval dataset, which is based on questions that are marked as duplicates on the Quora platform. Given a" - + " question, find other (duplicate) questions.", + description="NanoQuoraRetrieval is a smaller subset of the " + + "QuoraRetrieval dataset, which is based on questions that are marked as duplicates on the Quora platform. Given a" + + " question, find other (duplicate) questions.", reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs", dataset={ "path": "zeta-alpha-ai/NanoQuoraRetrieval", @@ -29,7 +30,7 @@ class NanoQuoraRetrieval(AbsTaskRetrieval): annotations_creators="human-annotated", dialect=[], sample_creation="found", - bibtex_citation="""@misc{quora-question-pairs, + bibtex_citation="""@misc{quora-question-pairs, author = {DataCanary, hilfialkaff, Lili Jiang, Meg Risdal, Nikhil Dandekar, tomtung}, title = {Quora Question Pairs}, publisher = {Kaggle}, @@ -64,20 +65,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py index 9c8db32871..b406419865 100644 --- a/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py @@ -5,11 +5,13 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata + class NanoSCIDOCSRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NanoSCIDOCSRetrieval", - description="NanoFiQA2018 is a smaller subset of " + "SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation" - + " prediction, to document classification and recommendation.", + description="NanoFiQA2018 is a smaller subset of " + + "SciDocs, a new evaluation benchmark consisting of seven document-level tasks ranging from citation" + + " prediction, to document classification and recommendation.", reference="https://allenai.org/data/scidocs", dataset={ "path": "zeta-alpha-ai/NanoSCIDOCS", @@ -62,20 +64,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) - self.data_loaded = True \ No newline at end of file + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py index 3f637a7134..994f11a62f 100644 --- a/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py @@ -5,6 +5,7 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata + class NanoSciFactRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NanoSciFactRetrieval", @@ -61,20 +62,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) - self.data_loaded = True \ No newline at end of file + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py index 46f543385c..dd4e5f37dc 100644 --- a/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py @@ -5,6 +5,7 @@ from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.TaskMetadata import TaskMetadata + class NanoTouche2020Retrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NanoTouche2020Retrieval", @@ -72,20 +73,10 @@ def load_data(self, **kwargs): self.relevant_docs = { split: { - sample["query-id"]: { - sample[ - "corpus-id" - ]: 1 # Assuming a score of 1 for relevant documents - } + sample["query-id"]: {sample["corpus-id"]: 1} for sample in self.relevant_docs[split] } for split in self.relevant_docs } - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) - self.data_loaded = True \ No newline at end of file + self.data_loaded = True diff --git a/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py b/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py index 4f0bc4e942..e1d4417771 100644 --- a/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py +++ b/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py @@ -1,6 +1,6 @@ - # print("corpus") - # print(self.corpus) - # print("queries") - # print(self.queries) - # print("relevant_docs") - # print(self.relevant_docs) \ No newline at end of file +# print("corpus") +# print(self.corpus) +# print("queries") +# print(self.queries) +# print("relevant_docs") +# print(self.relevant_docs) From f42d7bf5f28bdca95de710036d181e1dc68890f3 Mon Sep 17 00:00:00 2001 From: KGupta10 <92774828+KGupta10@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:32:38 -0800 Subject: [PATCH 5/9] Update mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py Co-authored-by: Roman Solomatin --- mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py b/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py index e1d4417771..e69de29bb2 100644 --- a/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py +++ b/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py @@ -1,6 +0,0 @@ -# print("corpus") -# print(self.corpus) -# print("queries") -# print(self.queries) -# print("relevant_docs") -# print(self.relevant_docs) From d3d394b0fc6b308e489cba74efd6beef71bbbb53 Mon Sep 17 00:00:00 2001 From: Kavya <92774828+KGupta10@users.noreply.github.com> Date: Mon, 16 Dec 2024 01:55:19 -0800 Subject: [PATCH 6/9] pin revision to commit and add datasets to benchmark.py --- mteb/benchmarks/benchmarks.py | 17 +++++++++++++++++ mteb/tasks/Retrieval/__init__.py | 1 + .../tasks/Retrieval/eng/NanoArguAnaRetrieval.py | 2 +- .../Retrieval/eng/NanoClimateFeverRetrieval.py | 2 +- .../tasks/Retrieval/eng/NanoDBPediaRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py | 2 +- .../Retrieval/eng/NanoFiQA2018Retrieval.py | 2 +- .../Retrieval/eng/NanoHotpotQARetrieval.py | 2 +- .../tasks/Retrieval/eng/NanoMSMARCORetrieval.py | 2 +- .../Retrieval/eng/NanoNFCorpusRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/NanoNQRetrieval.py | 2 +- mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py | 2 +- .../tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py | 2 +- .../tasks/Retrieval/eng/NanoSciFactRetrieval.py | 2 +- .../Retrieval/eng/NanoTouche2020Retrieval.py | 2 +- 15 files changed, 31 insertions(+), 13 deletions(-) diff --git a/mteb/benchmarks/benchmarks.py b/mteb/benchmarks/benchmarks.py index d5efbc092f..e2bb1d3976 100644 --- a/mteb/benchmarks/benchmarks.py +++ b/mteb/benchmarks/benchmarks.py @@ -93,6 +93,17 @@ def load_results( "MedrxivClusteringP2P.v2", "MedrxivClusteringS2S.v2", "MindSmallReranking", + "NanoArguAnaRetrieval", + "NanoClimateFeverRetrieval", + "NanoDBPediaRetrieval", + "NanoFEVERRetrieval", + "NanoFiQA2018Retrieval", + "NanoHotpotQARetrieval", + "NanoMSMARCORetrieval", + "NanoNQRetrieval", + "NanoQuoraRetrieval", + "NanoSCIDOCSRetrieval", + "NanoTouche2020Retrieval", "SCIDOCS", "SICK-R", "STS12", @@ -314,9 +325,11 @@ def load_results( tasks=[ "CUREv1", "NFCorpus", + "NanoNFCorpusRetrieval", "TRECCOVID", "TRECCOVID-PL", "SciFact", + "NanoSciFactRetrieval", "SciFact-PL", "MedicalQARetrieval", "PublicHealthQA", @@ -718,10 +731,12 @@ def load_results( "TwitterHjerneRetrieval", "AILAStatutes", "ArguAna", + "NanoArguAnaRetrieval", "HagridRetrieval", "LegalBenchCorporateLobbying", "LEMBPasskeyRetrieval", "SCIDOCS", + "NanoSCIDOCSRetrieval", "SpartQA", "TempReasonL1", "TRECCOVID", @@ -901,10 +916,12 @@ def load_results( "TwitterHjerneRetrieval", "LegalQuAD", "ArguAna", + "NanoArguAnaRetrieval", "HagridRetrieval", "LegalBenchCorporateLobbying", "LEMBPasskeyRetrieval", "SCIDOCS", + "NanoSCIDOCSRetrieval", "SpartQA", "TempReasonL1", "WinoGrande", diff --git a/mteb/tasks/Retrieval/__init__.py b/mteb/tasks/Retrieval/__init__.py index a588c94aff..d83df7ec5e 100644 --- a/mteb/tasks/Retrieval/__init__.py +++ b/mteb/tasks/Retrieval/__init__.py @@ -72,6 +72,7 @@ from .eng.NanoHotpotQARetrieval import * from .eng.NanoMSMARCORetrieval import * from .eng.NanoNFCorpusRetrieval import * +from .eng.NanoNQRetrieval import * from .eng.NanoQuoraRetrieval import * from .eng.NanoSCIDOCSRetrieval import * from .eng.NanoSciFactRetrieval import * diff --git a/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py index 6012574fb9..688da00a1a 100644 --- a/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py @@ -13,7 +13,7 @@ class NanoArguAnaRetrieval(AbsTaskRetrieval): reference="http://argumentation.bplaced.net/arguana/data", dataset={ "path": "zeta-alpha-ai/NanoArguAna", - "revision": "main", + "revision": "8f4a982d470a32c45817738b9d29042ca55d75ad", }, type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py index de326982a4..07c4cd9886 100644 --- a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py @@ -13,7 +13,7 @@ class NanoClimateFeverRetrieval(AbsTaskRetrieval): reference="https://arxiv.org/abs/2012.00614", dataset={ "path": "zeta-alpha-ai/NanoClimateFEVER", - "revision": "main", + "revision": "96741bfa30b9f56db8c9eb7d08e775ed6474f206", }, type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py index 00adb31db5..bd349715f3 100644 --- a/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py @@ -13,7 +13,7 @@ class NanoDBPediaRetrieval(AbsTaskRetrieval): reference="https://huggingface.co/datasets/zeta-alpha-ai/NanoDBPedia", dataset={ "path": "zeta-alpha-ai/NanoDBPedia", - "revision": "main", + "revision": "438f1c25129f05db6238699b5afdc9c6b58d2096", }, type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py index c2c0df3d4f..d2858d917e 100644 --- a/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py @@ -16,7 +16,7 @@ class NanoFEVERRetrieval(AbsTaskRetrieval): reference="https://fever.ai/", dataset={ "path": "zeta-alpha-ai/NanoFEVER", - "revision": "main", + "revision": "a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8", }, type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py index 71908c9714..acd12f25b7 100644 --- a/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py @@ -13,7 +13,7 @@ class NanoFiQA2018Retrieval(AbsTaskRetrieval): reference="https://sites.google.com/view/fiqa/", dataset={ "path": "zeta-alpha-ai/NanoFiQA2018", - "revision": "main", + "revision": "4163ba032953d5044a7a6244261413f609c14342", }, type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py index 16c5806a74..652f6a342e 100644 --- a/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py @@ -15,7 +15,7 @@ class NanoHotpotQARetrieval(AbsTaskRetrieval): reference="https://hotpotqa.github.io/", dataset={ "path": "zeta-alpha-ai/NanoHotpotQA", - "revision": "main", + "revision": "d79c0cdda980aba54842756770928035e1b61a51", }, type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py index 7124cec484..4cdea000b2 100644 --- a/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py @@ -13,7 +13,7 @@ class NanoMSMARCORetrieval(AbsTaskRetrieval): reference="https://microsoft.github.io/msmarco/", dataset={ "path": "zeta-alpha-ai/NanoMSMARCO", - "revision": "main", + "revision": "7b8ff22f2771dc65ac5b439f222eb19a1f56abda", }, type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py index 2173007864..e61fd85aff 100644 --- a/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py @@ -13,7 +13,7 @@ class NanoNFCorpusRetrieval(AbsTaskRetrieval): reference="https://www.cl.uni-heidelberg.de/statnlpgroup/nfcorpus/", dataset={ "path": "zeta-alpha-ai/NanoNFCorpus", - "revision": "main", + "revision": "dd542a7efb9ad2136b9e00768b60fca9038f8156", }, type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py index a8185667ff..92e8d3236d 100644 --- a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py @@ -13,7 +13,7 @@ class NanoNQRetrieval(AbsTaskRetrieval): reference="https://ai.google.com/research/NaturalQuestions", dataset={ "path": "zeta-alpha-ai/NanoNQ", - "revision": "main", + "revision": "77540146379abf95df8326a3c5bb9eb21c7146c3", }, type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py index 6a1a6c44b1..451d9a4b77 100644 --- a/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py @@ -15,7 +15,7 @@ class NanoQuoraRetrieval(AbsTaskRetrieval): reference="https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs", dataset={ "path": "zeta-alpha-ai/NanoQuoraRetrieval", - "revision": "main", + "revision": "2ab2d73e6c862026282808b913a34f4136928545", }, type="Retrieval", category="s2s", diff --git a/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py index b406419865..faa28052f0 100644 --- a/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py @@ -15,7 +15,7 @@ class NanoSCIDOCSRetrieval(AbsTaskRetrieval): reference="https://allenai.org/data/scidocs", dataset={ "path": "zeta-alpha-ai/NanoSCIDOCS", - "revision": "main", + "revision": "484eb90549fc3f0b9c42b3551e80ceb999515537", }, type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py index 994f11a62f..df6d1a6a9e 100644 --- a/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py @@ -13,7 +13,7 @@ class NanoSciFactRetrieval(AbsTaskRetrieval): reference="https://github.com/allenai/scifact", dataset={ "path": "zeta-alpha-ai/NanoSciFact", - "revision": "main", + "revision": "309f1d1ae3ae2e092444a8a0c25bed59b82318bc", }, type="Retrieval", category="s2p", diff --git a/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py index dd4e5f37dc..f0000801ea 100644 --- a/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py @@ -13,7 +13,7 @@ class NanoTouche2020Retrieval(AbsTaskRetrieval): reference="https://webis.de/events/touche-20/shared-task-1.html", dataset={ "path": "zeta-alpha-ai/NanoTouche2020", - "revision": "main", + "revision": "0d2f26ed8c5ad309f95c7f9499c70a40e140fccd", }, type="Retrieval", category="s2p", From 1f0c00938fdbed3524a2dc0fb42ccc49633e32ad Mon Sep 17 00:00:00 2001 From: Kavya <92774828+KGupta10@users.noreply.github.com> Date: Mon, 16 Dec 2024 02:22:49 -0800 Subject: [PATCH 7/9] create new benchmark for NanoBEIR --- mteb/benchmarks/benchmarks.py | 41 ++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/mteb/benchmarks/benchmarks.py b/mteb/benchmarks/benchmarks.py index e2bb1d3976..6494e169c7 100644 --- a/mteb/benchmarks/benchmarks.py +++ b/mteb/benchmarks/benchmarks.py @@ -93,17 +93,6 @@ def load_results( "MedrxivClusteringP2P.v2", "MedrxivClusteringS2S.v2", "MindSmallReranking", - "NanoArguAnaRetrieval", - "NanoClimateFeverRetrieval", - "NanoDBPediaRetrieval", - "NanoFEVERRetrieval", - "NanoFiQA2018Retrieval", - "NanoHotpotQARetrieval", - "NanoMSMARCORetrieval", - "NanoNQRetrieval", - "NanoQuoraRetrieval", - "NanoSCIDOCSRetrieval", - "NanoTouche2020Retrieval", "SCIDOCS", "SICK-R", "STS12", @@ -325,11 +314,9 @@ def load_results( tasks=[ "CUREv1", "NFCorpus", - "NanoNFCorpusRetrieval", "TRECCOVID", "TRECCOVID-PL", "SciFact", - "NanoSciFactRetrieval", "SciFact-PL", "MedicalQARetrieval", "PublicHealthQA", @@ -731,12 +718,10 @@ def load_results( "TwitterHjerneRetrieval", "AILAStatutes", "ArguAna", - "NanoArguAnaRetrieval", "HagridRetrieval", "LegalBenchCorporateLobbying", "LEMBPasskeyRetrieval", "SCIDOCS", - "NanoSCIDOCSRetrieval", "SpartQA", "TempReasonL1", "TRECCOVID", @@ -916,12 +901,10 @@ def load_results( "TwitterHjerneRetrieval", "LegalQuAD", "ArguAna", - "NanoArguAnaRetrieval", "HagridRetrieval", "LegalBenchCorporateLobbying", "LEMBPasskeyRetrieval", "SCIDOCS", - "NanoSCIDOCSRetrieval", "SpartQA", "TempReasonL1", "WinoGrande", @@ -995,3 +978,27 @@ def load_results( year={2024} }""", ) + +NANOBEIR = Benchmark( + name="NanoBEIR", + tasks=get_tasks( + tasks=[ + "NanoArguAnaRetrieval", + "NanoClimateFeverRetrieval", + "NanoDBPediaRetrieval", + "NanoFEVERRetrieval", + "NanoFiQA2018Retrieval", + "NanoHotpotQARetrieval", + "NanoMSMARCORetrieval", + "NanoNFCorpusRetrieval", + "NanoNQRetrieval", + "NanoQuoraRetrieval" + "NanoSCIDOCSRetrieval", + "NanoSciFactRetrieval", + "NanoTouche2020Retrieval", + ], + ), + description="A benchmark to evaluate with subsets of BEIR datasets to use less computational power", + reference="https://huggingface.co/collections/zeta-alpha-ai/nanobeir-66e1a0af21dfd93e620cd9f6", + citation=None, +) From 7fb48c19f23dd990959278e707999f316160bd72 Mon Sep 17 00:00:00 2001 From: Kavya <92774828+KGupta10@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:59:17 -0800 Subject: [PATCH 8/9] add revision when loading datasets --- mteb/benchmarks/benchmarks.py | 2 +- mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py | 8 ++++---- mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py | 6 +++--- mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py | 6 +++--- mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py | 6 +++--- mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py | 6 +++--- mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py | 6 +++--- mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py | 6 +++--- mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py | 6 +++--- mteb/tasks/Retrieval/eng/NanoNQRetrieval.py | 6 +++--- mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py | 6 +++--- mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py | 6 +++--- mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py | 6 +++--- mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py | 6 +++--- mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py | 0 15 files changed, 41 insertions(+), 41 deletions(-) delete mode 100644 mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py diff --git a/mteb/benchmarks/benchmarks.py b/mteb/benchmarks/benchmarks.py index 6494e169c7..e872143ee5 100644 --- a/mteb/benchmarks/benchmarks.py +++ b/mteb/benchmarks/benchmarks.py @@ -992,7 +992,7 @@ def load_results( "NanoMSMARCORetrieval", "NanoNFCorpusRetrieval", "NanoNQRetrieval", - "NanoQuoraRetrieval" + "NanoQuoraRetrieval", "NanoSCIDOCSRetrieval", "NanoSciFactRetrieval", "NanoTouche2020Retrieval", diff --git a/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py index 688da00a1a..a12826ef00 100644 --- a/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py @@ -44,10 +44,10 @@ class NanoArguAnaRetrieval(AbsTaskRetrieval): def load_data(self, **kwargs): if self.data_loaded: return - - self.corpus = load_dataset("zeta-alpha-ai/NanoArguAna", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoArguAna", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoArguAna", "qrels") + + self.corpus = load_dataset("zeta-alpha-ai/NanoArguAna", "corpus", revision="8f4a982d470a32c45817738b9d29042ca55d75ad") + self.queries = load_dataset("zeta-alpha-ai/NanoArguAna", "queries", revision="8f4a982d470a32c45817738b9d29042ca55d75ad") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoArguAna", "qrels", revision="8f4a982d470a32c45817738b9d29042ca55d75ad") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py index 07c4cd9886..c17a0efe5f 100644 --- a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py @@ -45,9 +45,9 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "qrels") + self.corpus = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "corpus", revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206") + self.queries = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "queries", revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "qrels", revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py index bd349715f3..5f5fd91650 100644 --- a/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py @@ -35,9 +35,9 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoDBPedia", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoDBPedia", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoDBPedia", "qrels") + self.corpus = load_dataset("zeta-alpha-ai/NanoDBPedia", "corpus", revision="438f1c25129f05db6238699b5afdc9c6b58d2096") + self.queries = load_dataset("zeta-alpha-ai/NanoDBPedia", "queries", revision="438f1c25129f05db6238699b5afdc9c6b58d2096") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoDBPedia", "qrels", revision="438f1c25129f05db6238699b5afdc9c6b58d2096") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py index d2858d917e..b57557672e 100644 --- a/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py @@ -59,9 +59,9 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoFEVER", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoFEVER", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoFEVER", "qrels") + self.corpus = load_dataset("zeta-alpha-ai/NanoFEVER", "corpus", revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8") + self.queries = load_dataset("zeta-alpha-ai/NanoFEVER", "queries", revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoFEVER", "qrels", revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py index acd12f25b7..582629cd03 100644 --- a/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py @@ -45,9 +45,9 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoFiQA2018", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoFiQA2018", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoFiQA2018", "qrels") + self.corpus = load_dataset("zeta-alpha-ai/NanoFiQA2018", "corpus", revision="4163ba032953d5044a7a6244261413f609c14342") + self.queries = load_dataset("zeta-alpha-ai/NanoFiQA2018", "queries", revision="4163ba032953d5044a7a6244261413f609c14342") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoFiQA2018", "qrels", revision="4163ba032953d5044a7a6244261413f609c14342") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py index 652f6a342e..250acf774e 100644 --- a/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py @@ -62,9 +62,9 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoHotpotQA", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoHotpotQA", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoHotpotQA", "qrels") + self.corpus = load_dataset("zeta-alpha-ai/NanoHotpotQA", "corpus", revision="d79c0cdda980aba54842756770928035e1b61a51") + self.queries = load_dataset("zeta-alpha-ai/NanoHotpotQA", "queries", revision="d79c0cdda980aba54842756770928035e1b61a51") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoHotpotQA", "qrels", revision="d79c0cdda980aba54842756770928035e1b61a51") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py index 4cdea000b2..80c7c37909 100644 --- a/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py @@ -57,9 +57,9 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoMSMARCO", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoMSMARCO", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoMSMARCO", "qrels") + self.corpus = load_dataset("zeta-alpha-ai/NanoMSMARCO", "corpus", revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda") + self.queries = load_dataset("zeta-alpha-ai/NanoMSMARCO", "queries", revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoMSMARCO", "qrels", revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py index e61fd85aff..337a140dc3 100644 --- a/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py @@ -47,9 +47,9 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoNFCorpus", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoNFCorpus", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoNFCorpus", "qrels") + self.corpus = load_dataset("zeta-alpha-ai/NanoNFCorpus", "corpus", revision="dd542a7efb9ad2136b9e00768b60fca9038f8156") + self.queries = load_dataset("zeta-alpha-ai/NanoNFCorpus", "queries", revision="dd542a7efb9ad2136b9e00768b60fca9038f8156") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoNFCorpus", "qrels", revision="dd542a7efb9ad2136b9e00768b60fca9038f8156") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py index 92e8d3236d..205c063a3c 100644 --- a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py @@ -43,9 +43,9 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoNQ", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoNQ", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoNQ", "qrels") + self.corpus = load_dataset("zeta-alpha-ai/NanoNQ", "corpus", revision="77540146379abf95df8326a3c5bb9eb21c7146c3") + self.queries = load_dataset("zeta-alpha-ai/NanoNQ", "queries", revision="77540146379abf95df8326a3c5bb9eb21c7146c3") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoNQ", "qrels", revision="77540146379abf95df8326a3c5bb9eb21c7146c3") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py index 451d9a4b77..3d0a060aba 100644 --- a/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py @@ -46,9 +46,9 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoQuoraRetrieval", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoQuoraRetrieval", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoQuoraRetrieval", "qrels") + self.corpus = load_dataset("zeta-alpha-ai/NanoQuoraRetrieval", "corpus", revision="2ab2d73e6c862026282808b913a34f4136928545") + self.queries = load_dataset("zeta-alpha-ai/NanoQuoraRetrieval", "queries", revision="2ab2d73e6c862026282808b913a34f4136928545") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoQuoraRetrieval", "qrels", revision="2ab2d73e6c862026282808b913a34f4136928545") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py index faa28052f0..fe92755b15 100644 --- a/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py @@ -45,9 +45,9 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoSCIDOCS", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoSCIDOCS", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoSCIDOCS", "qrels") + self.corpus = load_dataset("zeta-alpha-ai/NanoSCIDOCS", "corpus", revision="484eb90549fc3f0b9c42b3551e80ceb999515537") + self.queries = load_dataset("zeta-alpha-ai/NanoSCIDOCS", "queries", revision="484eb90549fc3f0b9c42b3551e80ceb999515537") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoSCIDOCS", "qrels", revision="484eb90549fc3f0b9c42b3551e80ceb999515537") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py index df6d1a6a9e..3e3e94440a 100644 --- a/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py @@ -43,9 +43,9 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoSciFact", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoSciFact", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoSciFact", "qrels") + self.corpus = load_dataset("zeta-alpha-ai/NanoSciFact", "corpus", revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc") + self.queries = load_dataset("zeta-alpha-ai/NanoSciFact", "queries", revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoSciFact", "qrels", revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py index f0000801ea..46c7dad1a5 100644 --- a/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py @@ -54,9 +54,9 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoTouche2020", "corpus") - self.queries = load_dataset("zeta-alpha-ai/NanoTouche2020", "queries") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoTouche2020", "qrels") + self.corpus = load_dataset("zeta-alpha-ai/NanoTouche2020", "corpus", revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd") + self.queries = load_dataset("zeta-alpha-ai/NanoTouche2020", "queries", revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd") + self.relevant_docs = load_dataset("zeta-alpha-ai/NanoTouche2020","qrels", revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd") self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py b/mteb/tasks/Retrieval/eng/tempCodeRunnerFile.py deleted file mode 100644 index e69de29bb2..0000000000 From 2631a4dfd39c5b0713632ead0a3f045a82bb2b84 Mon Sep 17 00:00:00 2001 From: isaac-chung Date: Wed, 18 Dec 2024 17:17:29 +0000 Subject: [PATCH 9/9] lint --- .../Retrieval/eng/NanoArguAnaRetrieval.py | 20 +++++++++++++++---- .../eng/NanoClimateFeverRetrieval.py | 18 ++++++++++++++--- .../Retrieval/eng/NanoDBPediaRetrieval.py | 18 ++++++++++++++--- .../tasks/Retrieval/eng/NanoFEVERRetrieval.py | 18 ++++++++++++++--- .../Retrieval/eng/NanoFiQA2018Retrieval.py | 18 ++++++++++++++--- .../Retrieval/eng/NanoHotpotQARetrieval.py | 18 ++++++++++++++--- .../Retrieval/eng/NanoMSMARCORetrieval.py | 18 ++++++++++++++--- .../Retrieval/eng/NanoNFCorpusRetrieval.py | 18 ++++++++++++++--- mteb/tasks/Retrieval/eng/NanoNQRetrieval.py | 18 ++++++++++++++--- .../tasks/Retrieval/eng/NanoQuoraRetrieval.py | 18 ++++++++++++++--- .../Retrieval/eng/NanoSCIDOCSRetrieval.py | 18 ++++++++++++++--- .../Retrieval/eng/NanoSciFactRetrieval.py | 18 ++++++++++++++--- .../Retrieval/eng/NanoTouche2020Retrieval.py | 18 ++++++++++++++--- 13 files changed, 196 insertions(+), 40 deletions(-) diff --git a/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py index a12826ef00..2230368b94 100644 --- a/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py @@ -44,10 +44,22 @@ class NanoArguAnaRetrieval(AbsTaskRetrieval): def load_data(self, **kwargs): if self.data_loaded: return - - self.corpus = load_dataset("zeta-alpha-ai/NanoArguAna", "corpus", revision="8f4a982d470a32c45817738b9d29042ca55d75ad") - self.queries = load_dataset("zeta-alpha-ai/NanoArguAna", "queries", revision="8f4a982d470a32c45817738b9d29042ca55d75ad") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoArguAna", "qrels", revision="8f4a982d470a32c45817738b9d29042ca55d75ad") + + self.corpus = load_dataset( + "zeta-alpha-ai/NanoArguAna", + "corpus", + revision="8f4a982d470a32c45817738b9d29042ca55d75ad", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoArguAna", + "queries", + revision="8f4a982d470a32c45817738b9d29042ca55d75ad", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoArguAna", + "qrels", + revision="8f4a982d470a32c45817738b9d29042ca55d75ad", + ) self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py index c17a0efe5f..0185a454d3 100644 --- a/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py @@ -45,9 +45,21 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "corpus", revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206") - self.queries = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "queries", revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "qrels", revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206") + self.corpus = load_dataset( + "zeta-alpha-ai/NanoClimateFEVER", + "corpus", + revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoClimateFEVER", + "queries", + revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoClimateFEVER", + "qrels", + revision="96741bfa30b9f56db8c9eb7d08e775ed6474f206", + ) self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py index 5f5fd91650..caa638743c 100644 --- a/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py @@ -35,9 +35,21 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoDBPedia", "corpus", revision="438f1c25129f05db6238699b5afdc9c6b58d2096") - self.queries = load_dataset("zeta-alpha-ai/NanoDBPedia", "queries", revision="438f1c25129f05db6238699b5afdc9c6b58d2096") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoDBPedia", "qrels", revision="438f1c25129f05db6238699b5afdc9c6b58d2096") + self.corpus = load_dataset( + "zeta-alpha-ai/NanoDBPedia", + "corpus", + revision="438f1c25129f05db6238699b5afdc9c6b58d2096", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoDBPedia", + "queries", + revision="438f1c25129f05db6238699b5afdc9c6b58d2096", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoDBPedia", + "qrels", + revision="438f1c25129f05db6238699b5afdc9c6b58d2096", + ) self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py index b57557672e..6bdd0ab4cf 100644 --- a/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py @@ -59,9 +59,21 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoFEVER", "corpus", revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8") - self.queries = load_dataset("zeta-alpha-ai/NanoFEVER", "queries", revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoFEVER", "qrels", revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8") + self.corpus = load_dataset( + "zeta-alpha-ai/NanoFEVER", + "corpus", + revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoFEVER", + "queries", + revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoFEVER", + "qrels", + revision="a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8", + ) self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py index 582629cd03..1a3467c1d7 100644 --- a/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoFiQA2018Retrieval.py @@ -45,9 +45,21 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoFiQA2018", "corpus", revision="4163ba032953d5044a7a6244261413f609c14342") - self.queries = load_dataset("zeta-alpha-ai/NanoFiQA2018", "queries", revision="4163ba032953d5044a7a6244261413f609c14342") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoFiQA2018", "qrels", revision="4163ba032953d5044a7a6244261413f609c14342") + self.corpus = load_dataset( + "zeta-alpha-ai/NanoFiQA2018", + "corpus", + revision="4163ba032953d5044a7a6244261413f609c14342", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoFiQA2018", + "queries", + revision="4163ba032953d5044a7a6244261413f609c14342", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoFiQA2018", + "qrels", + revision="4163ba032953d5044a7a6244261413f609c14342", + ) self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py index 250acf774e..4389aeafa8 100644 --- a/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoHotpotQARetrieval.py @@ -62,9 +62,21 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoHotpotQA", "corpus", revision="d79c0cdda980aba54842756770928035e1b61a51") - self.queries = load_dataset("zeta-alpha-ai/NanoHotpotQA", "queries", revision="d79c0cdda980aba54842756770928035e1b61a51") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoHotpotQA", "qrels", revision="d79c0cdda980aba54842756770928035e1b61a51") + self.corpus = load_dataset( + "zeta-alpha-ai/NanoHotpotQA", + "corpus", + revision="d79c0cdda980aba54842756770928035e1b61a51", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoHotpotQA", + "queries", + revision="d79c0cdda980aba54842756770928035e1b61a51", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoHotpotQA", + "qrels", + revision="d79c0cdda980aba54842756770928035e1b61a51", + ) self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py index 80c7c37909..8a2f51e7fd 100644 --- a/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoMSMARCORetrieval.py @@ -57,9 +57,21 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoMSMARCO", "corpus", revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda") - self.queries = load_dataset("zeta-alpha-ai/NanoMSMARCO", "queries", revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoMSMARCO", "qrels", revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda") + self.corpus = load_dataset( + "zeta-alpha-ai/NanoMSMARCO", + "corpus", + revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoMSMARCO", + "queries", + revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoMSMARCO", + "qrels", + revision="7b8ff22f2771dc65ac5b439f222eb19a1f56abda", + ) self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py index 337a140dc3..0f6ac8533a 100644 --- a/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoNFCorpusRetrieval.py @@ -47,9 +47,21 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoNFCorpus", "corpus", revision="dd542a7efb9ad2136b9e00768b60fca9038f8156") - self.queries = load_dataset("zeta-alpha-ai/NanoNFCorpus", "queries", revision="dd542a7efb9ad2136b9e00768b60fca9038f8156") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoNFCorpus", "qrels", revision="dd542a7efb9ad2136b9e00768b60fca9038f8156") + self.corpus = load_dataset( + "zeta-alpha-ai/NanoNFCorpus", + "corpus", + revision="dd542a7efb9ad2136b9e00768b60fca9038f8156", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoNFCorpus", + "queries", + revision="dd542a7efb9ad2136b9e00768b60fca9038f8156", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoNFCorpus", + "qrels", + revision="dd542a7efb9ad2136b9e00768b60fca9038f8156", + ) self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py index 205c063a3c..5aa831f799 100644 --- a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py @@ -43,9 +43,21 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoNQ", "corpus", revision="77540146379abf95df8326a3c5bb9eb21c7146c3") - self.queries = load_dataset("zeta-alpha-ai/NanoNQ", "queries", revision="77540146379abf95df8326a3c5bb9eb21c7146c3") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoNQ", "qrels", revision="77540146379abf95df8326a3c5bb9eb21c7146c3") + self.corpus = load_dataset( + "zeta-alpha-ai/NanoNQ", + "corpus", + revision="77540146379abf95df8326a3c5bb9eb21c7146c3", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoNQ", + "queries", + revision="77540146379abf95df8326a3c5bb9eb21c7146c3", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoNQ", + "qrels", + revision="77540146379abf95df8326a3c5bb9eb21c7146c3", + ) self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py index 3d0a060aba..1391d12b93 100644 --- a/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoQuoraRetrieval.py @@ -46,9 +46,21 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoQuoraRetrieval", "corpus", revision="2ab2d73e6c862026282808b913a34f4136928545") - self.queries = load_dataset("zeta-alpha-ai/NanoQuoraRetrieval", "queries", revision="2ab2d73e6c862026282808b913a34f4136928545") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoQuoraRetrieval", "qrels", revision="2ab2d73e6c862026282808b913a34f4136928545") + self.corpus = load_dataset( + "zeta-alpha-ai/NanoQuoraRetrieval", + "corpus", + revision="2ab2d73e6c862026282808b913a34f4136928545", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoQuoraRetrieval", + "queries", + revision="2ab2d73e6c862026282808b913a34f4136928545", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoQuoraRetrieval", + "qrels", + revision="2ab2d73e6c862026282808b913a34f4136928545", + ) self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py index fe92755b15..2d27e1a2dc 100644 --- a/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoSCIDOCSRetrieval.py @@ -45,9 +45,21 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoSCIDOCS", "corpus", revision="484eb90549fc3f0b9c42b3551e80ceb999515537") - self.queries = load_dataset("zeta-alpha-ai/NanoSCIDOCS", "queries", revision="484eb90549fc3f0b9c42b3551e80ceb999515537") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoSCIDOCS", "qrels", revision="484eb90549fc3f0b9c42b3551e80ceb999515537") + self.corpus = load_dataset( + "zeta-alpha-ai/NanoSCIDOCS", + "corpus", + revision="484eb90549fc3f0b9c42b3551e80ceb999515537", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoSCIDOCS", + "queries", + revision="484eb90549fc3f0b9c42b3551e80ceb999515537", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoSCIDOCS", + "qrels", + revision="484eb90549fc3f0b9c42b3551e80ceb999515537", + ) self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py index 3e3e94440a..aff949d319 100644 --- a/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoSciFactRetrieval.py @@ -43,9 +43,21 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoSciFact", "corpus", revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc") - self.queries = load_dataset("zeta-alpha-ai/NanoSciFact", "queries", revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoSciFact", "qrels", revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc") + self.corpus = load_dataset( + "zeta-alpha-ai/NanoSciFact", + "corpus", + revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoSciFact", + "queries", + revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoSciFact", + "qrels", + revision="309f1d1ae3ae2e092444a8a0c25bed59b82318bc", + ) self.corpus = { split: { diff --git a/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py index 46c7dad1a5..656b5494a0 100644 --- a/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoTouche2020Retrieval.py @@ -54,9 +54,21 @@ def load_data(self, **kwargs): if self.data_loaded: return - self.corpus = load_dataset("zeta-alpha-ai/NanoTouche2020", "corpus", revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd") - self.queries = load_dataset("zeta-alpha-ai/NanoTouche2020", "queries", revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd") - self.relevant_docs = load_dataset("zeta-alpha-ai/NanoTouche2020","qrels", revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd") + self.corpus = load_dataset( + "zeta-alpha-ai/NanoTouche2020", + "corpus", + revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd", + ) + self.queries = load_dataset( + "zeta-alpha-ai/NanoTouche2020", + "queries", + revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd", + ) + self.relevant_docs = load_dataset( + "zeta-alpha-ai/NanoTouche2020", + "qrels", + revision="0d2f26ed8c5ad309f95c7f9499c70a40e140fccd", + ) self.corpus = { split: {