Add NanoBEIR Datasets

mteb/benchmarks/benchmarks.py

@@ -93,6 +93,17 @@ def load_results(
             "MedrxivClusteringP2P.v2",
             "MedrxivClusteringS2S.v2",
             "MindSmallReranking",
+            "NanoArguAnaRetrieval",
+            "NanoClimateFeverRetrieval",
+            "NanoDBPediaRetrieval",
+            "NanoFEVERRetrieval",
+            "NanoFiQA2018Retrieval",
+            "NanoHotpotQARetrieval",
+            "NanoMSMARCORetrieval",
+            "NanoNQRetrieval",
+            "NanoQuoraRetrieval",
+            "NanoSCIDOCSRetrieval",
+            "NanoTouche2020Retrieval",
             "SCIDOCS",
             "SICK-R",
             "STS12",
@@ -314,9 +325,11 @@ def load_results(
         tasks=[
             "CUREv1",
             "NFCorpus",
+            "NanoNFCorpusRetrieval",
             "TRECCOVID",
             "TRECCOVID-PL",
             "SciFact",
+            "NanoSciFactRetrieval",
             "SciFact-PL",
             "MedicalQARetrieval",
             "PublicHealthQA",
@@ -718,10 +731,12 @@ def load_results(
             "TwitterHjerneRetrieval",
             "AILAStatutes",
             "ArguAna",
+            "NanoArguAnaRetrieval",
             "HagridRetrieval",
             "LegalBenchCorporateLobbying",
             "LEMBPasskeyRetrieval",
             "SCIDOCS",
+            "NanoSCIDOCSRetrieval",
             "SpartQA",
             "TempReasonL1",
             "TRECCOVID",
@@ -901,10 +916,12 @@ def load_results(
             "TwitterHjerneRetrieval",
             "LegalQuAD",
             "ArguAna",
+            "NanoArguAnaRetrieval",
             "HagridRetrieval",
             "LegalBenchCorporateLobbying",
             "LEMBPasskeyRetrieval",
             "SCIDOCS",
+            "NanoSCIDOCSRetrieval",
             "SpartQA",
             "TempReasonL1",
             "WinoGrande",

mteb/models/misc_models.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from mteb.model_meta import ModelMeta
 
 Haon_Chen__speed_embedding_7b_instruct = ModelMeta(

mteb/tasks/Retrieval/__init__.py

@@ -64,6 +64,19 @@
 from .eng.MLQuestions import *
 from .eng.MSMARCORetrieval import *
 from .eng.MSMARCOv2Retrieval import *
+from .eng.NanoArguAnaRetrieval import *
+from .eng.NanoClimateFeverRetrieval import *
+from .eng.NanoDBPediaRetrieval import *
+from .eng.NanoFEVERRetrieval import *
+from .eng.NanoFiQA2018Retrieval import *
+from .eng.NanoHotpotQARetrieval import *
+from .eng.NanoMSMARCORetrieval import *
+from .eng.NanoNFCorpusRetrieval import *
+from .eng.NanoNQRetrieval import *
+from .eng.NanoQuoraRetrieval import *
+from .eng.NanoSCIDOCSRetrieval import *
+from .eng.NanoSciFactRetrieval import *
+from .eng.NanoTouche2020Retrieval import *
 from .eng.NarrativeQARetrieval import *
 from .eng.NFCorpusRetrieval import *
 from .eng.NQRetrieval import *

mteb/tasks/Retrieval/eng/NanoArguAnaRetrieval.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from datasets import load_dataset
+
+from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+
+class NanoArguAnaRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="NanoArguAnaRetrieval",
+        description="NanoArguAna is a smaller subset of ArguAna, a dataset for argument retrieval in debate contexts.",
+        reference="http://argumentation.bplaced.net/arguana/data",
+        dataset={
+            "path": "zeta-alpha-ai/NanoArguAna",
+            "revision": "8f4a982d470a32c45817738b9d29042ca55d75ad",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["train"],
+        eval_langs=["eng-Latn"],
+        main_score="ndcg_at_10",
+        date=["2020-01-01", "2020-12-31"],
+        domains=["Medical", "Written"],
+        task_subtypes=["Discourse coherence"],
+        license="cc-by-4.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="""@inproceedings{boteva2016,
+  author = {Boteva, Vera and Gholipour, Demian and Sokolov, Artem and Riezler, Stefan},
+  title = {A Full-Text Learning to Rank Dataset for Medical Information Retrieval},
+  journal = {Proceedings of the 38th European Conference on Information Retrieval},
+  journal-abbrev = {ECIR},
+  year = {2016},
+  city = {Padova},
+  country = {Italy},
+  url = {http://www.cl.uni-heidelberg.de/~riezler/publications/papers/ECIR2016.pdf}
+}""",
+        prompt={"query": "Given a claim, find documents that refute the claim"},
+    )
+
+    def load_data(self, **kwargs):
+        if self.data_loaded:
+            return
+
+        self.corpus = load_dataset("zeta-alpha-ai/NanoArguAna", "corpus")
+        self.queries = load_dataset("zeta-alpha-ai/NanoArguAna", "queries")
+        self.relevant_docs = load_dataset("zeta-alpha-ai/NanoArguAna", "qrels")
+
+        self.corpus = {
+            split: {
+                sample["_id"]: {"_id": sample["_id"], "text": sample["text"]}
+                for sample in self.corpus[split]
+            }
+            for split in self.corpus
+        }
+
+        self.queries = {
+            split: {sample["_id"]: sample["text"] for sample in self.queries[split]}
+            for split in self.queries
+        }
+
+        self.relevant_docs = {
+            split: {
+                sample["query-id"]: {sample["corpus-id"]: 1}
+                for sample in self.relevant_docs[split]
+            }
+            for split in self.relevant_docs
+        }
+
+        self.data_loaded = True

mteb/tasks/Retrieval/eng/NanoClimateFeverRetrieval.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from datasets import load_dataset
+
+from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+
+class NanoClimateFeverRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="NanoClimateFeverRetrieval",
+        description="NanoClimateFever is a small version of the BEIR dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change.",
+        reference="https://arxiv.org/abs/2012.00614",
+        dataset={
+            "path": "zeta-alpha-ai/NanoClimateFEVER",
+            "revision": "96741bfa30b9f56db8c9eb7d08e775ed6474f206",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["train"],
+        eval_langs=["eng-Latn"],
+        main_score="ndcg_at_10",
+        date=["2020-01-01", "2020-12-31"],
+        domains=["Non-fiction", "Academic", "News"],
+        task_subtypes=["Claim verification"],
+        license="cc-by-4.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="""@misc{diggelmann2021climatefever,
+      title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, 
+      author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold},
+      year={2021},
+      eprint={2012.00614},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}""",
+        prompt={
+            "query": "Given a claim about climate change, retrieve documents that support or refute the claim"
+        },
+    )
+
+    def load_data(self, **kwargs):
+        if self.data_loaded:
+            return
+
+        self.corpus = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "corpus")
+        self.queries = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "queries")
+        self.relevant_docs = load_dataset("zeta-alpha-ai/NanoClimateFEVER", "qrels")
+
+        self.corpus = {
+            split: {
+                sample["_id"]: {"_id": sample["_id"], "text": sample["text"]}
+                for sample in self.corpus[split]
+            }
+            for split in self.corpus
+        }
+
+        self.queries = {
+            split: {sample["_id"]: sample["text"] for sample in self.queries[split]}
+            for split in self.queries
+        }
+
+        self.relevant_docs = {
+            split: {
+                sample["query-id"]: {sample["corpus-id"]: 1}
+                for sample in self.relevant_docs[split]
+            }
+            for split in self.relevant_docs
+        }
+
+        self.data_loaded = True

mteb/tasks/Retrieval/eng/NanoDBPediaRetrieval.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from datasets import load_dataset
+
+from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+
+class NanoDBPediaRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="NanoDBPediaRetrieval",
+        description="NanoDBPediaRetrieval is a small version of the standard test collection for entity search over the DBpedia knowledge base.",
+        reference="https://huggingface.co/datasets/zeta-alpha-ai/NanoDBPedia",
+        dataset={
+            "path": "zeta-alpha-ai/NanoDBPedia",
+            "revision": "438f1c25129f05db6238699b5afdc9c6b58d2096",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["train"],
+        eval_langs=["eng-Latn"],
+        main_score="ndcg_at_10",
+        date=["2015-01-01", "2015-12-31"],
+        domains=["Encyclopaedic"],
+        task_subtypes=["Topic classification"],
+        license="cc-by-4.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="""@article{lehmann2015dbpedia, title={DBpedia: A large-scale, multilingual knowledge base extracted from Wikipedia}, author={Lehmann, Jens and et al.}, journal={Semantic Web}, year={2015}}""",
+    )
+
+    def load_data(self, **kwargs):
+        if self.data_loaded:
+            return
+
+        self.corpus = load_dataset("zeta-alpha-ai/NanoDBPedia", "corpus")
+        self.queries = load_dataset("zeta-alpha-ai/NanoDBPedia", "queries")
+        self.relevant_docs = load_dataset("zeta-alpha-ai/NanoDBPedia", "qrels")
+
+        self.corpus = {
+            split: {
+                sample["_id"]: {"_id": sample["_id"], "text": sample["text"]}
+                for sample in self.corpus[split]
+            }
+            for split in self.corpus
+        }
+
+        self.queries = {
+            split: {sample["_id"]: sample["text"] for sample in self.queries[split]}
+            for split in self.queries
+        }
+
+        self.relevant_docs = {
+            split: {
+                sample["query-id"]: {sample["corpus-id"]: 1}
+                for sample in self.relevant_docs[split]
+            }
+            for split in self.relevant_docs
+        }
+
+        self.data_loaded = True

mteb/tasks/Retrieval/eng/NanoFEVERRetrieval.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+from datasets import load_dataset
+
+from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval
+from mteb.abstasks.TaskMetadata import TaskMetadata
+
+
+class NanoFEVERRetrieval(AbsTaskRetrieval):
+    metadata = TaskMetadata(
+        name="NanoFEVERRetrieval",
+        description="NanoFEVER is a smaller version of "
+        + "FEVER (Fact Extraction and VERification), which consists of 185,445 claims generated by altering sentences"
+        + " extracted from Wikipedia and subsequently verified without knowledge of the sentence they were"
+        + " derived from.",
+        reference="https://fever.ai/",
+        dataset={
+            "path": "zeta-alpha-ai/NanoFEVER",
+            "revision": "a8bfdf1bf15181167a7e22e69cf8754bdea9b4c8",
+        },
+        type="Retrieval",
+        category="s2p",
+        modalities=["text"],
+        eval_splits=["train"],
+        eval_langs=["eng-Latn"],
+        main_score="ndcg_at_10",
+        date=["2018-01-01", "2018-12-31"],
+        domains=["Academic", "Encyclopaedic"],
+        task_subtypes=["Claim verification"],
+        license="cc-by-4.0",
+        annotations_creators="expert-annotated",
+        dialect=[],
+        sample_creation="found",
+        bibtex_citation="""@inproceedings{thorne-etal-2018-fever,
+    title = "{FEVER}: a Large-scale Dataset for Fact Extraction and {VER}ification",
+    author = "Thorne, James  and
+      Vlachos, Andreas  and
+      Christodoulopoulos, Christos  and
+      Mittal, Arpit",
+    editor = "Walker, Marilyn  and
+      Ji, Heng  and
+      Stent, Amanda",
+    booktitle = "Proceedings of the 2018 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)",
+    month = jun,
+    year = "2018",
+    address = "New Orleans, Louisiana",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/N18-1074",
+    doi = "10.18653/v1/N18-1074",
+    pages = "809--819",
+    abstract = "In this paper we introduce a new publicly available dataset for verification against textual sources, FEVER: Fact Extraction and VERification. It consists of 185,445 claims generated by altering sentences extracted from Wikipedia and subsequently verified without knowledge of the sentence they were derived from. The claims are classified as Supported, Refuted or NotEnoughInfo by annotators achieving 0.6841 in Fleiss kappa. For the first two classes, the annotators also recorded the sentence(s) forming the necessary evidence for their judgment. To characterize the challenge of the dataset presented, we develop a pipeline approach and compare it to suitably designed oracles. The best accuracy we achieve on labeling a claim accompanied by the correct evidence is 31.87{\%}, while if we ignore the evidence we achieve 50.91{\%}. Thus we believe that FEVER is a challenging testbed that will help stimulate progress on claim verification against textual sources.",
+}""",
+        prompt={
+            "query": "Given a claim, retrieve documents that support or refute the claim"
+        },
+    )
+
+    def load_data(self, **kwargs):
+        if self.data_loaded:
+            return
+
+        self.corpus = load_dataset("zeta-alpha-ai/NanoFEVER", "corpus")
+        self.queries = load_dataset("zeta-alpha-ai/NanoFEVER", "queries")
+        self.relevant_docs = load_dataset("zeta-alpha-ai/NanoFEVER", "qrels")
+
+        self.corpus = {
+            split: {
+                sample["_id"]: {"_id": sample["_id"], "text": sample["text"]}
+                for sample in self.corpus[split]
+            }
+            for split in self.corpus
+        }
+
+        self.queries = {
+            split: {sample["_id"]: sample["text"] for sample in self.queries[split]}
+            for split in self.queries
+        }
+
+        self.relevant_docs = {
+            split: {
+                sample["query-id"]: {sample["corpus-id"]: 1}
+                for sample in self.relevant_docs[split]
+            }
+            for split in self.relevant_docs
+        }
+
+        self.data_loaded = True