diff --git a/docs/adding_a_dataset.md b/docs/adding_a_dataset.md index f2167f0fd8..ec55274b9d 100644 --- a/docs/adding_a_dataset.md +++ b/docs/adding_a_dataset.md @@ -76,7 +76,7 @@ evaluation = MTEB(tasks=[SciDocsReranking()]) evaluation.run(model) ``` -> **Note:** for multilingual / crosslingual tasks, make sure your class also inherits from the `MultilingualTask` class like in [this](https://github.com/embeddings-benchmark/mteb/blob/main/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py) example. +> **Note:** for multilingual / crosslingual tasks, make sure you've specified `eval_langs` as a dictionary, as shown in [this example](../mteb/tasks/Classification/multilingual/MTOPIntentClassification.py). diff --git a/mteb/abstasks/AbsTask.py b/mteb/abstasks/AbsTask.py index 678b9a6086..d3342104b3 100644 --- a/mteb/abstasks/AbsTask.py +++ b/mteb/abstasks/AbsTask.py @@ -74,6 +74,9 @@ class AbsTask(ABC): abstask_prompt: The potential prompt of the abstask superseded_by: Denotes the task that this task is superseeded by. Used to issue warning to users of outdated datasets, while maintaining reproducibility of existing benchmarks. + fast_loading: (Not recommended to use) Denotes if the task should be loaded using the fast loading method. + This is only possible if the dataset have a "default" config. We don't recommend to use this method, and suggest to use different subsets for loading datasets. + This was used only for historical reasons and will be removed in the future. """ metadata: TaskMetadata @@ -82,8 +85,8 @@ class AbsTask(ABC): superseded_by: str | None = None dataset: dict[HFSubset, DatasetDict] | None = None # type: ignore data_loaded: bool = False - is_multilingual: bool = False hf_subsets: list[HFSubset] | None = None + fast_loading: bool = False def __init__(self, seed: int = 42, **kwargs: Any): """The init function. This is called primarily to set the seed. @@ -96,6 +99,7 @@ def __init__(self, seed: int = 42, **kwargs: Any): self.seed = seed self.rng_state, self.np_rng = set_seed(seed) + self.hf_subsets = self.metadata.hf_subsets def check_if_dataset_is_superseded(self): """Check if the dataset is superseded by a newer version""" @@ -224,10 +228,43 @@ def load_data(self, **kwargs): """ if self.data_loaded: return - self.dataset = datasets.load_dataset(**self.metadata.dataset) # type: ignore + if self.metadata.is_multilingual: + if self.fast_loading: + self.fast_load() + else: + self.dataset = {} + for hf_subset in self.hf_subsets: + self.dataset[hf_subset] = datasets.load_dataset( + name=hf_subset, + **self.metadata.dataset, + ) + else: + # some of monolingual datasets explicitly adding the split name to the dataset name + self.dataset = datasets.load_dataset(**self.metadata.dataset) # type: ignore self.dataset_transform() self.data_loaded = True + def fast_load(self, **kwargs): + # todo remove + """Load all subsets at once, then group by language with Polars. Using fast loading has two requirements: + - Each row in the dataset should have a 'lang' feature giving the corresponding language/language pair + - The datasets must have a 'default' config that loads all the subsets of the dataset (see https://huggingface.co/docs/datasets/en/repository_structure#configurations) + """ + self.dataset = {} + merged_dataset = datasets.load_dataset( + **self.metadata.dataset + ) # load "default" subset + for split in merged_dataset.keys(): + df_split = merged_dataset[split].to_polars() + df_grouped = dict(df_split.group_by(["lang"])) + for lang in set(df_split["lang"].unique()) & set(self.hf_subsets): + self.dataset.setdefault(lang, {}) + self.dataset[lang][split] = datasets.Dataset.from_polars( + df_grouped[(lang,)].drop("lang") + ) # Remove lang column and convert back to HF datasets, not strictly necessary but better for compatibility + for lang, subset in self.dataset.items(): + self.dataset[lang] = datasets.DatasetDict(subset) + def calculate_metadata_metrics( self, overwrite_results: bool = False ) -> dict[str, DescriptiveStatistics | dict[str, DescriptiveStatistics]]: @@ -248,14 +285,14 @@ def calculate_metadata_metrics( for split in pbar_split: pbar_split.set_postfix_str(f"Split: {split}") logger.info(f"Processing metadata for split {split}") - if self.is_multilingual: + if self.metadata.is_multilingual: descriptive_stats[split] = self._calculate_metrics_from_split( split, compute_overall=True ) descriptive_stats[split][hf_subset_stat] = {} pbar_subsets = tqdm.tqdm( - self.metadata.hf_subsets_to_langscripts, + self.metadata.hf_subsets, desc="Processing Languages...", ) for hf_subset in pbar_subsets: @@ -345,7 +382,7 @@ def _add_main_score(self, scores: dict[HFSubset, ScoresDict]) -> None: scores["main_score"] = scores[self.metadata.main_score] def _upload_dataset_to_hub(self, repo_name: str, fields: list[str]) -> None: - if self.is_multilingual: + if self.metadata.is_multilingual: for config in self.metadata.eval_langs: logger.info(f"Converting {config} of {self.metadata.name}") sentences = {} diff --git a/mteb/abstasks/AbsTaskBitextMining.py b/mteb/abstasks/AbsTaskBitextMining.py index 19d380f3fb..9decf615fa 100644 --- a/mteb/abstasks/AbsTaskBitextMining.py +++ b/mteb/abstasks/AbsTaskBitextMining.py @@ -73,7 +73,7 @@ def evaluate( if not self.data_loaded: self.load_data() - hf_subsets = list(self.dataset) if self.is_multilingual else ["default"] + hf_subsets = self.hf_subsets # If subsets_to_run is specified, filter the hf_subsets accordingly if subsets_to_run is not None: @@ -191,7 +191,7 @@ def _calculate_metrics_from_split( ) def _push_dataset_to_hub(self, repo_name: str) -> None: - if self.is_multilingual: + if self.metadata.is_multilingual: for config in self.metadata.eval_langs: logger.info(f"Converting {config} of {self.metadata.name}") diff --git a/mteb/abstasks/AbsTaskClassification.py b/mteb/abstasks/AbsTaskClassification.py index afa9ad6a99..b68c60173c 100644 --- a/mteb/abstasks/AbsTaskClassification.py +++ b/mteb/abstasks/AbsTaskClassification.py @@ -89,7 +89,7 @@ def evaluate( self.load_data() scores = {} - hf_subsets = list(self.dataset) if self.is_multilingual else ["default"] + hf_subsets = self.hf_subsets if subsets_to_run is not None: hf_subsets = [s for s in hf_subsets if s in subsets_to_run] diff --git a/mteb/abstasks/AbsTaskReranking.py b/mteb/abstasks/AbsTaskReranking.py index 5467d3b326..75a52b9634 100644 --- a/mteb/abstasks/AbsTaskReranking.py +++ b/mteb/abstasks/AbsTaskReranking.py @@ -99,7 +99,7 @@ def transform_old_dataset_format(self, given_dataset=None): self.relevant_docs = defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) self.top_ranked = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) - hf_subsets = list(self.hf_subsets) if self.is_multilingual else ["default"] + hf_subsets = self.hf_subsets for hf_subset in hf_subsets: if given_dataset: diff --git a/mteb/abstasks/AbsTaskRetrieval.py b/mteb/abstasks/AbsTaskRetrieval.py index 631a580191..a806abefd4 100644 --- a/mteb/abstasks/AbsTaskRetrieval.py +++ b/mteb/abstasks/AbsTaskRetrieval.py @@ -178,7 +178,7 @@ def process_data(split: str, lang: str | None = None): else: self.top_ranked[split] = top_ranked - if self.is_multilingual: + if self.metadata.is_multilingual: for lang in self.metadata.eval_langs: for split in eval_splits: process_data(split, lang) @@ -204,7 +204,7 @@ def evaluate( ) scores = {} - hf_subsets = list(self.hf_subsets) if self.is_multilingual else ["default"] + hf_subsets = self.hf_subsets if subsets_to_run is not None: hf_subsets = [s for s in hf_subsets if s in subsets_to_run] @@ -515,7 +515,7 @@ def format_text_field(text): return text return f"{text.get('title', '')} {text.get('text', '')}".strip() - if self.is_multilingual: + if self.metadata.is_multilingual: for config in self.queries: logger.info(f"Converting {config} of {self.metadata.name}") diff --git a/mteb/abstasks/MultilingualTask.py b/mteb/abstasks/MultilingualTask.py deleted file mode 100644 index 09c823d457..0000000000 --- a/mteb/abstasks/MultilingualTask.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -import datasets - -from .AbsTask import AbsTask - - -class MultilingualTask(AbsTask): - def __init__(self, hf_subsets: list[str] | None = None, **kwargs): - super().__init__(**kwargs) - if isinstance(hf_subsets, list): - hf_subsets = [ - lang for lang in hf_subsets if lang in self.metadata.eval_langs - ] - if hf_subsets is not None and len(hf_subsets) > 0: - self.hf_subsets = hf_subsets - else: - self.hf_subsets = self.metadata.eval_langs - self.is_multilingual = True - - def load_data(self, **kwargs): - """Load dataset containing multiple subsets from HuggingFace hub""" - if self.data_loaded: - return - - if hasattr(self, "fast_loading") and self.fast_loading: - self.fast_load() - else: - self.slow_load() - - self.dataset_transform() - self.data_loaded = True - - def fast_load(self, **kwargs): - """Load all subsets at once, then group by language with Polars. Using fast loading has two requirements: - - Each row in the dataset should have a 'lang' feature giving the corresponding language/language pair - - The datasets must have a 'default' config that loads all the subsets of the dataset (see https://huggingface.co/docs/datasets/en/repository_structure#configurations) - """ - self.dataset = {} - merged_dataset = datasets.load_dataset( - **self.metadata.dataset - ) # load "default" subset - for split in merged_dataset.keys(): - df_split = merged_dataset[split].to_polars() - df_grouped = dict(df_split.group_by(["lang"])) - for lang in set(df_split["lang"].unique()) & set(self.hf_subsets): - self.dataset.setdefault(lang, {}) - self.dataset[lang][split] = datasets.Dataset.from_polars( - df_grouped[(lang,)].drop("lang") - ) # Remove lang column and convert back to HF datasets, not strictly necessary but better for compatibility - for lang, subset in self.dataset.items(): - self.dataset[lang] = datasets.DatasetDict(subset) - - def slow_load(self, **kwargs): - """Load each subsets iteratively""" - self.dataset = {} - for lang in self.hf_subsets: - self.dataset[lang] = datasets.load_dataset( - name=lang, - **self.metadata.dataset, - ) diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 35f59fab13..3695d0c337 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -446,5 +446,15 @@ def n_samples(self) -> dict[str, int] | None: n_samples[subset] = subset_value["num_samples"] return n_samples + @property + def hf_subsets(self) -> list[str]: + """Return the huggingface subsets.""" + return list(self.hf_subsets_to_langscripts.keys()) + + @property + def is_multilingual(self) -> bool: + """Check if the task is multilingual.""" + return isinstance(self.eval_langs, dict) + def __hash__(self) -> int: return hash(self.model_dump_json()) diff --git a/mteb/abstasks/__init__.py b/mteb/abstasks/__init__.py index a95eeaf09d..3d96459192 100644 --- a/mteb/abstasks/__init__.py +++ b/mteb/abstasks/__init__.py @@ -12,7 +12,6 @@ from .AbsTaskSpeedTask import AbsTaskSpeedTask from .AbsTaskSTS import AbsTaskSTS from .AbsTaskSummarization import AbsTaskSummarization -from .MultilingualTask import MultilingualTask from .TaskMetadata import TaskMetadata __all__ = [ @@ -28,6 +27,5 @@ "AbsTaskSpeedTask", "AbsTaskSTS", "AbsTaskSummarization", - "MultilingualTask", "TaskMetadata", ] diff --git a/mteb/evaluation/MTEB.py b/mteb/evaluation/MTEB.py index 377f8b72eb..966f9187b2 100644 --- a/mteb/evaluation/MTEB.py +++ b/mteb/evaluation/MTEB.py @@ -98,7 +98,7 @@ def _display_tasks(self, task_list: Iterable[AbsTask], name: str | None = None): category = f", [italic grey39]{task.metadata.category}[/]" multilingual = ( f", [italic red]multilingual {len(task.hf_subsets)} / {len(task.metadata.eval_langs)} Subsets[/]" - if task.is_multilingual + if task.metadata.is_multilingual else "" ) console.print(f"{prefix}{name}{category}{multilingual}") @@ -326,7 +326,7 @@ def run( task_eval_splits = ( eval_splits if eval_splits is not None else task.eval_splits ) - task_subsets = list(task.metadata.hf_subsets_to_langscripts.keys()) + task_subsets = task.metadata.hf_subsets existing_results = None save_path = None diff --git a/mteb/load_results/task_results.py b/mteb/load_results/task_results.py index d940202499..cb5f883bc8 100644 --- a/mteb/load_results/task_results.py +++ b/mteb/load_results/task_results.py @@ -540,7 +540,7 @@ def validate_and_filter_scores(self, task: AbsTask | None = None) -> TaskResult: if task is None: task = get_task(self.task_name) splits = task.metadata.eval_splits - if task.is_multilingual: + if task.metadata.is_multilingual: hf_subsets = getattr( task, "hf_subsets", task.metadata.hf_subsets_to_langscripts.keys() ) diff --git a/mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py b/mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py index c5bf62f5cf..d8821db13d 100644 --- a/mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py +++ b/mteb/tasks/BitextMining/kat/TbilisiCityHallBitextMining.py @@ -3,7 +3,6 @@ from datasets import DatasetDict, load_dataset from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -18,7 +17,7 @@ _EVAL_SPLIT = "test" -class TbilisiCityHallBitextMining(AbsTaskBitextMining, MultilingualTask): +class TbilisiCityHallBitextMining(AbsTaskBitextMining): metadata = TaskMetadata( name="TbilisiCityHallBitextMining", dataset={ diff --git a/mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py b/mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py index 8c2563bbf3..0a6f548bf7 100644 --- a/mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/BUCCBitextMining.py @@ -3,7 +3,6 @@ import logging from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -19,7 +18,7 @@ logger = logging.getLogger(__name__) -class BUCCBitextMining(AbsTaskBitextMining, MultilingualTask): +class BUCCBitextMining(AbsTaskBitextMining): superseded_by = "BUCC.v2" metadata = TaskMetadata( name="BUCC", diff --git a/mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py b/mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py index f6ab4a20d5..35e5285550 100644 --- a/mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py +++ b/mteb/tasks/BitextMining/multilingual/BUCCBitextMiningFast.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -15,7 +14,7 @@ _SPLITS = ["test"] -class BUCCBitextMiningFast(AbsTaskBitextMining, MultilingualTask): +class BUCCBitextMiningFast(AbsTaskBitextMining): fast_loading = True metadata = TaskMetadata( name="BUCC.v2", diff --git a/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py b/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py index 670723a38e..67517e2f71 100644 --- a/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/BibleNLPBitextMining.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = [ @@ -859,7 +858,7 @@ def extend_lang_pairs_english_centric() -> dict[str, list[str]]: _LANGUAGES_MAPPING = extend_lang_pairs_english_centric() -class BibleNLPBitextMining(AbsTaskBitextMining, MultilingualTask): +class BibleNLPBitextMining(AbsTaskBitextMining): metadata = TaskMetadata( name="BibleNLPBitextMining", dataset={ diff --git a/mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py b/mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py index 67a561d206..83d9b85580 100644 --- a/mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/DiaBLaBitextMining.py @@ -3,11 +3,10 @@ import datasets from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata -class DiaBLaBitextMining(AbsTaskBitextMining, MultilingualTask): +class DiaBLaBitextMining(AbsTaskBitextMining): metadata = TaskMetadata( name="DiaBlaBitextMining", dataset={ diff --git a/mteb/tasks/BitextMining/multilingual/FloresBitextMining.py b/mteb/tasks/BitextMining/multilingual/FloresBitextMining.py index 25d489f5d6..b342187d97 100644 --- a/mteb/tasks/BitextMining/multilingual/FloresBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/FloresBitextMining.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = [ @@ -235,7 +234,7 @@ def extend_lang_pairs() -> dict[str, list[str]]: _LANGUAGES_MAPPING = extend_lang_pairs() -class FloresBitextMining(AbsTaskBitextMining, MultilingualTask): +class FloresBitextMining(AbsTaskBitextMining): parallel_subsets = True metadata = TaskMetadata( name="FloresBitextMining", diff --git a/mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py b/mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py index 94b7f7cc02..618614326a 100644 --- a/mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/IN22ConvBitextMining.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = [ @@ -67,7 +66,7 @@ def check_uniques(example, uniques): return False -class IN22ConvBitextMining(AbsTaskBitextMining, MultilingualTask): +class IN22ConvBitextMining(AbsTaskBitextMining): parallel_subsets = True metadata = TaskMetadata( name="IN22ConvBitextMining", diff --git a/mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py b/mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py index 3a196780a3..e45c385000 100644 --- a/mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/IN22GenBitextMining.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = [ @@ -53,7 +52,7 @@ def extend_lang_pairs() -> dict[str, list[str]]: _LANGUAGES_MAPPING = extend_lang_pairs() -class IN22GenBitextMining(AbsTaskBitextMining, MultilingualTask): +class IN22GenBitextMining(AbsTaskBitextMining): parallel_subsets = True metadata = TaskMetadata( name="IN22GenBitextMining", diff --git a/mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py b/mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py index dfe4e4bfa7..65b2e78b1b 100644 --- a/mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/IWSLT2017BitextMining.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -34,7 +33,7 @@ _SPLITS = ["validation"] -class IWSLT2017BitextMining(AbsTaskBitextMining, MultilingualTask): +class IWSLT2017BitextMining(AbsTaskBitextMining): metadata = TaskMetadata( name="IWSLT2017BitextMining", dataset={ diff --git a/mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py b/mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py index 843ed35f40..22bf41091e 100644 --- a/mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/IndicGenBenchFloresBitextMining.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = [ @@ -97,7 +96,7 @@ def get_lang_pairs() -> dict[str, list[str]]: _LANGUAGES_MAPPING = get_lang_pairs() -class IndicGenBenchFloresBitextMining(AbsTaskBitextMining, MultilingualTask): +class IndicGenBenchFloresBitextMining(AbsTaskBitextMining): metadata = TaskMetadata( name="IndicGenBenchFloresBitextMining", dataset={ diff --git a/mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py b/mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py index 8abb8ce1ff..61ec93f108 100644 --- a/mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/LinceMTBitextMining.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -9,7 +8,7 @@ } -class LinceMTBitextMining(AbsTaskBitextMining, MultilingualTask): +class LinceMTBitextMining(AbsTaskBitextMining): metadata = TaskMetadata( name="LinceMTBitextMining", dataset={ diff --git a/mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py b/mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py index 0137d9330d..56e9e29248 100644 --- a/mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NTREXBitextMining.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _BRIDGE_LANGUAGES = ( @@ -245,7 +244,7 @@ def extend_lang_pairs() -> dict[str, list[str]]: _EVAL_LANGS = extend_lang_pairs() -class NTREXBitextMining(AbsTaskBitextMining, MultilingualTask): +class NTREXBitextMining(AbsTaskBitextMining): parallel_subsets = True metadata = TaskMetadata( name="NTREXBitextMining", diff --git a/mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py b/mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py index 4662833008..5383725879 100644 --- a/mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NollySentiBitextMining.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -12,7 +11,7 @@ } -class NollySentiBitextMining(AbsTaskBitextMining, MultilingualTask): +class NollySentiBitextMining(AbsTaskBitextMining): metadata = TaskMetadata( name="NollySentiBitextMining", dataset={ diff --git a/mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py b/mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py index c328461746..e3a47fda2f 100644 --- a/mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NusaTranslationBitextMining.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -19,7 +18,7 @@ } -class NusaTranslationBitextMining(AbsTaskBitextMining, MultilingualTask): +class NusaTranslationBitextMining(AbsTaskBitextMining): metadata = TaskMetadata( name="NusaTranslationBitextMining", dataset={ diff --git a/mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py b/mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py index 2f49c3acc4..307352801a 100644 --- a/mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/NusaXBitextMining.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -19,7 +18,7 @@ } -class NusaXBitextMining(AbsTaskBitextMining, MultilingualTask): +class NusaXBitextMining(AbsTaskBitextMining): metadata = TaskMetadata( name="NusaXBitextMining", dataset={ diff --git a/mteb/tasks/BitextMining/multilingual/PhincBitextMining.py b/mteb/tasks/BitextMining/multilingual/PhincBitextMining.py index c7fec75637..932225dc15 100644 --- a/mteb/tasks/BitextMining/multilingual/PhincBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/PhincBitextMining.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -9,7 +8,7 @@ } -class PhincBitextMining(AbsTaskBitextMining, MultilingualTask): +class PhincBitextMining(AbsTaskBitextMining): metadata = TaskMetadata( name="PhincBitextMining", dataset={ diff --git a/mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py b/mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py index 87c2f59ef8..be15e7d597 100644 --- a/mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/RomaTalesBitextMining.py @@ -3,11 +3,10 @@ import datasets from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata -class RomaTalesBitextMining(AbsTaskBitextMining, MultilingualTask): +class RomaTalesBitextMining(AbsTaskBitextMining): metadata = TaskMetadata( name="RomaTalesBitextMining", dataset={ diff --git a/mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py b/mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py index 4312332022..2f31e21bfa 100644 --- a/mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py +++ b/mteb/tasks/BitextMining/multilingual/TatoebaBitextMining.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -120,7 +119,7 @@ } -class TatoebaBitextMining(AbsTaskBitextMining, MultilingualTask): +class TatoebaBitextMining(AbsTaskBitextMining): fast_loading = True metadata = TaskMetadata( name="Tatoeba", diff --git a/mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py b/mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py index 5adb51888b..1a8caea300 100644 --- a/mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py +++ b/mteb/tasks/BitextMining/srn/SRNCorpusBitextMining.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -23,7 +22,7 @@ } -class SRNCorpusBitextMining(AbsTaskBitextMining, MultilingualTask): +class SRNCorpusBitextMining(AbsTaskBitextMining): metadata = TaskMetadata( name="SRNCorpusBitextMining", dataset={ diff --git a/mteb/tasks/Classification/multilingual/AfriSentiClassification.py b/mteb/tasks/Classification/multilingual/AfriSentiClassification.py index 9c1bd5daf7..ba438fab06 100644 --- a/mteb/tasks/Classification/multilingual/AfriSentiClassification.py +++ b/mteb/tasks/Classification/multilingual/AfriSentiClassification.py @@ -3,7 +3,6 @@ import datasets from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata @@ -14,7 +13,7 @@ def _transform(dataset, lang): return dataset -class AfriSentiClassification(MultilingualTask, AbsTaskClassification): +class AfriSentiClassification(AbsTaskClassification): metadata = TaskMetadata( name="AfriSentiClassification", description="AfriSenti is the largest sentiment analysis dataset for under-represented African languages.", diff --git a/mteb/tasks/Classification/multilingual/AmazonCounterfactualClassification.py b/mteb/tasks/Classification/multilingual/AmazonCounterfactualClassification.py index 168f990d8d..ceb2f4815c 100644 --- a/mteb/tasks/Classification/multilingual/AmazonCounterfactualClassification.py +++ b/mteb/tasks/Classification/multilingual/AmazonCounterfactualClassification.py @@ -1,11 +1,10 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata -class AmazonCounterfactualClassification(MultilingualTask, AbsTaskClassification): +class AmazonCounterfactualClassification(AbsTaskClassification): metadata = TaskMetadata( name="AmazonCounterfactualClassification", dataset={ diff --git a/mteb/tasks/Classification/multilingual/AmazonReviewsClassification.py b/mteb/tasks/Classification/multilingual/AmazonReviewsClassification.py index a3db9785bb..ae1511a588 100644 --- a/mteb/tasks/Classification/multilingual/AmazonReviewsClassification.py +++ b/mteb/tasks/Classification/multilingual/AmazonReviewsClassification.py @@ -1,11 +1,10 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata -class AmazonReviewsClassification(MultilingualTask, AbsTaskClassification): +class AmazonReviewsClassification(AbsTaskClassification): metadata = TaskMetadata( name="AmazonReviewsClassification", dataset={ diff --git a/mteb/tasks/Classification/multilingual/CataloniaTweetClassification.py b/mteb/tasks/Classification/multilingual/CataloniaTweetClassification.py index c21fee9cfa..42952b430d 100644 --- a/mteb/tasks/Classification/multilingual/CataloniaTweetClassification.py +++ b/mteb/tasks/Classification/multilingual/CataloniaTweetClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGS = { @@ -10,7 +9,7 @@ } -class CataloniaTweetClassification(MultilingualTask, AbsTaskClassification): +class CataloniaTweetClassification(AbsTaskClassification): metadata = TaskMetadata( name="CataloniaTweetClassification", description="""This dataset contains two corpora in Spanish and Catalan that consist of annotated Twitter diff --git a/mteb/tasks/Classification/multilingual/IndicNLPNewsClassification.py b/mteb/tasks/Classification/multilingual/IndicNLPNewsClassification.py index 3995917696..07c3011abf 100644 --- a/mteb/tasks/Classification/multilingual/IndicNLPNewsClassification.py +++ b/mteb/tasks/Classification/multilingual/IndicNLPNewsClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -16,7 +15,7 @@ } -class IndicNLPNewsClassification(MultilingualTask, AbsTaskClassification): +class IndicNLPNewsClassification(AbsTaskClassification): metadata = TaskMetadata( name="IndicNLPNewsClassification", dataset={ diff --git a/mteb/tasks/Classification/multilingual/IndicSentimentClassification.py b/mteb/tasks/Classification/multilingual/IndicSentimentClassification.py index e70acd9850..709e0993d6 100644 --- a/mteb/tasks/Classification/multilingual/IndicSentimentClassification.py +++ b/mteb/tasks/Classification/multilingual/IndicSentimentClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -21,7 +20,7 @@ } -class IndicSentimentClassification(AbsTaskClassification, MultilingualTask): +class IndicSentimentClassification(AbsTaskClassification): fast_loading = True metadata = TaskMetadata( name="IndicSentimentClassification", diff --git a/mteb/tasks/Classification/multilingual/MTOPDomainClassification.py b/mteb/tasks/Classification/multilingual/MTOPDomainClassification.py index eb8713fd6d..24ec91816d 100644 --- a/mteb/tasks/Classification/multilingual/MTOPDomainClassification.py +++ b/mteb/tasks/Classification/multilingual/MTOPDomainClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -14,7 +13,7 @@ } -class MTOPDomainClassification(MultilingualTask, AbsTaskClassification): +class MTOPDomainClassification(AbsTaskClassification): metadata = TaskMetadata( name="MTOPDomainClassification", dataset={ diff --git a/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py b/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py index 52863107b6..7f49eb1a92 100644 --- a/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py +++ b/mteb/tasks/Classification/multilingual/MTOPIntentClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -14,7 +13,7 @@ } -class MTOPIntentClassification(MultilingualTask, AbsTaskClassification): +class MTOPIntentClassification(AbsTaskClassification): metadata = TaskMetadata( name="MTOPIntentClassification", dataset={ diff --git a/mteb/tasks/Classification/multilingual/MasakhaNEWSClassification.py b/mteb/tasks/Classification/multilingual/MasakhaNEWSClassification.py index e5152a84d9..8ab8a1d2e0 100644 --- a/mteb/tasks/Classification/multilingual/MasakhaNEWSClassification.py +++ b/mteb/tasks/Classification/multilingual/MasakhaNEWSClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -24,7 +23,7 @@ } -class MasakhaNEWSClassification(AbsTaskClassification, MultilingualTask): +class MasakhaNEWSClassification(AbsTaskClassification): metadata = TaskMetadata( name="MasakhaNEWSClassification", dataset={ diff --git a/mteb/tasks/Classification/multilingual/MassiveIntentClassification.py b/mteb/tasks/Classification/multilingual/MassiveIntentClassification.py index e790b27663..1aa3030f64 100644 --- a/mteb/tasks/Classification/multilingual/MassiveIntentClassification.py +++ b/mteb/tasks/Classification/multilingual/MassiveIntentClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -59,7 +58,7 @@ } -class MassiveIntentClassification(MultilingualTask, AbsTaskClassification): +class MassiveIntentClassification(AbsTaskClassification): fast_loading = True metadata = TaskMetadata( name="MassiveIntentClassification", diff --git a/mteb/tasks/Classification/multilingual/MassiveScenarioClassification.py b/mteb/tasks/Classification/multilingual/MassiveScenarioClassification.py index 80e8583ecc..90ab22fb3e 100644 --- a/mteb/tasks/Classification/multilingual/MassiveScenarioClassification.py +++ b/mteb/tasks/Classification/multilingual/MassiveScenarioClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -59,7 +58,7 @@ } -class MassiveScenarioClassification(MultilingualTask, AbsTaskClassification): +class MassiveScenarioClassification(AbsTaskClassification): fast_loading = True metadata = TaskMetadata( name="MassiveScenarioClassification", diff --git a/mteb/tasks/Classification/multilingual/MultiHateClassification.py b/mteb/tasks/Classification/multilingual/MultiHateClassification.py index f20ba592c1..de97cc8126 100644 --- a/mteb/tasks/Classification/multilingual/MultiHateClassification.py +++ b/mteb/tasks/Classification/multilingual/MultiHateClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -19,7 +18,7 @@ } -class MultiHateClassification(MultilingualTask, AbsTaskClassification): +class MultiHateClassification(AbsTaskClassification): fast_loading = True metadata = TaskMetadata( name="MultiHateClassification", diff --git a/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py b/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py index 1108dd7cf8..abee3937a2 100644 --- a/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py +++ b/mteb/tasks/Classification/multilingual/MultilingualSentimentClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -39,7 +38,7 @@ } -class MultilingualSentimentClassification(AbsTaskClassification, MultilingualTask): +class MultilingualSentimentClassification(AbsTaskClassification): fast_loading = True metadata = TaskMetadata( name="MultilingualSentimentClassification", diff --git a/mteb/tasks/Classification/multilingual/NaijaSenti.py b/mteb/tasks/Classification/multilingual/NaijaSenti.py index 347f84771d..76e0733c2b 100644 --- a/mteb/tasks/Classification/multilingual/NaijaSenti.py +++ b/mteb/tasks/Classification/multilingual/NaijaSenti.py @@ -5,11 +5,10 @@ import datasets from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata -class NaijaSenti(AbsTaskClassification, MultilingualTask): +class NaijaSenti(AbsTaskClassification): metadata = TaskMetadata( name="NaijaSenti", description="NaijaSenti is the first large-scale human-annotated Twitter sentiment dataset for the four most widely spoken languages in Nigeria — Hausa, Igbo, Nigerian-Pidgin, and Yorùbá — consisting of around 30,000 annotated tweets per language, including a significant fraction of code-mixed tweets.", diff --git a/mteb/tasks/Classification/multilingual/NusaParagraphEmotionClassification.py b/mteb/tasks/Classification/multilingual/NusaParagraphEmotionClassification.py index fca11b365c..ff4a6738e6 100644 --- a/mteb/tasks/Classification/multilingual/NusaParagraphEmotionClassification.py +++ b/mteb/tasks/Classification/multilingual/NusaParagraphEmotionClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -18,7 +17,7 @@ } -class NusaParagraphEmotionClassification(MultilingualTask, AbsTaskClassification): +class NusaParagraphEmotionClassification(AbsTaskClassification): metadata = TaskMetadata( name="NusaParagraphEmotionClassification", dataset={ diff --git a/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py b/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py index effd257709..da8a1a6ace 100644 --- a/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py +++ b/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -18,7 +17,7 @@ } -class NusaParagraphTopicClassification(MultilingualTask, AbsTaskClassification): +class NusaParagraphTopicClassification(AbsTaskClassification): metadata = TaskMetadata( name="NusaParagraphTopicClassification", dataset={ diff --git a/mteb/tasks/Classification/multilingual/NusaXSenti.py b/mteb/tasks/Classification/multilingual/NusaXSenti.py index 1b9fa2460a..3158568190 100644 --- a/mteb/tasks/Classification/multilingual/NusaXSenti.py +++ b/mteb/tasks/Classification/multilingual/NusaXSenti.py @@ -1,11 +1,10 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata -class NusaXSentiClassification(AbsTaskClassification, MultilingualTask): +class NusaXSentiClassification(AbsTaskClassification): metadata = TaskMetadata( name="NusaX-senti", description="NusaX is a high-quality multilingual parallel corpus that covers 12 languages, Indonesian, English, and 10 Indonesian local languages, namely Acehnese, Balinese, Banjarese, Buginese, Madurese, Minangkabau, Javanese, Ngaju, Sundanese, and Toba Batak. NusaX-Senti is a 3-labels (positive, neutral, negative) sentiment analysis dataset for 10 Indonesian local languages + Indonesian and English.", diff --git a/mteb/tasks/Classification/multilingual/SIB200Classification.py b/mteb/tasks/Classification/multilingual/SIB200Classification.py index 88e5d4b9c8..43cd636375 100644 --- a/mteb/tasks/Classification/multilingual/SIB200Classification.py +++ b/mteb/tasks/Classification/multilingual/SIB200Classification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGS = { @@ -205,7 +204,7 @@ } -class SIB200Classification(MultilingualTask, AbsTaskClassification): +class SIB200Classification(AbsTaskClassification): metadata = TaskMetadata( name="SIB200Classification", description="""SIB-200 is the largest publicly available topic classification diff --git a/mteb/tasks/Classification/multilingual/ScalaClassification.py b/mteb/tasks/Classification/multilingual/ScalaClassification.py index 4d055d3578..d3ebce30b4 100644 --- a/mteb/tasks/Classification/multilingual/ScalaClassification.py +++ b/mteb/tasks/Classification/multilingual/ScalaClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGS = { @@ -12,7 +11,7 @@ } -class ScalaClassification(AbsTaskClassification, MultilingualTask): +class ScalaClassification(AbsTaskClassification): metadata = TaskMetadata( name="ScalaClassification", description="""ScaLa a linguistic acceptability dataset for the mainland Scandinavian languages automatically constructed from dependency annotations in Universal Dependencies Treebanks. diff --git a/mteb/tasks/Classification/multilingual/SwissJudgementClassification.py b/mteb/tasks/Classification/multilingual/SwissJudgementClassification.py index 92aa43268c..7408c821b4 100644 --- a/mteb/tasks/Classification/multilingual/SwissJudgementClassification.py +++ b/mteb/tasks/Classification/multilingual/SwissJudgementClassification.py @@ -1,11 +1,10 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata -class SwissJudgementClassification(MultilingualTask, AbsTaskClassification): +class SwissJudgementClassification(AbsTaskClassification): metadata = TaskMetadata( name="SwissJudgementClassification", description="Multilingual, diachronic dataset of Swiss Federal Supreme Court cases annotated with the respective binarized judgment outcome (approval/dismissal)", diff --git a/mteb/tasks/Classification/multilingual/TurkicClassification.py b/mteb/tasks/Classification/multilingual/TurkicClassification.py index 3ab0684df1..03736afe2a 100644 --- a/mteb/tasks/Classification/multilingual/TurkicClassification.py +++ b/mteb/tasks/Classification/multilingual/TurkicClassification.py @@ -6,11 +6,10 @@ from datasets import DatasetDict from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata -class TurkicClassification(MultilingualTask, AbsTaskClassification): +class TurkicClassification(AbsTaskClassification): metadata = TaskMetadata( name="TurkicClassification", description="A dataset of news classification in three Turkic languages.", diff --git a/mteb/tasks/Classification/multilingual/TweetSentimentClassification.py b/mteb/tasks/Classification/multilingual/TweetSentimentClassification.py index 4105f975a9..65688b68eb 100644 --- a/mteb/tasks/Classification/multilingual/TweetSentimentClassification.py +++ b/mteb/tasks/Classification/multilingual/TweetSentimentClassification.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClassification import AbsTaskClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -16,7 +15,7 @@ } -class TweetSentimentClassification(MultilingualTask, AbsTaskClassification): +class TweetSentimentClassification(AbsTaskClassification): fast_loading = True metadata = TaskMetadata( name="TweetSentimentClassification", diff --git a/mteb/tasks/Clustering/multilingual/IndicReviewsClusteringP2P.py b/mteb/tasks/Clustering/multilingual/IndicReviewsClusteringP2P.py index 8310f023df..fda9a8b513 100644 --- a/mteb/tasks/Clustering/multilingual/IndicReviewsClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/IndicReviewsClusteringP2P.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskClustering import AbsTaskClustering -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -21,7 +20,7 @@ } -class IndicReviewsClusteringP2P(AbsTaskClustering, MultilingualTask): +class IndicReviewsClusteringP2P(AbsTaskClustering): metadata = TaskMetadata( name="IndicReviewsClusteringP2P", dataset={ diff --git a/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py b/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py index 4699fdae85..a6784bbb52 100644 --- a/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/MLSUMClusteringP2P.py @@ -6,7 +6,6 @@ from mteb.abstasks.AbsTaskClustering import AbsTaskClustering from mteb.abstasks.AbsTaskClusteringFast import AbsTaskClusteringFast -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -21,7 +20,7 @@ N_SAMPLES = 2048 -class MLSUMClusteringP2P(AbsTaskClustering, MultilingualTask): +class MLSUMClusteringP2P(AbsTaskClustering): superseded_by = "MLSUMClusteringP2P.v2" metadata = TaskMetadata( name="MLSUMClusteringP2P", @@ -89,7 +88,7 @@ def dataset_transform(self, lang): self.dataset[lang] = _dataset -class MLSUMClusteringP2PFast(AbsTaskClusteringFast, MultilingualTask): +class MLSUMClusteringP2PFast(AbsTaskClusteringFast): max_document_to_embed = N_SAMPLES max_fraction_of_documents_to_embed = None diff --git a/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py b/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py index 787391a7f4..62f48955ef 100644 --- a/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py +++ b/mteb/tasks/Clustering/multilingual/MLSUMClusteringS2S.py @@ -6,7 +6,6 @@ from mteb.abstasks.AbsTaskClustering import AbsTaskClustering from mteb.abstasks.AbsTaskClusteringFast import AbsTaskClusteringFast -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -21,7 +20,7 @@ N_SAMPLES = 2048 -class MLSUMClusteringS2S(AbsTaskClustering, MultilingualTask): +class MLSUMClusteringS2S(AbsTaskClustering): superseded_by = "MLSUMClusteringS2S.v2" metadata = TaskMetadata( name="MLSUMClusteringS2S", @@ -84,7 +83,7 @@ def dataset_transform(self, lang): self.dataset[lang] = _dataset -class MLSUMClusteringS2SFast(AbsTaskClusteringFast, MultilingualTask): +class MLSUMClusteringS2SFast(AbsTaskClusteringFast): max_document_to_embed = N_SAMPLES max_fraction_of_documents_to_embed = None diff --git a/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py b/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py index 788c334bba..1ebc8f3009 100644 --- a/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringP2P.py @@ -4,7 +4,6 @@ import numpy as np from mteb.abstasks.AbsTaskClustering import AbsTaskClustering -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -27,7 +26,7 @@ } -class MasakhaNEWSClusteringP2P(AbsTaskClustering, MultilingualTask): +class MasakhaNEWSClusteringP2P(AbsTaskClustering): metadata = TaskMetadata( name="MasakhaNEWSClusteringP2P", description="Clustering of news article headlines and texts from MasakhaNEWS dataset. Clustering of 10 sets on the news article label.", diff --git a/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringS2S.py b/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringS2S.py index 56bd8f3890..252127841e 100644 --- a/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringS2S.py +++ b/mteb/tasks/Clustering/multilingual/MasakhaNEWSClusteringS2S.py @@ -4,7 +4,6 @@ import numpy as np from mteb.abstasks.AbsTaskClustering import AbsTaskClustering -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -27,7 +26,7 @@ } -class MasakhaNEWSClusteringS2S(AbsTaskClustering, MultilingualTask): +class MasakhaNEWSClusteringS2S(AbsTaskClustering): metadata = TaskMetadata( name="MasakhaNEWSClusteringS2S", dataset={ diff --git a/mteb/tasks/Clustering/multilingual/SIB200ClusteringS2S.py b/mteb/tasks/Clustering/multilingual/SIB200ClusteringS2S.py index 8569b55cd5..709730e89b 100644 --- a/mteb/tasks/Clustering/multilingual/SIB200ClusteringS2S.py +++ b/mteb/tasks/Clustering/multilingual/SIB200ClusteringS2S.py @@ -3,7 +3,6 @@ from datasets import Dataset, DatasetDict from mteb.abstasks.AbsTaskClusteringFast import AbsTaskClusteringFast -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGS = { @@ -207,7 +206,7 @@ } -class SIB200ClusteringFast(MultilingualTask, AbsTaskClusteringFast): +class SIB200ClusteringFast(AbsTaskClusteringFast): max_document_to_embed = 1004 max_fraction_of_documents_to_embed = None diff --git a/mteb/tasks/Clustering/multilingual/WikiClusteringP2P.py b/mteb/tasks/Clustering/multilingual/WikiClusteringP2P.py index 77e86a100e..f2db1a08d1 100644 --- a/mteb/tasks/Clustering/multilingual/WikiClusteringP2P.py +++ b/mteb/tasks/Clustering/multilingual/WikiClusteringP2P.py @@ -7,7 +7,6 @@ from mteb.abstasks.AbsTaskClustering import AbsTaskClustering from mteb.abstasks.AbsTaskClusteringFast import AbsTaskClusteringFast -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -28,7 +27,7 @@ } -class WikiClusteringP2P(AbsTaskClustering, MultilingualTask): +class WikiClusteringP2P(AbsTaskClustering): superseded_by = "WikiClusteringP2P.v2" metadata = TaskMetadata( name="WikiClusteringP2P", @@ -55,7 +54,7 @@ class WikiClusteringP2P(AbsTaskClustering, MultilingualTask): ) -class WikiClusteringFastP2P(AbsTaskClusteringFast, MultilingualTask): +class WikiClusteringFastP2P(AbsTaskClusteringFast): max_document_to_embed = 2048 max_fraction_of_documents_to_embed = None diff --git a/mteb/tasks/InstructionReranking/multilingual/mFollowIR.py b/mteb/tasks/InstructionReranking/multilingual/mFollowIR.py index 9df4ba4aae..0de3ebe7ce 100644 --- a/mteb/tasks/InstructionReranking/multilingual/mFollowIR.py +++ b/mteb/tasks/InstructionReranking/multilingual/mFollowIR.py @@ -4,7 +4,6 @@ import datasets -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskReranking import AbsTaskReranking @@ -129,7 +128,7 @@ def load_data( return (corpus, queries, instructions, relevant_docs, top_ranked) -class mFollowIRCrossLingual(MultilingualTask, AbsTaskReranking): +class mFollowIRCrossLingual(AbsTaskReranking): metadata = TaskMetadata( name="mFollowIRCrossLingual", description="This tasks measures retrieval instruction following ability on NeuCLIR narratives for the mFollowIR benchmark on the Farsi, Russian, and Chinese languages with English queries/instructions.", @@ -180,7 +179,7 @@ def load_data(self, **kwargs): self.data_loaded = True -class mFollowIR(AbsTaskReranking, MultilingualTask): +class mFollowIR(AbsTaskReranking): metadata = TaskMetadata( name="mFollowIR", description="This tasks measures retrieval instruction following ability on NeuCLIR narratives for the mFollowIR benchmark on the Farsi, Russian, and Chinese languages.", diff --git a/mteb/tasks/MultiLabelClassification/multilingual/MultiEURLEXMultilabelClassification.py b/mteb/tasks/MultiLabelClassification/multilingual/MultiEURLEXMultilabelClassification.py index 0aeff946aa..cd5c9f0ead 100644 --- a/mteb/tasks/MultiLabelClassification/multilingual/MultiEURLEXMultilabelClassification.py +++ b/mteb/tasks/MultiLabelClassification/multilingual/MultiEURLEXMultilabelClassification.py @@ -3,13 +3,10 @@ from mteb.abstasks.AbsTaskMultilabelClassification import ( AbsTaskMultilabelClassification, ) -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata -class MultiEURLEXMultilabelClassification( - MultilingualTask, AbsTaskMultilabelClassification -): +class MultiEURLEXMultilabelClassification(AbsTaskMultilabelClassification): metadata = TaskMetadata( name="MultiEURLEXMultilabelClassification", dataset={ diff --git a/mteb/tasks/PairClassification/multilingual/IndicXnliPairClassification.py b/mteb/tasks/PairClassification/multilingual/IndicXnliPairClassification.py index 2c6ef5f2c1..d1d2a6ecc0 100644 --- a/mteb/tasks/PairClassification/multilingual/IndicXnliPairClassification.py +++ b/mteb/tasks/PairClassification/multilingual/IndicXnliPairClassification.py @@ -1,6 +1,5 @@ from __future__ import annotations -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskPairClassification import AbsTaskPairClassification @@ -20,7 +19,7 @@ } -class IndicXnliPairClassification(AbsTaskPairClassification, MultilingualTask): +class IndicXnliPairClassification(AbsTaskPairClassification): metadata = TaskMetadata( name="IndicXnliPairClassification", dataset={ diff --git a/mteb/tasks/PairClassification/multilingual/OpusparcusPC.py b/mteb/tasks/PairClassification/multilingual/OpusparcusPC.py index 8fb328aaf4..25f02983d8 100644 --- a/mteb/tasks/PairClassification/multilingual/OpusparcusPC.py +++ b/mteb/tasks/PairClassification/multilingual/OpusparcusPC.py @@ -3,7 +3,6 @@ import datasets from mteb.abstasks.AbsTaskPairClassification import AbsTaskPairClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -16,7 +15,7 @@ } -class OpusparcusPC(AbsTaskPairClassification, MultilingualTask): +class OpusparcusPC(AbsTaskPairClassification): metadata = TaskMetadata( name="OpusparcusPC", dataset={ diff --git a/mteb/tasks/PairClassification/multilingual/PawsXPairClassification.py b/mteb/tasks/PairClassification/multilingual/PawsXPairClassification.py index 8864e8394c..91c99b5882 100644 --- a/mteb/tasks/PairClassification/multilingual/PawsXPairClassification.py +++ b/mteb/tasks/PairClassification/multilingual/PawsXPairClassification.py @@ -1,12 +1,11 @@ from __future__ import annotations -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskPairClassification import AbsTaskPairClassification -class PawsXPairClassification(MultilingualTask, AbsTaskPairClassification): +class PawsXPairClassification(AbsTaskPairClassification): metadata = TaskMetadata( name="PawsXPairClassification", dataset={ diff --git a/mteb/tasks/PairClassification/multilingual/RTE3.py b/mteb/tasks/PairClassification/multilingual/RTE3.py index 9a03fedb4f..6bf85340eb 100644 --- a/mteb/tasks/PairClassification/multilingual/RTE3.py +++ b/mteb/tasks/PairClassification/multilingual/RTE3.py @@ -3,7 +3,6 @@ import datasets from mteb.abstasks.AbsTaskPairClassification import AbsTaskPairClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGS = { @@ -14,7 +13,7 @@ } -class RTE3(MultilingualTask, AbsTaskPairClassification): +class RTE3(AbsTaskPairClassification): metadata = TaskMetadata( name="RTE3", dataset={ diff --git a/mteb/tasks/PairClassification/multilingual/XNLI.py b/mteb/tasks/PairClassification/multilingual/XNLI.py index c72d1694a3..3d1c008560 100644 --- a/mteb/tasks/PairClassification/multilingual/XNLI.py +++ b/mteb/tasks/PairClassification/multilingual/XNLI.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskPairClassification import AbsTaskPairClassification -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGS = { @@ -22,7 +21,7 @@ } -class XNLI(MultilingualTask, AbsTaskPairClassification): +class XNLI(AbsTaskPairClassification): metadata = TaskMetadata( name="XNLI", dataset={ @@ -106,7 +105,7 @@ def dataset_transform(self): } -class XNLIV2(MultilingualTask, AbsTaskPairClassification): +class XNLIV2(AbsTaskPairClassification): metadata = TaskMetadata( name="XNLIV2", dataset={ diff --git a/mteb/tasks/PairClassification/multilingual/XStance.py b/mteb/tasks/PairClassification/multilingual/XStance.py index 3c94ec4b60..a8567417c0 100644 --- a/mteb/tasks/PairClassification/multilingual/XStance.py +++ b/mteb/tasks/PairClassification/multilingual/XStance.py @@ -2,13 +2,12 @@ from datasets import load_dataset -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskPairClassification import AbsTaskPairClassification -class XStance(MultilingualTask, AbsTaskPairClassification): +class XStance(AbsTaskPairClassification): metadata = TaskMetadata( name="XStance", dataset={ diff --git a/mteb/tasks/Reranking/multilingual/ESCIReranking.py b/mteb/tasks/Reranking/multilingual/ESCIReranking.py index e3587dcb03..346d093442 100644 --- a/mteb/tasks/Reranking/multilingual/ESCIReranking.py +++ b/mteb/tasks/Reranking/multilingual/ESCIReranking.py @@ -3,7 +3,6 @@ import logging from mteb.abstasks.AbsTaskReranking import AbsTaskReranking -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata logger = logging.getLogger(__name__) @@ -24,7 +23,7 @@ }""" -class ESCIReranking(AbsTaskReranking, MultilingualTask): +class ESCIReranking(AbsTaskReranking): metadata = TaskMetadata( name="ESCIReranking", description="", diff --git a/mteb/tasks/Reranking/multilingual/MIRACLReranking.py b/mteb/tasks/Reranking/multilingual/MIRACLReranking.py index cd87ec3200..75f65d3bc3 100644 --- a/mteb/tasks/Reranking/multilingual/MIRACLReranking.py +++ b/mteb/tasks/Reranking/multilingual/MIRACLReranking.py @@ -5,7 +5,6 @@ import datasets -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskReranking import AbsTaskReranking @@ -47,7 +46,7 @@ }""" -class MIRACLReranking(AbsTaskReranking, MultilingualTask): +class MIRACLReranking(AbsTaskReranking): metadata = TaskMetadata( name="MIRACLReranking", description="MIRACL (Multilingual Information Retrieval Across a Continuum of Languages) is a multilingual retrieval dataset that focuses on search across 18 different languages.", @@ -121,8 +120,7 @@ def load_data(self, **kwargs): self.relevant_docs = defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) self.top_ranked = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) - hf_subsets = list(self.hf_subsets) if self.is_multilingual else ["default"] - for hf_subset in hf_subsets: + for hf_subset in self.hf_subsets: if "name" in self.metadata.dataset: cur_dataset = datasets.load_dataset(**self.metadata.dataset) # type: ignore assert ( diff --git a/mteb/tasks/Reranking/multilingual/WikipediaRerankingMultilingual.py b/mteb/tasks/Reranking/multilingual/WikipediaRerankingMultilingual.py index 8189d92f4e..31047c45fc 100644 --- a/mteb/tasks/Reranking/multilingual/WikipediaRerankingMultilingual.py +++ b/mteb/tasks/Reranking/multilingual/WikipediaRerankingMultilingual.py @@ -1,6 +1,5 @@ from __future__ import annotations -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskReranking import AbsTaskReranking @@ -25,7 +24,7 @@ } -class WikipediaRerankingMultilingual(AbsTaskReranking, MultilingualTask): +class WikipediaRerankingMultilingual(AbsTaskReranking): metadata = TaskMetadata( name="WikipediaRerankingMultilingual", description="The dataset is derived from Cohere's wikipedia-2023-11 dataset and contains synthetically generated queries.", diff --git a/mteb/tasks/Retrieval/code/COIRCodeSearchNetRetrieval.py b/mteb/tasks/Retrieval/code/COIRCodeSearchNetRetrieval.py index 315b749aac..0eaba4bc37 100644 --- a/mteb/tasks/Retrieval/code/COIRCodeSearchNetRetrieval.py +++ b/mteb/tasks/Retrieval/code/COIRCodeSearchNetRetrieval.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGS = ["python", "javascript", "go", "ruby", "java", "php"] @@ -73,7 +72,7 @@ def _load_code_search_code_retrieval( return corpus, queries, relevant_docs -class COIRCodeSearchNetRetrieval(MultilingualTask, AbsTaskRetrieval): +class COIRCodeSearchNetRetrieval(AbsTaskRetrieval): _EVAL_SPLIT = "test" metadata = TaskMetadata( name="COIRCodeSearchNetRetrieval", diff --git a/mteb/tasks/Retrieval/code/CodeEditSearchRetrieval.py b/mteb/tasks/Retrieval/code/CodeEditSearchRetrieval.py index 2e6100c9a6..14f26c3f4f 100644 --- a/mteb/tasks/Retrieval/code/CodeEditSearchRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeEditSearchRetrieval.py @@ -3,7 +3,6 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGS = [ @@ -23,7 +22,7 @@ ] -class CodeEditSearchRetrieval(MultilingualTask, AbsTaskRetrieval): +class CodeEditSearchRetrieval(AbsTaskRetrieval): _EVAL_SPLIT = "train" metadata = TaskMetadata( name="CodeEditSearchRetrieval", diff --git a/mteb/tasks/Retrieval/code/CodeSearchNetCCRetrieval.py b/mteb/tasks/Retrieval/code/CodeSearchNetCCRetrieval.py index 974c34a16a..9a25ce806c 100644 --- a/mteb/tasks/Retrieval/code/CodeSearchNetCCRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeSearchNetCCRetrieval.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGS = ["python", "javascript", "go", "ruby", "java", "php"] @@ -73,7 +72,7 @@ def _load_code_search_code_retrieval( return corpus, queries, relevant_docs -class CodeSearchNetCCRetrieval(MultilingualTask, AbsTaskRetrieval): +class CodeSearchNetCCRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="CodeSearchNetCCRetrieval", description="The dataset is a collection of code snippets. The task is to retrieve the most relevant code snippet for a given code snippet.", diff --git a/mteb/tasks/Retrieval/code/CodeSearchNetRetrieval.py b/mteb/tasks/Retrieval/code/CodeSearchNetRetrieval.py index 60c6e5b4d9..80fedd195b 100644 --- a/mteb/tasks/Retrieval/code/CodeSearchNetRetrieval.py +++ b/mteb/tasks/Retrieval/code/CodeSearchNetRetrieval.py @@ -3,13 +3,12 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGS = ["python", "javascript", "go", "ruby", "java", "php"] -class CodeSearchNetRetrieval(MultilingualTask, AbsTaskRetrieval): +class CodeSearchNetRetrieval(AbsTaskRetrieval): _EVAL_SPLIT = "test" metadata = TaskMetadata( name="CodeSearchNetRetrieval", diff --git a/mteb/tasks/Retrieval/eng/BrightRetrieval.py b/mteb/tasks/Retrieval/eng/BrightRetrieval.py index a5c03cdd6b..e81ed070e3 100644 --- a/mteb/tasks/Retrieval/eng/BrightRetrieval.py +++ b/mteb/tasks/Retrieval/eng/BrightRetrieval.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata DOMAINS_LONG = [ @@ -32,7 +31,7 @@ EVAL_SPLITS = ["standard", "long"] -class BrightRetrieval(MultilingualTask, AbsTaskRetrieval): +class BrightRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="BrightRetrieval", dataset={ diff --git a/mteb/tasks/Retrieval/eng/NQRetrieval.py b/mteb/tasks/Retrieval/eng/NQRetrieval.py index 661bf3e0e2..12f4690d92 100644 --- a/mteb/tasks/Retrieval/eng/NQRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NQRetrieval.py @@ -27,11 +27,11 @@ class NQ(AbsTaskRetrieval): annotations_creators=None, dialect=None, sample_creation=None, - bibtex_citation="""@article{47761,title = {Natural Questions: a Benchmark for Question Answering Research}, - author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh + bibtex_citation="""@article{47761,title = {Natural Questions: a Benchmark for Question Answering Research}, + author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le - and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational + and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational Linguistics}}""", prompt={ "query": "Given a question, retrieve Wikipedia passages that answer the question" @@ -61,10 +61,10 @@ class NQHardNegatives(AbsTaskRetrieval): annotations_creators=None, dialect=None, sample_creation=None, - bibtex_citation="""@article{47761,title = {Natural Questions: a Benchmark for Question Answering Research}, - author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh + bibtex_citation="""@article{47761,title = {Natural Questions: a Benchmark for Question Answering Research}, + author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le - and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational + and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational Linguistics}}""", ) diff --git a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py index 538a0881fa..5715a2a026 100644 --- a/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py +++ b/mteb/tasks/Retrieval/eng/NanoNQRetrieval.py @@ -30,11 +30,11 @@ class NanoNQRetrieval(AbsTaskRetrieval): annotations_creators="human-annotated", dialect=[], sample_creation="found", - bibtex_citation="""@article{47761,title = {Natural Questions: a Benchmark for Question Answering Research}, - author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh + bibtex_citation="""@article{47761,title = {Natural Questions: a Benchmark for Question Answering Research}, + author = {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le - and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational + and Slav Petrov},year = {2019},journal = {Transactions of the Association of Computational Linguistics}}""", prompt={ "query": "Given a question, retrieve Wikipedia passages that answer the question" diff --git a/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py b/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py index 2a45205cfd..e7a79a869f 100644 --- a/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/BelebeleRetrieval.py @@ -3,7 +3,6 @@ from datasets import load_dataset from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _EVAL_SPLIT = "test" @@ -177,7 +176,7 @@ def get_lang_pairs() -> dict[str, list[str]]: _LANGUAGES_MAPPING = get_lang_pairs() -class BelebeleRetrieval(MultilingualTask, AbsTaskRetrieval): +class BelebeleRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="BelebeleRetrieval", dataset={ diff --git a/mteb/tasks/Retrieval/multilingual/CUREv1Retrieval.py b/mteb/tasks/Retrieval/multilingual/CUREv1Retrieval.py index 9f935d1a9a..31ee5bccac 100644 --- a/mteb/tasks/Retrieval/multilingual/CUREv1Retrieval.py +++ b/mteb/tasks/Retrieval/multilingual/CUREv1Retrieval.py @@ -7,7 +7,6 @@ from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from ....abstasks.MultilingualTask import MultilingualTask _LANGUAGES = { "en": ["eng-Latn", "eng-Latn"], @@ -34,7 +33,7 @@ def names(cls) -> list[str]: return sorted(cls._member_names_) -class CUREv1Retrieval(MultilingualTask, AbsTaskRetrieval): +class CUREv1Retrieval(AbsTaskRetrieval): metadata = TaskMetadata( dataset={ "path": "clinia/CUREv1", diff --git a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py index 07097468e8..cc5eb7043f 100644 --- a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py +++ b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT19.py @@ -3,7 +3,6 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -30,7 +29,7 @@ def extend_lang_pairs() -> dict[str, list[str]]: _EVAL_LANGS = extend_lang_pairs() -class CrossLingualSemanticDiscriminationWMT19(AbsTaskRetrieval, MultilingualTask): +class CrossLingualSemanticDiscriminationWMT19(AbsTaskRetrieval): metadata = TaskMetadata( name="CrossLingualSemanticDiscriminationWMT19", dataset={ diff --git a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py index 682bdbe7c1..c0b7800b31 100644 --- a/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py +++ b/mteb/tasks/Retrieval/multilingual/CrossLingualSemanticDiscriminationWMT21.py @@ -3,7 +3,6 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -30,7 +29,7 @@ def extend_lang_pairs() -> dict[str, list[str]]: _EVAL_LANGS = extend_lang_pairs() -class CrossLingualSemanticDiscriminationWMT21(AbsTaskRetrieval, MultilingualTask): +class CrossLingualSemanticDiscriminationWMT21(AbsTaskRetrieval): metadata = TaskMetadata( name="CrossLingualSemanticDiscriminationWMT21", dataset={ diff --git a/mteb/tasks/Retrieval/multilingual/IndicQARetrieval.py b/mteb/tasks/Retrieval/multilingual/IndicQARetrieval.py index c0e2ef4cc3..7f30f0b4d5 100644 --- a/mteb/tasks/Retrieval/multilingual/IndicQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/IndicQARetrieval.py @@ -1,6 +1,5 @@ from __future__ import annotations -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval @@ -20,7 +19,7 @@ } -class IndicQARetrieval(AbsTaskRetrieval, MultilingualTask): +class IndicQARetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="IndicQARetrieval", dataset={ diff --git a/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py b/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py index beec172071..f5ed20f9b3 100644 --- a/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MIRACLRetrieval.py @@ -2,7 +2,6 @@ import datasets -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval @@ -97,7 +96,7 @@ def _load_miracl_data( return corpus, queries, relevant_docs -class MIRACLRetrieval(MultilingualTask, AbsTaskRetrieval): +class MIRACLRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="MIRACLRetrieval", description="MIRACL (Multilingual Information Retrieval Across a Continuum of Languages) is a multilingual retrieval dataset that focuses on search across 18 different languages.", @@ -285,7 +284,7 @@ def _load_miracl_data_hard_negatives( return corpus, queries, relevant_docs -class MIRACLRetrievalHardNegatives(MultilingualTask, AbsTaskRetrieval): +class MIRACLRetrievalHardNegatives(AbsTaskRetrieval): metadata = TaskMetadata( name="MIRACLRetrievalHardNegatives", description="MIRACL (Multilingual Information Retrieval Across a Continuum of Languages) is a multilingual retrieval dataset that focuses on search across 18 different languages. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", diff --git a/mteb/tasks/Retrieval/multilingual/MLQARetrieval.py b/mteb/tasks/Retrieval/multilingual/MLQARetrieval.py index b8fd1a6ff1..3a1d74a867 100644 --- a/mteb/tasks/Retrieval/multilingual/MLQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MLQARetrieval.py @@ -3,7 +3,6 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -77,7 +76,7 @@ def extend_lang_pairs() -> dict[str, list[str]]: _EVAL_LANGS = extend_lang_pairs() -class MLQARetrieval(AbsTaskRetrieval, MultilingualTask): +class MLQARetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="MLQARetrieval", description="""MLQA (MultiLingual Question Answering) is a benchmark dataset for evaluating cross-lingual question answering performance. diff --git a/mteb/tasks/Retrieval/multilingual/MintakaRetrieval.py b/mteb/tasks/Retrieval/multilingual/MintakaRetrieval.py index b8a2eec9af..2f3058e31f 100644 --- a/mteb/tasks/Retrieval/multilingual/MintakaRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MintakaRetrieval.py @@ -2,7 +2,6 @@ import datasets -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval @@ -64,7 +63,7 @@ def _load_mintaka_data( return corpus, queries, relevant_docs -class MintakaRetrieval(MultilingualTask, AbsTaskRetrieval): +class MintakaRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="MintakaRetrieval", description="We introduce Mintaka, a complex, natural, and multilingual dataset designed for experimenting with end-to-end question-answering models. Mintaka is composed of 20,000 question-answer pairs collected in English, annotated with Wikidata entities, and translated into Arabic, French, German, Hindi, Italian, Japanese, Portuguese, and Spanish for a total of 180,000 samples. Mintaka includes 8 types of complex questions, including superlative, intersection, and multi-hop questions, which were naturally elicited from crowd workers. ", diff --git a/mteb/tasks/Retrieval/multilingual/MrTidyRetrieval.py b/mteb/tasks/Retrieval/multilingual/MrTidyRetrieval.py index 7fe6c764d0..7888ab0910 100644 --- a/mteb/tasks/Retrieval/multilingual/MrTidyRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MrTidyRetrieval.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _EVAL_LANGS = { @@ -85,7 +84,7 @@ def _load_data_retrieval( return corpus, queries, relevant_docs -class MrTidyRetrieval(MultilingualTask, AbsTaskRetrieval): +class MrTidyRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="MrTidyRetrieval", description="Mr. TyDi is a multi-lingual benchmark dataset built on TyDi, covering eleven typologically diverse languages. It is designed for monolingual retrieval, specifically to evaluate ranking with learned dense representations.", diff --git a/mteb/tasks/Retrieval/multilingual/MultiLongDocRetrieval.py b/mteb/tasks/Retrieval/multilingual/MultiLongDocRetrieval.py index 998895e894..7b000d1178 100644 --- a/mteb/tasks/Retrieval/multilingual/MultiLongDocRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/MultiLongDocRetrieval.py @@ -3,7 +3,6 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskRetrieval import * @@ -60,7 +59,7 @@ def load_mldr_data( return corpus, queries, relevant_docs -class MultiLongDocRetrieval(MultilingualTask, AbsTaskRetrieval): +class MultiLongDocRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="MultiLongDocRetrieval", description="""Multi Long Doc Retrieval (MLDR) 'is curated by the multilingual articles from Wikipedia, Wudao and mC4 (see Table 7), and NarrativeQA (Kocˇisky ́ et al., 2018; Gu ̈nther et al., 2023), which is only for English.' (Chen et al., 2024). diff --git a/mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py b/mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py index 94c7bf39fb..33a46e06dd 100644 --- a/mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py +++ b/mteb/tasks/Retrieval/multilingual/NeuCLIR2022Retrieval.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskRetrieval import * @@ -57,7 +56,7 @@ def load_neuclir_data( return corpus, queries, relevant_docs -class NeuCLIR2022Retrieval(MultilingualTask, AbsTaskRetrieval): +class NeuCLIR2022Retrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NeuCLIR2022Retrieval", description="The task involves identifying and retrieving the documents that are relevant to the queries.", @@ -170,7 +169,7 @@ def load_neuclir_data_hard_negatives( return corpus, queries, relevant_docs -class NeuCLIR2022RetrievalHardNegatives(MultilingualTask, AbsTaskRetrieval): +class NeuCLIR2022RetrievalHardNegatives(AbsTaskRetrieval): metadata = TaskMetadata( name="NeuCLIR2022RetrievalHardNegatives", description="The task involves identifying and retrieving the documents that are relevant to the queries. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", diff --git a/mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py b/mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py index 5f0626146e..6923865721 100644 --- a/mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py +++ b/mteb/tasks/Retrieval/multilingual/NeuCLIR2023Retrieval.py @@ -5,7 +5,6 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskRetrieval import * @@ -56,7 +55,7 @@ def load_neuclir_data( return corpus, queries, relevant_docs -class NeuCLIR2023Retrieval(MultilingualTask, AbsTaskRetrieval): +class NeuCLIR2023Retrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="NeuCLIR2023Retrieval", description="The task involves identifying and retrieving the documents that are relevant to the queries.", @@ -171,7 +170,7 @@ def load_neuclir_data_hard_negatives( return corpus, queries, relevant_docs -class NeuCLIR2023RetrievalHardNegatives(MultilingualTask, AbsTaskRetrieval): +class NeuCLIR2023RetrievalHardNegatives(AbsTaskRetrieval): metadata = TaskMetadata( name="NeuCLIR2023RetrievalHardNegatives", description="The task involves identifying and retrieving the documents that are relevant to the queries. The hard negative version has been created by pooling the 250 top documents per query from BM25, e5-multilingual-large and e5-mistral-instruct.", diff --git a/mteb/tasks/Retrieval/multilingual/PublicHealthQARetrieval.py b/mteb/tasks/Retrieval/multilingual/PublicHealthQARetrieval.py index bf5e786a74..db398f078c 100644 --- a/mteb/tasks/Retrieval/multilingual/PublicHealthQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/PublicHealthQARetrieval.py @@ -2,7 +2,6 @@ import datasets -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval @@ -63,7 +62,7 @@ def _load_publichealthqa_data( return corpus, queries, relevant_docs -class PublicHealthQARetrieval(MultilingualTask, AbsTaskRetrieval): +class PublicHealthQARetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="PublicHealthQA", description="A multilingual dataset for public health question answering, based on FAQ sourced from CDC and WHO.", @@ -87,12 +86,12 @@ class PublicHealthQARetrieval(MultilingualTask, AbsTaskRetrieval): sample_creation="found", bibtex_citation=""" @misc {xing_han_lu_2024, - author = { {Xing Han Lu} }, - title = { publichealth-qa (Revision 3b67b6b) }, - year = 2024, - url = { https://huggingface.co/datasets/xhluca/publichealth-qa }, - doi = { 10.57967/hf/2247 }, - publisher = { Hugging Face } + author = { {Xing Han Lu} }, + title = { publichealth-qa (Revision 3b67b6b) }, + year = 2024, + url = { https://huggingface.co/datasets/xhluca/publichealth-qa }, + doi = { 10.57967/hf/2247 }, + publisher = { Hugging Face } } """, ) diff --git a/mteb/tasks/Retrieval/multilingual/StatcanDialogueDatasetRetrieval.py b/mteb/tasks/Retrieval/multilingual/StatcanDialogueDatasetRetrieval.py index 98e8a3aad7..3621759f35 100644 --- a/mteb/tasks/Retrieval/multilingual/StatcanDialogueDatasetRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/StatcanDialogueDatasetRetrieval.py @@ -4,7 +4,6 @@ import datasets -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval @@ -63,7 +62,7 @@ def _load_statcan_data( return corpus, queries, relevant_docs -class StatcanDialogueDatasetRetrieval(MultilingualTask, AbsTaskRetrieval): +class StatcanDialogueDatasetRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="StatcanDialogueDatasetRetrieval", description="A Dataset for Retrieving Data Tables through Conversations with Genuine Intents, available in English and French.", diff --git a/mteb/tasks/Retrieval/multilingual/WikipediaRetrievalMultilingual.py b/mteb/tasks/Retrieval/multilingual/WikipediaRetrievalMultilingual.py index f51ebac2f2..09f533cff3 100644 --- a/mteb/tasks/Retrieval/multilingual/WikipediaRetrievalMultilingual.py +++ b/mteb/tasks/Retrieval/multilingual/WikipediaRetrievalMultilingual.py @@ -1,6 +1,5 @@ from __future__ import annotations -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval @@ -25,7 +24,7 @@ } -class WikipediaRetrievalMultilingual(AbsTaskRetrieval, MultilingualTask): +class WikipediaRetrievalMultilingual(AbsTaskRetrieval): metadata = TaskMetadata( name="WikipediaRetrievalMultilingual", description="The dataset is derived from Cohere's wikipedia-2023-11 dataset and contains synthetically generated queries.", diff --git a/mteb/tasks/Retrieval/multilingual/XMarketRetrieval.py b/mteb/tasks/Retrieval/multilingual/XMarketRetrieval.py index bb0a5b63b1..aada66d529 100644 --- a/mteb/tasks/Retrieval/multilingual/XMarketRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/XMarketRetrieval.py @@ -2,7 +2,6 @@ import datasets -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval @@ -64,7 +63,7 @@ def _load_xmarket_data( return corpus, queries, relevant_docs -class XMarket(MultilingualTask, AbsTaskRetrieval): +class XMarket(AbsTaskRetrieval): metadata = TaskMetadata( name="XMarket", description="XMarket", diff --git a/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py b/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py index b53049ac90..7896ea5d66 100644 --- a/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/XPQARetrieval.py @@ -3,7 +3,6 @@ import datasets from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _EVAL_LANGS = { @@ -62,7 +61,7 @@ } -class XPQARetrieval(AbsTaskRetrieval, MultilingualTask): +class XPQARetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="XPQARetrieval", description="XPQARetrieval", diff --git a/mteb/tasks/Retrieval/multilingual/XQuADRetrieval.py b/mteb/tasks/Retrieval/multilingual/XQuADRetrieval.py index ff2591ae7e..764e0eabd7 100644 --- a/mteb/tasks/Retrieval/multilingual/XQuADRetrieval.py +++ b/mteb/tasks/Retrieval/multilingual/XQuADRetrieval.py @@ -4,7 +4,6 @@ import datasets -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata from ....abstasks.AbsTaskRetrieval import AbsTaskRetrieval @@ -25,7 +24,7 @@ } -class XQuADRetrieval(MultilingualTask, AbsTaskRetrieval): +class XQuADRetrieval(AbsTaskRetrieval): metadata = TaskMetadata( name="XQuADRetrieval", dataset={ diff --git a/mteb/tasks/STS/multilingual/IndicCrosslingualSTS.py b/mteb/tasks/STS/multilingual/IndicCrosslingualSTS.py index 2c2f12f294..9782b14cac 100644 --- a/mteb/tasks/STS/multilingual/IndicCrosslingualSTS.py +++ b/mteb/tasks/STS/multilingual/IndicCrosslingualSTS.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskSTS import AbsTaskSTS -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -20,7 +19,7 @@ } -class IndicCrosslingualSTS(AbsTaskSTS, MultilingualTask): +class IndicCrosslingualSTS(AbsTaskSTS): metadata = TaskMetadata( name="IndicCrosslingualSTS", dataset={ diff --git a/mteb/tasks/STS/multilingual/STS17CrosslingualSTS.py b/mteb/tasks/STS/multilingual/STS17CrosslingualSTS.py index 793357e2eb..bdcb63065d 100644 --- a/mteb/tasks/STS/multilingual/STS17CrosslingualSTS.py +++ b/mteb/tasks/STS/multilingual/STS17CrosslingualSTS.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskSTS import AbsTaskSTS -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -19,7 +18,7 @@ } -class STS17Crosslingual(AbsTaskSTS, MultilingualTask): +class STS17Crosslingual(AbsTaskSTS): fast_loading = True metadata = TaskMetadata( name="STS17", diff --git a/mteb/tasks/STS/multilingual/STS22CrosslingualSTS.py b/mteb/tasks/STS/multilingual/STS22CrosslingualSTS.py index 05bd05dd39..af73fe11cf 100644 --- a/mteb/tasks/STS/multilingual/STS22CrosslingualSTS.py +++ b/mteb/tasks/STS/multilingual/STS22CrosslingualSTS.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskSTS import AbsTaskSTS -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -26,7 +25,7 @@ } -class STS22CrosslingualSTSv2(AbsTaskSTS, MultilingualTask): +class STS22CrosslingualSTSv2(AbsTaskSTS): fast_loading = True metadata = TaskMetadata( name="STS22.v2", @@ -83,7 +82,7 @@ class STS22CrosslingualSTSv2(AbsTaskSTS, MultilingualTask): max_score = 4 -class STS22CrosslingualSTS(AbsTaskSTS, MultilingualTask): +class STS22CrosslingualSTS(AbsTaskSTS): superseded_by = "STS22.v2" fast_loading = True metadata = TaskMetadata( diff --git a/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py b/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py index ad15498247..540e1ccca4 100644 --- a/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py +++ b/mteb/tasks/STS/multilingual/STSBenchmarkMultilingualSTS.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskSTS import AbsTaskSTS -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -20,7 +19,7 @@ _SPLITS = ["dev", "test"] -class STSBenchmarkMultilingualSTS(AbsTaskSTS, MultilingualTask): +class STSBenchmarkMultilingualSTS(AbsTaskSTS): fast_loading = True metadata = TaskMetadata( name="STSBenchmarkMultilingualSTS", diff --git a/mteb/tasks/STS/multilingual/SemRel24STS.py b/mteb/tasks/STS/multilingual/SemRel24STS.py index 905f5cb332..8f06587d38 100644 --- a/mteb/tasks/STS/multilingual/SemRel24STS.py +++ b/mteb/tasks/STS/multilingual/SemRel24STS.py @@ -1,7 +1,6 @@ from __future__ import annotations from mteb.abstasks.AbsTaskSTS import AbsTaskSTS -from mteb.abstasks.MultilingualTask import MultilingualTask from mteb.abstasks.TaskMetadata import TaskMetadata _LANGUAGES = { @@ -22,7 +21,7 @@ _SPLITS = ["test"] -class SemRel24STS(AbsTaskSTS, MultilingualTask): +class SemRel24STS(AbsTaskSTS): metadata = TaskMetadata( name="SemRel24STS", dataset={ diff --git a/scripts/generate_imports.py b/scripts/generate_imports.py index ae331de9df..602d6eef0f 100644 --- a/scripts/generate_imports.py +++ b/scripts/generate_imports.py @@ -27,7 +27,6 @@ def find_task_classes_in_module(full_module_name): and issubclass(obj, AbsTask) and obj is not AbsTask and not obj.__name__.startswith("AbsTask") - and not obj.__name__ == "MultilingualTask" ): task_classes.append(name) return task_classes diff --git a/tests/test_TaskMetadata.py b/tests/test_TaskMetadata.py index 701abab729..f3e6b48260 100644 --- a/tests/test_TaskMetadata.py +++ b/tests/test_TaskMetadata.py @@ -523,15 +523,3 @@ def test_empty_descriptive_stat_in_new_datasets(task: AbsTask): task.metadata.descriptive_stats is not None ), f"Dataset {task.metadata.name} should have descriptive stats. You can add metadata to your task by running `YorTask().calculate_metadata_metrics()`" assert task.metadata.n_samples is not None - - -@pytest.mark.parametrize("task", get_tasks()) -def test_eval_langs_correctly_specified(task: AbsTask): - if task.is_multilingual: - assert isinstance( - task.metadata.eval_langs, dict - ), f"{task.metadata.name} should have eval_langs as a dict" - else: - assert isinstance( - task.metadata.eval_langs, list - ), f"{task.metadata.name} should have eval_langs as a list" diff --git a/tests/test_benchmark/mock_tasks.py b/tests/test_benchmark/mock_tasks.py index 99c24b756f..b9e86f1e13 100644 --- a/tests/test_benchmark/mock_tasks.py +++ b/tests/test_benchmark/mock_tasks.py @@ -4,7 +4,6 @@ from datasets import Dataset, DatasetDict -from mteb.abstasks import MultilingualTask from mteb.abstasks.AbsTaskBitextMining import AbsTaskBitextMining from mteb.abstasks.AbsTaskClassification import AbsTaskClassification from mteb.abstasks.AbsTaskClustering import AbsTaskClustering @@ -110,7 +109,7 @@ def load_data(self, **kwargs): self.data_loaded = True -class MockMultilingualClassificationTask(AbsTaskClassification, MultilingualTask): +class MockMultilingualClassificationTask(AbsTaskClassification): expected_stats = { "test": { "num_samples": 4, @@ -282,7 +281,7 @@ def load_data(self, **kwargs): self.data_loaded = True -class MockMultilingualBitextMiningTask(AbsTaskBitextMining, MultilingualTask): +class MockMultilingualBitextMiningTask(AbsTaskBitextMining): expected_stats = { "test": { "num_samples": 4, @@ -358,7 +357,7 @@ def load_data(self, **kwargs): self.data_loaded = True -class MockMultilingualParallelBitextMiningTask(AbsTaskBitextMining, MultilingualTask): +class MockMultilingualParallelBitextMiningTask(AbsTaskBitextMining): parallel_subsets = True expected_stats = { "test": { @@ -482,7 +481,7 @@ def load_data(self, **kwargs): self.data_loaded = True -class MockMultilingualClusteringTask(AbsTaskClustering, MultilingualTask): +class MockMultilingualClusteringTask(AbsTaskClustering): expected_stats = { "test": { "num_samples": 2, @@ -609,7 +608,7 @@ def load_data(self, **kwargs): self.data_loaded = True -class MockMultilingualClusteringFastTask(AbsTaskClusteringFast, MultilingualTask): +class MockMultilingualClusteringFastTask(AbsTaskClusteringFast): max_document_to_embed = 3 max_fraction_of_documents_to_embed = None expected_stats = { @@ -739,9 +738,7 @@ def load_data(self, **kwargs): self.data_loaded = True -class MockMultilingualPairClassificationTask( - AbsTaskPairClassification, MultilingualTask -): +class MockMultilingualPairClassificationTask(AbsTaskPairClassification): expected_stats = { "test": { "num_samples": 4, @@ -879,7 +876,7 @@ def load_data(self, **kwargs): max_score = 1 -class MockMultilingualSTSTask(AbsTaskSTS, MultilingualTask): +class MockMultilingualSTSTask(AbsTaskSTS): expected_stats = { "test": { "num_samples": 4, @@ -1028,7 +1025,7 @@ def load_data(self, **kwargs): max_score = 1 -class MockMultilingualSummarizationTask(AbsTaskSummarization, MultilingualTask): +class MockMultilingualSummarizationTask(AbsTaskSummarization): expected_stats = { "test": { "num_samples": 4, @@ -1203,7 +1200,7 @@ def load_data(self, **kwargs): self.data_loaded = True -class MockMultilingualRerankingTask(AbsTaskReranking, MultilingualTask): +class MockMultilingualRerankingTask(AbsTaskReranking): expected_stats = { "test": { "num_samples": 8, @@ -1451,7 +1448,7 @@ def load_data(self, **kwargs): self.data_loaded = True -class MockMultilingualRetrievalTask(AbsTaskRetrieval, MultilingualTask): +class MockMultilingualRetrievalTask(AbsTaskRetrieval): expected_stats = { "val": { "num_samples": 8, @@ -1747,9 +1744,7 @@ def load_data(self, **kwargs): self.data_loaded = True -class MockMultilingualMultilabelClassification( - AbsTaskMultilabelClassification, MultilingualTask -): +class MockMultilingualMultilabelClassification(AbsTaskMultilabelClassification): expected_stats = { "test": { "num_samples": 12, @@ -2023,7 +2018,7 @@ def load_data(self, **kwargs): self.data_loaded = True -class MockMultilingualInstructionRetrieval(AbsTaskRetrieval, MultilingualTask): +class MockMultilingualInstructionRetrieval(AbsTaskRetrieval): expected_stats = { "test": { "num_samples": 8, @@ -2170,7 +2165,7 @@ def load_data(self, **kwargs): self.top_ranked = None -class MockMultilingualInstructionReranking(AbsTaskReranking, MultilingualTask): +class MockMultilingualInstructionReranking(AbsTaskReranking): expected_stats = { "test": { "num_samples": 8, diff --git a/tests/test_overview.py b/tests/test_overview.py index 6136af1ea5..38f6dcbd88 100644 --- a/tests/test_overview.py +++ b/tests/test_overview.py @@ -76,7 +76,7 @@ def test_get_tasks_filtering(): tasks = get_tasks(languages=["eng"]) for task in tasks: - if task.is_multilingual: + if task.metadata.is_multilingual: assert isinstance(task.metadata.eval_langs, dict) for hf_subset in task.hf_subsets: diff --git a/tests/test_tasks/test_all_abstasks.py b/tests/test_tasks/test_all_abstasks.py index 91a7b95070..bdc6e300f6 100644 --- a/tests/test_tasks/test_all_abstasks.py +++ b/tests/test_tasks/test_all_abstasks.py @@ -8,7 +8,7 @@ import pytest import mteb -from mteb.abstasks import AbsTask, MultilingualTask +from mteb.abstasks import AbsTask from mteb.abstasks.AbsTaskReranking import AbsTaskReranking from mteb.abstasks.AbsTaskRetrieval import AbsTaskRetrieval from mteb.abstasks.AbsTaskSpeedTask import AbsTaskSpeedTask @@ -32,7 +32,7 @@ def test_load_data( isinstance(task, AbsTaskRetrieval) or isinstance(task, AbsTaskReranking) or isinstance(task, AbsTaskSpeedTask) - or isinstance(task, MultilingualTask) + or task.metadata.is_multilingual ): pytest.skip() with patch.object(task, "dataset_transform") as mock_dataset_transform: @@ -40,7 +40,7 @@ def test_load_data( mock_load_dataset.assert_called() # They don't yet but should they so they can be expanded more easily? - if not task.is_multilingual: + if not task.metadata.is_multilingual: mock_dataset_transform.assert_called_once()