From e93ab3665e7a657eb523ae0c83dee3814b5eb7e4 Mon Sep 17 00:00:00 2001 From: silky1708 Date: Mon, 10 Mar 2025 16:18:38 -0700 Subject: [PATCH 1/7] add GTZANGenre dataset --- mteb/abstasks/TaskMetadata.py | 31 +----------- .../Audio/AudioClassification/__init__.py | 1 + .../AudioClassification/eng/GTZANGenre.py | 49 +++++++++++++++++++ 3 files changed, 51 insertions(+), 30 deletions(-) create mode 100644 mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 57cbb157e4..b769d8ee08 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -58,6 +58,7 @@ "Environment Sound Classification", "Gunshot Audio Classification", "Instrument Source Classification", + "Music Genre Classification", "Gender Clustering", "Rendered semantic textual similarity", ] @@ -125,38 +126,8 @@ "AudioMultilabelClassification", "AudioZeroshotClassification", "AudioClassification", - "AudioCrossFoldClassification", ) -TASK_TYPE = ( - ( - "BitextMining", - "Classification", - "MultilabelClassification", - "Clustering", - "PairClassification", - "Reranking", - "Retrieval", - "STS", - "Summarization", - "InstructionRetrieval", - "Speed", - "Any2AnyMultiChoice", - "Any2AnyRetrieval", - "Any2TextMutipleChoice", - "ImageClustering", - "ImageClassification", - "ImageMultilabelClassification", - "ImageTextPairClassification", - "VisualSTS", - "ZeroShotClassification", - ) - + MIEB_TASK_TYPE - + MAEB_TASK_TYPE -) - -TASK_TYPE = Literal[TASK_TYPE] - TASK_CATEGORY = Literal[ "s2s", # Sentence-to-sentence diff --git a/mteb/tasks/Audio/AudioClassification/__init__.py b/mteb/tasks/Audio/AudioClassification/__init__.py index 5cd37da3a5..03a29e0987 100644 --- a/mteb/tasks/Audio/AudioClassification/__init__.py +++ b/mteb/tasks/Audio/AudioClassification/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations from .eng.ESC50 import * +from .eng.GTZANGenre import * from .eng.GunshotTriangulation import * from .eng.NSynth import * diff --git a/mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py b/mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py new file mode 100644 index 0000000000..a9cdff4816 --- /dev/null +++ b/mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from mteb.abstasks.Audio.AbsTaskAudioClassification import ( + AbsTaskAudioClassification, +) +from mteb.abstasks.TaskMetadata import TaskMetadata + + +class GTZANGenre(AbsTaskAudioClassification): + metadata = TaskMetadata( + name="GTZANGenre", + description="Music Genre Classification (10 classes)", + reference="https://huggingface.co/datasets/sanchit-gandhi/gtzan", + dataset={ + "path": "sanchit-gandhi/gtzan", + "revision": "4bd857132cb0e731bef3ec68558e7acc0a85f144", + }, + type="AudioClassification", + category="a2t", + eval_splits=["train"], + eval_langs=["eng-Latn"], + main_score="accuracy", + date=("2023-06-23", "2023-06-23"), + domains=["Music"], + task_subtypes=["Music Genre Classification"], + license="not specified", + annotations_creators="human-annotated", + dialect=[], + modalities=["audio"], + sample_creation="found", + bibtex_citation="""@ARTICLE{1021072, + author={Tzanetakis, G. and Cook, P.}, + journal={IEEE Transactions on Speech and Audio Processing}, + title={Musical genre classification of audio signals}, + year={2002}, + volume={10}, + number={5}, + pages={293-302}, + keywords={Humans;Music information retrieval;Instruments;Computer science;Multiple signal classification;Signal analysis;Pattern recognition;Feature extraction;Wavelet analysis;Cultural differences}, + doi={10.1109/TSA.2002.800560}}""", + descriptive_stats={ + "n_samples": {"train": 1000}, + }, + ) + + audio_column_name: str = "audio" + label_column_name: str = "genre" + samples_per_label: int = 10 + is_cross_validation: bool = True From fda7ac1f6c3814dcc1a7ab915876002a556ae560 Mon Sep 17 00:00:00 2001 From: silky1708 Date: Thu, 13 Mar 2025 13:07:02 -0700 Subject: [PATCH 2/7] reupload gtzan dataset --- mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py b/mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py index a9cdff4816..b224811a4e 100644 --- a/mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py +++ b/mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py @@ -10,17 +10,17 @@ class GTZANGenre(AbsTaskAudioClassification): metadata = TaskMetadata( name="GTZANGenre", description="Music Genre Classification (10 classes)", - reference="https://huggingface.co/datasets/sanchit-gandhi/gtzan", + reference="https://huggingface.co/datasets/silky1708/GTZAN-Genre", dataset={ - "path": "sanchit-gandhi/gtzan", - "revision": "4bd857132cb0e731bef3ec68558e7acc0a85f144", + "path": "silky1708/GTZAN-Genre", + "revision": "5efdda59d0d185bfe17ada9b54d233349d0e0168", }, type="AudioClassification", category="a2t", eval_splits=["train"], eval_langs=["eng-Latn"], main_score="accuracy", - date=("2023-06-23", "2023-06-23"), + date=("2025-03-13", "2025-03-13"), domains=["Music"], task_subtypes=["Music Genre Classification"], license="not specified", @@ -44,6 +44,6 @@ class GTZANGenre(AbsTaskAudioClassification): ) audio_column_name: str = "audio" - label_column_name: str = "genre" + label_column_name: str = "label" samples_per_label: int = 10 is_cross_validation: bool = True From 99c694eab775e164d529d275bfd43e1bd622394f Mon Sep 17 00:00:00 2001 From: silky1708 Date: Thu, 13 Mar 2025 13:42:21 -0700 Subject: [PATCH 3/7] update TaskMetadata from mteb:maeb --- mteb/abstasks/TaskMetadata.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index b769d8ee08..ce1a8fd6e3 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -58,7 +58,6 @@ "Environment Sound Classification", "Gunshot Audio Classification", "Instrument Source Classification", - "Music Genre Classification", "Gender Clustering", "Rendered semantic textual similarity", ] @@ -126,8 +125,38 @@ "AudioMultilabelClassification", "AudioZeroshotClassification", "AudioClassification", + "AudioCrossFoldClassification", ) +TASK_TYPE = ( + ( + "BitextMining", + "Classification", + "MultilabelClassification", + "Clustering", + "PairClassification", + "Reranking", + "Retrieval", + "STS", + "Summarization", + "InstructionRetrieval", + "Speed", + "Any2AnyMultiChoice", + "Any2AnyRetrieval", + "Any2TextMutipleChoice", + "ImageClustering", + "ImageClassification", + "ImageMultilabelClassification", + "ImageTextPairClassification", + "VisualSTS", + "ZeroShotClassification", + ) + + MIEB_TASK_TYPE + + MAEB_TASK_TYPE +) + +TASK_TYPE = Literal[TASK_TYPE] + TASK_CATEGORY = Literal[ "s2s", # Sentence-to-sentence @@ -457,4 +486,4 @@ def __hash__(self) -> int: @property def revision(self) -> str: - return self.dataset["revision"] + return self.dataset["revision"] \ No newline at end of file From ceb3d2b3a3a575702028a24ba23754b1a2c0d1e2 Mon Sep 17 00:00:00 2001 From: silky1708 Date: Thu, 13 Mar 2025 13:42:56 -0700 Subject: [PATCH 4/7] make pr --- mteb/abstasks/AbsTaskClusteringFast.py | 12 +++---- mteb/abstasks/AbsTaskInstructionRetrieval.py | 6 ++-- .../Audio/AbsTaskAudioClassification.py | 6 ++-- mteb/abstasks/TaskMetadata.py | 2 +- mteb/evaluation/MTEB.py | 6 ++-- mteb/models/rerankers_custom.py | 6 ++-- .../BitextMining/vie/VieMedEVBitextMining.py | 12 +++---- scripts/running_model/check_results.py | 6 ++-- tests/test_TaskMetadata.py | 36 +++++++++---------- tests/test_cli.py | 36 +++++++++---------- tests/test_overview.py | 6 ++-- tests/test_tasks/test_all_abstasks.py | 6 ++-- 12 files changed, 70 insertions(+), 70 deletions(-) diff --git a/mteb/abstasks/AbsTaskClusteringFast.py b/mteb/abstasks/AbsTaskClusteringFast.py index 0a41c4cbc2..40e36d29e2 100644 --- a/mteb/abstasks/AbsTaskClusteringFast.py +++ b/mteb/abstasks/AbsTaskClusteringFast.py @@ -320,9 +320,9 @@ def convert_to_fast( # check that it is the same distribution row_label_set = set(lab) - assert row_label_set.issubset(all_labels_set), ( - "The clusters are not sampled from the same distribution as they have different labels." - ) + assert row_label_set.issubset( + all_labels_set + ), "The clusters are not sampled from the same distribution as they have different labels." for l, s in zip(lab, sents): if s not in sent_set: @@ -353,6 +353,6 @@ def check_label_distribution(ds: DatasetDict) -> None: # check that it is the same distribution row_label_set = set(lab) - assert row_label_set.issubset(all_labels_set), ( - "The clusters are not sampled from the same distribution as they have different labels." - ) + assert row_label_set.issubset( + all_labels_set + ), "The clusters are not sampled from the same distribution as they have different labels." diff --git a/mteb/abstasks/AbsTaskInstructionRetrieval.py b/mteb/abstasks/AbsTaskInstructionRetrieval.py index 1022a01bce..eed22167fa 100644 --- a/mteb/abstasks/AbsTaskInstructionRetrieval.py +++ b/mteb/abstasks/AbsTaskInstructionRetrieval.py @@ -379,9 +379,9 @@ def load_data(self, **kwargs): doc["id"]: {"title": doc["title"], "text": doc["text"]} for doc in corpus } - assert len(top_ranked) == len(queries), ( - f"Top ranked not loaded properly! Expected {len(self.queries)} but got {len(self.top_ranked)}." - ) + assert ( + len(top_ranked) == len(queries) + ), f"Top ranked not loaded properly! Expected {len(self.queries)} but got {len(self.top_ranked)}." ( self.corpus[split], diff --git a/mteb/abstasks/Audio/AbsTaskAudioClassification.py b/mteb/abstasks/Audio/AbsTaskAudioClassification.py index 9aac7827fa..f349da5292 100644 --- a/mteb/abstasks/Audio/AbsTaskAudioClassification.py +++ b/mteb/abstasks/Audio/AbsTaskAudioClassification.py @@ -121,9 +121,9 @@ def _evaluate_subset_cross_validation( encode_kwargs: dict[str, Any] = {}, **kwargs, ) -> ScoresDict: - assert train_split == eval_split, ( - f"Performing {self.n_splits}-fold cross validation, but the dataset has a train (`{train_split}`) and test split (`{eval_split}`)! Set `is_cross_validation` to False, and retry." - ) + assert ( + train_split == eval_split + ), f"Performing {self.n_splits}-fold cross validation, but the dataset has a train (`{train_split}`) and test split (`{eval_split}`)! Set `is_cross_validation` to False, and retry." logger.info( f"Performing {self.n_splits}-fold cross-validation on the entire dataset!" ) diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index ce1a8fd6e3..57cbb157e4 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -486,4 +486,4 @@ def __hash__(self) -> int: @property def revision(self) -> str: - return self.dataset["revision"] \ No newline at end of file + return self.dataset["revision"] diff --git a/mteb/evaluation/MTEB.py b/mteb/evaluation/MTEB.py index a6bacc189e..96a1a97ad5 100644 --- a/mteb/evaluation/MTEB.py +++ b/mteb/evaluation/MTEB.py @@ -71,9 +71,9 @@ def __init__( if isinstance(tasks[0], Benchmark): self.benchmarks = tasks self._tasks = self._tasks = list(chain.from_iterable(tasks)) # type: ignore - assert task_types is None and task_categories is None, ( - "Cannot specify both `tasks` and `task_types`/`task_categories`" - ) + assert ( + task_types is None and task_categories is None + ), "Cannot specify both `tasks` and `task_types`/`task_categories`" else: self._task_types = task_types self._task_categories = task_categories diff --git a/mteb/models/rerankers_custom.py b/mteb/models/rerankers_custom.py index 83c054c9c9..0061ffc1c3 100644 --- a/mteb/models/rerankers_custom.py +++ b/mteb/models/rerankers_custom.py @@ -85,9 +85,9 @@ def predict(self, input_to_rerank, **kwargs): assert len(queries) == len(passages) query_passage_tuples = list(zip(queries, passages)) scores = self.model.compute_score(query_passage_tuples, normalize=True) - assert len(scores) == len(queries), ( - f"Expected {len(queries)} scores, got {len(scores)}" - ) + assert len(scores) == len( + queries + ), f"Expected {len(queries)} scores, got {len(scores)}" return scores diff --git a/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py b/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py index ab32025167..8dea762f86 100644 --- a/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py +++ b/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py @@ -54,9 +54,9 @@ def dataset_transform(self): # Pairs are in two halves en_sentences = all_texts[:mid_index] vie_sentences = all_texts[mid_index:] - assert len(en_sentences) == len(vie_sentences), ( - "The split does not result in equal halves." - ) + assert len(en_sentences) == len( + vie_sentences + ), "The split does not result in equal halves." # Downsample indices = list(range(len(en_sentences))) @@ -64,9 +64,9 @@ def dataset_transform(self): sample_indices = indices[:TEST_SAMPLES] en_sentences = [en_sentences[i] for i in sample_indices] vie_sentences = [vie_sentences[i] for i in sample_indices] - assert len(en_sentences) == len(vie_sentences) == TEST_SAMPLES, ( - f"Exceeded {TEST_SAMPLES} samples for 'test' split." - ) + assert ( + len(en_sentences) == len(vie_sentences) == TEST_SAMPLES + ), f"Exceeded {TEST_SAMPLES} samples for 'test' split." # Return dataset ds["test"] = datasets.Dataset.from_dict( diff --git a/scripts/running_model/check_results.py b/scripts/running_model/check_results.py index e518b77efd..c410fb5be7 100644 --- a/scripts/running_model/check_results.py +++ b/scripts/running_model/check_results.py @@ -174,9 +174,9 @@ def normalize_results(results): # [t.task_name for t in mteb_results['GritLM/GritLM-7B']["13f00a0e36500c80ce12870ea513846a066004af"] if t.task_name == "SemRel24STS"] # it is there -assert [len(revisions.keys()) == 1 for model, revisions in mteb_results.items()], ( - "Some models have more than one revision" -) +assert [ + len(revisions.keys()) == 1 for model, revisions in mteb_results.items() +], "Some models have more than one revision" results_df = results_to_dataframe(mteb_results) diff --git a/tests/test_TaskMetadata.py b/tests/test_TaskMetadata.py index 241ec536ea..54e074c2b4 100644 --- a/tests/test_TaskMetadata.py +++ b/tests/test_TaskMetadata.py @@ -518,17 +518,17 @@ def test_disallow_trust_remote_code_in_new_datasets(): "SwednClusteringS2S", ] - assert 135 == len(exceptions), ( - "The number of exceptions has changed. Please do not add new datasets to this list." - ) + assert ( + 135 == len(exceptions) + ), "The number of exceptions has changed. Please do not add new datasets to this list." exceptions = [] for task in get_tasks(): if task.metadata.dataset.get("trust_remote_code", False): - assert task.metadata.name not in exceptions, ( - f"Dataset {task.metadata.name} should not trust remote code" - ) + assert ( + task.metadata.name not in exceptions + ), f"Dataset {task.metadata.name} should not trust remote code" def test_empy_descriptive_stat_in_new_datasets(): @@ -1090,17 +1090,17 @@ def test_empy_descriptive_stat_in_new_datasets(): "SummEvalFrSummarization.v2", ] - assert 553 == len(exceptions), ( - "The number of exceptions has changed. Please do not add new datasets to this list." - ) + assert ( + 553 == len(exceptions) + ), "The number of exceptions has changed. Please do not add new datasets to this list." exceptions = [] for task in get_tasks(): if task.metadata.descriptive_stats is None: - assert task.metadata.name not in exceptions, ( - f"Dataset {task.metadata.name} should have descriptive stats" - ) + assert ( + task.metadata.name not in exceptions + ), f"Dataset {task.metadata.name} should have descriptive stats" @pytest.mark.parametrize("task", get_tasks()) @@ -1109,10 +1109,10 @@ def test_eval_langs_correctly_specified(task: AbsTask): return if task.is_multilingual: - assert isinstance(task.metadata.eval_langs, dict), ( - f"{task.metadata.name} should have eval_langs as a dict" - ) + assert isinstance( + task.metadata.eval_langs, dict + ), f"{task.metadata.name} should have eval_langs as a dict" else: - assert isinstance(task.metadata.eval_langs, list), ( - f"{task.metadata.name} should have eval_langs as a list" - ) + assert isinstance( + task.metadata.eval_langs, list + ), f"{task.metadata.name} should have eval_langs as a list" diff --git a/tests/test_cli.py b/tests/test_cli.py index 743e080ba9..5825c7fae9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -17,18 +17,18 @@ def test_available_tasks(): command = f"{sys.executable} -m mteb available_tasks" result = subprocess.run(command, shell=True, capture_output=True, text=True) assert result.returncode == 0, "Command failed" - assert "Banking77Classification" in result.stdout, ( - "Sample task Banking77Classification task not found in available tasks" - ) + assert ( + "Banking77Classification" in result.stdout + ), "Sample task Banking77Classification task not found in available tasks" def test_available_benchmarks(): command = f"{sys.executable} -m mteb available_benchmarks" result = subprocess.run(command, shell=True, capture_output=True, text=True) assert result.returncode == 0, "Command failed" - assert "MTEB(eng, v1)" in result.stdout, ( - "Sample benchmark MTEB(eng, v1) task not found in available benchmarks" - ) + assert ( + "MTEB(eng, v1)" in result.stdout + ), "Sample benchmark MTEB(eng, v1) task not found in available benchmarks" run_task_fixures = [ @@ -75,12 +75,12 @@ def test_run_task( f"tests/results/test_model/{model_name_as_path}/{model_revision}" ) assert results_path.exists(), "Output folder not created" - assert "model_meta.json" in [f.name for f in list(results_path.glob("*.json"))], ( - "model_meta.json not found in output folder" - ) - assert f"{task_name}.json" in [f.name for f in list(results_path.glob("*.json"))], ( - f"{task_name} not found in output folder" - ) + assert "model_meta.json" in [ + f.name for f in list(results_path.glob("*.json")) + ], "model_meta.json not found in output folder" + assert f"{task_name}.json" in [ + f.name for f in list(results_path.glob("*.json")) + ], f"{task_name} not found in output folder" def test_create_meta(): @@ -117,9 +117,9 @@ def test_create_meta(): for key in frontmatter_gold: assert key in frontmatter, f"Key {key} not found in output" - assert frontmatter[key] == frontmatter_gold[key], ( - f"Value for {key} does not match" - ) + assert ( + frontmatter[key] == frontmatter_gold[key] + ), f"Value for {key} does not match" # ensure that the command line interface works as well command = f"{sys.executable} -m mteb create_meta --results_folder {results} --output_path {output_path} --overwrite" @@ -178,9 +178,9 @@ def test_create_meta_from_existing(existing_readme_name: str, gold_readme_name: for key in frontmatter_gold: assert key in frontmatter, f"Key {key} not found in output" - assert frontmatter[key] == frontmatter_gold[key], ( - f"Value for {key} does not match" - ) + assert ( + frontmatter[key] == frontmatter_gold[key] + ), f"Value for {key} does not match" assert readme_output == gold_readme # ensure that the command line interface works as well command = f"{sys.executable} -m mteb create_meta --results_folder {results} --output_path {output_path} --from_existing {existing_readme} --overwrite" diff --git a/tests/test_overview.py b/tests/test_overview.py index 4486bc1136..6b7c37c88a 100644 --- a/tests/test_overview.py +++ b/tests/test_overview.py @@ -126,9 +126,9 @@ def test_get_tasks_filtering(): assert isinstance(task.metadata.eval_langs, dict) for hf_subset in task.hf_subsets: - assert "eng-Latn" in task.metadata.eval_langs[hf_subset], ( - f"{task.metadata.name}" - ) + assert ( + "eng-Latn" in task.metadata.eval_langs[hf_subset] + ), f"{task.metadata.name}" @pytest.mark.parametrize("script", [["Latn"], ["Cyrl"], None]) diff --git a/tests/test_tasks/test_all_abstasks.py b/tests/test_tasks/test_all_abstasks.py index 0e79434372..ef52b3ddfc 100644 --- a/tests/test_tasks/test_all_abstasks.py +++ b/tests/test_tasks/test_all_abstasks.py @@ -96,6 +96,6 @@ def test_superseded_dataset_exists(): tasks = mteb.get_tasks(exclude_superseded=False) for task in tasks: if task.superseded_by: - assert task.superseded_by in TASKS_REGISTRY, ( - f"{task} is superseded by {task.superseded_by} but {task.superseded_by} is not in the TASKS_REGISTRY" - ) + assert ( + task.superseded_by in TASKS_REGISTRY + ), f"{task} is superseded by {task.superseded_by} but {task.superseded_by} is not in the TASKS_REGISTRY" From e84a4e176977d4a6ba3d078caceff906d1e8557e Mon Sep 17 00:00:00 2001 From: silky1708 Date: Thu, 13 Mar 2025 13:46:14 -0700 Subject: [PATCH 5/7] add task subtype --- mteb/abstasks/TaskMetadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mteb/abstasks/TaskMetadata.py b/mteb/abstasks/TaskMetadata.py index 57cbb157e4..e300526033 100644 --- a/mteb/abstasks/TaskMetadata.py +++ b/mteb/abstasks/TaskMetadata.py @@ -58,6 +58,7 @@ "Environment Sound Classification", "Gunshot Audio Classification", "Instrument Source Classification", + "Music Genre Classification", "Gender Clustering", "Rendered semantic textual similarity", ] From 335ea1a1bc9b5e33a2cd3d7eaac25345e8cbb720 Mon Sep 17 00:00:00 2001 From: silky1708 Date: Thu, 13 Mar 2025 15:07:08 -0700 Subject: [PATCH 6/7] update date for gtzan dataset --- mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py b/mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py index b224811a4e..5aef540a96 100644 --- a/mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py +++ b/mteb/tasks/Audio/AudioClassification/eng/GTZANGenre.py @@ -20,7 +20,7 @@ class GTZANGenre(AbsTaskAudioClassification): eval_splits=["train"], eval_langs=["eng-Latn"], main_score="accuracy", - date=("2025-03-13", "2025-03-13"), + date=("2000-01-01", "2001-12-31"), domains=["Music"], task_subtypes=["Music Genre Classification"], license="not specified", From 050e5f838b0b45d49faba0fbfcf72f005a95274a Mon Sep 17 00:00:00 2001 From: silky1708 Date: Thu, 13 Mar 2025 15:26:48 -0700 Subject: [PATCH 7/7] update ruff to 0.9.7; make lint --- mteb/abstasks/AbsTaskClusteringFast.py | 12 +++---- mteb/abstasks/AbsTaskInstructionRetrieval.py | 6 ++-- .../Audio/AbsTaskAudioClassification.py | 6 ++-- mteb/evaluation/MTEB.py | 6 ++-- mteb/models/rerankers_custom.py | 6 ++-- .../BitextMining/vie/VieMedEVBitextMining.py | 12 +++---- scripts/running_model/check_results.py | 6 ++-- tests/test_TaskMetadata.py | 36 +++++++++---------- tests/test_cli.py | 36 +++++++++---------- tests/test_overview.py | 6 ++-- tests/test_tasks/test_all_abstasks.py | 6 ++-- 11 files changed, 69 insertions(+), 69 deletions(-) diff --git a/mteb/abstasks/AbsTaskClusteringFast.py b/mteb/abstasks/AbsTaskClusteringFast.py index 40e36d29e2..0a41c4cbc2 100644 --- a/mteb/abstasks/AbsTaskClusteringFast.py +++ b/mteb/abstasks/AbsTaskClusteringFast.py @@ -320,9 +320,9 @@ def convert_to_fast( # check that it is the same distribution row_label_set = set(lab) - assert row_label_set.issubset( - all_labels_set - ), "The clusters are not sampled from the same distribution as they have different labels." + assert row_label_set.issubset(all_labels_set), ( + "The clusters are not sampled from the same distribution as they have different labels." + ) for l, s in zip(lab, sents): if s not in sent_set: @@ -353,6 +353,6 @@ def check_label_distribution(ds: DatasetDict) -> None: # check that it is the same distribution row_label_set = set(lab) - assert row_label_set.issubset( - all_labels_set - ), "The clusters are not sampled from the same distribution as they have different labels." + assert row_label_set.issubset(all_labels_set), ( + "The clusters are not sampled from the same distribution as they have different labels." + ) diff --git a/mteb/abstasks/AbsTaskInstructionRetrieval.py b/mteb/abstasks/AbsTaskInstructionRetrieval.py index eed22167fa..1022a01bce 100644 --- a/mteb/abstasks/AbsTaskInstructionRetrieval.py +++ b/mteb/abstasks/AbsTaskInstructionRetrieval.py @@ -379,9 +379,9 @@ def load_data(self, **kwargs): doc["id"]: {"title": doc["title"], "text": doc["text"]} for doc in corpus } - assert ( - len(top_ranked) == len(queries) - ), f"Top ranked not loaded properly! Expected {len(self.queries)} but got {len(self.top_ranked)}." + assert len(top_ranked) == len(queries), ( + f"Top ranked not loaded properly! Expected {len(self.queries)} but got {len(self.top_ranked)}." + ) ( self.corpus[split], diff --git a/mteb/abstasks/Audio/AbsTaskAudioClassification.py b/mteb/abstasks/Audio/AbsTaskAudioClassification.py index f349da5292..9aac7827fa 100644 --- a/mteb/abstasks/Audio/AbsTaskAudioClassification.py +++ b/mteb/abstasks/Audio/AbsTaskAudioClassification.py @@ -121,9 +121,9 @@ def _evaluate_subset_cross_validation( encode_kwargs: dict[str, Any] = {}, **kwargs, ) -> ScoresDict: - assert ( - train_split == eval_split - ), f"Performing {self.n_splits}-fold cross validation, but the dataset has a train (`{train_split}`) and test split (`{eval_split}`)! Set `is_cross_validation` to False, and retry." + assert train_split == eval_split, ( + f"Performing {self.n_splits}-fold cross validation, but the dataset has a train (`{train_split}`) and test split (`{eval_split}`)! Set `is_cross_validation` to False, and retry." + ) logger.info( f"Performing {self.n_splits}-fold cross-validation on the entire dataset!" ) diff --git a/mteb/evaluation/MTEB.py b/mteb/evaluation/MTEB.py index 96a1a97ad5..a6bacc189e 100644 --- a/mteb/evaluation/MTEB.py +++ b/mteb/evaluation/MTEB.py @@ -71,9 +71,9 @@ def __init__( if isinstance(tasks[0], Benchmark): self.benchmarks = tasks self._tasks = self._tasks = list(chain.from_iterable(tasks)) # type: ignore - assert ( - task_types is None and task_categories is None - ), "Cannot specify both `tasks` and `task_types`/`task_categories`" + assert task_types is None and task_categories is None, ( + "Cannot specify both `tasks` and `task_types`/`task_categories`" + ) else: self._task_types = task_types self._task_categories = task_categories diff --git a/mteb/models/rerankers_custom.py b/mteb/models/rerankers_custom.py index 0061ffc1c3..83c054c9c9 100644 --- a/mteb/models/rerankers_custom.py +++ b/mteb/models/rerankers_custom.py @@ -85,9 +85,9 @@ def predict(self, input_to_rerank, **kwargs): assert len(queries) == len(passages) query_passage_tuples = list(zip(queries, passages)) scores = self.model.compute_score(query_passage_tuples, normalize=True) - assert len(scores) == len( - queries - ), f"Expected {len(queries)} scores, got {len(scores)}" + assert len(scores) == len(queries), ( + f"Expected {len(queries)} scores, got {len(scores)}" + ) return scores diff --git a/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py b/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py index 8dea762f86..ab32025167 100644 --- a/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py +++ b/mteb/tasks/BitextMining/vie/VieMedEVBitextMining.py @@ -54,9 +54,9 @@ def dataset_transform(self): # Pairs are in two halves en_sentences = all_texts[:mid_index] vie_sentences = all_texts[mid_index:] - assert len(en_sentences) == len( - vie_sentences - ), "The split does not result in equal halves." + assert len(en_sentences) == len(vie_sentences), ( + "The split does not result in equal halves." + ) # Downsample indices = list(range(len(en_sentences))) @@ -64,9 +64,9 @@ def dataset_transform(self): sample_indices = indices[:TEST_SAMPLES] en_sentences = [en_sentences[i] for i in sample_indices] vie_sentences = [vie_sentences[i] for i in sample_indices] - assert ( - len(en_sentences) == len(vie_sentences) == TEST_SAMPLES - ), f"Exceeded {TEST_SAMPLES} samples for 'test' split." + assert len(en_sentences) == len(vie_sentences) == TEST_SAMPLES, ( + f"Exceeded {TEST_SAMPLES} samples for 'test' split." + ) # Return dataset ds["test"] = datasets.Dataset.from_dict( diff --git a/scripts/running_model/check_results.py b/scripts/running_model/check_results.py index c410fb5be7..e518b77efd 100644 --- a/scripts/running_model/check_results.py +++ b/scripts/running_model/check_results.py @@ -174,9 +174,9 @@ def normalize_results(results): # [t.task_name for t in mteb_results['GritLM/GritLM-7B']["13f00a0e36500c80ce12870ea513846a066004af"] if t.task_name == "SemRel24STS"] # it is there -assert [ - len(revisions.keys()) == 1 for model, revisions in mteb_results.items() -], "Some models have more than one revision" +assert [len(revisions.keys()) == 1 for model, revisions in mteb_results.items()], ( + "Some models have more than one revision" +) results_df = results_to_dataframe(mteb_results) diff --git a/tests/test_TaskMetadata.py b/tests/test_TaskMetadata.py index 54e074c2b4..241ec536ea 100644 --- a/tests/test_TaskMetadata.py +++ b/tests/test_TaskMetadata.py @@ -518,17 +518,17 @@ def test_disallow_trust_remote_code_in_new_datasets(): "SwednClusteringS2S", ] - assert ( - 135 == len(exceptions) - ), "The number of exceptions has changed. Please do not add new datasets to this list." + assert 135 == len(exceptions), ( + "The number of exceptions has changed. Please do not add new datasets to this list." + ) exceptions = [] for task in get_tasks(): if task.metadata.dataset.get("trust_remote_code", False): - assert ( - task.metadata.name not in exceptions - ), f"Dataset {task.metadata.name} should not trust remote code" + assert task.metadata.name not in exceptions, ( + f"Dataset {task.metadata.name} should not trust remote code" + ) def test_empy_descriptive_stat_in_new_datasets(): @@ -1090,17 +1090,17 @@ def test_empy_descriptive_stat_in_new_datasets(): "SummEvalFrSummarization.v2", ] - assert ( - 553 == len(exceptions) - ), "The number of exceptions has changed. Please do not add new datasets to this list." + assert 553 == len(exceptions), ( + "The number of exceptions has changed. Please do not add new datasets to this list." + ) exceptions = [] for task in get_tasks(): if task.metadata.descriptive_stats is None: - assert ( - task.metadata.name not in exceptions - ), f"Dataset {task.metadata.name} should have descriptive stats" + assert task.metadata.name not in exceptions, ( + f"Dataset {task.metadata.name} should have descriptive stats" + ) @pytest.mark.parametrize("task", get_tasks()) @@ -1109,10 +1109,10 @@ def test_eval_langs_correctly_specified(task: AbsTask): return if task.is_multilingual: - assert isinstance( - task.metadata.eval_langs, dict - ), f"{task.metadata.name} should have eval_langs as a dict" + assert isinstance(task.metadata.eval_langs, dict), ( + f"{task.metadata.name} should have eval_langs as a dict" + ) else: - assert isinstance( - task.metadata.eval_langs, list - ), f"{task.metadata.name} should have eval_langs as a list" + assert isinstance(task.metadata.eval_langs, list), ( + f"{task.metadata.name} should have eval_langs as a list" + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 5825c7fae9..743e080ba9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -17,18 +17,18 @@ def test_available_tasks(): command = f"{sys.executable} -m mteb available_tasks" result = subprocess.run(command, shell=True, capture_output=True, text=True) assert result.returncode == 0, "Command failed" - assert ( - "Banking77Classification" in result.stdout - ), "Sample task Banking77Classification task not found in available tasks" + assert "Banking77Classification" in result.stdout, ( + "Sample task Banking77Classification task not found in available tasks" + ) def test_available_benchmarks(): command = f"{sys.executable} -m mteb available_benchmarks" result = subprocess.run(command, shell=True, capture_output=True, text=True) assert result.returncode == 0, "Command failed" - assert ( - "MTEB(eng, v1)" in result.stdout - ), "Sample benchmark MTEB(eng, v1) task not found in available benchmarks" + assert "MTEB(eng, v1)" in result.stdout, ( + "Sample benchmark MTEB(eng, v1) task not found in available benchmarks" + ) run_task_fixures = [ @@ -75,12 +75,12 @@ def test_run_task( f"tests/results/test_model/{model_name_as_path}/{model_revision}" ) assert results_path.exists(), "Output folder not created" - assert "model_meta.json" in [ - f.name for f in list(results_path.glob("*.json")) - ], "model_meta.json not found in output folder" - assert f"{task_name}.json" in [ - f.name for f in list(results_path.glob("*.json")) - ], f"{task_name} not found in output folder" + assert "model_meta.json" in [f.name for f in list(results_path.glob("*.json"))], ( + "model_meta.json not found in output folder" + ) + assert f"{task_name}.json" in [f.name for f in list(results_path.glob("*.json"))], ( + f"{task_name} not found in output folder" + ) def test_create_meta(): @@ -117,9 +117,9 @@ def test_create_meta(): for key in frontmatter_gold: assert key in frontmatter, f"Key {key} not found in output" - assert ( - frontmatter[key] == frontmatter_gold[key] - ), f"Value for {key} does not match" + assert frontmatter[key] == frontmatter_gold[key], ( + f"Value for {key} does not match" + ) # ensure that the command line interface works as well command = f"{sys.executable} -m mteb create_meta --results_folder {results} --output_path {output_path} --overwrite" @@ -178,9 +178,9 @@ def test_create_meta_from_existing(existing_readme_name: str, gold_readme_name: for key in frontmatter_gold: assert key in frontmatter, f"Key {key} not found in output" - assert ( - frontmatter[key] == frontmatter_gold[key] - ), f"Value for {key} does not match" + assert frontmatter[key] == frontmatter_gold[key], ( + f"Value for {key} does not match" + ) assert readme_output == gold_readme # ensure that the command line interface works as well command = f"{sys.executable} -m mteb create_meta --results_folder {results} --output_path {output_path} --from_existing {existing_readme} --overwrite" diff --git a/tests/test_overview.py b/tests/test_overview.py index 6b7c37c88a..4486bc1136 100644 --- a/tests/test_overview.py +++ b/tests/test_overview.py @@ -126,9 +126,9 @@ def test_get_tasks_filtering(): assert isinstance(task.metadata.eval_langs, dict) for hf_subset in task.hf_subsets: - assert ( - "eng-Latn" in task.metadata.eval_langs[hf_subset] - ), f"{task.metadata.name}" + assert "eng-Latn" in task.metadata.eval_langs[hf_subset], ( + f"{task.metadata.name}" + ) @pytest.mark.parametrize("script", [["Latn"], ["Cyrl"], None]) diff --git a/tests/test_tasks/test_all_abstasks.py b/tests/test_tasks/test_all_abstasks.py index ef52b3ddfc..0e79434372 100644 --- a/tests/test_tasks/test_all_abstasks.py +++ b/tests/test_tasks/test_all_abstasks.py @@ -96,6 +96,6 @@ def test_superseded_dataset_exists(): tasks = mteb.get_tasks(exclude_superseded=False) for task in tasks: if task.superseded_by: - assert ( - task.superseded_by in TASKS_REGISTRY - ), f"{task} is superseded by {task.superseded_by} but {task.superseded_by} is not in the TASKS_REGISTRY" + assert task.superseded_by in TASKS_REGISTRY, ( + f"{task} is superseded by {task.superseded_by} but {task.superseded_by} is not in the TASKS_REGISTRY" + )