From b11d09664dd3f7d08a51687a46af45b2e9a8857c Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Fri, 17 Jan 2025 15:10:32 +0300 Subject: [PATCH 1/5] fix split evals --- mteb/abstasks/AbsTask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mteb/abstasks/AbsTask.py b/mteb/abstasks/AbsTask.py index 1d2e4fcb05..1ec1ebc4fc 100644 --- a/mteb/abstasks/AbsTask.py +++ b/mteb/abstasks/AbsTask.py @@ -118,7 +118,7 @@ def evaluate( hf_subsets = copy(self.hf_subsets) if subsets_to_run is not None: # allow overwrites of pre-filtering - hf_subsets = subsets_to_run + hf_subsets = [s for s in hf_subsets if s in subsets_to_run] for hf_subset in hf_subsets: logger.info( From 2708db40eae8eed1fe00459eeb270fe96c8133e1 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Fri, 17 Jan 2025 17:07:56 +0300 Subject: [PATCH 2/5] add test --- mteb/models/overview.py | 1 - tests/test_benchmark/mock_tasks.py | 10 ++++------ tests/test_evaluation/test_split_evaluation.py | 17 ++++++++++++++++- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/mteb/models/overview.py b/mteb/models/overview.py index 634530089f..868db8da44 100644 --- a/mteb/models/overview.py +++ b/mteb/models/overview.py @@ -30,7 +30,6 @@ llm2vec_models, misc_models, model2vec_models, - moka_models, mxbai_models, no_instruct_sentence_models, nomic_models, diff --git a/tests/test_benchmark/mock_tasks.py b/tests/test_benchmark/mock_tasks.py index d3a11b2a4b..7d3d2d7528 100644 --- a/tests/test_benchmark/mock_tasks.py +++ b/tests/test_benchmark/mock_tasks.py @@ -927,12 +927,10 @@ def load_data(self, **kwargs): ), } - self.dataset = DatasetDict( - { - "eng": data, - "fra": data, - } - ) + self.dataset = {} + for lang in self.hf_subsets: + self.dataset[lang] = data + self.data_loaded = True @property diff --git a/tests/test_evaluation/test_split_evaluation.py b/tests/test_evaluation/test_split_evaluation.py index 7ce3f512e0..87b21d555e 100644 --- a/tests/test_evaluation/test_split_evaluation.py +++ b/tests/test_evaluation/test_split_evaluation.py @@ -1,14 +1,18 @@ from __future__ import annotations +from argparse import Namespace + import pytest +import mteb from mteb import MTEB +from mteb.cli import run from tests.test_benchmark.mock_models import ( MockSentenceTransformer, ) from tests.test_benchmark.mock_tasks import ( MockMultilingualRetrievalTask, - MockRetrievalTask, + MockMultilingualSTSTask, MockRetrievalTask, ) @@ -362,3 +366,14 @@ def test_all_splits_subsets_evaluated_with_overwrite( for split in ["test"]: assert len(results2[0].scores[split]) == 2 assert sorted(results2[0].languages) == ["eng", "fra"] + + +def test_splits_evaluated_with_prefiltering(): + """Test that the evaluation only runs on the specified languages. Issue https://github.com/embeddings-benchmark/mteb/pull/1787#issuecomment-2598205049""" + task = MockMultilingualSTSTask().filter_languages(languages=["fra"]) + + evaluation = MTEB(tasks=[task]) + + evaluation.run( + MockSentenceTransformer(), overwrite_results=True + ) From 5cb4b436f97ffaab0ae334cb8c6f4fe5d6f270d3 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Fri, 17 Jan 2025 17:08:39 +0300 Subject: [PATCH 3/5] lint --- tests/test_evaluation/test_split_evaluation.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/test_evaluation/test_split_evaluation.py b/tests/test_evaluation/test_split_evaluation.py index 87b21d555e..1db0a20e01 100644 --- a/tests/test_evaluation/test_split_evaluation.py +++ b/tests/test_evaluation/test_split_evaluation.py @@ -1,18 +1,15 @@ from __future__ import annotations -from argparse import Namespace - import pytest -import mteb from mteb import MTEB -from mteb.cli import run from tests.test_benchmark.mock_models import ( MockSentenceTransformer, ) from tests.test_benchmark.mock_tasks import ( MockMultilingualRetrievalTask, - MockMultilingualSTSTask, MockRetrievalTask, + MockMultilingualSTSTask, + MockRetrievalTask, ) @@ -374,6 +371,4 @@ def test_splits_evaluated_with_prefiltering(): evaluation = MTEB(tasks=[task]) - evaluation.run( - MockSentenceTransformer(), overwrite_results=True - ) + evaluation.run(MockSentenceTransformer(), overwrite_results=True) From 03bd737f5fa897e5f16a7d2f3523f4a4e54895f5 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Fri, 17 Jan 2025 17:20:59 +0300 Subject: [PATCH 4/5] fix moka --- mteb/models/overview.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mteb/models/overview.py b/mteb/models/overview.py index 167c369da4..ea0fa1524c 100644 --- a/mteb/models/overview.py +++ b/mteb/models/overview.py @@ -30,6 +30,7 @@ llm2vec_models, misc_models, model2vec_models, + moka_models, mxbai_models, no_instruct_sentence_models, nomic_models, From 9f32889a7c69220a64c968bf3cb4e2cf9d1bdea3 Mon Sep 17 00:00:00 2001 From: Roman Solomatin <36135455+Samoed@users.noreply.github.com> Date: Fri, 17 Jan 2025 17:54:30 +0300 Subject: [PATCH 5/5] add assert --- tests/test_evaluation/test_split_evaluation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_evaluation/test_split_evaluation.py b/tests/test_evaluation/test_split_evaluation.py index 1db0a20e01..7db10e09d4 100644 --- a/tests/test_evaluation/test_split_evaluation.py +++ b/tests/test_evaluation/test_split_evaluation.py @@ -371,4 +371,10 @@ def test_splits_evaluated_with_prefiltering(): evaluation = MTEB(tasks=[task]) - evaluation.run(MockSentenceTransformer(), overwrite_results=True) + results = evaluation.run(MockSentenceTransformer(), overwrite_results=True) + result_scores = results[0].scores + + assert len(result_scores) == 1 + assert "test" in result_scores + assert len(result_scores["test"]) == 1 + assert result_scores["test"][0]["hf_subset"] == "fra"