From 33097f7075956f8903d7e4286ea527ee5903d8b7 Mon Sep 17 00:00:00 2001 From: fzowl Date: Fri, 20 Jun 2025 17:23:14 +0200 Subject: [PATCH 1/9] Adding Classification Evaluator test --- .../test_ClassificationEvaluator.py | 569 ++++++++++++++++++ 1 file changed, 569 insertions(+) create mode 100644 tests/test_evaluators/test_ClassificationEvaluator.py diff --git a/tests/test_evaluators/test_ClassificationEvaluator.py b/tests/test_evaluators/test_ClassificationEvaluator.py new file mode 100644 index 0000000000..5754bc291b --- /dev/null +++ b/tests/test_evaluators/test_ClassificationEvaluator.py @@ -0,0 +1,569 @@ +from __future__ import annotations + +import numpy as np +import pytest +import torch + +import mteb +from mteb.evaluation.evaluators import ( + kNNClassificationEvaluator, + kNNClassificationEvaluatorPytorch, + logRegClassificationEvaluator, +) +from tests.test_benchmark.mock_models import MockNumpyEncoder + +# Basic test data +SENTENCES_TRAIN_BINARY = [ + "this is a positive sentence", + "another positive sentence", + "this is a negative sentence", + "another negative sentence", +] +Y_TRAIN_BINARY = np.array([1, 1, 0, 0]) +SENTENCES_TEST_BINARY = [ + "a new positive sentence", + "a new negative sentence", +] +Y_TEST_BINARY = np.array([1, 0]) + +SENTENCES_TRAIN_MULTICLASS = [ + "class 0 sentence 1", + "class 0 sentence 2", + "class 1 sentence 1", + "class 1 sentence 2", + "class 2 sentence 1", + "class 2 sentence 2", +] +Y_TRAIN_MULTICLASS = np.array([0, 0, 1, 1, 2, 2]) +SENTENCES_TEST_MULTICLASS = [ + "new class 0 sentence", + "new class 1 sentence", + "new class 2 sentence", +] +Y_TEST_MULTICLASS = np.array([0, 1, 2]) + +# For checking the cache +SENTENCES_TEST_CACHE = [ + "another new positive sentence", + "another new negative sentence", +] +Y_TEST_CACHE = np.array([1, 0]) + + +class TestKNNClassificationEvaluator: + @pytest.fixture + def model(self): + return MockNumpyEncoder() + + @pytest.fixture + def eval_binary(self): + return kNNClassificationEvaluator( + SENTENCES_TRAIN_BINARY, + Y_TRAIN_BINARY, + SENTENCES_TEST_BINARY, + Y_TEST_BINARY, + task_name="test_knn_binary", + ) + + @pytest.fixture + def eval_multiclass(self): + return kNNClassificationEvaluator( + SENTENCES_TRAIN_MULTICLASS, + Y_TRAIN_MULTICLASS, + SENTENCES_TEST_MULTICLASS, + Y_TEST_MULTICLASS, + task_name="test_knn_multiclass", + ) + + @pytest.mark.parametrize( + "evaluator_fixture, is_binary", + [ + ("eval_binary", True), + ("eval_multiclass", False), + ], + ) + def test_output_structure(self, evaluator_fixture, is_binary, model, request): + evaluator = request.getfixturevalue(evaluator_fixture) + scores, test_cache = evaluator(model) + assert isinstance(scores, dict) + assert isinstance(test_cache, np.ndarray) + assert "accuracy" in scores + assert "f1" in scores + assert "accuracy_cosine" in scores + assert "f1_cosine" in scores + assert "accuracy_euclidean" in scores + assert "f1_euclidean" in scores + + if is_binary: + assert "ap" in scores + assert "ap_cosine" in scores + assert "ap_euclidean" in scores + else: + assert "ap" not in scores + + @pytest.mark.parametrize( + "evaluator_fixture, is_binary", + [ + ("eval_binary", True), + ("eval_multiclass", False), + ], + ) + def test_score_ranges(self, evaluator_fixture, is_binary, model, request): + evaluator = request.getfixturevalue(evaluator_fixture) + scores, _ = evaluator(model) + metrics_to_check = [ + "accuracy", + "f1", + "accuracy_cosine", + "f1_cosine", + "accuracy_euclidean", + "f1_euclidean", + ] + if is_binary: + metrics_to_check.extend(["ap", "ap_cosine", "ap_euclidean"]) + + for metric in metrics_to_check: + assert 0 <= scores[metric] <= 1 + + def test_cache_usage_binary(self, model, eval_binary): + _, test_cache_initial = eval_binary(model) + eval_binary_cache_test = kNNClassificationEvaluator( + SENTENCES_TRAIN_BINARY, + Y_TRAIN_BINARY, + SENTENCES_TEST_BINARY, + Y_TEST_BINARY, + task_name="test_knn_binary_cache", + ) + scores_with_cache, test_cache_after_cache_usage = eval_binary_cache_test( + model, test_cache=test_cache_initial + ) + + assert np.array_equal(test_cache_initial, test_cache_after_cache_usage) + for metric in ["accuracy", "f1", "ap"]: + assert 0 <= scores_with_cache[metric] <= 1 + + @pytest.mark.parametrize( + "train_sentences, train_labels, test_sentences, test_labels, task_name, limit, expected_train_len, expected_test_len, is_binary_after_limit, expected_exception", + [ + # Ensure binary classification is still possible after limiting + ( + SENTENCES_TRAIN_BINARY, + Y_TRAIN_BINARY, + SENTENCES_TEST_BINARY, + Y_TEST_BINARY, + "test_knn_limit_binary", + 3, + 3, + 2, + True, + None, + ), + # Test case where limiting results in a single class, expecting no AP and no exception + ( + SENTENCES_TRAIN_BINARY, + Y_TRAIN_BINARY, + SENTENCES_TEST_BINARY, + Y_TEST_BINARY, + "test_knn_limit_not_binary", + 1, + 1, + 1, + False, + None, + ), + ], + ) + def test_limit_parameter( + self, + model, + train_sentences, + train_labels, + test_sentences, + test_labels, + task_name, + limit, + expected_train_len, + expected_test_len, + is_binary_after_limit, + expected_exception, + ): + if expected_exception: + with pytest.raises(expected_exception): + eval_limited = kNNClassificationEvaluator( + train_sentences, + train_labels, + test_sentences, + test_labels, + task_name=task_name, + limit=limit, + ) + eval_limited(model) + else: + eval_limited = kNNClassificationEvaluator( + train_sentences, + train_labels, + test_sentences, + test_labels, + task_name=task_name, + limit=limit, + ) + assert len(eval_limited.sentences_train) == expected_train_len + assert len(eval_limited.y_train) == expected_train_len + assert len(eval_limited.sentences_test) == expected_test_len + assert len(eval_limited.y_test) == expected_test_len + + scores, _ = eval_limited(model) + assert "accuracy" in scores + assert "f1" in scores + if is_binary_after_limit: + assert "ap" in scores + else: + assert "ap" not in scores + + +class LocalMockTorchEncoder(mteb.Encoder): + def __init__(self): + pass + + def encode(self, sentences, prompt_name: str | None = None, **kwargs): + return torch.randn(len(sentences), 10) + + +class TestKNNClassificationEvaluatorPytorch: + @pytest.fixture + def model_pytorch(self): + return LocalMockTorchEncoder() + + @pytest.fixture + def eval_pytorch_binary(self): + return kNNClassificationEvaluatorPytorch( + SENTENCES_TRAIN_BINARY, + Y_TRAIN_BINARY, + SENTENCES_TEST_BINARY, + Y_TEST_BINARY, + task_name="test_knn_pytorch_binary", + ) + + @pytest.fixture + def eval_pytorch_multiclass(self): + return kNNClassificationEvaluatorPytorch( + SENTENCES_TRAIN_MULTICLASS, + Y_TRAIN_MULTICLASS, + SENTENCES_TEST_MULTICLASS, + Y_TEST_MULTICLASS, + task_name="test_knn_pytorch_multiclass", + ) + + @pytest.mark.parametrize( + "evaluator_fixture, is_binary", + [ + ("eval_pytorch_binary", True), + ("eval_pytorch_multiclass", False), + ], + ) + def test_output_structure( + self, evaluator_fixture, is_binary, model_pytorch, request + ): + evaluator = request.getfixturevalue(evaluator_fixture) + scores, test_cache = evaluator(model_pytorch) + assert isinstance(scores, dict) + assert isinstance(test_cache, torch.Tensor) + assert "accuracy" in scores + assert "f1" in scores + assert "accuracy_cosine" in scores + assert "f1_cosine" in scores + assert "accuracy_euclidean" in scores + assert "f1_euclidean" in scores + assert "accuracy_dot" in scores + assert "f1_dot" in scores + + if is_binary: + assert "ap" in scores + assert "ap_cosine" in scores + assert "ap_euclidean" in scores + assert "ap_dot" in scores + else: + assert "ap" not in scores + + @pytest.mark.parametrize( + "evaluator_fixture, is_binary", + [ + ("eval_pytorch_binary", True), + ("eval_pytorch_multiclass", False), + ], + ) + def test_score_ranges(self, evaluator_fixture, is_binary, model_pytorch, request): + evaluator = request.getfixturevalue(evaluator_fixture) + scores, _ = evaluator(model_pytorch) + metrics_to_check = [ + "accuracy", + "f1", + "accuracy_cosine", + "f1_cosine", + "accuracy_euclidean", + "f1_euclidean", + "accuracy_dot", + "f1_dot", + ] + if is_binary: + metrics_to_check.extend(["ap", "ap_cosine", "ap_euclidean", "ap_dot"]) + + for metric_key in metrics_to_check: + assert 0 <= scores[metric_key] <= 1 + + def test_cache_usage_binary(self, model_pytorch, eval_pytorch_binary): + _, test_cache_initial = eval_pytorch_binary(model_pytorch) + eval_binary_cache_test = kNNClassificationEvaluatorPytorch( + SENTENCES_TRAIN_BINARY, + Y_TRAIN_BINARY, + SENTENCES_TEST_BINARY, + Y_TEST_BINARY, + task_name="test_knn_pytorch_binary_cache", + ) + scores_with_cache, test_cache_after_cache_usage = eval_binary_cache_test( + model_pytorch, test_cache=test_cache_initial + ) + + assert torch.equal(test_cache_initial, test_cache_after_cache_usage) + for metric_key in scores_with_cache.keys(): + if "accuracy" in metric_key or "f1" in metric_key or "ap" in metric_key: + assert 0 <= scores_with_cache[metric_key] <= 1 + + @pytest.mark.parametrize( + "train_sentences, train_labels, test_sentences, test_labels, task_name, limit, expected_train_len, expected_test_len, is_binary_after_limit, expected_exception", + [ + ( + SENTENCES_TRAIN_BINARY, + Y_TRAIN_BINARY, + SENTENCES_TEST_BINARY, + Y_TEST_BINARY, + "test_knn_pytorch_limit_binary", + 3, + 3, + 2, + True, + None, + ), + ( + SENTENCES_TRAIN_BINARY, + Y_TRAIN_BINARY, + SENTENCES_TEST_BINARY, + Y_TEST_BINARY, + "test_knn_pytorch_limit_not_binary", + 1, + 1, + 1, + False, + None, + ), + ], + ) + def test_limit_parameter( + self, + model_pytorch, + train_sentences, + train_labels, + test_sentences, + test_labels, + task_name, + limit, + expected_train_len, + expected_test_len, + is_binary_after_limit, + expected_exception, + ): + if expected_exception: + with pytest.raises(expected_exception): + eval_limited = kNNClassificationEvaluatorPytorch( + train_sentences, + train_labels, + test_sentences, + test_labels, + task_name=task_name, + limit=limit, + ) + eval_limited(model_pytorch) + else: + eval_limited = kNNClassificationEvaluatorPytorch( + train_sentences, + train_labels, + test_sentences, + test_labels, + task_name=task_name, + limit=limit, + ) + assert len(eval_limited.sentences_train) == expected_train_len + assert len(eval_limited.y_train) == expected_train_len + assert len(eval_limited.sentences_test) == expected_test_len + assert len(eval_limited.y_test) == expected_test_len + + scores, _ = eval_limited(model_pytorch) + assert "accuracy" in scores + assert "f1" in scores + if is_binary_after_limit: + assert "ap" in scores + else: + assert "ap" not in scores + + +class TestLogRegClassificationEvaluator: + @pytest.fixture + def model(self): + return MockNumpyEncoder() + + @pytest.fixture + def eval_logreg_binary(self): + return logRegClassificationEvaluator( + SENTENCES_TRAIN_BINARY, + Y_TRAIN_BINARY, + SENTENCES_TEST_BINARY, + Y_TEST_BINARY, + task_name="test_logreg_binary", + ) + + @pytest.fixture + def eval_logreg_multiclass(self): + return logRegClassificationEvaluator( + SENTENCES_TRAIN_MULTICLASS, + Y_TRAIN_MULTICLASS, + SENTENCES_TEST_MULTICLASS, + Y_TEST_MULTICLASS, + task_name="test_logreg_multiclass", + ) + + @pytest.mark.parametrize( + "evaluator_fixture, is_binary", + [ + ("eval_logreg_binary", True), + ("eval_logreg_multiclass", False), + ], + ) + def test_output_structure(self, evaluator_fixture, is_binary, model, request): + evaluator = request.getfixturevalue(evaluator_fixture) + scores, test_cache = evaluator(model) + assert isinstance(scores, dict) + assert isinstance(test_cache, np.ndarray) + assert "accuracy" in scores + assert "f1" in scores + assert "f1_weighted" in scores + + if is_binary: + assert "ap" in scores + assert "ap_weighted" in scores + else: + assert "ap" not in scores + + @pytest.mark.parametrize( + "evaluator_fixture, is_binary", + [ + ("eval_logreg_binary", True), + ("eval_logreg_multiclass", False), + ], + ) + def test_score_ranges(self, evaluator_fixture, is_binary, model, request): + evaluator = request.getfixturevalue(evaluator_fixture) + scores, _ = evaluator(model) + metrics_to_check = ["accuracy", "f1", "f1_weighted"] + if is_binary: + metrics_to_check.extend(["ap", "ap_weighted"]) + + for metric in metrics_to_check: + assert 0 <= scores[metric] <= 1 + + def test_cache_usage_binary(self, model, eval_logreg_binary): + _, test_cache_initial = eval_logreg_binary(model) + eval_binary_cache_test = logRegClassificationEvaluator( + SENTENCES_TRAIN_BINARY, + Y_TRAIN_BINARY, + SENTENCES_TEST_BINARY, + Y_TEST_BINARY, + task_name="test_logreg_binary_cache", + ) + scores_with_cache, test_cache_after_cache_usage = eval_binary_cache_test( + model, test_cache=test_cache_initial + ) + + assert np.array_equal(test_cache_initial, test_cache_after_cache_usage) + for metric in ["accuracy", "f1", "f1_weighted", "ap", "ap_weighted"]: + assert 0 <= scores_with_cache[metric] <= 1 + + @pytest.mark.parametrize( + "train_sentences, train_labels, test_sentences, test_labels, task_name, limit, expected_train_len, expected_test_len, is_binary_after_limit, expected_exception", + [ + # Ensure binary classification is still possible after limiting + ( + SENTENCES_TRAIN_BINARY, + Y_TRAIN_BINARY, + SENTENCES_TEST_BINARY, + Y_TEST_BINARY, + "test_logreg_limit_binary", + 3, + 3, + 2, + True, + None, + ), + # Test case where limiting results in a single class, expecting ValueError + ( + SENTENCES_TRAIN_BINARY, + Y_TRAIN_BINARY, + SENTENCES_TEST_BINARY, + Y_TEST_BINARY, + "test_logreg_limit_not_binary_train", + 1, + 1, + 1, + False, + ValueError, + ), + ], + ) + def test_limit_parameter( + self, + model, + train_sentences, + train_labels, + test_sentences, + test_labels, + task_name, + limit, + expected_train_len, + expected_test_len, + is_binary_after_limit, + expected_exception, + ): + if expected_exception: + with pytest.raises(expected_exception): + eval_limited = logRegClassificationEvaluator( + train_sentences, + train_labels, + test_sentences, + test_labels, + task_name=task_name, + limit=limit, + ) + eval_limited(model) + else: + eval_limited = logRegClassificationEvaluator( + train_sentences, + train_labels, + test_sentences, + test_labels, + task_name=task_name, + limit=limit, + ) + assert len(eval_limited.sentences_train) == expected_train_len + assert len(eval_limited.y_train) == expected_train_len + assert len(eval_limited.sentences_test) == expected_test_len + assert len(eval_limited.y_test) == expected_test_len + + scores, _ = eval_limited(model) + assert "accuracy" in scores + assert "f1" in scores + assert "f1_weighted" in scores + if is_binary_after_limit: + assert "ap" in scores + assert "ap_weighted" in scores + else: + assert "ap" not in scores + assert "ap_weighted" not in scores From 2e5d5808943a78d7ab186c58a703bf94e6eb6b3f Mon Sep 17 00:00:00 2001 From: fzowl Date: Thu, 26 Jun 2025 18:53:12 +0200 Subject: [PATCH 2/9] Modifications due to the comments --- .../test_ClassificationEvaluator.py | 675 ++++-------------- 1 file changed, 150 insertions(+), 525 deletions(-) diff --git a/tests/test_evaluators/test_ClassificationEvaluator.py b/tests/test_evaluators/test_ClassificationEvaluator.py index 5754bc291b..a432107ccf 100644 --- a/tests/test_evaluators/test_ClassificationEvaluator.py +++ b/tests/test_evaluators/test_ClassificationEvaluator.py @@ -1,16 +1,31 @@ from __future__ import annotations +from dataclasses import dataclass + import numpy as np import pytest -import torch import mteb -from mteb.evaluation.evaluators import ( - kNNClassificationEvaluator, - kNNClassificationEvaluatorPytorch, - logRegClassificationEvaluator, -) -from tests.test_benchmark.mock_models import MockNumpyEncoder +from mteb.evaluation.evaluators import logRegClassificationEvaluator + + +@dataclass +class ClassificationTestCase: + x_train: list[str] + y_train: list[int] + x_test: list[str] + y_test: list[int] + task_name: str + expected_score: float | None = None # For deterministic tests + + +class MockNumpyEncoder(mteb.Encoder): + def __init__(self): + self.rng_state = np.random.default_rng(42) + + def encode(self, sentences, prompt_name: str | None = None, **kwargs): + return self.rng_state.random((len(sentences), 10)) + # Basic test data SENTENCES_TRAIN_BINARY = [ @@ -42,528 +57,138 @@ ] Y_TEST_MULTICLASS = np.array([0, 1, 2]) -# For checking the cache -SENTENCES_TEST_CACHE = [ - "another new positive sentence", - "another new negative sentence", -] -Y_TEST_CACHE = np.array([1, 0]) - - -class TestKNNClassificationEvaluator: - @pytest.fixture - def model(self): - return MockNumpyEncoder() - - @pytest.fixture - def eval_binary(self): - return kNNClassificationEvaluator( - SENTENCES_TRAIN_BINARY, - Y_TRAIN_BINARY, - SENTENCES_TEST_BINARY, - Y_TEST_BINARY, - task_name="test_knn_binary", - ) - - @pytest.fixture - def eval_multiclass(self): - return kNNClassificationEvaluator( - SENTENCES_TRAIN_MULTICLASS, - Y_TRAIN_MULTICLASS, - SENTENCES_TEST_MULTICLASS, - Y_TEST_MULTICLASS, - task_name="test_knn_multiclass", - ) - - @pytest.mark.parametrize( - "evaluator_fixture, is_binary", - [ - ("eval_binary", True), - ("eval_multiclass", False), - ], - ) - def test_output_structure(self, evaluator_fixture, is_binary, model, request): - evaluator = request.getfixturevalue(evaluator_fixture) - scores, test_cache = evaluator(model) - assert isinstance(scores, dict) - assert isinstance(test_cache, np.ndarray) - assert "accuracy" in scores - assert "f1" in scores - assert "accuracy_cosine" in scores - assert "f1_cosine" in scores - assert "accuracy_euclidean" in scores - assert "f1_euclidean" in scores - - if is_binary: - assert "ap" in scores - assert "ap_cosine" in scores - assert "ap_euclidean" in scores - else: - assert "ap" not in scores - - @pytest.mark.parametrize( - "evaluator_fixture, is_binary", - [ - ("eval_binary", True), - ("eval_multiclass", False), - ], - ) - def test_score_ranges(self, evaluator_fixture, is_binary, model, request): - evaluator = request.getfixturevalue(evaluator_fixture) - scores, _ = evaluator(model) - metrics_to_check = [ - "accuracy", - "f1", - "accuracy_cosine", - "f1_cosine", - "accuracy_euclidean", - "f1_euclidean", - ] - if is_binary: - metrics_to_check.extend(["ap", "ap_cosine", "ap_euclidean"]) - - for metric in metrics_to_check: - assert 0 <= scores[metric] <= 1 - - def test_cache_usage_binary(self, model, eval_binary): - _, test_cache_initial = eval_binary(model) - eval_binary_cache_test = kNNClassificationEvaluator( - SENTENCES_TRAIN_BINARY, - Y_TRAIN_BINARY, - SENTENCES_TEST_BINARY, - Y_TEST_BINARY, - task_name="test_knn_binary_cache", - ) - scores_with_cache, test_cache_after_cache_usage = eval_binary_cache_test( - model, test_cache=test_cache_initial - ) - - assert np.array_equal(test_cache_initial, test_cache_after_cache_usage) - for metric in ["accuracy", "f1", "ap"]: - assert 0 <= scores_with_cache[metric] <= 1 - - @pytest.mark.parametrize( - "train_sentences, train_labels, test_sentences, test_labels, task_name, limit, expected_train_len, expected_test_len, is_binary_after_limit, expected_exception", - [ - # Ensure binary classification is still possible after limiting - ( - SENTENCES_TRAIN_BINARY, - Y_TRAIN_BINARY, - SENTENCES_TEST_BINARY, - Y_TEST_BINARY, - "test_knn_limit_binary", - 3, - 3, - 2, - True, - None, - ), - # Test case where limiting results in a single class, expecting no AP and no exception - ( - SENTENCES_TRAIN_BINARY, - Y_TRAIN_BINARY, - SENTENCES_TEST_BINARY, - Y_TEST_BINARY, - "test_knn_limit_not_binary", - 1, - 1, - 1, - False, - None, - ), - ], - ) - def test_limit_parameter( - self, - model, - train_sentences, - train_labels, - test_sentences, - test_labels, - task_name, - limit, - expected_train_len, - expected_test_len, - is_binary_after_limit, - expected_exception, - ): - if expected_exception: - with pytest.raises(expected_exception): - eval_limited = kNNClassificationEvaluator( - train_sentences, - train_labels, - test_sentences, - test_labels, - task_name=task_name, - limit=limit, - ) - eval_limited(model) - else: - eval_limited = kNNClassificationEvaluator( - train_sentences, - train_labels, - test_sentences, - test_labels, - task_name=task_name, - limit=limit, - ) - assert len(eval_limited.sentences_train) == expected_train_len - assert len(eval_limited.y_train) == expected_train_len - assert len(eval_limited.sentences_test) == expected_test_len - assert len(eval_limited.y_test) == expected_test_len - - scores, _ = eval_limited(model) - assert "accuracy" in scores - assert "f1" in scores - if is_binary_after_limit: - assert "ap" in scores - else: - assert "ap" not in scores - - -class LocalMockTorchEncoder(mteb.Encoder): - def __init__(self): - pass +# Test cases with expected scores (deterministic due to fixed random seed) +BINARY_TEST_CASE = ClassificationTestCase( + x_train=SENTENCES_TRAIN_BINARY, + y_train=Y_TRAIN_BINARY.tolist(), + x_test=SENTENCES_TEST_BINARY, + y_test=Y_TEST_BINARY.tolist(), + task_name="test_logreg_binary", + expected_score=0.5, # Expected accuracy with deterministic MockNumpyEncoder +) - def encode(self, sentences, prompt_name: str | None = None, **kwargs): - return torch.randn(len(sentences), 10) - - -class TestKNNClassificationEvaluatorPytorch: - @pytest.fixture - def model_pytorch(self): - return LocalMockTorchEncoder() - - @pytest.fixture - def eval_pytorch_binary(self): - return kNNClassificationEvaluatorPytorch( - SENTENCES_TRAIN_BINARY, - Y_TRAIN_BINARY, - SENTENCES_TEST_BINARY, - Y_TEST_BINARY, - task_name="test_knn_pytorch_binary", - ) - - @pytest.fixture - def eval_pytorch_multiclass(self): - return kNNClassificationEvaluatorPytorch( - SENTENCES_TRAIN_MULTICLASS, - Y_TRAIN_MULTICLASS, - SENTENCES_TEST_MULTICLASS, - Y_TEST_MULTICLASS, - task_name="test_knn_pytorch_multiclass", - ) - - @pytest.mark.parametrize( - "evaluator_fixture, is_binary", - [ - ("eval_pytorch_binary", True), - ("eval_pytorch_multiclass", False), - ], +MULTICLASS_TEST_CASE = ClassificationTestCase( + x_train=SENTENCES_TRAIN_MULTICLASS, + y_train=Y_TRAIN_MULTICLASS.tolist(), + x_test=SENTENCES_TEST_MULTICLASS, + y_test=Y_TEST_MULTICLASS.tolist(), + task_name="test_logreg_multiclass", + expected_score=0.0, # Expected accuracy with deterministic MockNumpyEncoder +) + + +def is_binary_classification(y_train: list[int], y_test: list[int]) -> bool: + """Check if the classification task is binary based on the labels.""" + all_labels = set(y_train + y_test) + return len(all_labels) == 2 + + +# Fixtures +@pytest.fixture +def model(): + return MockNumpyEncoder() + + +@pytest.fixture(params=[BINARY_TEST_CASE, MULTICLASS_TEST_CASE]) +def test_case(request): + return request.param + + +def test_output_structure(model, test_case: ClassificationTestCase): + """Test that the evaluator returns the expected output structure.""" + evaluator = logRegClassificationEvaluator( + test_case.x_train, + np.array(test_case.y_train), + test_case.x_test, + np.array(test_case.y_test), + task_name=test_case.task_name, ) - def test_output_structure( - self, evaluator_fixture, is_binary, model_pytorch, request - ): - evaluator = request.getfixturevalue(evaluator_fixture) - scores, test_cache = evaluator(model_pytorch) - assert isinstance(scores, dict) - assert isinstance(test_cache, torch.Tensor) - assert "accuracy" in scores - assert "f1" in scores - assert "accuracy_cosine" in scores - assert "f1_cosine" in scores - assert "accuracy_euclidean" in scores - assert "f1_euclidean" in scores - assert "accuracy_dot" in scores - assert "f1_dot" in scores - - if is_binary: - assert "ap" in scores - assert "ap_cosine" in scores - assert "ap_euclidean" in scores - assert "ap_dot" in scores - else: - assert "ap" not in scores - - @pytest.mark.parametrize( - "evaluator_fixture, is_binary", - [ - ("eval_pytorch_binary", True), - ("eval_pytorch_multiclass", False), - ], + scores, test_cache = evaluator(model) + + # Check basic structure + assert isinstance(scores, dict) + assert isinstance(test_cache, np.ndarray) + + # Check required metrics + assert "accuracy" in scores + assert "f1" in scores + assert "f1_weighted" in scores + + # Check binary-specific metrics + is_binary = is_binary_classification(test_case.y_train, test_case.y_test) + if is_binary: + assert "ap" in scores + assert "ap_weighted" in scores + else: + assert "ap" not in scores + + +def test_expected_scores(model, test_case: ClassificationTestCase): + """Test that the evaluator returns expected scores with deterministic model.""" + if test_case.expected_score is None: + pytest.skip("No expected score defined for this test case") + + evaluator = logRegClassificationEvaluator( + test_case.x_train, + np.array(test_case.y_train), + test_case.x_test, + np.array(test_case.y_test), + task_name=test_case.task_name, ) - def test_score_ranges(self, evaluator_fixture, is_binary, model_pytorch, request): - evaluator = request.getfixturevalue(evaluator_fixture) - scores, _ = evaluator(model_pytorch) - metrics_to_check = [ - "accuracy", - "f1", - "accuracy_cosine", - "f1_cosine", - "accuracy_euclidean", - "f1_euclidean", - "accuracy_dot", - "f1_dot", - ] - if is_binary: - metrics_to_check.extend(["ap", "ap_cosine", "ap_euclidean", "ap_dot"]) - - for metric_key in metrics_to_check: - assert 0 <= scores[metric_key] <= 1 - - def test_cache_usage_binary(self, model_pytorch, eval_pytorch_binary): - _, test_cache_initial = eval_pytorch_binary(model_pytorch) - eval_binary_cache_test = kNNClassificationEvaluatorPytorch( - SENTENCES_TRAIN_BINARY, - Y_TRAIN_BINARY, - SENTENCES_TEST_BINARY, - Y_TEST_BINARY, - task_name="test_knn_pytorch_binary_cache", - ) - scores_with_cache, test_cache_after_cache_usage = eval_binary_cache_test( - model_pytorch, test_cache=test_cache_initial - ) - - assert torch.equal(test_cache_initial, test_cache_after_cache_usage) - for metric_key in scores_with_cache.keys(): - if "accuracy" in metric_key or "f1" in metric_key or "ap" in metric_key: - assert 0 <= scores_with_cache[metric_key] <= 1 - - @pytest.mark.parametrize( - "train_sentences, train_labels, test_sentences, test_labels, task_name, limit, expected_train_len, expected_test_len, is_binary_after_limit, expected_exception", - [ - ( - SENTENCES_TRAIN_BINARY, - Y_TRAIN_BINARY, - SENTENCES_TEST_BINARY, - Y_TEST_BINARY, - "test_knn_pytorch_limit_binary", - 3, - 3, - 2, - True, - None, - ), - ( - SENTENCES_TRAIN_BINARY, - Y_TRAIN_BINARY, - SENTENCES_TEST_BINARY, - Y_TEST_BINARY, - "test_knn_pytorch_limit_not_binary", - 1, - 1, - 1, - False, - None, - ), - ], + scores, _ = evaluator(model) + + # Check that accuracy matches expected value (with some tolerance for floating point) + assert abs(scores["accuracy"] - test_case.expected_score) < 1e-10, ( + f"Expected accuracy {test_case.expected_score}, got {scores['accuracy']}" ) - def test_limit_parameter( - self, - model_pytorch, - train_sentences, - train_labels, - test_sentences, - test_labels, - task_name, - limit, - expected_train_len, - expected_test_len, - is_binary_after_limit, - expected_exception, - ): - if expected_exception: - with pytest.raises(expected_exception): - eval_limited = kNNClassificationEvaluatorPytorch( - train_sentences, - train_labels, - test_sentences, - test_labels, - task_name=task_name, - limit=limit, - ) - eval_limited(model_pytorch) - else: - eval_limited = kNNClassificationEvaluatorPytorch( - train_sentences, - train_labels, - test_sentences, - test_labels, - task_name=task_name, - limit=limit, - ) - assert len(eval_limited.sentences_train) == expected_train_len - assert len(eval_limited.y_train) == expected_train_len - assert len(eval_limited.sentences_test) == expected_test_len - assert len(eval_limited.y_test) == expected_test_len - - scores, _ = eval_limited(model_pytorch) - assert "accuracy" in scores - assert "f1" in scores - if is_binary_after_limit: - assert "ap" in scores - else: - assert "ap" not in scores - - -class TestLogRegClassificationEvaluator: - @pytest.fixture - def model(self): - return MockNumpyEncoder() - - @pytest.fixture - def eval_logreg_binary(self): - return logRegClassificationEvaluator( - SENTENCES_TRAIN_BINARY, - Y_TRAIN_BINARY, - SENTENCES_TEST_BINARY, - Y_TEST_BINARY, - task_name="test_logreg_binary", - ) - - @pytest.fixture - def eval_logreg_multiclass(self): - return logRegClassificationEvaluator( - SENTENCES_TRAIN_MULTICLASS, - Y_TRAIN_MULTICLASS, - SENTENCES_TEST_MULTICLASS, - Y_TEST_MULTICLASS, - task_name="test_logreg_multiclass", - ) - - @pytest.mark.parametrize( - "evaluator_fixture, is_binary", - [ - ("eval_logreg_binary", True), - ("eval_logreg_multiclass", False), - ], + + +def test_cache_usage_binary(model): + """Test that embedding caching works correctly for binary classification. + + This test verifies the caching mechanism used to avoid re-encoding the same + sentences multiple times. The workflow is: + + 1. Run a first evaluation which encodes test sentences and returns embeddings cache + 2. Run a second evaluation with the same test sentences, passing the cache from step 1 + 3. Verify that the cache is preserved (not modified) during the second evaluation + 4. Verify that the second evaluation still produces valid classification results + + The cache contains the encoded embeddings for the test sentences, allowing the + evaluator to skip the encoding step when the same sentences are evaluated again. + This is particularly useful when running multiple evaluations on the same dataset + with different models or parameters. + """ + test_case = BINARY_TEST_CASE + + # First evaluation to generate cache + evaluator_initial = logRegClassificationEvaluator( + test_case.x_train, + np.array(test_case.y_train), + test_case.x_test, + np.array(test_case.y_test), + task_name=test_case.task_name, ) - def test_output_structure(self, evaluator_fixture, is_binary, model, request): - evaluator = request.getfixturevalue(evaluator_fixture) - scores, test_cache = evaluator(model) - assert isinstance(scores, dict) - assert isinstance(test_cache, np.ndarray) - assert "accuracy" in scores - assert "f1" in scores - assert "f1_weighted" in scores - - if is_binary: - assert "ap" in scores - assert "ap_weighted" in scores - else: - assert "ap" not in scores - - @pytest.mark.parametrize( - "evaluator_fixture, is_binary", - [ - ("eval_logreg_binary", True), - ("eval_logreg_multiclass", False), - ], + _, test_cache_initial = evaluator_initial(model) + + # Second evaluation using cache + evaluator_with_cache = logRegClassificationEvaluator( + test_case.x_train, + np.array(test_case.y_train), + test_case.x_test, + np.array(test_case.y_test), + task_name=f"{test_case.task_name}_cache", ) - def test_score_ranges(self, evaluator_fixture, is_binary, model, request): - evaluator = request.getfixturevalue(evaluator_fixture) - scores, _ = evaluator(model) - metrics_to_check = ["accuracy", "f1", "f1_weighted"] - if is_binary: - metrics_to_check.extend(["ap", "ap_weighted"]) - - for metric in metrics_to_check: - assert 0 <= scores[metric] <= 1 - - def test_cache_usage_binary(self, model, eval_logreg_binary): - _, test_cache_initial = eval_logreg_binary(model) - eval_binary_cache_test = logRegClassificationEvaluator( - SENTENCES_TRAIN_BINARY, - Y_TRAIN_BINARY, - SENTENCES_TEST_BINARY, - Y_TEST_BINARY, - task_name="test_logreg_binary_cache", - ) - scores_with_cache, test_cache_after_cache_usage = eval_binary_cache_test( - model, test_cache=test_cache_initial - ) - - assert np.array_equal(test_cache_initial, test_cache_after_cache_usage) - for metric in ["accuracy", "f1", "f1_weighted", "ap", "ap_weighted"]: - assert 0 <= scores_with_cache[metric] <= 1 - - @pytest.mark.parametrize( - "train_sentences, train_labels, test_sentences, test_labels, task_name, limit, expected_train_len, expected_test_len, is_binary_after_limit, expected_exception", - [ - # Ensure binary classification is still possible after limiting - ( - SENTENCES_TRAIN_BINARY, - Y_TRAIN_BINARY, - SENTENCES_TEST_BINARY, - Y_TEST_BINARY, - "test_logreg_limit_binary", - 3, - 3, - 2, - True, - None, - ), - # Test case where limiting results in a single class, expecting ValueError - ( - SENTENCES_TRAIN_BINARY, - Y_TRAIN_BINARY, - SENTENCES_TEST_BINARY, - Y_TEST_BINARY, - "test_logreg_limit_not_binary_train", - 1, - 1, - 1, - False, - ValueError, - ), - ], + scores_with_cache, test_cache_after_cache_usage = evaluator_with_cache( + model, test_cache=test_cache_initial ) - def test_limit_parameter( - self, - model, - train_sentences, - train_labels, - test_sentences, - test_labels, - task_name, - limit, - expected_train_len, - expected_test_len, - is_binary_after_limit, - expected_exception, - ): - if expected_exception: - with pytest.raises(expected_exception): - eval_limited = logRegClassificationEvaluator( - train_sentences, - train_labels, - test_sentences, - test_labels, - task_name=task_name, - limit=limit, - ) - eval_limited(model) - else: - eval_limited = logRegClassificationEvaluator( - train_sentences, - train_labels, - test_sentences, - test_labels, - task_name=task_name, - limit=limit, - ) - assert len(eval_limited.sentences_train) == expected_train_len - assert len(eval_limited.y_train) == expected_train_len - assert len(eval_limited.sentences_test) == expected_test_len - assert len(eval_limited.y_test) == expected_test_len - - scores, _ = eval_limited(model) - assert "accuracy" in scores - assert "f1" in scores - assert "f1_weighted" in scores - if is_binary_after_limit: - assert "ap" in scores - assert "ap_weighted" in scores - else: - assert "ap" not in scores - assert "ap_weighted" not in scores + + # Verify cache is preserved + assert np.array_equal(test_cache_initial, test_cache_after_cache_usage) + + # Verify that scores are returned (structure check only) + assert "accuracy" in scores_with_cache + assert "f1" in scores_with_cache + assert "f1_weighted" in scores_with_cache + assert "ap" in scores_with_cache + assert "ap_weighted" in scores_with_cache From da244587ac71f6b6f6a5c919db753a641da256c5 Mon Sep 17 00:00:00 2001 From: fzowl <160063452+fzowl@users.noreply.github.com> Date: Tue, 8 Jul 2025 23:04:50 +0200 Subject: [PATCH 3/9] Update tests/test_evaluators/test_ClassificationEvaluator.py Co-authored-by: Kenneth Enevoldsen --- .../test_ClassificationEvaluator.py | 55 ++++--------------- 1 file changed, 12 insertions(+), 43 deletions(-) diff --git a/tests/test_evaluators/test_ClassificationEvaluator.py b/tests/test_evaluators/test_ClassificationEvaluator.py index a432107ccf..20ed0daf22 100644 --- a/tests/test_evaluators/test_ClassificationEvaluator.py +++ b/tests/test_evaluators/test_ClassificationEvaluator.py @@ -27,53 +27,22 @@ def encode(self, sentences, prompt_name: str | None = None, **kwargs): return self.rng_state.random((len(sentences), 10)) -# Basic test data -SENTENCES_TRAIN_BINARY = [ - "this is a positive sentence", - "another positive sentence", - "this is a negative sentence", - "another negative sentence", -] -Y_TRAIN_BINARY = np.array([1, 1, 0, 0]) -SENTENCES_TEST_BINARY = [ - "a new positive sentence", - "a new negative sentence", -] -Y_TEST_BINARY = np.array([1, 0]) - -SENTENCES_TRAIN_MULTICLASS = [ - "class 0 sentence 1", - "class 0 sentence 2", - "class 1 sentence 1", - "class 1 sentence 2", - "class 2 sentence 1", - "class 2 sentence 2", -] -Y_TRAIN_MULTICLASS = np.array([0, 0, 1, 1, 2, 2]) -SENTENCES_TEST_MULTICLASS = [ - "new class 0 sentence", - "new class 1 sentence", - "new class 2 sentence", -] -Y_TEST_MULTICLASS = np.array([0, 1, 2]) - -# Test cases with expected scores (deterministic due to fixed random seed) BINARY_TEST_CASE = ClassificationTestCase( - x_train=SENTENCES_TRAIN_BINARY, - y_train=Y_TRAIN_BINARY.tolist(), - x_test=SENTENCES_TEST_BINARY, - y_test=Y_TEST_BINARY.tolist(), - task_name="test_logreg_binary", - expected_score=0.5, # Expected accuracy with deterministic MockNumpyEncoder + x_train=["pos1", "pos2", "neg1", "neg2"], + y_train=[1, 1, 0, 0], + x_test=["new pos", "new neg"], + y_test=[1, 0], + model= MockNumpyEncoder(), + expected_score=0.5, ) MULTICLASS_TEST_CASE = ClassificationTestCase( - x_train=SENTENCES_TRAIN_MULTICLASS, - y_train=Y_TRAIN_MULTICLASS.tolist(), - x_test=SENTENCES_TEST_MULTICLASS, - y_test=Y_TEST_MULTICLASS.tolist(), - task_name="test_logreg_multiclass", - expected_score=0.0, # Expected accuracy with deterministic MockNumpyEncoder + x_train=["cls 1", "still cls 1", "cls 2", "also cls 2", "cls 3", "cls 3 too"], + y_train=[0, 0, 1, 1, 2, 2], + x_test=["new cls 1", "new cls 2", "new cls 3"], + y_test=[0, 1, 2], + model= MockNumpyEncoder(), + expected_score=0.0, ) From 1321cbf8b84790f093457a531324753468be605c Mon Sep 17 00:00:00 2001 From: fzowl <160063452+fzowl@users.noreply.github.com> Date: Tue, 8 Jul 2025 23:05:03 +0200 Subject: [PATCH 4/9] Update tests/test_evaluators/test_ClassificationEvaluator.py Co-authored-by: Kenneth Enevoldsen --- tests/test_evaluators/test_ClassificationEvaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_evaluators/test_ClassificationEvaluator.py b/tests/test_evaluators/test_ClassificationEvaluator.py index 20ed0daf22..96adbe9786 100644 --- a/tests/test_evaluators/test_ClassificationEvaluator.py +++ b/tests/test_evaluators/test_ClassificationEvaluator.py @@ -62,7 +62,7 @@ def model(): def test_case(request): return request.param - +@pytest.fixture("test_case", [BINARY_TEST_CASE, MULTICLASS_TEST_CASE]) def test_output_structure(model, test_case: ClassificationTestCase): """Test that the evaluator returns the expected output structure.""" evaluator = logRegClassificationEvaluator( From 00e72e094720a6e098db4734d5b7f1570004a05b Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 9 Jul 2025 13:24:23 +0200 Subject: [PATCH 5/9] Modifications due to the comments --- tests/test_benchmark/mock_models.py | 9 ++++-- .../test_ClassificationEvaluator.py | 28 +++++++------------ 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/tests/test_benchmark/mock_models.py b/tests/test_benchmark/mock_models.py index 0ec55ea183..8fd45fd6bd 100644 --- a/tests/test_benchmark/mock_models.py +++ b/tests/test_benchmark/mock_models.py @@ -20,11 +20,14 @@ class MockNumpyEncoder(mteb.Encoder): - def __init__(self): - pass + def __init__(self, seed: int | None = None): + if seed is not None: + self.rng = np.random.default_rng(seed) + else: + self.rng = np.random.default_rng() def encode(self, sentences, prompt_name: str | None = None, **kwargs): - return np.random.rand(len(sentences), 10) + return self.rng.random((len(sentences), 10)) class MockTorchEncoder(mteb.Encoder): diff --git a/tests/test_evaluators/test_ClassificationEvaluator.py b/tests/test_evaluators/test_ClassificationEvaluator.py index 96adbe9786..7175af7938 100644 --- a/tests/test_evaluators/test_ClassificationEvaluator.py +++ b/tests/test_evaluators/test_ClassificationEvaluator.py @@ -5,8 +5,8 @@ import numpy as np import pytest -import mteb from mteb.evaluation.evaluators import logRegClassificationEvaluator +from tests.test_benchmark.mock_models import MockNumpyEncoder @dataclass @@ -15,24 +15,14 @@ class ClassificationTestCase: y_train: list[int] x_test: list[str] y_test: list[int] - task_name: str expected_score: float | None = None # For deterministic tests -class MockNumpyEncoder(mteb.Encoder): - def __init__(self): - self.rng_state = np.random.default_rng(42) - - def encode(self, sentences, prompt_name: str | None = None, **kwargs): - return self.rng_state.random((len(sentences), 10)) - - BINARY_TEST_CASE = ClassificationTestCase( x_train=["pos1", "pos2", "neg1", "neg2"], y_train=[1, 1, 0, 0], x_test=["new pos", "new neg"], y_test=[1, 0], - model= MockNumpyEncoder(), expected_score=0.5, ) @@ -41,7 +31,6 @@ def encode(self, sentences, prompt_name: str | None = None, **kwargs): y_train=[0, 0, 1, 1, 2, 2], x_test=["new cls 1", "new cls 2", "new cls 3"], y_test=[0, 1, 2], - model= MockNumpyEncoder(), expected_score=0.0, ) @@ -55,13 +44,14 @@ def is_binary_classification(y_train: list[int], y_test: list[int]) -> bool: # Fixtures @pytest.fixture def model(): - return MockNumpyEncoder() + return MockNumpyEncoder(seed=42) @pytest.fixture(params=[BINARY_TEST_CASE, MULTICLASS_TEST_CASE]) def test_case(request): return request.param + @pytest.fixture("test_case", [BINARY_TEST_CASE, MULTICLASS_TEST_CASE]) def test_output_structure(model, test_case: ClassificationTestCase): """Test that the evaluator returns the expected output structure.""" @@ -70,7 +60,7 @@ def test_output_structure(model, test_case: ClassificationTestCase): np.array(test_case.y_train), test_case.x_test, np.array(test_case.y_test), - task_name=test_case.task_name, + task_name="test_classification", ) scores, test_cache = evaluator(model) @@ -92,6 +82,7 @@ def test_output_structure(model, test_case: ClassificationTestCase): assert "ap" not in scores +@pytest.fixture("test_case", [BINARY_TEST_CASE, MULTICLASS_TEST_CASE]) def test_expected_scores(model, test_case: ClassificationTestCase): """Test that the evaluator returns expected scores with deterministic model.""" if test_case.expected_score is None: @@ -102,7 +93,7 @@ def test_expected_scores(model, test_case: ClassificationTestCase): np.array(test_case.y_train), test_case.x_test, np.array(test_case.y_test), - task_name=test_case.task_name, + task_name="test_classification", ) scores, _ = evaluator(model) @@ -112,7 +103,7 @@ def test_expected_scores(model, test_case: ClassificationTestCase): ) -def test_cache_usage_binary(model): +def test_cache_usage_binary(): """Test that embedding caching works correctly for binary classification. This test verifies the caching mechanism used to avoid re-encoding the same @@ -129,6 +120,7 @@ def test_cache_usage_binary(model): with different models or parameters. """ test_case = BINARY_TEST_CASE + model = MockNumpyEncoder(seed=42) # First evaluation to generate cache evaluator_initial = logRegClassificationEvaluator( @@ -136,7 +128,7 @@ def test_cache_usage_binary(model): np.array(test_case.y_train), test_case.x_test, np.array(test_case.y_test), - task_name=test_case.task_name, + task_name="test_binary_cache", ) _, test_cache_initial = evaluator_initial(model) @@ -146,7 +138,7 @@ def test_cache_usage_binary(model): np.array(test_case.y_train), test_case.x_test, np.array(test_case.y_test), - task_name=f"{test_case.task_name}_cache", + task_name="test_binary_cache_usage", ) scores_with_cache, test_cache_after_cache_usage = evaluator_with_cache( model, test_cache=test_cache_initial From 6794535fd3a535c63decea08ea08d26219d5a696 Mon Sep 17 00:00:00 2001 From: fzowl Date: Mon, 14 Jul 2025 23:45:09 +0200 Subject: [PATCH 6/9] Modifications due to the comments --- .../test_ClassificationEvaluator.py | 100 +++++++----------- 1 file changed, 40 insertions(+), 60 deletions(-) diff --git a/tests/test_evaluators/test_ClassificationEvaluator.py b/tests/test_evaluators/test_ClassificationEvaluator.py index 7175af7938..31b179be5b 100644 --- a/tests/test_evaluators/test_ClassificationEvaluator.py +++ b/tests/test_evaluators/test_ClassificationEvaluator.py @@ -1,38 +1,11 @@ from __future__ import annotations -from dataclasses import dataclass - import numpy as np import pytest from mteb.evaluation.evaluators import logRegClassificationEvaluator from tests.test_benchmark.mock_models import MockNumpyEncoder - - -@dataclass -class ClassificationTestCase: - x_train: list[str] - y_train: list[int] - x_test: list[str] - y_test: list[int] - expected_score: float | None = None # For deterministic tests - - -BINARY_TEST_CASE = ClassificationTestCase( - x_train=["pos1", "pos2", "neg1", "neg2"], - y_train=[1, 1, 0, 0], - x_test=["new pos", "new neg"], - y_test=[1, 0], - expected_score=0.5, -) - -MULTICLASS_TEST_CASE = ClassificationTestCase( - x_train=["cls 1", "still cls 1", "cls 2", "also cls 2", "cls 3", "cls 3 too"], - y_train=[0, 0, 1, 1, 2, 2], - x_test=["new cls 1", "new cls 2", "new cls 3"], - y_test=[0, 1, 2], - expected_score=0.0, -) +from tests.test_benchmark.mock_tasks import MockClassificationTask def is_binary_classification(y_train: list[int], y_test: list[int]) -> bool: @@ -47,19 +20,23 @@ def model(): return MockNumpyEncoder(seed=42) -@pytest.fixture(params=[BINARY_TEST_CASE, MULTICLASS_TEST_CASE]) -def test_case(request): - return request.param +@pytest.fixture +def mock_task(): + task = MockClassificationTask() + task.load_data() + return task -@pytest.fixture("test_case", [BINARY_TEST_CASE, MULTICLASS_TEST_CASE]) -def test_output_structure(model, test_case: ClassificationTestCase): +def test_output_structure(model, mock_task): """Test that the evaluator returns the expected output structure.""" + train_data = mock_task.dataset["train"] + test_data = mock_task.dataset["test"] + evaluator = logRegClassificationEvaluator( - test_case.x_train, - np.array(test_case.y_train), - test_case.x_test, - np.array(test_case.y_test), + train_data["text"], + np.array(train_data["label"]), + test_data["text"], + np.array(test_data["label"]), task_name="test_classification", ) scores, test_cache = evaluator(model) @@ -73,8 +50,8 @@ def test_output_structure(model, test_case: ClassificationTestCase): assert "f1" in scores assert "f1_weighted" in scores - # Check binary-specific metrics - is_binary = is_binary_classification(test_case.y_train, test_case.y_test) + # Check binary-specific metrics (MockClassificationTask is binary) + is_binary = is_binary_classification(train_data["label"], test_data["label"]) if is_binary: assert "ap" in scores assert "ap_weighted" in scores @@ -82,25 +59,24 @@ def test_output_structure(model, test_case: ClassificationTestCase): assert "ap" not in scores -@pytest.fixture("test_case", [BINARY_TEST_CASE, MULTICLASS_TEST_CASE]) -def test_expected_scores(model, test_case: ClassificationTestCase): +def test_expected_scores(model, mock_task): """Test that the evaluator returns expected scores with deterministic model.""" - if test_case.expected_score is None: - pytest.skip("No expected score defined for this test case") + train_data = mock_task.dataset["train"] + test_data = mock_task.dataset["test"] evaluator = logRegClassificationEvaluator( - test_case.x_train, - np.array(test_case.y_train), - test_case.x_test, - np.array(test_case.y_test), + train_data["text"], + np.array(train_data["label"]), + test_data["text"], + np.array(test_data["label"]), task_name="test_classification", ) scores, _ = evaluator(model) - # Check that accuracy matches expected value (with some tolerance for floating point) - assert abs(scores["accuracy"] - test_case.expected_score) < 1e-10, ( - f"Expected accuracy {test_case.expected_score}, got {scores['accuracy']}" - ) + # Check that we get reasonable scores (MockClassificationTask has deterministic data) + assert 0.0 <= scores["accuracy"] <= 1.0 + assert 0.0 <= scores["f1"] <= 1.0 + assert 0.0 <= scores["f1_weighted"] <= 1.0 def test_cache_usage_binary(): @@ -119,25 +95,29 @@ def test_cache_usage_binary(): This is particularly useful when running multiple evaluations on the same dataset with different models or parameters. """ - test_case = BINARY_TEST_CASE + mock_task = MockClassificationTask() + mock_task.load_data() + train_data = mock_task.dataset["train"] + test_data = mock_task.dataset["test"] + model = MockNumpyEncoder(seed=42) # First evaluation to generate cache evaluator_initial = logRegClassificationEvaluator( - test_case.x_train, - np.array(test_case.y_train), - test_case.x_test, - np.array(test_case.y_test), + train_data["text"], + np.array(train_data["label"]), + test_data["text"], + np.array(test_data["label"]), task_name="test_binary_cache", ) _, test_cache_initial = evaluator_initial(model) # Second evaluation using cache evaluator_with_cache = logRegClassificationEvaluator( - test_case.x_train, - np.array(test_case.y_train), - test_case.x_test, - np.array(test_case.y_test), + train_data["text"], + np.array(train_data["label"]), + test_data["text"], + np.array(test_data["label"]), task_name="test_binary_cache_usage", ) scores_with_cache, test_cache_after_cache_usage = evaluator_with_cache( From 112cdc7c0128a3b88a005d86434b602240d71619 Mon Sep 17 00:00:00 2001 From: fzowl Date: Wed, 16 Jul 2025 17:35:07 +0200 Subject: [PATCH 7/9] Adding STSEvaluator and SummarizationEvaluator tests --- mteb/models/kalm_models.py | 1 + tests/test_evaluators/test_STSEvaluator.py | 115 ++++++++++++++ .../test_SummarizationEvaluator.py | 145 ++++++++++++++++++ 3 files changed, 261 insertions(+) create mode 100644 tests/test_evaluators/test_STSEvaluator.py create mode 100644 tests/test_evaluators/test_SummarizationEvaluator.py diff --git a/mteb/models/kalm_models.py b/mteb/models/kalm_models.py index 68f17460c3..02251df44d 100644 --- a/mteb/models/kalm_models.py +++ b/mteb/models/kalm_models.py @@ -7,6 +7,7 @@ import numpy as np import torch + from mteb.encoder_interface import PromptType from mteb.model_meta import ModelMeta from mteb.models.instruct_wrapper import InstructSentenceTransformerWrapper diff --git a/tests/test_evaluators/test_STSEvaluator.py b/tests/test_evaluators/test_STSEvaluator.py new file mode 100644 index 0000000000..11541c5ee9 --- /dev/null +++ b/tests/test_evaluators/test_STSEvaluator.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import pytest + +from mteb.evaluation.evaluators.STSEvaluator import STSEvaluator +from tests.test_benchmark.mock_models import MockNumpyEncoder +from tests.test_benchmark.mock_tasks import MockSTSTask + + +# Fixtures +@pytest.fixture +def model(): + return MockNumpyEncoder(seed=42) + + +@pytest.fixture +def mock_task(): + task = MockSTSTask() + task.load_data() + return task + + +def test_output_structure(model, mock_task): + """Test that the evaluator returns the expected output structure.""" + test_data = mock_task.dataset["test"] + + evaluator = STSEvaluator( + sentences1=test_data["sentence1"], + sentences2=test_data["sentence2"], + gold_scores=test_data["score"], + task_name="test_sts", + ) + scores = evaluator(model) + + # Check basic structure + assert isinstance(scores, dict) + + # Check required metrics + assert "pearson" in scores + assert "spearman" in scores + assert "cosine_pearson" in scores + assert "cosine_spearman" in scores + assert "manhattan_pearson" in scores + assert "manhattan_spearman" in scores + assert "euclidean_pearson" in scores + assert "euclidean_spearman" in scores + + +def test_expected_scores(model, mock_task): + """Test that the evaluator returns expected scores with deterministic model.""" + test_data = mock_task.dataset["test"] + + evaluator = STSEvaluator( + sentences1=test_data["sentence1"], + sentences2=test_data["sentence2"], + gold_scores=test_data["score"], + task_name="test_sts", + ) + scores = evaluator(model) + + # Check that we get reasonable correlation scores (between -1 and 1) + assert -1.0 <= scores["pearson"] <= 1.0 + assert -1.0 <= scores["spearman"] <= 1.0 + assert -1.0 <= scores["cosine_pearson"] <= 1.0 + assert -1.0 <= scores["cosine_spearman"] <= 1.0 + assert -1.0 <= scores["manhattan_pearson"] <= 1.0 + assert -1.0 <= scores["manhattan_spearman"] <= 1.0 + assert -1.0 <= scores["euclidean_pearson"] <= 1.0 + assert -1.0 <= scores["euclidean_spearman"] <= 1.0 + + +def test_limit_parameter(model, mock_task): + """Test that the limit parameter works correctly.""" + test_data = mock_task.dataset["test"] + + # Test with limit parameter, but need at least 2 items for correlation + original_length = len(test_data["sentence1"]) + + # Only test if we have enough data, otherwise skip + if original_length >= 2: + evaluator_limited = STSEvaluator( + sentences1=test_data["sentence1"], + sentences2=test_data["sentence2"], + gold_scores=test_data["score"], + task_name="test_sts_limited", + limit=original_length, # Use all available data + ) + + # Check that limit is applied + assert len(evaluator_limited.sentences1) == original_length + assert len(evaluator_limited.sentences2) == original_length + assert len(evaluator_limited.gold_scores) == original_length + + # Test evaluation still works + scores = evaluator_limited(model) + assert isinstance(scores, dict) + assert "pearson" in scores + + +def test_basic_functionality(model, mock_task): + """Test basic functionality and proper initialization.""" + test_data = mock_task.dataset["test"] + + evaluator = STSEvaluator( + sentences1=test_data["sentence1"], + sentences2=test_data["sentence2"], + gold_scores=test_data["score"], + task_name="test_sts", + ) + + # Check that data is properly stored + assert evaluator.sentences1 == test_data["sentence1"] + assert evaluator.sentences2 == test_data["sentence2"] + assert evaluator.gold_scores == test_data["score"] + assert evaluator.task_name == "test_sts" diff --git a/tests/test_evaluators/test_SummarizationEvaluator.py b/tests/test_evaluators/test_SummarizationEvaluator.py new file mode 100644 index 0000000000..ee4a9ffa16 --- /dev/null +++ b/tests/test_evaluators/test_SummarizationEvaluator.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import pytest + +from mteb.evaluation.evaluators.SummarizationEvaluator import SummarizationEvaluator +from tests.test_benchmark.mock_models import MockNumpyEncoder +from tests.test_benchmark.mock_tasks import MockSummarizationTask + + +# Fixtures +@pytest.fixture +def model(): + return MockNumpyEncoder(seed=42) + + +@pytest.fixture +def mock_task(): + task = MockSummarizationTask() + task.load_data() + return task + + +def test_output_structure(model, mock_task): + """Test that the evaluator returns the expected output structure.""" + test_data = mock_task.dataset["test"] + + evaluator = SummarizationEvaluator( + human_summaries=test_data["human_summaries"], + machine_summaries=test_data["machine_summaries"], + gold_scores=test_data["relevance"], + task_name="test_summarization", + ) + scores = evaluator(model) + + # Check basic structure + assert isinstance(scores, dict) + + # Check required metrics + assert "pearson" in scores + assert "spearman" in scores + assert "cosine_pearson" in scores + assert "cosine_spearman" in scores + assert "dot_pearson" in scores + assert "dot_spearman" in scores + + +def test_expected_scores(model, mock_task): + """Test that the evaluator returns expected scores with deterministic model.""" + test_data = mock_task.dataset["test"] + + evaluator = SummarizationEvaluator( + human_summaries=test_data["human_summaries"], + machine_summaries=test_data["machine_summaries"], + gold_scores=test_data["relevance"], + task_name="test_summarization", + ) + scores = evaluator(model) + + # Check that we get reasonable correlation scores (between -1 and 1) + assert -1.0 <= scores["pearson"] <= 1.0 + assert -1.0 <= scores["spearman"] <= 1.0 + assert -1.0 <= scores["cosine_pearson"] <= 1.0 + assert -1.0 <= scores["cosine_spearman"] <= 1.0 + assert -1.0 <= scores["dot_pearson"] <= 1.0 + assert -1.0 <= scores["dot_spearman"] <= 1.0 + + +def test_basic_functionality(model, mock_task): + """Test basic functionality and proper initialization.""" + test_data = mock_task.dataset["test"] + + evaluator = SummarizationEvaluator( + human_summaries=test_data["human_summaries"], + machine_summaries=test_data["machine_summaries"], + gold_scores=test_data["relevance"], + task_name="test_summarization", + ) + + # Check that data is properly stored + assert evaluator.human_summaries == test_data["human_summaries"] + assert evaluator.machine_summaries == test_data["machine_summaries"] + assert evaluator.gold_scores == test_data["relevance"] + assert evaluator.task_name == "test_summarization" + + +def test_encode_kwargs_handling(model, mock_task): + """Test that encode_kwargs are properly handled.""" + test_data = mock_task.dataset["test"] + + evaluator = SummarizationEvaluator( + human_summaries=test_data["human_summaries"], + machine_summaries=test_data["machine_summaries"], + gold_scores=test_data["relevance"], + task_name="test_summarization", + ) + + # Test that the evaluator accepts encode_kwargs + scores = evaluator(model, encode_kwargs={"batch_size": 16}) + assert isinstance(scores, dict) + assert "pearson" in scores + + +def test_batch_size_parameter(model, mock_task): + """Test that the batch_size parameter in encode_kwargs works correctly.""" + test_data = mock_task.dataset["test"] + + evaluator = SummarizationEvaluator( + human_summaries=test_data["human_summaries"], + machine_summaries=test_data["machine_summaries"], + gold_scores=test_data["relevance"], + task_name="test_summarization", + ) + + # Test with custom batch size + scores = evaluator(model, encode_kwargs={"batch_size": 16}) + + # Check that evaluation still works with custom batch size + assert isinstance(scores, dict) + assert "pearson" in scores + assert -1.0 <= scores["pearson"] <= 1.0 + + +def test_empty_scores_handling(model, mock_task): + """Test that the evaluator handles cases where some samples have equal scores.""" + test_data = mock_task.dataset["test"] + + # Create a case where some gold scores are identical + modified_gold_scores = test_data["relevance"].copy() + if len(modified_gold_scores) > 0 and len(modified_gold_scores[0]) > 1: + # Make all scores in the first sample identical + modified_gold_scores[0] = [modified_gold_scores[0][0]] * len( + modified_gold_scores[0] + ) + + evaluator = SummarizationEvaluator( + human_summaries=test_data["human_summaries"], + machine_summaries=test_data["machine_summaries"], + gold_scores=modified_gold_scores, + task_name="test_summarization_equal_scores", + ) + + # Should still work even with some samples having equal scores + scores = evaluator(model) + assert isinstance(scores, dict) + assert "pearson" in scores From 51741f714139312d6d3bf5d07510186309c166a1 Mon Sep 17 00:00:00 2001 From: fzowl Date: Fri, 18 Jul 2025 11:02:35 +0200 Subject: [PATCH 8/9] Correcting due to the comments --- tests/test_evaluators/test_STSEvaluator.py | 122 ++++++++++++------ .../test_SummarizationEvaluator.py | 86 ++++++++---- 2 files changed, 139 insertions(+), 69 deletions(-) diff --git a/tests/test_evaluators/test_STSEvaluator.py b/tests/test_evaluators/test_STSEvaluator.py index 11541c5ee9..a1d37735d2 100644 --- a/tests/test_evaluators/test_STSEvaluator.py +++ b/tests/test_evaluators/test_STSEvaluator.py @@ -21,7 +21,7 @@ def mock_task(): def test_output_structure(model, mock_task): - """Test that the evaluator returns the expected output structure.""" + """Test that the evaluator returns the expected output structure and scores.""" test_data = mock_task.dataset["test"] evaluator = STSEvaluator( @@ -45,9 +45,19 @@ def test_output_structure(model, mock_task): assert "euclidean_pearson" in scores assert "euclidean_spearman" in scores + # Check exact score values with deterministic model + assert scores["pearson"] == -1.0 + assert scores["spearman"] == -0.9999999999999999 + assert scores["cosine_pearson"] == -1.0 + assert scores["cosine_spearman"] == -0.9999999999999999 + assert scores["manhattan_pearson"] == -1.0 + assert scores["manhattan_spearman"] == -0.9999999999999999 + assert scores["euclidean_pearson"] == -1.0 + assert scores["euclidean_spearman"] == -0.9999999999999999 -def test_expected_scores(model, mock_task): - """Test that the evaluator returns expected scores with deterministic model.""" + +def test_basic_functionality(model, mock_task): + """Test basic functionality and proper initialization.""" test_data = mock_task.dataset["test"] evaluator = STSEvaluator( @@ -56,49 +66,51 @@ def test_expected_scores(model, mock_task): gold_scores=test_data["score"], task_name="test_sts", ) - scores = evaluator(model) - # Check that we get reasonable correlation scores (between -1 and 1) - assert -1.0 <= scores["pearson"] <= 1.0 - assert -1.0 <= scores["spearman"] <= 1.0 - assert -1.0 <= scores["cosine_pearson"] <= 1.0 - assert -1.0 <= scores["cosine_spearman"] <= 1.0 - assert -1.0 <= scores["manhattan_pearson"] <= 1.0 - assert -1.0 <= scores["manhattan_spearman"] <= 1.0 - assert -1.0 <= scores["euclidean_pearson"] <= 1.0 - assert -1.0 <= scores["euclidean_spearman"] <= 1.0 + # Check that data is properly stored + assert evaluator.sentences1 == test_data["sentence1"] + assert evaluator.sentences2 == test_data["sentence2"] + assert evaluator.gold_scores == test_data["score"] + assert evaluator.task_name == "test_sts" -def test_limit_parameter(model, mock_task): - """Test that the limit parameter works correctly.""" +def test_encode_kwargs_handling(model, mock_task): + """Test that encode_kwargs are properly handled.""" test_data = mock_task.dataset["test"] - # Test with limit parameter, but need at least 2 items for correlation - original_length = len(test_data["sentence1"]) + evaluator = STSEvaluator( + sentences1=test_data["sentence1"], + sentences2=test_data["sentence2"], + gold_scores=test_data["score"], + task_name="test_sts", + ) - # Only test if we have enough data, otherwise skip - if original_length >= 2: - evaluator_limited = STSEvaluator( - sentences1=test_data["sentence1"], - sentences2=test_data["sentence2"], - gold_scores=test_data["score"], - task_name="test_sts_limited", - limit=original_length, # Use all available data - ) + # Test that the evaluator accepts encode_kwargs + scores = evaluator(model, encode_kwargs={"batch_size": 16}) + assert isinstance(scores, dict) + assert "pearson" in scores - # Check that limit is applied - assert len(evaluator_limited.sentences1) == original_length - assert len(evaluator_limited.sentences2) == original_length - assert len(evaluator_limited.gold_scores) == original_length - # Test evaluation still works - scores = evaluator_limited(model) - assert isinstance(scores, dict) - assert "pearson" in scores +def test_batch_size_parameter(mock_task): + """Test that the batch_size parameter in encode_kwargs works correctly.""" + from tests.test_benchmark.mock_models import MockNumpyEncoder + # Create a mock encoder that respects batch_size and tracks batch calls + class BatchTrackingMockEncoder(MockNumpyEncoder): + def __init__(self, seed=42): + super().__init__(seed) + self.batch_calls = [] # Track each batch call + + def encode(self, sentences, prompt_name=None, **kwargs): + batch_size = kwargs.get("batch_size", 32) + + # Track individual batch calls + for i in range(0, len(sentences), batch_size): + batch = sentences[i : i + batch_size] + self.batch_calls.append(len(batch)) + + return super().encode(sentences, prompt_name, **kwargs) -def test_basic_functionality(model, mock_task): - """Test basic functionality and proper initialization.""" test_data = mock_task.dataset["test"] evaluator = STSEvaluator( @@ -108,8 +120,36 @@ def test_basic_functionality(model, mock_task): task_name="test_sts", ) - # Check that data is properly stored - assert evaluator.sentences1 == test_data["sentence1"] - assert evaluator.sentences2 == test_data["sentence2"] - assert evaluator.gold_scores == test_data["score"] - assert evaluator.task_name == "test_sts" + # Test with batch_size=1 - should process texts one at a time + tracking_model_1 = BatchTrackingMockEncoder() + scores_batch_1 = evaluator(tracking_model_1, encode_kwargs={"batch_size": 1}) + + # Calculate expected batch calls + num_sentences1 = len(test_data["sentence1"]) + num_sentences2 = len(test_data["sentence2"]) + + # With batch_size=1, each text should be processed in its own batch + expected_batch_1_calls = num_sentences1 + num_sentences2 + assert len(tracking_model_1.batch_calls) == expected_batch_1_calls + assert all(batch_size == 1 for batch_size in tracking_model_1.batch_calls) + + # Test with batch_size=2 - should process texts in pairs (mostly) + tracking_model_2 = BatchTrackingMockEncoder() + scores_batch_2 = evaluator(tracking_model_2, encode_kwargs={"batch_size": 2}) + + # Calculate expected batch calls for batch_size=2 + import math + + expected_batch_2_calls = math.ceil(num_sentences1 / 2) + math.ceil( + num_sentences2 / 2 + ) + assert len(tracking_model_2.batch_calls) == expected_batch_2_calls + # Each batch should have at most 2 items (last batch might have 1) + assert all(batch_size <= 2 for batch_size in tracking_model_2.batch_calls) + assert all(batch_size >= 1 for batch_size in tracking_model_2.batch_calls) + + # Check that evaluation works with custom batch sizes + assert isinstance(scores_batch_1, dict) + assert isinstance(scores_batch_2, dict) + assert "pearson" in scores_batch_1 + assert "pearson" in scores_batch_2 diff --git a/tests/test_evaluators/test_SummarizationEvaluator.py b/tests/test_evaluators/test_SummarizationEvaluator.py index ee4a9ffa16..f58537d528 100644 --- a/tests/test_evaluators/test_SummarizationEvaluator.py +++ b/tests/test_evaluators/test_SummarizationEvaluator.py @@ -21,7 +21,7 @@ def mock_task(): def test_output_structure(model, mock_task): - """Test that the evaluator returns the expected output structure.""" + """Test that the evaluator returns the expected output structure and scores.""" test_data = mock_task.dataset["test"] evaluator = SummarizationEvaluator( @@ -43,26 +43,13 @@ def test_output_structure(model, mock_task): assert "dot_pearson" in scores assert "dot_spearman" in scores - -def test_expected_scores(model, mock_task): - """Test that the evaluator returns expected scores with deterministic model.""" - test_data = mock_task.dataset["test"] - - evaluator = SummarizationEvaluator( - human_summaries=test_data["human_summaries"], - machine_summaries=test_data["machine_summaries"], - gold_scores=test_data["relevance"], - task_name="test_summarization", - ) - scores = evaluator(model) - - # Check that we get reasonable correlation scores (between -1 and 1) - assert -1.0 <= scores["pearson"] <= 1.0 - assert -1.0 <= scores["spearman"] <= 1.0 - assert -1.0 <= scores["cosine_pearson"] <= 1.0 - assert -1.0 <= scores["cosine_spearman"] <= 1.0 - assert -1.0 <= scores["dot_pearson"] <= 1.0 - assert -1.0 <= scores["dot_spearman"] <= 1.0 + # Check exact score values with deterministic model + assert scores["pearson"] == -1.0 + assert scores["spearman"] == -0.9999999999999999 + assert scores["cosine_pearson"] == -1.0 + assert scores["cosine_spearman"] == -0.9999999999999999 + assert scores["dot_pearson"] == 0.0 + assert scores["dot_spearman"] == 0.0 def test_basic_functionality(model, mock_task): @@ -100,8 +87,26 @@ def test_encode_kwargs_handling(model, mock_task): assert "pearson" in scores -def test_batch_size_parameter(model, mock_task): +def test_batch_size_parameter(mock_task): """Test that the batch_size parameter in encode_kwargs works correctly.""" + from tests.test_benchmark.mock_models import MockNumpyEncoder + + # Create a mock encoder that respects batch_size and tracks batch calls + class BatchTrackingMockEncoder(MockNumpyEncoder): + def __init__(self, seed=42): + super().__init__(seed) + self.batch_calls = [] # Track each batch call + + def encode(self, sentences, prompt_name=None, **kwargs): + batch_size = kwargs.get("batch_size", 32) + + # Track individual batch calls + for i in range(0, len(sentences), batch_size): + batch = sentences[i : i + batch_size] + self.batch_calls.append(len(batch)) + + return super().encode(sentences, prompt_name, **kwargs) + test_data = mock_task.dataset["test"] evaluator = SummarizationEvaluator( @@ -111,13 +116,38 @@ def test_batch_size_parameter(model, mock_task): task_name="test_summarization", ) - # Test with custom batch size - scores = evaluator(model, encode_kwargs={"batch_size": 16}) + # Test with batch_size=1 - should process texts one at a time + tracking_model_1 = BatchTrackingMockEncoder() + scores_batch_1 = evaluator(tracking_model_1, encode_kwargs={"batch_size": 1}) - # Check that evaluation still works with custom batch size - assert isinstance(scores, dict) - assert "pearson" in scores - assert -1.0 <= scores["pearson"] <= 1.0 + # Calculate expected batch calls + total_human_texts = sum(len(hs) for hs in test_data["human_summaries"]) + total_machine_texts = sum(len(ms) for ms in test_data["machine_summaries"]) + + # With batch_size=1, each text should be processed in its own batch + expected_batch_1_calls = total_human_texts + total_machine_texts + assert len(tracking_model_1.batch_calls) == expected_batch_1_calls + assert all(batch_size == 1 for batch_size in tracking_model_1.batch_calls) + + # Test with batch_size=2 - should process texts in pairs (mostly) + tracking_model_2 = BatchTrackingMockEncoder() + scores_batch_2 = evaluator(tracking_model_2, encode_kwargs={"batch_size": 2}) + + # Calculate expected batch calls for batch_size=2 + import math + + expected_batch_2_calls = math.ceil(total_human_texts / 2) + math.ceil( + total_machine_texts / 2 + ) + assert len(tracking_model_2.batch_calls) == expected_batch_2_calls + # Each batch should have at most 2 items (last batch might have 1) + assert all(1 <= batch_size <= 2 for batch_size in tracking_model_2.batch_calls) + + # Check that evaluation works with custom batch sizes + assert isinstance(scores_batch_1, dict) + assert isinstance(scores_batch_2, dict) + assert "pearson" in scores_batch_1 + assert "pearson" in scores_batch_2 def test_empty_scores_handling(model, mock_task): From ca0239804f28bc86f7d82e0928d9ff421cd783a4 Mon Sep 17 00:00:00 2001 From: fzowl Date: Fri, 18 Jul 2025 14:00:35 +0200 Subject: [PATCH 9/9] Correcting due to the comments --- tests/test_evaluators/test_STSEvaluator.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/test_evaluators/test_STSEvaluator.py b/tests/test_evaluators/test_STSEvaluator.py index a1d37735d2..aee3963724 100644 --- a/tests/test_evaluators/test_STSEvaluator.py +++ b/tests/test_evaluators/test_STSEvaluator.py @@ -74,23 +74,6 @@ def test_basic_functionality(model, mock_task): assert evaluator.task_name == "test_sts" -def test_encode_kwargs_handling(model, mock_task): - """Test that encode_kwargs are properly handled.""" - test_data = mock_task.dataset["test"] - - evaluator = STSEvaluator( - sentences1=test_data["sentence1"], - sentences2=test_data["sentence2"], - gold_scores=test_data["score"], - task_name="test_sts", - ) - - # Test that the evaluator accepts encode_kwargs - scores = evaluator(model, encode_kwargs={"batch_size": 16}) - assert isinstance(scores, dict) - assert "pearson" in scores - - def test_batch_size_parameter(mock_task): """Test that the batch_size parameter in encode_kwargs works correctly.""" from tests.test_benchmark.mock_models import MockNumpyEncoder