diff --git a/docs/create_benchmarks_table.py b/docs/create_benchmarks_table.py index f22d2638bc..287e072a8c 100644 --- a/docs/create_benchmarks_table.py +++ b/docs/create_benchmarks_table.py @@ -3,8 +3,9 @@ from collections import Counter from pathlib import Path +from create_tasks_table import insert_tables + import mteb -from docs.create_tasks_table import insert_tables def benchmark_to_markdown_row(b: mteb.Benchmark) -> str: diff --git a/tests/scripts/test_generate_model_meta.py b/tests/scripts/test_generate_model_meta.py deleted file mode 100644 index c930eff497..0000000000 --- a/tests/scripts/test_generate_model_meta.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import annotations - -import importlib.util -from pathlib import Path - -import pytest - -from scripts.generate_metadata import get_base_model -from scripts.generate_metadata import main as generate_metadata_main - - -def test_create_model_meta_embedding_models_from_hf(tmp_path: Path): - models = ["intfloat/multilingual-e5-large", "intfloat/multilingual-e5-small"] - tmp_path = tmp_path / "new_models.py" - generate_metadata_main(tmp_path, models) - - assert tmp_path.exists() - assert tmp_path.read_text().startswith("from mteb.model_meta import ModelMeta") - - spec = importlib.util.spec_from_file_location("new_models", tmp_path) - new_models = importlib.util.module_from_spec(spec) - spec.loader.exec_module(new_models) - - assert hasattr(new_models, "intfloat__multilingual_e5_large") - assert hasattr(new_models, "intfloat__multilingual_e5_small") - - assert ( - new_models.intfloat__multilingual_e5_large.name - == "intfloat/multilingual-e5-large" - ) - assert ( - new_models.intfloat__multilingual_e5_small.name - == "intfloat/multilingual-e5-small" - ) - - -def test_get_base_model_name_is_the_same(): - model_name = "jinaai/jina-embeddings-v3" - model = get_base_model(model_name) - assert model is None - - -@pytest.mark.skip(reason="No support for cross-encoder models") -def test_create_model_meta_cross_encoder_models_from_hf(tmp_path: Path): - models = ["intfloat/multilingual-e5-cross-encoder"] - tmp_path = tmp_path / "new_models.py" - generate_metadata_main(tmp_path, models) - assert True