From cd4ccba4d2caba0d14c7bfb0da64ec95eb1f62b0 Mon Sep 17 00:00:00 2001 From: Nikolay Banar Date: Wed, 3 Dec 2025 17:09:28 +0100 Subject: [PATCH 1/2] e5-nl models --- .../models/model_implementations/e5_models.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/mteb/models/model_implementations/e5_models.py b/mteb/models/model_implementations/e5_models.py index a989bcdca5..44782ab0bc 100644 --- a/mteb/models/model_implementations/e5_models.py +++ b/mteb/models/model_implementations/e5_models.py @@ -126,6 +126,18 @@ } """ +E5_NL_CITATION = """ +@misc{banar2025mtebnle5nlembeddingbenchmark, + archiveprefix = {arXiv}, + author = {Nikolay Banar and Ehsan Lotfi and Jens Van Nooten and Cristina Arhiliuc and Marija Kliocaite and Walter Daelemans}, + eprint = {2509.12340}, + primaryclass = {cs.CL}, + title = {MTEB-NL and E5-NL: Embedding Benchmark and Models for Dutch}, + url = {https://arxiv.org/abs/2509.12340}, + year = {2025}, +} +""" + model_prompts = { PromptType.query.value: "query: ", PromptType.document.value: "passage: ", @@ -399,3 +411,81 @@ training_datasets=E5_TRAINING_DATA, citation=E5_CITATION, ) + +e5_nl_small = ModelMeta( + loader=sentence_transformers_loader, + loader_kwargs=dict( + model_prompts=model_prompts, + ), + name="clips/e5-small-trm-nl", + languages=["nld-Latn"], + open_weights=True, + revision="0243664a6c5e12eef854b091eb283e51833c3e9f", + release_date="2025-09-23", + n_parameters=40_800_000, + memory_usage_mb=78, + embed_dim=384, + license="mit", + max_tokens=512, + reference="https://huggingface.co/clips/e5-small-trm-nl", + similarity_fn_name=ScoringFunction.COSINE, + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code="https://github.com/ELotfi/e5-nl", # couldn't find + public_training_data=None, + training_datasets=ME5_TRAINING_DATA, # mMARCO-NL, HotpotQA-NL, FEVER-NL, and LLM generated data + adapted_from="intfloat/multilingual-e5-small", + citation=E5_NL_CITATION, +) + +e5_nl_base = ModelMeta( + loader=sentence_transformers_loader, + loader_kwargs=dict( + model_prompts=model_prompts, + ), + name="clips/e5-base-trm-nl", + languages=["nld-Latn"], + open_weights=True, + revision="6bd5722f236da48b4b8bcb28cc1fc478f7089956", + release_date="2025-09-23", + n_parameters=124_400_000, + memory_usage_mb=237, + embed_dim=768, + license="mit", + max_tokens=514, + reference="https://huggingface.co/clips/e5-base-trm-nl", + similarity_fn_name=ScoringFunction.COSINE, + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code="https://github.com/ELotfi/e5-nl", + public_training_data=None, + adapted_from="intfloat/multilingual-e5-base", + training_datasets=ME5_TRAINING_DATA, # mMARCO-NL, HotpotQA-NL, FEVER-NL, and LLM generated data + citation=E5_NL_CITATION, +) + +e5_nl_large = ModelMeta( + loader=sentence_transformers_loader, + loader_kwargs=dict( + model_prompts=model_prompts, + ), + name="clips/e5-large-trm-nl", + languages=["nld-Latn"], + open_weights=True, + revision="683333f86ed9eb3699b5567f0fdabeb958d412b0", + release_date="2025-09-23", + n_parameters=355_000_000, + memory_usage_mb=1355, + embed_dim=1024, + license="mit", + max_tokens=514, + reference="https://huggingface.co/clips/e5-large-trm-nl", + similarity_fn_name=ScoringFunction.COSINE, + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code="https://github.com/ELotfi/e5-nl", + public_training_data=None, + training_datasets=ME5_TRAINING_DATA, # mMARCO-NL, HotpotQA-NL, FEVER-NL, and LLM generated data + adapted_from="intfloat/multilingual-e5-large", + citation=E5_NL_CITATION, +) From 9809420d5c8e06214bfba1366fd2ce40b33c6115 Mon Sep 17 00:00:00 2001 From: Nikolay Banar Date: Wed, 3 Dec 2025 18:11:55 +0100 Subject: [PATCH 2/2] e5-nl moved to a new file --- .../model_implementations/clips_models.py | 97 +++++++++++++++++++ .../models/model_implementations/e5_models.py | 90 ----------------- 2 files changed, 97 insertions(+), 90 deletions(-) create mode 100644 mteb/models/model_implementations/clips_models.py diff --git a/mteb/models/model_implementations/clips_models.py b/mteb/models/model_implementations/clips_models.py new file mode 100644 index 0000000000..17073d05cd --- /dev/null +++ b/mteb/models/model_implementations/clips_models.py @@ -0,0 +1,97 @@ +from mteb.models.model_meta import ( + ModelMeta, + ScoringFunction, +) +from mteb.models.sentence_transformer_wrapper import sentence_transformers_loader + +from .e5_models import ME5_TRAINING_DATA, model_prompts + +E5_NL_CITATION = """ +@misc{banar2025mtebnle5nlembeddingbenchmark, + archiveprefix = {arXiv}, + author = {Nikolay Banar and Ehsan Lotfi and Jens Van Nooten and Cristina Arhiliuc and Marija Kliocaite and Walter Daelemans}, + eprint = {2509.12340}, + primaryclass = {cs.CL}, + title = {MTEB-NL and E5-NL: Embedding Benchmark and Models for Dutch}, + url = {https://arxiv.org/abs/2509.12340}, + year = {2025}, +} +""" + +e5_nl_small = ModelMeta( + loader=sentence_transformers_loader, + loader_kwargs=dict( + model_prompts=model_prompts, + ), + name="clips/e5-small-trm-nl", + languages=["nld-Latn"], + open_weights=True, + revision="0243664a6c5e12eef854b091eb283e51833c3e9f", + release_date="2025-09-23", + n_parameters=40_800_000, + memory_usage_mb=78, + embed_dim=384, + license="mit", + max_tokens=512, + reference="https://huggingface.co/clips/e5-small-trm-nl", + similarity_fn_name=ScoringFunction.COSINE, + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code="https://github.com/ELotfi/e5-nl", + public_training_data="https://huggingface.co/collections/clips/beir-nl", + training_datasets=ME5_TRAINING_DATA, # mMARCO-NL, HotpotQA-NL, FEVER-NL, and LLM generated data + adapted_from="intfloat/multilingual-e5-small", + citation=E5_NL_CITATION, +) + +e5_nl_base = ModelMeta( + loader=sentence_transformers_loader, + loader_kwargs=dict( + model_prompts=model_prompts, + ), + name="clips/e5-base-trm-nl", + languages=["nld-Latn"], + open_weights=True, + revision="6bd5722f236da48b4b8bcb28cc1fc478f7089956", + release_date="2025-09-23", + n_parameters=124_400_000, + memory_usage_mb=237, + embed_dim=768, + license="mit", + max_tokens=514, + reference="https://huggingface.co/clips/e5-base-trm-nl", + similarity_fn_name=ScoringFunction.COSINE, + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code="https://github.com/ELotfi/e5-nl", + public_training_data="https://huggingface.co/collections/clips/beir-nl", + adapted_from="intfloat/multilingual-e5-base", + training_datasets=ME5_TRAINING_DATA, # mMARCO-NL, HotpotQA-NL, FEVER-NL, and LLM generated data + citation=E5_NL_CITATION, +) + +e5_nl_large = ModelMeta( + loader=sentence_transformers_loader, + loader_kwargs=dict( + model_prompts=model_prompts, + ), + name="clips/e5-large-trm-nl", + languages=["nld-Latn"], + open_weights=True, + revision="683333f86ed9eb3699b5567f0fdabeb958d412b0", + release_date="2025-09-23", + n_parameters=355_000_000, + memory_usage_mb=1355, + embed_dim=1024, + license="mit", + max_tokens=514, + reference="https://huggingface.co/clips/e5-large-trm-nl", + similarity_fn_name=ScoringFunction.COSINE, + framework=["Sentence Transformers", "PyTorch"], + use_instructions=True, + public_training_code="https://github.com/ELotfi/e5-nl", + public_training_data="https://huggingface.co/collections/clips/beir-nl", + training_datasets=ME5_TRAINING_DATA, # mMARCO-NL, HotpotQA-NL, FEVER-NL, and LLM generated data + adapted_from="intfloat/multilingual-e5-large", + citation=E5_NL_CITATION, +) diff --git a/mteb/models/model_implementations/e5_models.py b/mteb/models/model_implementations/e5_models.py index 44782ab0bc..a989bcdca5 100644 --- a/mteb/models/model_implementations/e5_models.py +++ b/mteb/models/model_implementations/e5_models.py @@ -126,18 +126,6 @@ } """ -E5_NL_CITATION = """ -@misc{banar2025mtebnle5nlembeddingbenchmark, - archiveprefix = {arXiv}, - author = {Nikolay Banar and Ehsan Lotfi and Jens Van Nooten and Cristina Arhiliuc and Marija Kliocaite and Walter Daelemans}, - eprint = {2509.12340}, - primaryclass = {cs.CL}, - title = {MTEB-NL and E5-NL: Embedding Benchmark and Models for Dutch}, - url = {https://arxiv.org/abs/2509.12340}, - year = {2025}, -} -""" - model_prompts = { PromptType.query.value: "query: ", PromptType.document.value: "passage: ", @@ -411,81 +399,3 @@ training_datasets=E5_TRAINING_DATA, citation=E5_CITATION, ) - -e5_nl_small = ModelMeta( - loader=sentence_transformers_loader, - loader_kwargs=dict( - model_prompts=model_prompts, - ), - name="clips/e5-small-trm-nl", - languages=["nld-Latn"], - open_weights=True, - revision="0243664a6c5e12eef854b091eb283e51833c3e9f", - release_date="2025-09-23", - n_parameters=40_800_000, - memory_usage_mb=78, - embed_dim=384, - license="mit", - max_tokens=512, - reference="https://huggingface.co/clips/e5-small-trm-nl", - similarity_fn_name=ScoringFunction.COSINE, - framework=["Sentence Transformers", "PyTorch"], - use_instructions=True, - public_training_code="https://github.com/ELotfi/e5-nl", # couldn't find - public_training_data=None, - training_datasets=ME5_TRAINING_DATA, # mMARCO-NL, HotpotQA-NL, FEVER-NL, and LLM generated data - adapted_from="intfloat/multilingual-e5-small", - citation=E5_NL_CITATION, -) - -e5_nl_base = ModelMeta( - loader=sentence_transformers_loader, - loader_kwargs=dict( - model_prompts=model_prompts, - ), - name="clips/e5-base-trm-nl", - languages=["nld-Latn"], - open_weights=True, - revision="6bd5722f236da48b4b8bcb28cc1fc478f7089956", - release_date="2025-09-23", - n_parameters=124_400_000, - memory_usage_mb=237, - embed_dim=768, - license="mit", - max_tokens=514, - reference="https://huggingface.co/clips/e5-base-trm-nl", - similarity_fn_name=ScoringFunction.COSINE, - framework=["Sentence Transformers", "PyTorch"], - use_instructions=True, - public_training_code="https://github.com/ELotfi/e5-nl", - public_training_data=None, - adapted_from="intfloat/multilingual-e5-base", - training_datasets=ME5_TRAINING_DATA, # mMARCO-NL, HotpotQA-NL, FEVER-NL, and LLM generated data - citation=E5_NL_CITATION, -) - -e5_nl_large = ModelMeta( - loader=sentence_transformers_loader, - loader_kwargs=dict( - model_prompts=model_prompts, - ), - name="clips/e5-large-trm-nl", - languages=["nld-Latn"], - open_weights=True, - revision="683333f86ed9eb3699b5567f0fdabeb958d412b0", - release_date="2025-09-23", - n_parameters=355_000_000, - memory_usage_mb=1355, - embed_dim=1024, - license="mit", - max_tokens=514, - reference="https://huggingface.co/clips/e5-large-trm-nl", - similarity_fn_name=ScoringFunction.COSINE, - framework=["Sentence Transformers", "PyTorch"], - use_instructions=True, - public_training_code="https://github.com/ELotfi/e5-nl", - public_training_data=None, - training_datasets=ME5_TRAINING_DATA, # mMARCO-NL, HotpotQA-NL, FEVER-NL, and LLM generated data - adapted_from="intfloat/multilingual-e5-large", - citation=E5_NL_CITATION, -)