fix: Add Model2vec

mteb/load_results/task_results.py

@@ -511,8 +511,8 @@ def validate_and_filter_scores(self, task: AbsTask | None = None) -> AbsTask:
                 new_scores[split].append(_scores)
                 seen_subsets.add(_scores["hf_subset"])
             if seen_subsets != hf_subsets:
-                raise ValueError(
-                    f"Missing subsets {hf_subsets - seen_subsets} for split {split}"
+                logger.warning(
+                    f"{task.metadata.name}: Missing subsets {hf_subsets - seen_subsets} for split {split}"
                 )
             seen_splits.add(split)
         if seen_splits != set(splits):

mteb/models/model2vec_models.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Sequence
+from functools import partial
+from typing import Any
+
+import numpy as np
+
+from mteb.encoder_interface import PromptType
+from mteb.model_meta import ModelMeta
+
+from .wrapper import Wrapper
+
+logger = logging.getLogger(__name__)
+
+
+class Model2VecWrapper(Wrapper):
+    def __init__(
+        self,
+        model_name: str,
+        **kwargs,
+    ) -> None:
+        """Wrapper for Model2Vec models.
+
+        Args:
+            model: The Model2Vec model to use. Can be a string (model name), a SentenceTransformer model, or a CrossEncoder model.
+        """
+        try:
+            from model2vec import StaticModel
+        except ModuleNotFoundError as e:
+            raise ModuleNotFoundError(
+                "To use the Model2Vec models `model2vec` is required. Please install it with `pip install mteb[model2vec]`."
+            ) from e
+
+        self.model_name = model_name
+        self.static_model = StaticModel.from_pretrained(self.model)
+
+    def encode(
+        self,
+        sentences: Sequence[str],
+        **kwargs: Any,
+    ) -> np.ndarray:
+        """Encodes the given sentences using the encoder.
+
+        Args:
+            sentences: The sentences to encode.
+            **kwargs: Additional arguments to pass to the encoder.
+
+        Returns:
+            The encoded sentences.
+        """
+        return self.static_model.encode(sentences)
+
+
+m2v_base_glove_subword = ModelMeta(
+    loader=partial(
+        Model2VecWrapper,
+        model_name="minishlab/M2V_base_glove_subword",
+    ),
+    name="minishlab/M2V_base_glove_subword",
+    languages=["eng_Latn"],
+    open_weights=True,
+    revision="5f4f5ca159b7321a8b39739bba0794fa0debddf4",
+    release_date="2024-10-14",  # blog post date
+    n_parameters=103 * 1e6,
+    max_tokens=1000000,  # I took this from the current leaderboard, I'm not sure if it's unlimited in theory, tough
+    embed_dim=256,
+    license="mit",
+    similarity_fn_name="cosine",
+    framework=["PyTorch"],
+    reference="https://huggingface.co/minishlab/M2V_base_glove_subword",
+    use_instructions=False,
+    adapted_from="BAAI/bge-base-en-v1.5",
+    superseded_by=None,
+)
+
+
+m2v_base_glove = ModelMeta(
+    loader=partial(
+        Model2VecWrapper,
+        model_name="minishlab/M2V_base_glove",
+    ),
+    name="minishlab/M2V_base_glove",
+    languages=["eng_Latn"],
+    open_weights=True,
+    revision="38ebd7f10f71e67fa8db898290f92b82e9cfff2b",
+    release_date="2024-10-14",  # blog post date
+    n_parameters=102 * 1e6,
+    max_tokens=1000000,
+    embed_dim=256,
+    license="mit",
+    similarity_fn_name="cosine",
+    framework=["PyTorch"],
+    reference="https://huggingface.co/minishlab/M2V_base_glove",
+    use_instructions=False,
+    adapted_from="BAAI/bge-base-en-v1.5",
+    superseded_by=None,
+)
+
+m2v_base_output = ModelMeta(
+    loader=partial(
+        Model2VecWrapper,
+        model_name="minishlab/M2V_base_output",
+    ),
+    name="minishlab/M2V_base_output",
+    languages=["eng_Latn"],
+    open_weights=True,
+    revision="02460ae401a22b09d2c6652e23371398329551e2",
+    release_date="2024-10-14",  # blog post date
+    n_parameters=7.56 * 1e6,
+    max_tokens=1000000,
+    embed_dim=256,
+    license="mit",
+    similarity_fn_name="cosine",
+    framework=["PyTorch"],
+    reference="https://huggingface.co/minishlab/M2V_base_output",
+    use_instructions=False,
+    adapted_from="BAAI/bge-base-en-v1.5",
+    superseded_by=None,
+)
+
+m2v_multilingual_output = ModelMeta(
+    loader=partial(
+        Model2VecWrapper,
+        model_name="minishlab/M2V_multilingual_output",
+    ),
+    name="minishlab/M2V_multilingual_output",
+    languages=["eng_Latn"],
+    open_weights=True,
+    revision="2cf4ec4e1f51aeca6c55cf9b93097d00711a6305",
+    release_date="2024-10-14",  # blog post date
+    n_parameters=128 * 1e6,
+    max_tokens=1000000,
+    embed_dim=256,
+    license="mit",
+    similarity_fn_name="cosine",
+    framework=["PyTorch"],
+    reference="https://huggingface.co/minishlab/M2V_multilingual_output",
+    use_instructions=False,
+    adapted_from="sentence-transformers/LaBSE",
+    superseded_by=None,
+)
+
+potion_base_2m = ModelMeta(
+    loader=partial(
+        Model2VecWrapper,
+        model_name="minishlab/potion-base-2M",
+    ),
+    name="minishlab/potion-base-2M",
+    languages=["eng_Latn"],
+    open_weights=True,
+    revision="86db093558fbced2072b929eb1690bce5272bd4b",
+    release_date="2024-10-14",  # blog post date
+    n_parameters=2 * 1e6,
+    max_tokens=1000000,
+    embed_dim=256,
+    license="mit",
+    similarity_fn_name="cosine",
+    framework=["PyTorch"],
+    reference="https://huggingface.co/minishlab/potion-base-2M",
+    use_instructions=False,
+    adapted_from="BAAI/bge-base-en-v1.5",
+    superseded_by=None,
+)
+
+potion_base_4m = ModelMeta(
+    loader=partial(
+        Model2VecWrapper,
+        model_name="minishlab/potion-base-4M",
+    ),
+    name="minishlab/potion-base-4M",
+    languages=["eng_Latn"],
+    open_weights=True,
+    revision="81b1802ada41afcd0987a37dc15e569c9fa76f04",
+    release_date="2024-10-14",  # blog post date
+    n_parameters=3.78 * 1e6,
+    max_tokens=1000000,
+    embed_dim=256,
+    license="mit",
+    similarity_fn_name="cosine",
+    framework=["PyTorch"],
+    reference="https://huggingface.co/minishlab/potion-base-4M",
+    use_instructions=False,
+    adapted_from="BAAI/bge-base-en-v1.5",
+    superseded_by=None,
+)
+
+potion_base_8m = ModelMeta(
+    loader=partial(
+        Model2VecWrapper,
+        model_name="minishlab/potion-base-8M",
+    ),
+    name="minishlab/potion-base-8M",
+    languages=["eng_Latn"],
+    open_weights=True,
+    revision="dcbec7aa2d52fc76754ac6291803feedd8c619ce",
+    release_date="2024-10-14",  # blog post date
+    n_parameters=7.56 * 1e6,
+    max_tokens=1000000,
+    embed_dim=256,
+    license="mit",
+    similarity_fn_name="cosine",
+    framework=["PyTorch"],
+    reference="https://huggingface.co/minishlab/potion-base-8M",
+    use_instructions=False,
+    adapted_from="BAAI/bge-base-en-v1.5",
+    superseded_by=None,
+)

mteb/models/overview.py

@@ -22,6 +22,7 @@
     gte_models,
     jina_models,
     llm2vec_models,
+    model2vec_models,
     mxbai_models,
     nomic_models,
     openai_models,
@@ -51,6 +52,7 @@
     gte_models,
     llm2vec_models,
     mxbai_models,
+    model2vec_models,
     nomic_models,
     openai_models,
     ru_sentence_models,

pyproject.toml

@@ -62,6 +62,7 @@ flagembedding = ["FlagEmbedding"]
 jina = ["einops>=0.8.0"]
 flash_attention = ["flash-attn>=2.6.3"]
 openai = ["openai>=1.41.0", "tiktoken>=0.8.0"]
+model2vec = ["model2vec>=0.3.0"]
 
 
 [tool.coverage.report]