feat: Add .from_hf_hub() and similar methods to ModelMeta

mteb/deprecated_evaluator.py

@@ -13,21 +13,11 @@
 from time import time
 from typing import TYPE_CHECKING, Any
 
-from mteb.abstasks.task_metadata import TaskCategory, TaskType
-from mteb.models.get_model_meta import (
-    _model_meta_from_cross_encoder,
-    _model_meta_from_sentence_transformers,
-)
-
-if sys.version_info >= (3, 13):
-    from warnings import deprecated
-else:
-    from typing_extensions import deprecated
-
 import datasets
 
 import mteb
 from mteb.abstasks import AbsTask
+from mteb.abstasks.task_metadata import TaskCategory, TaskType
 from mteb.benchmarks import Benchmark
 from mteb.models import (
     CrossEncoderWrapper,
@@ -39,6 +29,11 @@
 from mteb.results import TaskResult
 from mteb.types import ScoresDict
 
+if sys.version_info >= (3, 13):
+    from warnings import deprecated
+else:
+    from typing_extensions import deprecated
+
 if TYPE_CHECKING:
     from sentence_transformers import CrossEncoder, SentenceTransformer
 
@@ -669,9 +664,9 @@ def _get_model_meta(model: EncoderProtocol) -> ModelMeta:
         from sentence_transformers import CrossEncoder, SentenceTransformer
 
         if isinstance(model, CrossEncoder):
-            meta = _model_meta_from_cross_encoder(model)
+            meta = ModelMeta.from_cross_encoder(model)
         elif isinstance(model, SentenceTransformer):
-            meta = _model_meta_from_sentence_transformers(model)
+            meta = ModelMeta.from_sentence_transformer_model(model)
         else:
             meta = ModelMeta(
                 loader=None,

mteb/evaluate.py

@@ -2,7 +2,6 @@
 
 import logging
 from collections.abc import Iterable
-from copy import deepcopy
 from pathlib import Path
 from time import time
 from typing import TYPE_CHECKING, Any, cast
@@ -53,36 +52,6 @@ class OverwriteStrategy(HelpfulStrEnum):
     ONLY_CACHE = "only-cache"
 
 
-_empty_model_meta = ModelMeta(
-    loader=None,
-    name=None,
-    revision=None,
-    release_date=None,
-    languages=None,
-    framework=[],
-    similarity_fn_name=None,
-    n_parameters=None,
-    memory_usage_mb=None,
-    max_tokens=None,
-    embed_dim=None,
-    license=None,
-    open_weights=None,
-    public_training_code=None,
-    public_training_data=None,
-    use_instructions=None,
-    training_datasets=None,
-    modalities=[],
-)
-
-
-def _create_empty_model_meta() -> ModelMeta:
-    logger.warning("Model metadata is missing. Using empty metadata.")
-    meta = deepcopy(_empty_model_meta)
-    meta.revision = "no_revision_available"
-    meta.name = "no_model_name_available"
-    return meta
-
-
 def _sanitize_model(
     model: ModelMeta | MTEBModels | SentenceTransformer | CrossEncoder,
 ) -> tuple[MTEBModels | ModelMeta, ModelMeta, ModelName, Revision]:
@@ -101,9 +70,11 @@ def _sanitize_model(
     elif hasattr(model, "mteb_model_meta"):
         meta = model.mteb_model_meta  # type: ignore[attr-defined]
         if not isinstance(meta, ModelMeta):
-            meta = _create_empty_model_meta()
+            meta = ModelMeta.from_hf_hub(None)
     else:
-        meta = _create_empty_model_meta() if not isinstance(model, ModelMeta) else model
+        meta = (
+            ModelMeta.from_hf_hub(None) if not isinstance(model, ModelMeta) else model
+        )
 
     model_name = cast(str, meta.name)
     model_revision = cast(str, meta.revision)

mteb/models/get_model_meta.py

@@ -1,26 +1,15 @@
-from __future__ import annotations
-
 import difflib
 import logging
-import warnings
 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any
-
-from huggingface_hub import ModelCard
-from huggingface_hub.errors import RepositoryNotFoundError
+from typing import Any
 
 from mteb.abstasks import AbsTask
 from mteb.models import (
-    CrossEncoderWrapper,
     ModelMeta,
     MTEBModels,
-    sentence_transformers_loader,
 )
 from mteb.models.model_implementations import MODEL_REGISTRY
 
-if TYPE_CHECKING:
-    from sentence_transformers import CrossEncoder, SentenceTransformer
-
 logger = logging.getLogger(__name__)
 
 
@@ -101,24 +90,9 @@ def get_model(
     Returns:
         A model object
     """
-    from sentence_transformers import CrossEncoder, SentenceTransformer
-
     meta = get_model_meta(model_name, revision)
     model = meta.load_model(**kwargs)
 
-    # If revision not available in the modelmeta, try to extract it from sentence-transformers
-    if hasattr(model, "model") and isinstance(model.model, SentenceTransformer):  # type: ignore
-        _meta = _model_meta_from_sentence_transformers(model.model)  # type: ignore
-        if meta.revision is None:
-            meta.revision = _meta.revision if _meta.revision else meta.revision
-        if not meta.similarity_fn_name:
-            meta.similarity_fn_name = _meta.similarity_fn_name
-
-    elif isinstance(model, CrossEncoder):
-        _meta = _model_meta_from_cross_encoder(model.model)
-        if meta.revision is None:
-            meta.revision = _meta.revision if _meta.revision else meta.revision
-
     model.mteb_model_meta = meta  # type: ignore
     return model
 
@@ -148,12 +122,8 @@ def get_model_meta(
         logger.info(
             "Model not found in model registry. Attempting to extract metadata by loading the model ({model_name}) using HuggingFace."
         )
-        try:
-            meta = _model_meta_from_hf_hub(model_name)
-            meta.revision = revision
-            return meta
-        except RepositoryNotFoundError:
-            pass
+        meta = ModelMeta.from_hf_hub(model_name, revision)
+        return meta
 
     not_found_msg = f"Model '{model_name}' not found in MTEB registry"
     not_found_msg += " nor on the Huggingface Hub." if fetch_from_hf else "."
@@ -171,93 +141,3 @@ def get_model_meta(
             suggestion = f" Did you mean: '{close_matches[0]}'?"
 
     raise KeyError(not_found_msg + suggestion)
-
-
-def _model_meta_from_hf_hub(model_name: str) -> ModelMeta:
-    card = ModelCard.load(model_name)
-    card_data = card.data.to_dict()
-    frameworks = ["PyTorch"]
-    loader = None
-    if card_data.get("library_name", None) == "sentence-transformers":
-        frameworks.append("Sentence Transformers")
-        loader = sentence_transformers_loader
-    else:
-        msg = (
-            "Model library not recognized, defaulting to Sentence Transformers loader."
-        )
-        logger.warning(msg)
-        warnings.warn(msg)
-        loader = sentence_transformers_loader
-
-    revision = card_data.get("base_model_revision", None)
-    license = card_data.get("license", None)
-    return ModelMeta(
-        loader=loader,
-        name=model_name,
-        revision=revision,
-        release_date=None,
-        languages=None,
-        license=license,
-        framework=frameworks,  # type: ignore
-        training_datasets=None,
-        similarity_fn_name=None,
-        n_parameters=None,
-        memory_usage_mb=None,
-        max_tokens=None,
-        embed_dim=None,
-        open_weights=True,
-        public_training_code=None,
-        public_training_data=None,
-        use_instructions=None,
-    )
-
-
-def _model_meta_from_cross_encoder(model: CrossEncoder) -> ModelMeta:
-    return ModelMeta(
-        loader=CrossEncoderWrapper,
-        name=model.model.name_or_path,
-        revision=model.config._commit_hash,
-        release_date=None,
-        languages=None,
-        framework=["Sentence Transformers"],
-        similarity_fn_name=None,
-        n_parameters=None,
-        memory_usage_mb=None,
-        max_tokens=None,
-        embed_dim=None,
-        license=None,
-        open_weights=True,
-        public_training_code=None,
-        public_training_data=None,
-        use_instructions=None,
-        training_datasets=None,
-    )
-
-
-def _model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMeta:
-    name: str | None = (
-        model.model_card_data.model_name
-        if model.model_card_data.model_name
-        else model.model_card_data.base_model
-    )
-    embeddings_dim = model.get_sentence_embedding_dimension()
-    meta = ModelMeta(
-        loader=sentence_transformers_loader,
-        name=name,
-        revision=model.model_card_data.base_model_revision,
-        release_date=None,
-        languages=None,
-        framework=["Sentence Transformers"],
-        similarity_fn_name=None,
-        n_parameters=None,
-        memory_usage_mb=None,
-        max_tokens=None,
-        embed_dim=embeddings_dim,
-        license=None,
-        open_weights=True,
-        public_training_code=None,
-        public_training_data=None,
-        use_instructions=None,
-        training_datasets=None,
-    )
-    return meta