From 4a1be8b90a83e09ad373798ce4e368a9a07eea83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Kardos?= Date: Tue, 26 Nov 2024 09:04:09 +0100 Subject: [PATCH 1/8] Made get_scores error tolerant --- mteb/load_results/benchmark_results.py | 56 ++++++++++++++++---------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/mteb/load_results/benchmark_results.py b/mteb/load_results/benchmark_results.py index bf3fa5fe92..e267b24774 100644 --- a/mteb/load_results/benchmark_results.py +++ b/mteb/load_results/benchmark_results.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import warnings from collections import defaultdict from collections.abc import Iterable from pathlib import Path @@ -10,7 +11,8 @@ from pydantic import BaseModel, ConfigDict from mteb.abstasks.AbsTask import AbsTask, ScoresDict -from mteb.abstasks.TaskMetadata import ISO_LANGUAGE_SCRIPT, TASK_DOMAIN, TASK_TYPE +from mteb.abstasks.TaskMetadata import (ISO_LANGUAGE_SCRIPT, TASK_DOMAIN, + TASK_TYPE) from mteb.languages import ISO_LANGUAGE from mteb.load_results.task_results import TaskResult from mteb.models.overview import get_model_metas @@ -225,33 +227,43 @@ def get_scores( entries = [] if format == "wide": for model_res in self: - model_scores = model_res.get_scores( - splits=splits, - languages=languages, - scripts=scripts, - getter=getter, - aggregation=aggregation, - format="wide", - ) - entries.append( - { - "model": model_res.model_name, - "revision": model_res.model_revision, - **model_scores, - } - ) - if format == "long": - for model_res in self: - entries.extend( - model_res.get_scores( + try: + model_scores = model_res.get_scores( splits=splits, languages=languages, scripts=scripts, getter=getter, aggregation=aggregation, - format="long", + format="wide", + ) + entries.append( + { + "model": model_res.model_name, + "revision": model_res.model_revision, + **model_scores, + } + ) + except Exception as e: + warnings.warn( + f"Couldn't get scores for {model_res.model_name}({model_res.model_revision}), due to: {e}" + ) + if format == "long": + for model_res in self: + try: + entries.extend( + model_res.get_scores( + splits=splits, + languages=languages, + scripts=scripts, + getter=getter, + aggregation=aggregation, + format="long", + ) + ) + except Exception as e: + warnings.warn( + f"Couldn't get scores for {model_res.model_name}({model_res.model_revision}), due to: {e}" ) - ) return entries def __iter__(self): From b9285094e52f3b99c415d199cb7fb5b083f52ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Kardos?= Date: Wed, 27 Nov 2024 13:47:44 +0100 Subject: [PATCH 2/8] Added join_revisions, made get_scores failsafe --- mteb/load_results/benchmark_results.py | 122 +++++++++++++++++++------ 1 file changed, 93 insertions(+), 29 deletions(-) diff --git a/mteb/load_results/benchmark_results.py b/mteb/load_results/benchmark_results.py index e267b24774..68d246966d 100644 --- a/mteb/load_results/benchmark_results.py +++ b/mteb/load_results/benchmark_results.py @@ -5,9 +5,11 @@ from collections import defaultdict from collections.abc import Iterable from pathlib import Path -from typing import Any, Callable, Literal +from typing import Any, Callable, Literal, Optional import numpy as np +import pandas as pd +from packaging.version import InvalidVersion, Version from pydantic import BaseModel, ConfigDict from mteb.abstasks.AbsTask import AbsTask, ScoresDict @@ -91,36 +93,45 @@ def get_scores( format: Literal["wide", "long"] = "wide", ) -> dict | list: if format == "wide": - scores = { - res.task_name: res.get_score( - splits=splits, - languages=languages, - scripts=scripts, - getter=getter, - aggregation=aggregation, - ) - for res in self.task_results - } - return scores - if format == "long": - entries = [] - for task_res in self.task_results: - entry = dict( # noqa - model_name=self.model_name, - model_revision=self.model_revision, - task_name=task_res.task_name, - score=task_res.get_score( + scores = {} + for res in self.task_results: + try: + scores[res.task_name] = res.get_score( splits=splits, languages=languages, + scripts=scripts, getter=getter, aggregation=aggregation, - ), - mteb_version=task_res.mteb_version, - dataset_revision=task_res.dataset_revision, - evaluation_time=task_res.evaluation_time, - kg_co2_emissions=task_res.kg_co2_emissions, - ) - entries.append(entry) + ) + except Exception as e: + warnings.warn( + f"Couldn't get scores for {res.task_name} due to {e}." + ) + return scores + if format == "long": + entries = [] + for task_res in self.task_results: + try: + entry = dict( # noqa + model_name=self.model_name, + model_revision=self.model_revision, + task_name=task_res.task_name, + score=task_res.get_score( + splits=splits, + languages=languages, + getter=getter, + aggregation=aggregation, + ), + mteb_version=task_res.mteb_version, + dataset_revision=task_res.dataset_revision, + evaluation_time=task_res.evaluation_time, + kg_co2_emissions=task_res.kg_co2_emissions, + ) + entries.append(entry) + except Exception as e: + warnings.warn( + f"Couldn't get scores for {task_res.task_name} due to {e}." + ) return entries def __iter__(self): @@ -200,6 +211,8 @@ def filter_models( n_parameters_range: tuple[int | None, int | None] = (None, None), use_instructions: bool | None = None, ) -> BenchmarkResults: + if model_names is None: + model_names = [model_res.model_name for model_res in self] model_metas = get_model_metas( model_names=model_names, languages=languages, @@ -208,13 +221,64 @@ def filter_models( n_parameters_range=n_parameters_range, use_instructions=use_instructions, ) - model_revision_pairs = {(meta.name, meta.revision) for meta in model_metas} + models = {meta.name for meta in model_metas} + # model_revision_pairs = {(meta.name, meta.revision) for meta in model_metas} new_model_results = [] for model_res in self: - if (model_res.model_name, model_res.model_revision) in model_revision_pairs: + if model_res.model_name in models: new_model_results.append(model_res) return type(self).model_construct(model_results=new_model_results) + def join_revisions(self): + def parse_version(version_str: str) -> Optional[Version]: + try: + return Version(version_str) + except (InvalidVersion, TypeError): + return None + + def keep_best(group: pd.DataFrame) -> pd.DataFrame: + is_main_revision = group["revision"] == group["main_revision"] + if is_main_revision.sum() == 1: + return group[is_main_revision] + if group["mteb_version"].notna().any(): + group = group.dropna(subset=["mteb_version"]) + group = group.sort_values("mteb_version", ascending=False) + return group.head(n=1) + return group.head(n=1) + + records = [] + for model_result in self: + for task_result in model_result: + records.append( + dict( + model=model_result.model_name, + revision=model_result.model_revision, + task_name=task_result.task_name, + mteb_version=task_result.mteb_version, + task_result=task_result, + ) + ) + task_df = pd.DataFrame.from_records(records) + model_to_main_revision = { + meta.name: meta.revision for meta in get_model_metas() + } + task_df["main_revision"] = task_df["model"].map(model_to_main_revision) + task_df["mteb_version"] = task_df["mteb_version"].map(parse_version) + task_df = ( + task_df.groupby(["model", "task_name"]) + .apply(keep_best) + .reset_index(drop=True) + ) + model_results = [] + for (model, model_revision), group in task_df.groupby(["model", "revision"]): + model_result = ModelResult.model_construct( + model_name=model, + model_revision=model_revision, + task_results=list(group["task_result"]), + ) + model_results.append(model_result) + return BenchmarkResults.model_construct(model_results=model_results) + def get_scores( self, splits: list[Split] | None = None, From ddee7907c5ab848824cc60238c1a253726bb29ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Kardos?= Date: Wed, 27 Nov 2024 13:48:25 +0100 Subject: [PATCH 3/8] Fetching metadata fixed fr HF models --- mteb/models/overview.py | 74 +++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/mteb/models/overview.py b/mteb/models/overview.py index 91b84e38d8..20ca1012b0 100644 --- a/mteb/models/overview.py +++ b/mteb/models/overview.py @@ -2,37 +2,22 @@ import logging from collections.abc import Iterable +from functools import lru_cache from typing import Any +from huggingface_hub import ModelCard from sentence_transformers import SentenceTransformer from mteb.encoder_interface import Encoder from mteb.model_meta import ModelMeta -from mteb.models import ( - bge_models, - bm25, - cohere_models, - e5_instruct, - e5_models, - google_models, - gritlm_models, - gte_models, - jina_models, - llm2vec_models, - mxbai_models, - nomic_models, - openai_models, - promptriever_models, - repllama_models, - rerankers_custom, - rerankers_monot5_based, - ru_sentence_models, - salesforce_models, - sentence_transformers_models, - stella_models, - uae_models, - voyage_models, -) +from mteb.models import (bge_models, bm25, cohere_models, e5_instruct, + e5_models, google_models, gritlm_models, gte_models, + jina_models, llm2vec_models, mxbai_models, + nomic_models, openai_models, promptriever_models, + repllama_models, rerankers_custom, + rerankers_monot5_based, ru_sentence_models, + salesforce_models, sentence_transformers_models, + stella_models, uae_models, voyage_models) logger = logging.getLogger(__name__) @@ -152,21 +137,46 @@ def get_model_meta(model_name: str, revision: str | None = None) -> ModelMeta: return MODEL_REGISTRY[model_name] else: # assume it is a sentence-transformers model logger.info( - "Model not found in model registry, assuming it is a sentence-transformers model." + "Model not found in model registry, assuming it is on HF Hub model." ) logger.info( - f"Attempting to extract metadata by loading the model ({model_name}) using sentence-transformers." - ) - model = SentenceTransformer( - model_name, revision=revision, trust_remote_code=True + f"Attempting to extract metadata by loading the model ({model_name}) using HuggingFace." ) - meta = model_meta_from_sentence_transformers(model) - + meta = model_meta_from_hf_hub(model_name) meta.revision = revision meta.name = model_name return meta +@lru_cache +def model_meta_from_hf_hub(model_name: str) -> ModelMeta: + try: + card = ModelCard.load(model_name) + card_data = card.data.to_dict() + frameworks = ["PyTorch"] + if card_data.get("library_name", None) == "sentence-transformers": + frameworks.append("Sentence Transformers") + return ModelMeta( + name=model_name, + revision=None, + # TODO + release_date=None, + # TODO: We need a mapping between conflicting language codes + languages=None, + license=card_data.get("license", None), + framework=frameworks, + public_training_data=bool(card_data.get("datasets", None)), + ) + except Exception as e: + logger.warning(f"Failed to extract metadata from model: {e}.") + return ModelMeta( + name=None, + revision=None, + languages=None, + release_date=None, + ) + + def model_meta_from_sentence_transformers(model: SentenceTransformer) -> ModelMeta: try: name = ( From 48637d4e988a731d7711ebbb55e0c3fb8b5f7108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Kardos?= Date: Wed, 27 Nov 2024 13:49:04 +0100 Subject: [PATCH 4/8] Added failsafe metadata fetching to leaderboard code --- mteb/leaderboard/table.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mteb/leaderboard/table.py b/mteb/leaderboard/table.py index c965a7f682..734de2c238 100644 --- a/mteb/leaderboard/table.py +++ b/mteb/leaderboard/table.py @@ -98,6 +98,13 @@ def get_means_per_types(df: pd.DataFrame) -> pd.DataFrame: return pd.DataFrame.from_records(records) +def failsafe_get_model_meta(model_name): + try: + return get_model_meta(model_name) + except Exception as e: + return None + + def scores_to_tables( scores_long: list[dict], search_query: str | None = None ) -> tuple[gr.DataFrame, gr.DataFrame]: @@ -132,7 +139,8 @@ def scores_to_tables( joint_table["borda_rank"] = get_borda_rank(per_task) joint_table = joint_table.reset_index() joint_table = joint_table.drop(columns=["model_revision"]) - model_metas = joint_table["model_name"].map(get_model_meta) + model_metas = joint_table["model_name"].map(failsafe_get_model_meta) + joint_table = joint_table[model_metas.notna()] joint_table["model_link"] = model_metas.map(lambda m: m.reference) joint_table.insert( 1, From 731945bbe4f1f55c858ae587cd5321d96039559a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Kardos?= Date: Wed, 27 Nov 2024 13:49:27 +0100 Subject: [PATCH 5/8] Added revision joining to leaderboard app --- mteb/leaderboard/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mteb/leaderboard/app.py b/mteb/leaderboard/app.py index 8a5eb961c1..913b23c829 100644 --- a/mteb/leaderboard/app.py +++ b/mteb/leaderboard/app.py @@ -81,7 +81,7 @@ def update_task_info(task_names: str) -> gr.DataFrame: return gr.DataFrame(df, datatype=["markdown"] + ["str"] * (len(df.columns) - 1)) -all_results = load_results().filter_models() +all_results = load_results().join_revisions() # Model sizes in million parameters min_model_size, max_model_size = 0, 10_000 From ff9124681675eb00db1444a1336d4ab2c893736c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Kardos?= Date: Wed, 27 Nov 2024 14:43:23 +0100 Subject: [PATCH 6/8] fix From c9aa2c0e3368b00367d69008c66fd3c161c97548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Kardos?= Date: Wed, 27 Nov 2024 14:44:59 +0100 Subject: [PATCH 7/8] Only show models that have metadata, when filter_models is called --- mteb/load_results/benchmark_results.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mteb/load_results/benchmark_results.py b/mteb/load_results/benchmark_results.py index 68d246966d..77f7fa02f6 100644 --- a/mteb/load_results/benchmark_results.py +++ b/mteb/load_results/benchmark_results.py @@ -211,8 +211,8 @@ def filter_models( n_parameters_range: tuple[int | None, int | None] = (None, None), use_instructions: bool | None = None, ) -> BenchmarkResults: - if model_names is None: - model_names = [model_res.model_name for model_res in self] + # if model_names is None: + # model_names = [model_res.model_name for model_res in self] model_metas = get_model_metas( model_names=model_names, languages=languages, From 20a62cdaf756faa2c207c41b9fe517d19b8069b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Kardos?= Date: Wed, 27 Nov 2024 14:52:17 +0100 Subject: [PATCH 8/8] Ran linting --- mteb/load_results/benchmark_results.py | 3 +-- mteb/models/overview.py | 33 +++++++++++++++++++------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/mteb/load_results/benchmark_results.py b/mteb/load_results/benchmark_results.py index 77f7fa02f6..756024a4e6 100644 --- a/mteb/load_results/benchmark_results.py +++ b/mteb/load_results/benchmark_results.py @@ -13,8 +13,7 @@ from pydantic import BaseModel, ConfigDict from mteb.abstasks.AbsTask import AbsTask, ScoresDict -from mteb.abstasks.TaskMetadata import (ISO_LANGUAGE_SCRIPT, TASK_DOMAIN, - TASK_TYPE) +from mteb.abstasks.TaskMetadata import ISO_LANGUAGE_SCRIPT, TASK_DOMAIN, TASK_TYPE from mteb.languages import ISO_LANGUAGE from mteb.load_results.task_results import TaskResult from mteb.models.overview import get_model_metas diff --git a/mteb/models/overview.py b/mteb/models/overview.py index 20ca1012b0..f54a085d02 100644 --- a/mteb/models/overview.py +++ b/mteb/models/overview.py @@ -10,14 +10,31 @@ from mteb.encoder_interface import Encoder from mteb.model_meta import ModelMeta -from mteb.models import (bge_models, bm25, cohere_models, e5_instruct, - e5_models, google_models, gritlm_models, gte_models, - jina_models, llm2vec_models, mxbai_models, - nomic_models, openai_models, promptriever_models, - repllama_models, rerankers_custom, - rerankers_monot5_based, ru_sentence_models, - salesforce_models, sentence_transformers_models, - stella_models, uae_models, voyage_models) +from mteb.models import ( + bge_models, + bm25, + cohere_models, + e5_instruct, + e5_models, + google_models, + gritlm_models, + gte_models, + jina_models, + llm2vec_models, + mxbai_models, + nomic_models, + openai_models, + promptriever_models, + repllama_models, + rerankers_custom, + rerankers_monot5_based, + ru_sentence_models, + salesforce_models, + sentence_transformers_models, + stella_models, + uae_models, + voyage_models, +) logger = logging.getLogger(__name__)