diff --git a/mteb/cache.py b/mteb/cache.py index 0b582d2337..2cfb327741 100644 --- a/mteb/cache.py +++ b/mteb/cache.py @@ -351,8 +351,20 @@ def _download_cached_results_from_branch( response = requests.get(url, timeout=timeout) response.raise_for_status() - # Validate content-type header + # Check if this is a Git LFS pointer file content_type = response.headers.get("content-type", "").lower() + if ( + content_type == "text/plain; charset=utf-8" + and b"git-lfs" in response.content + ): + # Try Git LFS media URL instead + media_url = f"https://media.githubusercontent.com/media/{repo_path}/{branch}/{filename}" + logger.info(f"Detected Git LFS file, trying media URL: {media_url}") + response = requests.get(media_url, timeout=timeout) + response.raise_for_status() + content_type = response.headers.get("content-type", "").lower() + + # Validate content-type header expected_content_types = [ "application/gzip", "application/octet-stream", diff --git a/mteb/leaderboard/app.py b/mteb/leaderboard/app.py index 3e985b64e8..838056bc85 100644 --- a/mteb/leaderboard/app.py +++ b/mteb/leaderboard/app.py @@ -1154,6 +1154,13 @@ def update_tables( logging.getLogger("mteb.results.benchmark_results").setLevel(logging.ERROR) warnings.filterwarnings("ignore", message="Couldn't get scores for .* due to .*") + warnings.filterwarnings("ignore", message="Could not get source model: .*") + warnings.filterwarnings( + "ignore", message="No scores data available. Returning empty DataFrame." + ) + warnings.filterwarnings("ignore", message="Main score .* not found in scores") + warnings.filterwarnings("ignore", message=".*: Missing subsets .* for split .*") + warnings.filterwarnings("ignore", message=".*: Missing splits .*") app = get_leaderboard_app() diff --git a/uv.lock b/uv.lock index 200cca8101..aba127a6a1 100644 --- a/uv.lock +++ b/uv.lock @@ -5802,7 +5802,7 @@ wheels = [ [[package]] name = "mteb" -version = "2.5.3" +version = "2.6.1" source = { editable = "." } dependencies = [ { name = "datasets" }, @@ -5945,6 +5945,12 @@ dev = [ { name = "bibtexparser" }, { name = "gitpython" }, { name = "iso639" }, + { name = "mkdocs" }, + { name = "mkdocs-bibtex" }, + { name = "mkdocs-exclude" }, + { name = "mkdocs-include-dir-to-nav" }, + { name = "mkdocs-material" }, + { name = "mkdocstrings-python" }, { name = "mypy" }, { name = "pandas-stubs" }, { name = "pillow" }, @@ -5956,6 +5962,7 @@ dev = [ { name = "ruff" }, { name = "scipy-stubs", version = "1.15.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-4-mteb-blip2' and extra == 'extra-4-mteb-timm') or (extra == 'extra-4-mteb-colpali-engine' and extra == 'extra-4-mteb-llama-embed-nemotron') or (extra == 'extra-4-mteb-colpali-engine' and extra == 'extra-4-mteb-llm2vec') or (extra == 'extra-4-mteb-colpali-engine' and extra == 'extra-4-mteb-pylate') or (extra == 'extra-4-mteb-colqwen3' and extra == 'extra-4-mteb-llama-embed-nemotron') or (extra == 'extra-4-mteb-colqwen3' and extra == 'extra-4-mteb-llm2vec') or (extra == 'extra-4-mteb-colqwen3' and extra == 'extra-4-mteb-pylate') or (extra == 'extra-4-mteb-jina-v4' and extra == 'extra-4-mteb-llama-embed-nemotron') or (extra == 'extra-4-mteb-jina-v4' and extra == 'extra-4-mteb-llm2vec') or (extra == 'extra-4-mteb-llama-embed-nemotron' and extra == 'extra-4-mteb-llm2vec') or (extra == 'extra-4-mteb-llm2vec' and extra == 'extra-4-mteb-model2vec') or (extra == 'extra-4-mteb-llm2vec' and extra == 'extra-4-mteb-pylate')" }, { name = "scipy-stubs", version = "1.16.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-4-mteb-blip2' and extra == 'extra-4-mteb-timm') or (extra == 'extra-4-mteb-colpali-engine' and extra == 'extra-4-mteb-llama-embed-nemotron') or (extra == 'extra-4-mteb-colpali-engine' and extra == 'extra-4-mteb-llm2vec') or (extra == 'extra-4-mteb-colpali-engine' and extra == 'extra-4-mteb-pylate') or (extra == 'extra-4-mteb-colqwen3' and extra == 'extra-4-mteb-llama-embed-nemotron') or (extra == 'extra-4-mteb-colqwen3' and extra == 'extra-4-mteb-llm2vec') or (extra == 'extra-4-mteb-colqwen3' and extra == 'extra-4-mteb-pylate') or (extra == 'extra-4-mteb-jina-v4' and extra == 'extra-4-mteb-llama-embed-nemotron') or (extra == 'extra-4-mteb-jina-v4' and extra == 'extra-4-mteb-llm2vec') or (extra == 'extra-4-mteb-llama-embed-nemotron' and extra == 'extra-4-mteb-llm2vec') or (extra == 'extra-4-mteb-llm2vec' and extra == 'extra-4-mteb-model2vec') or (extra == 'extra-4-mteb-llm2vec' and extra == 'extra-4-mteb-pylate')" }, + { name = "tabulate" }, { name = "types-cachetools" }, { name = "types-colorama" }, { name = "types-defusedxml" }, @@ -6000,6 +6007,7 @@ test = [ { name = "pytest-coverage" }, { name = "pytest-rerunfailures" }, { name = "pytest-xdist" }, + { name = "tabulate" }, ] typing = [ { name = "mypy" }, @@ -6097,6 +6105,12 @@ dev = [ { name = "bibtexparser", specifier = ">=1.4.3" }, { name = "gitpython", specifier = ">=3.0.0" }, { name = "iso639", specifier = ">=0.1.4" }, + { name = "mkdocs", specifier = ">=1.6.1" }, + { name = "mkdocs-bibtex", specifier = ">=2.16.2" }, + { name = "mkdocs-exclude", specifier = ">=1.0.2" }, + { name = "mkdocs-include-dir-to-nav", specifier = ">=1.2.0" }, + { name = "mkdocs-material", specifier = ">=9.5.47" }, + { name = "mkdocstrings-python", specifier = ">=1.18.2" }, { name = "mypy", specifier = "==1.19.1" }, { name = "pandas-stubs", specifier = ">=2.3.2.250926" }, { name = "pillow", specifier = ">=12.0.0" }, @@ -6107,6 +6121,7 @@ dev = [ { name = "pytest-xdist", specifier = ">=3.6.1,<3.7.0" }, { name = "ruff", specifier = "==0.14.0" }, { name = "scipy-stubs", specifier = ">=1.15.3.0" }, + { name = "tabulate", specifier = ">=0.9.0" }, { name = "types-cachetools", specifier = ">=6.2.0.20250827" }, { name = "types-colorama", specifier = ">=0.4.15.20250801" }, { name = "types-defusedxml", specifier = ">=0.7.0.20250822" }, @@ -6151,6 +6166,7 @@ test = [ { name = "pytest-coverage", specifier = ">=0.0" }, { name = "pytest-rerunfailures", specifier = ">=15.0,<16.0" }, { name = "pytest-xdist", specifier = ">=3.6.1,<3.7.0" }, + { name = "tabulate", specifier = ">=0.9.0" }, ] typing = [ { name = "mypy", specifier = "==1.19.1" },