Upgrade to MTEB v2

.github/ISSUE_TEMPLATE/enhancement.yaml

@@ -8,4 +8,3 @@ body:
       description: Please provide a clear and concise description of the feature you would like to see added.
     validations:
       required: true
-

.github/ISSUE_TEMPLATE/eval_request.yaml

@@ -19,7 +19,7 @@ body:
     id: contribute
     attributes:
       label: Are you interested in contributing to the evaluation of this model?
-      description: By default MTEB maintainters will only handle evaluation on private subsets due to resource constraints. If you are interested in contributing to the evaluation, please let us know. 
+      description: By default MTEB maintainers will only handle evaluation on private subsets due to resource constraints. If you are interested in contributing to the evaluation, please let us know.
       options:
         - "Yes"
         - "No"
@@ -30,4 +30,4 @@ body:
       description: If you are unsure, please check using mteb model registry (e.g. using `mteb.get_model_meta("model_id")`).
       options:
         - "Yes"
-        - "No"
+        - "No"

.github/workflows/dataset_loading.yml

@@ -1,27 +1,34 @@
 name: Datasets available on HuggingFace
-
 on:
   push:
     branches: [main]
+  pull_request:
+    paths:
+      - "mteb/tasks/**.py"
 
 jobs:
-  extract-and-run:
+  dataset-loading-check:
     runs-on: ubuntu-latest
-
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
-
+      uses: actions/checkout@v6
+    
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v6
       with:
-        python-version: '3.10'
+        python-version: '3.11'
         cache: 'pip'
-
+    
     - name: Install dependencies
       run: |
-          make install-for-tests
-
+        make install-for-tests
+    
     - name: Run dataset loading tests
+      env:
+        HF_TOKEN: ${{ secrets.HF_TOKEN }}
       run: |
-        make dataset-load-test
+        if [ "${{ github.event_name }}" == "pull_request" ]; then
+          make dataset-load-test-pr BASE_BRANCH=${{ github.event.pull_request.base.ref }}
+        else
+          make dataset-load-test
+        fi

.github/workflows/documentation.yml

@@ -0,0 +1,33 @@
+name: Documentation
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.10"
+
+      - name: Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e . --group docs
+
+      - name: Build and Deploy
+        if: github.event_name == 'push'
+        run: |
+          make build-docs-overview
+          mkdocs gh-deploy --force
+
+      - name: Build
+        if: github.event_name == 'pull_request'
+        run: make build-docs

.github/workflows/leaderboard_build.yml

@@ -4,18 +4,17 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
 
 jobs:
   leaderboard:
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6
         with:
           python-version: '3.10'
           cache: 'pip'
@@ -26,4 +25,4 @@ jobs:
 
       - name: Run leaderboard build test
         run: |
-          make leaderboard-build-test
+          make leaderboard-build-test

.github/workflows/lint.yml

@@ -11,9 +11,9 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.10"
           cache: "pip"

.github/workflows/model_loading.yml

@@ -11,10 +11,10 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6
         with:
           python-version: "3.10"
           cache: "pip"

.github/workflows/release.yml

@@ -24,7 +24,7 @@ jobs:
 
     if: ${{ github.ref == 'refs/heads/main' && github.event.workflow_run.conclusion == 'success'}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
           token: ${{ secrets.RELEASE }}

.github/workflows/test.yml

@@ -8,21 +8,25 @@ on:
     branches: [main]
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest] #, macos-latest, windows-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
         include:
-          # Add Windows with Python 3.8 only to avoid tests taking too long
+          # Add Windows with Python 3.10 only to avoid tests taking too long
           - os: windows-latest
-            python-version: "3.9"
+            python-version: "3.10"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6
 
       - name: Cache Hugging Face
         id: cache-hf
@@ -32,7 +36,7 @@ jobs:
           key: ${{ runner.os }}-hf
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"

.gitignore

@@ -143,7 +143,6 @@ sb.ipynb
 tests/create_meta/model_card.md
 
 # removed results from mteb repo they are now available at: https://github.com/embeddings-benchmark/results
-results/
 uv.lock
 
 # model loading tests
@@ -152,3 +151,13 @@ mteb/leaderboard/__cached_results.json
 
 # gradio
 .gradio/
+
+# codecarbon
+powermetrics_log.txt
+
+# vscode
+.vscode/launch.json
+
+/docs/overview/available_models/
+/docs/overview/available_tasks/
+/docs/overview/available_benchmarks.md

.pre-commit-config.yaml

@@ -19,6 +19,11 @@ repos:
     -   id: end-of-file-fixer # generated a lot of changes
     -   id: trailing-whitespace
     -   id: check-toml
+  - repo: https://github.com/crate-ci/typos
+    rev: v1.38.1
+    hooks:
+    - id: typos
+      args: ["--diff"]
 
   - repo: local
     hooks:

Makefile

@@ -6,18 +6,20 @@ install:
 install-for-tests:
 	@echo "--- 🚀 Installing project dependencies for test ---"
 	@echo "This ensures that the project is not installed in editable mode"
-	pip install ".[image]" --group dev
+	pip install ".[bm25s,pylate,image,codecarbon,faiss-cpu]" --group dev
 
 lint:
 	@echo "--- 🧹 Running linters ---"
 	ruff format . 			# running ruff formatting
 	ruff check . --fix --exit-non-zero-on-fix  	# running ruff linting # --exit-non-zero-on-fix is used for the pre-commit hook to work
+	typos
 
 lint-check:
 	@echo "--- 🧹 Check is project is linted ---"
 	# Required for CI to work, otherwise it will just pass
 	ruff format . --check						    # running ruff formatting
 	ruff check .    						        # running ruff linting
+	typos --diff
 
 test:
 	@echo "--- 🧪 Running tests ---"
@@ -33,12 +35,21 @@ pr:
 	make lint
 	make test
 
-
-build-docs:
+build-docs: build-docs-overview
 	@echo "--- 📚 Building documentation ---"
-	# since we do not have a documentation site, this just build tables for the .md files
-	python docs/create_tasks_table.py
-	python docs/create_benchmarks_table.py
+	python -m mkdocs build
+
+
+build-docs-overview:
+	@echo "--- 📚 Building documentation overview ---"
+	python docs/overview/create_available_tasks.py
+	python docs/overview/create_available_models.py
+	python docs/overview/create_available_benchmarks.py
+
+
+serve-docs:
+	@echo "--- 📚 Serving documentation ---"
+	python -m mkdocs serve
 
 
 model-load-test:
@@ -74,3 +85,8 @@ format-citations:
 check: ## Run code quality tools.
 	@echo "--- 🧹 Running code quality tools ---"
 	@pre-commit run -a
+
+.PHONY: typecheck
+typecheck:
+	@echo "--- 🔍 Running type checks ---"
+	mypy mteb