feat: Introduce MAEB

.github/workflows/test.yml

@@ -61,11 +61,54 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      # required for evaluation the audio subset
+      - name: Install FFmpeg (Ubuntu)
+        if: runner.os != 'Windows'
+        shell: bash
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y software-properties-common
+          sudo add-apt-repository -y ppa:ubuntuhandbook1/ffmpeg8
+          sudo apt-get update
+          sudo apt-get install -y ffmpeg
+
+      - name: Setup Miniconda (Windows)
+        if: runner.os == 'Windows'
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          miniconda-version: "latest"
+          activate-environment: ffmpeg
+
+      - name: Install FFmpeg (Windows)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        # using conda to install ffmpeg on windows, to avoid issues with dlls
+        run: |
+          conda install -y "ffmpeg=8.0.1" -c conda-forge
+
+      - name: Check FFmpeg version
+        run: ffmpeg -version
+
       - name: Install dependencies
         shell: bash
         run: |
           make install-for-tests
 
+      - name: Setup Pytorch dll PATH (Windows)
+        # otherwise pytorch audio cannot find pytorch dlls on windows
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          $torchLib = "D:\a\mteb\mteb\.venv\Lib\site-packages\torch\lib"
+          if (Test-Path $torchLib) {
+            echo "$torchLib" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+            $env:PATH = "$torchLib;$env:PATH"
+            echo "PATH=$env:PATH" >> $env:GITHUB_ENV
+          } else {
+            Write-Host "Torch lib path not found: $torchLib"
+          }
+
       - name: Run tests
         if: runner.os != 'Windows'
         shell: bash
@@ -77,9 +120,14 @@ jobs:
         # this step will run the workflow twice since we have experienced
         # failures when running on windows when loading the datasets
         if: runner.os == 'Windows'
-        shell: bash
+        shell: pwsh
         run: |
           # run the test once and if it fails, run it again
           # if it fails again, the workflow will fail.
           # If it passes the first time the test will not run again
-          make test || make test
+          try {
+            make test
+          } catch {
+            Write-Host "First test run failed, retrying..."
+            make test
+          }

.gitignore

@@ -163,3 +163,5 @@ powermetrics_log.txt
 /docs/overview/available_benchmarks.md
 
 CLAUDE.md
+
+*.tex

Makefile

@@ -5,7 +5,7 @@ install:
 install-for-tests:
 	@echo "--- 🚀 Installing project dependencies for test ---"
 	@echo "This ensures that the project is not installed in editable mode"
-	uv sync --no-editable --extra bm25s --extra pylate --extra image --extra codecarbon --extra leaderboard --extra faiss-cpu --group dev
+	uv sync --no-editable --extra bm25s --extra image --extra audio --extra codecarbon --extra leaderboard --extra faiss-cpu --group dev
 
 lint:
 	@echo "--- 🧹 Running linters ---"
@@ -77,7 +77,7 @@ leaderboard-test-all:
 
 run-leaderboard:
 	@echo "--- 🚀 Running leaderboard locally ---"
-	uv run --extra leaderboard python -m mteb.leaderboard.app
+	uv run --no-sync --extra leaderboard python -m mteb.leaderboard.app
 
 format-citations:
 	@echo "--- 🧹 Formatting citations ---"

docs/installation.md

@@ -28,6 +28,46 @@ If you want to run certain models implemented within mteb you will often need so
 
 If a specific model requires a dependency it will raise an error with the recommended installation. To see full list of available models you can look at the [models overview](./overview/available_models/text.md).
 
+## Audio Tasks
+
+If you want to run audio tasks, install the audio dependencies:
+
+=== "pip"
+    ```bash
+    pip install mteb[audio]
+    ```
+
+=== "uv"
+    ```bash
+    uv add "mteb[audio]"
+    ```
+
+### Additional Requirements for `datasets>=4`
+
+If you are using `datasets>=4`, you will need to:
+
+1. **Install FFmpeg**: The `datasets` library version 4+ uses `torchcodec` for audio processing, which requires FFmpeg to be installed on your system.
+
+    === "macOS"
+        ```bash
+        brew install ffmpeg
+        ```
+
+    === "Ubuntu/Debian"
+        ```bash
+        sudo apt-get install ffmpeg
+        ```
+
+    === "Windows"
+        Download from [ffmpeg.org](https://ffmpeg.org/download.html) and add to your PATH.
+
+2. **Use `transformers>=4.57.6`**: Due to compatibility issues with `datasets>=4`, you need a recent version of transformers:
+    ```bash
+    pip install "transformers>=4.57.6"
+    ```
+
+If you are using `datasets<4`, no additional requirements are needed beyond the `mteb[audio]` installation.
+
 ## Migrating to uv (for Contributors)
 
 If you're a contributor currently using pip, here's how to migrate to uv for faster dependency management: