[MAEB] merge from main again

.github/workflows/dataset_loading.yml

@@ -33,15 +33,19 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v6
 
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
         python-version: '3.11'
-        cache: 'pip'
 
     - name: Install dependencies
       run: |
-        make install-for-tests
+        make install
 
     - name: Run dataset loading tests
       env:

.github/workflows/documentation.yml

@@ -33,20 +33,19 @@ jobs:
           docker system prune -af
 
       - uses: actions/checkout@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
       - uses: actions/setup-python@v6
         with:
           python-version: "3.10"
 
-      - name: Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e . --group docs
-
       - name: Build and Deploy
         if: github.event_name == 'push'
         run: |
           make build-docs-overview
-          mkdocs gh-deploy --force
+          uv run --group docs mkdocs gh-deploy --force
 
       - name: Build
         if: github.event_name == 'pull_request'

.github/workflows/hf_space_docker.yml

@@ -0,0 +1,73 @@
+name: Build HF Space Docker Image
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'Dockerfile.hf-space'
+      - '.github/workflows/hf_space_docker.yml'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'Dockerfile.hf-space'
+      - '.github/workflows/hf_space_docker.yml'
+  workflow_dispatch:
+
+jobs:
+  build-hf-space:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Build HF Space Docker image
+      run: |
+        docker build -f Dockerfile.hf-space -t mteb-hf-space:test .
+        echo "✅ Docker image built successfully"
+
+    - name: Test Docker image
+      run: |
+        # Test that the image can be created and the container starts
+        docker run -d --name mteb-test -p 7860:7860 mteb-hf-space:test
+
+        # Give the container a moment to start
+        sleep 5
+
+        # Check if container is running
+        if docker ps | grep -q mteb-test; then
+          echo "✅ Container is running"
+          docker logs mteb-test
+        else
+          echo "❌ Container failed to start"
+          docker logs mteb-test
+          exit 1
+        fi
+
+        # Clean up
+        docker stop mteb-test
+        docker rm mteb-test
+
+    - name: Validate HF Space configuration
+      run: |
+        # Check that required environment variables and ports are set
+        docker run --rm mteb-hf-space:test sh -c '
+          if [ "$GRADIO_SERVER_NAME" = "0.0.0.0" ]; then
+            echo "✅ GRADIO_SERVER_NAME is correctly set"
+          else
+            echo "❌ GRADIO_SERVER_NAME is not set to 0.0.0.0"
+            exit 1
+          fi
+        '
+
+        # Check exposed port
+        EXPOSED_PORT=$(docker inspect mteb-hf-space:test --format='{{range $key, $value := .Config.ExposedPorts}}{{$key}}{{end}}' | grep -o '7860')
+        if [ "$EXPOSED_PORT" = "7860" ]; then
+          echo "✅ Port 7860 is exposed"
+        else
+          echo "❌ Port 7860 is not exposed"
+          exit 1
+        fi

.github/workflows/leaderboard_build.yml

@@ -33,15 +33,15 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v6
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
           python-version: '3.10'
-          cache: 'pip'
-
-      - name: Install dependencies (incl. leaderboard extra)
-        run: |
-          pip install ".[leaderboard]" --group dev
 
       - name: Run leaderboard build test
         run: |

.github/workflows/leaderboard_docker.yml

@@ -0,0 +1,166 @@
+name: Test Leaderboard Dockerfile
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'Dockerfile'
+      - '.github/workflows/leaderboard_docker.yml'
+      - 'pyproject.toml'
+      - 'uv.lock'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'Dockerfile'
+      - '.github/workflows/leaderboard_docker.yml'
+      - 'pyproject.toml'
+      - 'uv.lock'
+  workflow_dispatch:
+
+jobs:
+  test-dockerfile:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Free disk space
+      run: |
+        sudo rm -rf \
+          "$AGENT_TOOLSDIRECTORY" \
+          /opt/ghc \
+          /opt/google/chrome \
+          /opt/microsoft/msedge \
+          /opt/microsoft/powershell \
+          /opt/pipx \
+          /usr/lib/mono \
+          /usr/local/julia* \
+          /usr/local/lib/android \
+          /usr/local/lib/node_modules \
+          /usr/local/share/chromium \
+          /usr/local/share/powershell \
+          /usr/share/dotnet \
+          /usr/share/swift
+        docker system prune -af
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Log in to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build and push Docker image
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./Dockerfile
+        push: ${{ github.ref == 'refs/heads/main' }}
+        tags: |
+          ghcr.io/${{ github.repository }}/leaderboard:${{ github.sha }}
+          ghcr.io/${{ github.repository }}/leaderboard:latest
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        load: true
+
+    - name: Prepare image for local testing
+      run: |
+        docker tag ghcr.io/${{ github.repository }}/leaderboard:${{ github.sha }} mteb-leaderboard:test
+
+    - name: Test container can start and run
+      timeout-minutes: 6
+      run: |
+        # Start the container in background
+        docker run -d --name mteb-test -p 7860:7860 mteb-leaderboard:test
+
+        # Monitor container with smart exit conditions
+        echo "Starting leaderboard container..."
+        START_TIME=$(date +%s)
+        STARTUP_DETECTED=false
+        PROGRESS_DETECTED=false
+        INIT_COMPLETE_DETECTED=false
+
+        while true; do
+          CURRENT_TIME=$(date +%s)
+          ELAPSED=$((CURRENT_TIME - START_TIME))
+
+          # Maximum timeout: 5.5 minutes - always fail at timeout
+          if [ $ELAPSED -gt 330 ]; then
+            echo "❌ Timeout reached after ${ELAPSED}s - container failed to complete initialization"
+            docker logs --tail 20 mteb-test
+            docker stop mteb-test
+            docker rm mteb-test
+            exit 1
+          fi
+
+          # Check if container is still running
+          if ! docker ps | grep -q mteb-test; then
+            # Container exited, check exit code and logs
+            EXIT_CODE=$(docker inspect mteb-test --format='{{.State.ExitCode}}')
+            echo "Container exited with code: $EXIT_CODE after ${ELAPSED}s"
+
+            # Check if we made good progress before exit
+            LOGS=$(docker logs mteb-test 2>&1)
+            if echo "$LOGS" | grep -q "=== Leaderboard app initialization complete" || \
+               echo "$LOGS" | grep -q "Running on.*http"; then
+              echo "✅ Container completed significant startup progress successfully"
+              docker rm mteb-test
+              exit 0
+            else
+              echo "❌ Container failed before completing startup initialization"
+              echo "Last logs:"
+              echo "$LOGS" | tail -10
+              docker rm mteb-test
+              exit 1
+            fi
+          fi
+
+          # Get current logs and check for progress indicators
+          LOGS=$(docker logs mteb-test 2>&1)
+
+          # Check for startup detection
+          if [ "$STARTUP_DETECTED" = "false" ] && echo "$LOGS" | grep -q "Starting leaderboard app in process"; then
+            echo "✅ Detected leaderboard app startup"
+            STARTUP_DETECTED=true
+          fi
+
+          # Check for full initialization complete
+          if [ "$INIT_COMPLETE_DETECTED" = "false" ] && echo "$LOGS" | grep -q "=== Leaderboard app initialization complete"; then
+            echo "✅ Detected full app initialization complete!"
+            INIT_COMPLETE_DETECTED=true
+            echo "🔄 Testing if server is responding..."
+            # Give it a moment for the server to fully start
+            sleep 3
+            # Try to ping the server
+            if curl -s --max-time 5 http://localhost:7860/ > /dev/null; then
+              echo "✅ Server is responding - container fully operational!"
+              docker stop mteb-test
+              docker rm mteb-test
+              exit 0
+            else
+              echo "⏳ Server not yet responding, continuing to wait..."
+              PROGRESS_DETECTED=true
+            fi
+          fi
+
+          # Check for Gradio server ready
+          if echo "$LOGS" | grep -q "Running on.*http"; then
+            echo "✅ Detected Gradio server ready - container fully operational!"
+            docker stop mteb-test
+            docker rm mteb-test
+            exit 0
+          fi
+
+          # Show progress every 30 seconds
+          if [ $((ELAPSED % 30)) -eq 0 ] && [ $ELAPSED -gt 0 ]; then
+            echo "Container still running after ${ELAPSED}s... (startup: $STARTUP_DETECTED, init_complete: $INIT_COMPLETE_DETECTED)"
+            # Show last few lines of logs for progress
+            echo "$LOGS" | grep -E "(Step [0-9]/7|Starting leaderboard|initialization complete|Running on)" | tail -2
+          fi
+
+          sleep 5
+        done

.github/workflows/lint.yml

@@ -33,13 +33,14 @@ jobs:
 
       - uses: actions/checkout@v6
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
       - uses: actions/setup-python@v6
         with:
           python-version: "3.10"
-          cache: "pip"
-
-      - name: Install dependencies
-        run: make install
 
       - name: Lint
         id: lint

.github/workflows/model_loading.yml

@@ -33,11 +33,15 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v6
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
           python-version: "3.10"
-          cache: "pip"
 
       - name: Install dependencies and run tests
         run: |

.github/workflows/test.yml

@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest] #, macos-latest, windows-latest]
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"] #, "3.14"]
         include:
           # Add Windows with Python 3.10 only to avoid tests taking too long
           - os: windows-latest
@@ -54,13 +54,12 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.cache/huggingface
-          key: ${{ runner.os }}-hf
+          key: ${{ runner.os }}-hf-maeb
 
-      - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v6
+      - name: Install uv and set the Python version
+        uses: astral-sh/setup-uv@v7
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
 
       - name: Install dependencies
         shell: bash