diff --git a/.github/workflows/leaderboard_build.yml b/.github/workflows/leaderboard_build.yml
new file mode 100644
index 0000000000..90d52c70a2
--- /dev/null
+++ b/.github/workflows/leaderboard_build.yml
@@ -0,0 +1,29 @@
+name: Leaderboard Build Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  leaderboard:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: Install dependencies (incl. leaderboard extra)
+        run: |
+          pip install ".[dev,leaderboard]"
+
+      - name: Run leaderboard build test
+        run: |
+          make leaderboard-build-test
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 5bd3eb7033..2e51e2aff7 100644
--- a/Makefile
+++ b/Makefile
@@ -21,7 +21,7 @@ lint-check:
 
 test:
 	@echo "--- 🧪 Running tests ---"
-	pytest -n auto -m "not test_datasets"
+	pytest -n auto -m "not (test_datasets or leaderboard_stability)"
 
 
 test-with-coverage:
@@ -52,6 +52,9 @@ dataset-load-test:
 	@echo "--- 🚀 Running dataset load test ---"
 	pytest -n auto -m test_datasets
 
+leaderboard-build-test:
+	@echo "--- 🚀 Running leaderboard build test ---"
+	pytest -n auto -m leaderboard_stability
 
 run-leaderboard:
 	@echo "--- 🚀 Running leaderboard locally ---"
diff --git a/tests/test_leaderboard.py b/tests/test_leaderboard.py
new file mode 100644
index 0000000000..a232244a9d
--- /dev/null
+++ b/tests/test_leaderboard.py
@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import multiprocessing
+import time
+
+import pytest
+
+TIMEOUT = 300
+
+
+def run_leaderboard_app():
+    """Function to launch the leaderboard app."""
+    from mteb.leaderboard.app import get_leaderboard_app
+
+    app = get_leaderboard_app()
+    app.launch(server_name="0.0.0.0", server_port=7860, prevent_thread_lock=True)
+
+
+@pytest.mark.timeout(TIMEOUT)
+@pytest.mark.leaderboard_stability
+def test_leaderboard_app_does_not_crash():
+    """Test to ensure the leaderboard app does not crash within the first 5 minutes."""
+    process = multiprocessing.Process(target=run_leaderboard_app)
+    process.start()
+
+    try:
+        for _ in range(TIMEOUT):
+            if not process.is_alive():
+                pytest.fail("Leaderboard app crashed during the test.")
+            time.sleep(1)
+    finally:
+        process.terminate()
+        process.join()