From fc0f0cb605e309543d9d319dcda4ab950dab7f9c Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:05:05 +0200 Subject: [PATCH 1/4] Split deterministic v1 tests from live E2Es --- .github/workflows/test.yml | 49 +++++++++++++++++++++++++++++++------- tests/v1/conftest.py | 7 +++--- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac7169e365..6419e889e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,10 +23,6 @@ jobs: test: name: Verifiers runs-on: ubuntu-latest - env: - PRIME_API_KEY: ${{ secrets.PRIME_API_KEY }} - PRIME_TEAM_ID: ${{ secrets.PRIME_TEAM_ID }} - HF_TOKEN: ${{ secrets.HF_TOKEN }} strategy: fail-fast: false matrix: @@ -56,13 +52,48 @@ jobs: - name: Install dependencies run: uv sync - - name: Install prime - run: uv pip install prime - - name: Run tests run: | uv run pytest tests/ -v --ignore=tests/v1 -m "not prime" --cov=verifiers --cov-report=xml --cov-report=term - - name: Run v1 tests + - name: Run deterministic v1 tests + run: | + uv run pytest tests/v1 -vv -n auto -m "not e2e" --cov=verifiers --cov-append --cov-report=xml --cov-report=term + + v1-e2e: + name: V1 live E2E (Python 3.12) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + PRIME_API_KEY: ${{ secrets.PRIME_API_KEY }} + PRIME_TEAM_ID: ${{ secrets.PRIME_TEAM_ID }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Set up Python 3.12 + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Cache Hugging Face tokenizers + uses: actions/cache@v4 + with: + path: ${{ env.HF_HOME }} + key: hf-tokenizers-${{ runner.os }}-${{ hashFiles('uv.lock', 'tests/test_renderer_client.py', 'tests/test_renderer_e2e.py') }} + restore-keys: | + hf-tokenizers-${{ runner.os }}- + + - name: Install dependencies + run: uv sync + + - name: Run live v1 E2Es run: | - uv run pytest tests/v1 -vv -n auto -m "not prime and not modal" --cov=verifiers --cov-append --cov-report=xml --cov-report=term + uv run pytest tests/v1 -vv -n 2 -m "e2e and not prime and not modal" --cov=verifiers --cov-report=xml --cov-report=term diff --git a/tests/v1/conftest.py b/tests/v1/conftest.py index 79aee17565..23c742fe87 100644 --- a/tests/v1/conftest.py +++ b/tests/v1/conftest.py @@ -17,7 +17,8 @@ Every combination carries its axes' pytest marks, so subsets select with `-m`: uv run pytest tests/v1 -n auto # everything (needs modal setup) - uv run pytest tests/v1 -n auto -m "not prime and not modal" # the CI set (host + docker only) + uv run pytest tests/v1 -n auto -m "not e2e" # deterministic CI matrix + uv run pytest tests/v1 -n 2 -m "e2e and not prime and not modal" # live CI job uv run pytest tests/v1 -n auto -m docker # any case touching the docker runtime uv run pytest tests/v1 -n auto -m bash # only the bash harness uv run pytest tests/v1 -n auto -m prime # only prime (real sandboxes; local) @@ -27,8 +28,8 @@ harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `pool` / `codex` / `claude_code`. A mark is applied per axis, so it selects every case touching that value on ANY axis; for one exact combination use `-k` on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). -prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only -— CI runs `-m "not prime and not modal"`. +prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only. +CI runs deterministic tests across the Python matrix and the remaining live E2Es once. """ import os From fd09d254bd977b3d188e987a7b3b153b1621abf4 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:19:57 +0200 Subject: [PATCH 2/4] Address workflow review feedback --- .github/workflows/test.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6419e889e1..7a3907b6a6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,9 @@ on: env: HF_HOME: ${{ github.workspace }}/.cache/huggingface +permissions: + contents: read + jobs: test: name: Verifiers @@ -62,7 +65,10 @@ jobs: v1-e2e: name: V1 live E2E (Python 3.12) - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + if: >- + github.event_name != 'pull_request' || + (github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.user.login != 'dependabot[bot]') runs-on: ubuntu-latest timeout-minutes: 120 env: From 28314f0d252cdd86543c5030ec990bfba7f591ab Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:27:39 +0200 Subject: [PATCH 3/4] Restore HF token for renderer tests --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7a3907b6a6..e2ff5c2812 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,6 +26,8 @@ jobs: test: name: Verifiers runs-on: ubuntu-latest + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} strategy: fail-fast: false matrix: From 4f1bbc93aa640a58b3d54d4614d4b2949b4f2507 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:05:17 +0200 Subject: [PATCH 4/4] Restore automatic live E2E concurrency --- .github/workflows/test.yml | 2 +- tests/v1/conftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e2ff5c2812..2ef2499195 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -104,4 +104,4 @@ jobs: - name: Run live v1 E2Es run: | - uv run pytest tests/v1 -vv -n 2 -m "e2e and not prime and not modal" --cov=verifiers --cov-report=xml --cov-report=term + uv run pytest tests/v1 -vv -n auto -m "e2e and not prime and not modal" --cov=verifiers --cov-report=xml --cov-report=term diff --git a/tests/v1/conftest.py b/tests/v1/conftest.py index 23c742fe87..3330a3a27d 100644 --- a/tests/v1/conftest.py +++ b/tests/v1/conftest.py @@ -18,7 +18,7 @@ uv run pytest tests/v1 -n auto # everything (needs modal setup) uv run pytest tests/v1 -n auto -m "not e2e" # deterministic CI matrix - uv run pytest tests/v1 -n 2 -m "e2e and not prime and not modal" # live CI job + uv run pytest tests/v1 -n auto -m "e2e and not prime and not modal" # live CI job uv run pytest tests/v1 -n auto -m docker # any case touching the docker runtime uv run pytest tests/v1 -n auto -m bash # only the bash harness uv run pytest tests/v1 -n auto -m prime # only prime (real sandboxes; local)