diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac7169e365..2ef2499195 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,13 +19,14 @@ on: env: HF_HOME: ${{ github.workspace }}/.cache/huggingface +permissions: + contents: read + jobs: test: name: Verifiers runs-on: ubuntu-latest env: - PRIME_API_KEY: ${{ secrets.PRIME_API_KEY }} - PRIME_TEAM_ID: ${{ secrets.PRIME_TEAM_ID }} HF_TOKEN: ${{ secrets.HF_TOKEN }} strategy: fail-fast: false @@ -56,13 +57,51 @@ jobs: - name: Install dependencies run: uv sync - - name: Install prime - run: uv pip install prime - - name: Run tests run: | uv run pytest tests/ -v --ignore=tests/v1 -m "not prime" --cov=verifiers --cov-report=xml --cov-report=term - - name: Run v1 tests + - name: Run deterministic v1 tests + run: | + uv run pytest tests/v1 -vv -n auto -m "not e2e" --cov=verifiers --cov-append --cov-report=xml --cov-report=term + + v1-e2e: + name: V1 live E2E (Python 3.12) + if: >- + github.event_name != 'pull_request' || + (github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.user.login != 'dependabot[bot]') + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + PRIME_API_KEY: ${{ secrets.PRIME_API_KEY }} + PRIME_TEAM_ID: ${{ secrets.PRIME_TEAM_ID }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Set up Python 3.12 + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Cache Hugging Face tokenizers + uses: actions/cache@v4 + with: + path: ${{ env.HF_HOME }} + key: hf-tokenizers-${{ runner.os }}-${{ hashFiles('uv.lock', 'tests/test_renderer_client.py', 'tests/test_renderer_e2e.py') }} + restore-keys: | + hf-tokenizers-${{ runner.os }}- + + - name: Install dependencies + run: uv sync + + - name: Run live v1 E2Es run: | - uv run pytest tests/v1 -vv -n auto -m "not prime and not modal" --cov=verifiers --cov-append --cov-report=xml --cov-report=term + uv run pytest tests/v1 -vv -n auto -m "e2e and not prime and not modal" --cov=verifiers --cov-report=xml --cov-report=term diff --git a/tests/v1/conftest.py b/tests/v1/conftest.py index 79aee17565..3330a3a27d 100644 --- a/tests/v1/conftest.py +++ b/tests/v1/conftest.py @@ -17,7 +17,8 @@ Every combination carries its axes' pytest marks, so subsets select with `-m`: uv run pytest tests/v1 -n auto # everything (needs modal setup) - uv run pytest tests/v1 -n auto -m "not prime and not modal" # the CI set (host + docker only) + uv run pytest tests/v1 -n auto -m "not e2e" # deterministic CI matrix + uv run pytest tests/v1 -n auto -m "e2e and not prime and not modal" # live CI job uv run pytest tests/v1 -n auto -m docker # any case touching the docker runtime uv run pytest tests/v1 -n auto -m bash # only the bash harness uv run pytest tests/v1 -n auto -m prime # only prime (real sandboxes; local) @@ -27,8 +28,8 @@ harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `pool` / `codex` / `claude_code`. A mark is applied per axis, so it selects every case touching that value on ANY axis; for one exact combination use `-k` on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). -prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only -— CI runs `-m "not prime and not modal"`. +prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only. +CI runs deterministic tests across the Python matrix and the remaining live E2Es once. """ import os