add the prime-runs SDK

.github/workflows/ci.yml

@@ -41,6 +41,7 @@ jobs:
           "packages/prime-sandboxes/src/prime_sandboxes/__init__.py"
           "packages/prime-evals/src/prime_evals/__init__.py"
           "packages/prime-traces/src/prime_traces/__init__.py"
+          "packages/prime-runs/src/prime_runs/__init__.py"
         )
 
         BUMPED_FILES=""
@@ -160,6 +161,33 @@ jobs:
       working-directory: packages/prime-traces
       run: uv run pytest tests/ -v
 
+  test-runs:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v4
+      with:
+        version: "latest"
+
+    - name: Install dependencies
+      working-directory: packages/prime-runs
+      run: uv sync --all-extras
+
+    # No secrets: the runs tests are hermetic (httpx.MockTransport + tmp dirs).
+    - name: Run tests
+      working-directory: packages/prime-runs
+      run: uv run pytest tests/ -v
+
   test-prime:
     runs-on: ubuntu-latest
     strategy:

.github/workflows/release-runs.yml

@@ -0,0 +1,78 @@
+name: Release prime-runs
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'packages/prime-runs/**'
+      - '.github/workflows/release-runs.yml'
+  workflow_dispatch:
+    inputs:
+      force_release:
+        description: 'Force release even if tag exists'
+        required: false
+        default: 'false'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    environment: pypi-prod
+    permissions:
+      contents: write
+      packages: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Get version
+        id: version
+        working-directory: packages/prime-runs
+        run: |
+          VERSION=$(grep -E "^__version__\s*=\s*" src/prime_runs/__init__.py | sed -E 's/^__version__[[:space:]]*=[[:space:]]*"([^"]+)".*$/\1/')
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "Version: $VERSION"
+
+      - name: Check for existing tag
+        id: check_tag
+        run: |
+          TAG="prime-runs-v${{ steps.version.outputs.version }}"
+          if git tag -l "$TAG" | grep -q .; then
+            echo "exists=true" >> $GITHUB_OUTPUT
+            echo "Tag $TAG already exists"
+          else
+            echo "exists=false" >> $GITHUB_OUTPUT
+            echo "Tag $TAG does not exist"
+          fi
+
+      - name: Set up Python
+        if: steps.check_tag.outputs.exists != 'true' || inputs.force_release == 'true'
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+
+      - name: Install uv
+        if: steps.check_tag.outputs.exists != 'true' || inputs.force_release == 'true'
+        uses: astral-sh/setup-uv@v4
+
+      - name: Build package
+        if: steps.check_tag.outputs.exists != 'true' || inputs.force_release == 'true'
+        working-directory: packages/prime-runs
+        run: |
+          uv build --out-dir dist
+
+      - name: Create tag
+        if: steps.check_tag.outputs.exists != 'true'
+        run: |
+          TAG="prime-runs-v${{ steps.version.outputs.version }}"
+          git config user.name 'github-actions[bot]'
+          git config user.email 'github-actions[bot]@users.noreply.github.com'
+          git tag -a "$TAG" -m "Release $TAG"
+          git push origin "$TAG"
+
+      - name: Publish to PyPI
+        if: steps.check_tag.outputs.exists != 'true' || inputs.force_release == 'true'
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+        with:
+          packages-dir: packages/prime-runs/dist

packages/prime-runs/README.md

@@ -0,0 +1,161 @@
+# Prime Runs SDK
+
+Track eval and training runs on the Prime Intellect platform.
+
+```bash
+pip install prime-runs
+```
+
+## Quick start
+
+```python
+import prime_runs as pr
+
+run = pr.init(
+    name="gsm8k-qwen3-8b",
+    environments=["gsm8k"],
+    model="Qwen/Qwen3-8B",
+    framework="verifiers",
+    config={"num_rollouts": 4, "max_tokens": 2048},
+)
+print(run.url)   # https://app.primeintellect.ai/dashboard/evaluations/eval-...
+
+for episode in rollouts:          # episodes carry run.id — see "Identity" below
+    run.log_traces([episode])
+    run.log({"reward": episode.reward})
+
+run.finish(summary=pr.metrics.from_episodes(episodes))
+```
+
+`init()` opens the run and returns a handle carrying its ID and dashboard URL.
+Records stream out on a background thread while the run proceeds — the dashboard
+fills in as rollouts land, rather than all at once at the end. `finish()` closes
+the run out.
+
+Prefer a `with` block, which also reports a terminal status on the paths where
+you never reach `finish()`:
+
+```python
+with pr.init(name="gsm8k-qwen3-8b", environments=["gsm8k"]) as run:
+    ...
+```
+
+## Identity
+
+`init()` is called **before** the first rollout, and the ID it returns is *the*
+run ID everywhere — including inside every trace document you write, and
+including the local archive. Stamp it once at rollout time:
+
+```python
+run = pr.init(...)
+trace.record_run(EvalRunInfo(id=run.id))     # verifiers
+```
+
+Nothing is re-stamped afterwards and no record of yours is rewritten. The
+ingestion service extracts `run.id` from the trace body into an indexed column
+with a delete-by-run path, so "every trace for this run" is a fast query rather
+than a scan over upload metadata.
+
+`init()` also exports `PRIME_RUN_ID`, so forked workers and subprocess launchers
+join the run their parent opened instead of each opening their own.
+
+## Modes
+
+| mode | what happens |
+| --- | --- |
+| `online` | the run lives on the platform (default when an API key is present) |
+| `offline` | the run lives in a local directory, ready to sync later |
+| `disabled` | every call is a no-op, with the same object shape |
+
+An offline run is a real run: a real ID, a status, a config, a summary, a metrics
+stream, and records written in the JSONL wire format the traces service accepts.
+That is why producers do not need a `--no-push` branch — the call sites are
+identical either way, and a missing API key degrades to offline rather than
+skipping the run.
+
+Set the mode explicitly, or through `$PRIME_RUNS_MODE`.
+
+## What the run handle does for you
+
+- **Streams instead of buffering.** Records go out as they are produced, so a
+  run with a hundred thousand episodes does not hold them all in memory.
+- **Contains its own errors.** With the default `on_error="warn"`, nothing the
+  platform raises escapes into your loop. Use `on_error="raise"` in tests and CI,
+  where a silent upload failure is the bug.
+- **Applies backpressure.** The upload queue is bounded; if a producer durably
+  outruns the uploader, records are dropped and counted (`run.dropped_records`)
+  rather than stalling the run.
+- **Waits for its own uploads.** `finish()` gives queued records the same budget
+  a single upload gets (300s, `finish_timeout=`) and says so in a warning if
+  they do not drain, rather than finalizing over records still in flight.
+- **Survives forks.** A forked child gets a fresh uploader, a fresh connection
+  pool and fresh file handles instead of writing the parent's — which would
+  interleave two processes into one HTTP stream and flush the parent's buffered
+  records a second time. It also joins the parent's run rather than opening
+  its own.
+- **Reports a terminal status.** Context manager, `atexit` and signal handlers
+  all route to the same idempotent `finish()`. A run the producer decided had
+  failed is `failed`; one stopped from outside its control flow — Ctrl-C,
+  SIGTERM, an exit that never reached `finish()` — is `crashed`. Neither is
+  left running forever.
+- **Knows about ranks.** Rank 0 owns creation and finalization; other ranks join
+  through `PRIME_RUN_ID` and upload their own records.
+
+## Configuration
+
+Resolved from environment variables first, then `~/.prime/config.json`:
+
+| setting | env var | default |
+| --- | --- | --- |
+| API key | `PRIME_API_KEY` | — |
+| team | `PRIME_TEAM_ID` | — |
+| platform API | `PRIME_API_BASE_URL` | `https://api.primeintellect.ai` |
+| dashboard | `PRIME_FRONTEND_URL` | `https://app.primeintellect.ai` |
+| traces service | `PRIME_TRACES_URL` | resolved by `prime-traces` |
+| offline runs | `PRIME_RUNS_DIR` | `./prime-runs` |
+
+Or pass them to `init()` directly (`api_key=`, `base_url=`, `team_id=`, `dir=`).
+
+## Backends and sinks
+
+Two independent axes:
+
+- **Backends** own run *lifecycle* — `EvalsBackend` (`/api/v1/evaluations/*`),
+  `OfflineBackend` (a local directory). Selected by `kind`.
+- **Sinks** own sample *transport* — `TracesSink` (primary; streaming,
+  episode-aware, content-addressed and therefore idempotent on retry) and
+  `EvalSamplesSink` (the flat v0 sample table today's viewer reads).
+
+Both sinks run during the transition, because Prime Traces is in closed beta and
+a traces-only client would leave non-allowlisted accounts with an empty
+dashboard. When the Viewer API reads traces natively, the default sink list drops
+one entry — and no producer changes.
+
+Turn either off with `pr.init(traces=False)` / `pr.init(samples=False)`, or pass
+`sinks=[...]` to supply your own.
+
+## Also here
+
+`prime_runs.projection` holds `trace_to_sample` / `build_samples`, the projection
+from native traces onto the platform's v0 eval-sample format. It lives here
+because it is knowledge about a platform wire format, not about any one eval
+framework. `prime_runs.metrics.from_episodes` is the run-level aggregation the
+eval dashboard reads — opt-in, because what a run's headline number means is a
+judgement that belongs next to the producer.
+
+Both are duck-typed: verifiers `Trace`/`Episode` and prime-rl `Rollout` satisfy
+them structurally, and no producer package is imported. This is a leaf package by
+design — the `prime` CLI depends on `verifiers`, so verifiers can never depend on
+`prime`.
+
+## Status
+
+Eval runs are supported. Training runs (`kind="train"`, over
+`/api/v1/rft/external-runs`) are next; `pr.init(kind="train")` raises a clear
+error until then.
+
+One platform gap is worth knowing about: there is currently no producer-facing
+way to mark an evaluation **failed**. The SDK calls the status endpoint it needs,
+treats its absence as expected, and records the terminal state in the run's
+metadata as a fallback — a failed run will keep showing as running on the
+dashboard, and the SDK says so in a warning.

packages/prime-runs/pyproject.toml

@@ -0,0 +1,70 @@
+[project]
+name = "prime-runs"
+# Version is single-sourced from src/prime_runs/__init__.py via Hatch
+dynamic = ["version"]
+description = "Prime Intellect Runs SDK - Track eval and training runs on the Prime platform"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+authors = [
+    { name = "Prime Intellect", email = "contact@primeintellect.ai" }
+]
+# Deliberately a leaf package. The `prime` CLI depends on `verifiers`, so
+# verifiers can never depend on `prime` — and verifiers is a first-class
+# consumer of this SDK. Nothing here may pull in prime, verifiers, typer,
+# rich or textual, directly or transitively.
+dependencies = [
+    "httpx>=0.25.0",
+    "pydantic>=2.0.0",
+    "tenacity>=9.1.2",
+    "prime-traces>=0.0.2",
+]
+keywords = ["evals", "evaluations", "training", "runs", "traces", "observability"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence"
+]
+
+[project.urls]
+Homepage = "https://github.com/PrimeIntellect-ai/prime"
+Documentation = "https://github.com/PrimeIntellect-ai/prime/tree/main/packages/prime-runs"
+Repository = "https://github.com/PrimeIntellect-ai/prime.git"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-asyncio>=0.23.0",
+    "ruff>=0.13.1",
+]
+
+[tool.uv.sources]
+prime-traces = { workspace = true }
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "src/prime_runs/__init__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/prime_runs"]
+
+[tool.pytest.ini_options]
+addopts = "-v"
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+extend-select = ["E", "F", "I"]