add the prime-runs SDK

.github/workflows/ci.yml

@@ -41,6 +41,7 @@ jobs:
           "packages/prime-sandboxes/src/prime_sandboxes/__init__.py"
           "packages/prime-evals/src/prime_evals/__init__.py"
           "packages/prime-traces/src/prime_traces/__init__.py"
+          "packages/prime-runs/src/prime_runs/__init__.py"
         )
 
         BUMPED_FILES=""
@@ -168,6 +169,33 @@ jobs:
       working-directory: packages/prime-traces
       run: uv run pytest tests/ -v
 
+  test-runs:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v4
+      with:
+        version: "latest"
+
+    - name: Install dependencies
+      working-directory: packages/prime-runs
+      run: uv sync --all-extras
+
+    # No secrets: the runs tests are hermetic (httpx.MockTransport + tmp dirs).
+    - name: Run tests
+      working-directory: packages/prime-runs
+      run: uv run pytest tests/ -v
+
   test-prime:
     runs-on: ubuntu-latest
     strategy:

.github/workflows/release-runs.yml

@@ -0,0 +1,91 @@
+name: Release prime-runs
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'packages/prime-runs/**'
+      - '.github/workflows/release-runs.yml'
+  workflow_dispatch:
+    inputs:
+      force_release:
+        description: 'Force release even if tag exists'
+        required: false
+        default: 'false'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    environment: pypi-prod
+    permissions:
+      contents: write
+      packages: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Get version
+        id: version
+        working-directory: packages/prime-runs
+        run: |
+          VERSION=$(grep -E "^__version__\s*=\s*" src/prime_runs/__init__.py | sed -E 's/^__version__[[:space:]]*=[[:space:]]*"([^"]+)".*$/\1/')
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "Version: $VERSION"
+
+      - name: Check for existing tag
+        id: check_tag
+        run: |
+          TAG="prime-runs-v${{ steps.version.outputs.version }}"
+          if git tag -l "$TAG" | grep -q .; then
+            echo "exists=true" >> $GITHUB_OUTPUT
+            echo "Tag $TAG already exists"
+          else
+            echo "exists=false" >> $GITHUB_OUTPUT
+            echo "Tag $TAG does not exist"
+          fi
+
+      - name: Refuse untagged releases outside main
+        if: github.event_name == 'workflow_dispatch' && steps.check_tag.outputs.exists != 'true' && github.ref != 'refs/heads/main'
+        run: |
+          echo "::error title=Invalid release ref::New prime-runs releases must be dispatched from main, not ${{ github.ref }}."
+          exit 1
+
+      # A forced release must build the tagged source, not whatever the
+      # dispatch ref currently points at.
+      - name: Check out existing tag
+        if: steps.check_tag.outputs.exists == 'true' && inputs.force_release == 'true'
+        run: git checkout --detach "prime-runs-v${{ steps.version.outputs.version }}"
+
+      - name: Set up Python
+        if: steps.check_tag.outputs.exists != 'true' || inputs.force_release == 'true'
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+
+      - name: Install uv
+        if: steps.check_tag.outputs.exists != 'true' || inputs.force_release == 'true'
+        uses: astral-sh/setup-uv@v4
+
+      - name: Build package
+        if: steps.check_tag.outputs.exists != 'true' || inputs.force_release == 'true'
+        working-directory: packages/prime-runs
+        run: |
+          uv build --out-dir dist
+
+      - name: Publish to PyPI
+        if: steps.check_tag.outputs.exists != 'true' || inputs.force_release == 'true'
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+        with:
+          packages-dir: packages/prime-runs/dist
+          skip-existing: ${{ inputs.force_release == 'true' }}
+
+      - name: Create tag
+        if: steps.check_tag.outputs.exists != 'true'
+        run: |
+          TAG="prime-runs-v${{ steps.version.outputs.version }}"
+          git config user.name 'github-actions[bot]'
+          git config user.email 'github-actions[bot]@users.noreply.github.com'
+          git tag -a "$TAG" -m "Release $TAG"
+          git push origin "$TAG"

packages/prime-runs/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Prime Intellect
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/prime-runs/README.md

@@ -0,0 +1,124 @@
+# Prime Runs SDK
+
+Track evaluation and training runs on the Prime Intellect platform: `init()`
+opens the run, records stream out while it proceeds, and `finish()` closes it
+out with a terminal status.
+
+## Install
+
+```bash
+uv add prime-runs            # or: pip install prime-runs
+uv add 'prime-runs[train]'   # training runs: adds pyarrow for the sample table
+```
+
+## Eval runs
+
+```python
+import prime_runs as pr
+
+run = pr.init(
+    name="gsm8k-qwen3-8b",
+    environments=["gsm8k"],      # hub names (get-or-create) or owner/name slugs
+    model="Qwen/Qwen3-8B",
+    framework="verifiers",
+    config="eval.toml",          # the launched file, stored byte for byte
+)
+print(run.url)                   # https://app.primeintellect.ai/dashboard/evaluations/...
+
+for episode in rollouts:
+    run.log_episodes([episode])  # a queue put; bare traces: log_traces()
+
+run.finish(summary=pr.metrics.from_episodes(episodes))
+```
+
+`init()` is called before the first rollout. Every record the run uploads is
+keyed to it: the SDK sets `run.id` and `run.type` on the uploaded copy of each
+trace and episode, over whatever run id the producer recorded locally, and
+keeps the rest of that block (`name`, `work`). A producer never needs to know
+the platform's id; `run.url` is the handle. A `with run:` block finishes for
+you: an exception marks the run `failed`, Ctrl-C `cancelled`, and a process
+that exits without finishing is reported `crashed` by an atexit hook.
+
+`config=` takes the path to the launched file (kept verbatim under
+`config_source`, comments and all) or a mapping stored as given; put a file
+under `pr.CONFIG_SOURCE_KEY` in the mapping to send both. Nothing is redacted.
+
+## Training runs
+
+```python
+run = pr.init(
+    kind="train",
+    name="qwen3-8b-gsm8k-rl",
+    model="Qwen/Qwen3-8B",                    # the base model
+    environments=["primeintellect/gsm8k"],    # hub ids, passed through
+    training=pr.TrainingSpec(max_steps=1000, batch_size=64, rollouts_per_example=8),
+    config=train_config.model_dump(),
+    team_id="team_...",                       # external runs belong to a team
+)
+
+for step, (episodes, metrics) in enumerate(training_loop):
+    run.log_episodes(episodes)          # episodes carry run.work.step (TrainRunInfo)
+    run.log_metrics(metrics, step=step)
+
+run.finish()
+```
+
+- The platform enables external runs per team; a team outside the allowlist
+  gets a `ForbiddenError` from `init()`.
+- `init(kind="train", id=os.environ["RUN_ID"])` attaches to an external run a
+  launcher already created: nothing is registered, the platform keeps the run's
+  failure marking, and a clean `finish()` still completes it.
+- Metrics are one row per `log_metrics` call, on their own uploader. The sample
+  table gets one Parquet object per upload, every 10th step, keyed by the step
+  an episode was dispatched at; a step logged in several calls gets several
+  objects, and the viewer shows their union.
+- The status vocabulary is `completed | failed`; `cancelled` and `crashed`
+  arrive as `failed` with the reason in `error_message`.
+
+## How it behaves
+
+- **Streams.** Records go out on a background thread as they are logged;
+  whatever queues up during one request goes out as the next.
+- **Contains its errors.** With the default `on_error="warn"` nothing the
+  platform raises escapes into your loop; `on_error="raise"` surfaces the first
+  failure from `flush()` or `finish()`, for tests and CI. Platform errors are
+  the `prime_traces` exception family.
+- **Degrades.** A transient failure costs its batch, three in a row retire the
+  sink (a training run pauses it for five minutes instead), and a full queue
+  drops records rather than stalling the run. Losses are counted in
+  `run.dropped_records` and `run.failed_records`.
+- **Drains on exit.** `finish()` gives queued uploads up to `finish_timeout`
+  (300 s) before closing the run out; an abort path can pass
+  `finish(timeout=...)`.
+
+An online run writes to Prime Traces (the system of record, gated to an
+allowlist; outside it that sink turns itself off quietly) and to the sample
+table today's viewer reads. `log_*()` are queue puts, safe inside a coroutine;
+`init()` and `finish()` do network I/O.
+
+## Configuration
+
+| Source                 | Meaning                                                                |
+| ---------------------- | ---------------------------------------------------------------------- |
+| `PRIME_API_KEY`        | Platform API token                                                     |
+| `PRIME_TEAM_ID`        | Team context; required for training runs                               |
+| `PRIME_API_BASE_URL`   | Platform API; defaults to `https://api.primeintellect.ai`              |
+| `PRIME_FRONTEND_URL`   | Dashboard; defaults to `https://app.primeintellect.ai`                 |
+| `PRIME_TRACES_URL`     | Prime Traces service, resolved by `prime-traces`                       |
+| `PRIME_RUNS_MODE`      | `online` or `disabled`; unset means online when there is an API key    |
+| `~/.prime/config.json` | Shared prime CLI config (`api_key`, `team_id`, `base_url`)             |
+
+Precedence is `init()` argument → environment variable → config file. A missing
+API key disables the run with a warning.
+
+## Not yet available
+
+- **Failed or cancelled evaluations on the dashboard.** The evaluations API has
+  no producer-facing status endpoint yet, so the terminal state is recorded
+  under `metadata.prime_runs` and the run keeps showing as running.
+
+## Related packages
+
+- [prime-traces](https://github.com/PrimeIntellect-ai/prime/tree/main/packages/prime-traces) — Prime Traces SDK
+- [prime](https://github.com/PrimeIntellect-ai/prime/tree/main/packages/prime) — Prime CLI
+- [prime-evals](https://github.com/PrimeIntellect-ai/prime/tree/main/packages/prime-evals) — Evals SDK

packages/prime-runs/pyproject.toml

@@ -0,0 +1,79 @@
+[project]
+name = "prime-runs"
+# Version is single-sourced from src/prime_runs/__init__.py via Hatch
+dynamic = ["version"]
+description = "Prime Intellect Runs SDK - Track eval and training runs on the Prime platform"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+authors = [
+    { name = "Prime Intellect", email = "contact@primeintellect.ai" }
+]
+# Deliberately a leaf package. The `prime` CLI depends on `verifiers`, so
+# verifiers can never depend on `prime` — and verifiers is a first-class
+# consumer of this SDK. Nothing here may pull in prime, verifiers, typer,
+# rich or textual, directly or transitively.
+#
+# Kept to what is actually imported. This package models no response bodies
+# (so no direct `pydantic`; it arrives transitively through prime-traces) and
+# takes its error mapping and backoff from `prime_traces.core` rather than
+# `tenacity`. Nothing is added to what verifiers and prime-rl already carry.
+dependencies = [
+    "httpx>=0.25.0",
+    "prime-traces>=0.0.3",
+]
+keywords = ["evals", "evaluations", "training", "runs", "traces", "observability"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence"
+]
+
+[project.urls]
+Homepage = "https://github.com/PrimeIntellect-ai/prime"
+Documentation = "https://github.com/PrimeIntellect-ai/prime/tree/main/packages/prime-runs"
+Repository = "https://github.com/PrimeIntellect-ai/prime.git"
+
+[project.optional-dependencies]
+# The training sample table is Parquet. Optional: an eval producer never needs
+# it, and prime-rl already carries pyarrow.
+train = [
+    "pyarrow>=15.0.0",
+]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-asyncio>=0.23.0",
+    "ruff>=0.13.1",
+]
+
+[tool.uv.sources]
+prime-traces = { workspace = true }
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.version]
+path = "src/prime_runs/__init__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/prime_runs"]
+
+[tool.pytest.ini_options]
+addopts = "-v"
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+extend-select = ["E", "F", "I"]